From d758efe1d9add9f229064768ad448e8938ba49d4 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Wed, 15 Jul 2026 16:30:58 -0400 Subject: [PATCH 01/14] Store known span tags densely in TagMap by tag-id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Known tags (keyOf resolves to a stored id) are held in insertion-ordered parallel arrays (knownIds/knownValues) with NO per-tag Entry object — the allocation lever. Lazily allocated on the first known-tag write; custom tags stay in the hash buckets. Disjoint by construction (known-ness is global), so read-through shadow checks stay within-region and the bucket path is unchanged. - KnownTagCodec (id encoding + resolver) + hand-written KnownTags (keyOf substrate over StringIndex). Off-by-default: dormant until a resolver registers, so production is byte-identical. - CoreTracer flips it live behind `-Ddd.trace.dense.tags.enabled` for A/B. - Sizing is a generous fixed stopgap (KNOWN_INIT_CAP=12, the per-type max); exact per-type sizing comes with the tag registry. Reconciled onto the level-split stack (fold + read-through + StringIndex); built on the folded final-class TagMap. Co-Authored-By: Claude Opus 4.8 --- .../java/datadog/trace/core/CoreTracer.java | 9 + .../trace/api/DenseStoreAllocBenchmark.java | 148 +++++ .../java/datadog/trace/api/KnownTagCodec.java | 177 ++++++ .../java/datadog/trace/api/KnownTags.java | 328 +++++++++++ .../main/java/datadog/trace/api/TagMap.java | 528 +++++++++++++++--- .../java/datadog/trace/api/KnownTagsTest.java | 152 +++++ .../trace/api/TagMapDenseForkedTest.java | 273 +++++++++ .../trace/api/TagMapDenseFuzzForkedTest.java | 202 +++++++ 8 files changed, 1753 insertions(+), 64 deletions(-) create mode 100644 internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java create mode 100644 internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java create mode 100644 internal-api/src/main/java/datadog/trace/api/KnownTags.java create mode 100644 internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java create mode 100644 internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java create mode 100644 internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 120ba6e8df8..4e32410d76c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -40,6 +40,7 @@ import datadog.trace.api.EndpointTracker; import datadog.trace.api.IdGenerationStrategy; import datadog.trace.api.InstrumenterConfig; +import datadog.trace.api.KnownTags; import datadog.trace.api.Pair; import datadog.trace.api.TagMap; import datadog.trace.api.TraceConfig; @@ -656,6 +657,14 @@ private CoreTracer( // preload this enum to avoid triggering classloading on the hot path TraceCollector.PublishState.values(); + // Dense known-tag store (experimental, OFF by default): registering the KnownTagCodec resolver + // flips the dense store live so known tags store without a per-tag Entry. Gated by a system + // property for A/B benchmarking; when off, keyOf stays a no-op and tag storage is byte-identical + // to today. Promote to a Config flag if this becomes a permanent rollout. + if (Boolean.getBoolean("dd.trace.dense.tags.enabled")) { + KnownTags.init(); + } + if (reportInTracerFlare) { TracerFlare.addReporter(this); } diff --git a/internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java new file mode 100644 index 00000000000..6c11f8bc212 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java @@ -0,0 +1,148 @@ +package datadog.trace.api; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Deterministic allocation A/B for the dense known-tag store, using the REAL {@link KnownTags} + * resolver (a {@code StringIndex} probe + a constant-returning {@code switch} — allocation-free, + * exactly like production). An earlier synthetic prefix resolver allocated in {@code keyOf} + * (substring) and {@code nameOf} (concat), contaminating the dense arm; this measures the store, + * not the resolver. + * + *

Models how a real span's tags route: {@code today} = all custom (what ships now — every tag + * buckets, since nothing is registered as known), {@code dense} = the same tag count with a + * realistic fraction routed to the dense store (real known tag names) and the rest custom. Run with + * {@code -prof gc}; the {@code gc.alloc.rate.norm} (B/op) delta at the same {@code tagCount} is + * what enabling the dense store does to a real span's per-build allocation. + * + *

Results — buildMap, JDK 17 (Zulu 17.0.7, Apple Silicon), {@code -prof gc -f 1 -wi 2 -i 3}, + * 2026-07-08. Allocation is deterministic (±0.001 B/op); throughput on this run is NOT + * trustworthy (single fork, short) — read B/op only. + * + * 

{@code
+ * scenario    tagCount=7   tagCount=12
+ * today          408 B/op     704 B/op
+ * dense          376 B/op     416 B/op
+ * allKnown       176 B/op     400 B/op
+ * }
+ * + *

Gate met: {@code dense < today} at both counts (the over-provision artifact is gone). The + * Entry-less win scales with the known-tag fraction — ~8% at 7 tags (~70% known), ~41% at 12; + * {@code allKnown} (the codegen endgame / read-through parent shape) reaches ~57% at 7. + * + *

Serialize paths (same run, B/op). {@code buildAndSerialize} (alloc-free {@code forEach} + * flyweight) adds a flat +16 B/op over {@code buildMap} in every scenario (7: 392, 12: 432 dense). + * {@code buildAndSerializeViaIterator} — the {@code EntryReader} enhanced-for modeling the count + * pre-pass at {@code TraceMapperV0_4:95} — adds a CONSTANT per-call cost (+56 custom / +80 dense, + * identical at 7 and 12 tags): that flat-vs-tagCount signature is the {@code EntryReaderIterator} + * OBJECT, NOT per-tag Entry — the iterator reuses a dense flyweight (TagMap:2182/2652). So the + * dense win SURVIVES serialization; the only nit is {@code iterator()} allocating one Iterator per + * call, which {@code forEach} avoids and which can be recycled away. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 2, time = 2) +@Measurement(iterations = 3, time = 2) +@Fork(1) +@Threads(1) +public class DenseStoreAllocBenchmark { + + // Real stored (dense-routed) tag names — a realistic web/db span's known set. + static final String[] KNOWN = + new String[] { + DDTags.BASE_SERVICE, + Tags.VERSION, + Tags.COMPONENT, + Tags.SPAN_KIND, + Tags.HTTP_METHOD, + Tags.HTTP_ROUTE, + Tags.DB_TYPE, + Tags.DB_INSTANCE, + Tags.PEER_HOSTNAME, + Tags.DB_USER, + DDTags.LANGUAGE_TAG_KEY, + Tags.PEER_PORT, + }; + + // today = all custom (all bucket, what ships now); dense = ~70% known + custom (a real span); + // allKnown = 100% known (the trace-tier read-through parent's shape — exercises lazy buckets). + @Param({"today", "dense", "allKnown"}) + String scenario; + + @Param({"7", "12"}) + int tagCount; + + private String[] keys; + private String[] values; + + @Setup(Level.Trial) + public void setup() { + KnownTags.init(); // registers the real (allocation-free) resolver + int knownCount; + if ("allKnown".equals(scenario)) { + knownCount = tagCount; // 100% known (<= KNOWN.length) + } else if ("dense".equals(scenario)) { + knownCount = (tagCount * 7) / 10; // ~70% known + custom + } else { + knownCount = 0; // today: all custom (all bucket) + } + this.keys = new String[tagCount]; + this.values = new String[tagCount]; + for (int i = 0; i < tagCount; i++) { + this.keys[i] = i < knownCount ? KNOWN[i] : "custom.tag." + i; + this.values[i] = "value-" + i; + } + } + + @Benchmark + public TagMap buildMap() { + TagMap m = TagMap.create(16); + for (int i = 0; i < tagCount; i++) { + m.set(keys[i], values[i]); + } + return m; + } + + @Benchmark + public void buildAndSerialize(Blackhole bh) { + TagMap m = TagMap.create(16); + for (int i = 0; i < tagCount; i++) { + m.set(keys[i], values[i]); + } + // forEach: the alloc-free flyweight emit for dense + m.forEach(reader -> bh.consume(reader.objectValue())); + bh.consume(m); + } + + @Benchmark + public void buildAndSerializeViaIterator(Blackhole bh) { + TagMap m = TagMap.create(16); + for (int i = 0; i < tagCount; i++) { + m.set(keys[i], values[i]); + } + // models the REAL serializer's count pre-pass (TraceMapperV0_4:95). The EntryReader iterator + // uses a reused dense flyweight (NO per-tag Entry alloc — TagMap:2182/2652), so the dense win + // SURVIVES; the only extra cost vs forEach is the EntryReaderIterator object itself (a fixed + // per-call cost, constant across tagCount — not per-tag). forEach avoids even that. + for (TagMap.EntryReader reader : m) { + bh.consume(reader.objectValue()); + } + bh.consume(m); + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java new file mode 100644 index 00000000000..7ed9dff633a --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -0,0 +1,177 @@ +package datadog.trace.api; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * Registry for generated tag ID ↔ name resolution. The code generator populates this at tracer init + * via {@link #register(Resolver)}. Once registered, HotSpot CHA devirtualizes and inlines the + * resolver's switch, making {@link #nameOf}/{@link #keyOf} effectively zero-overhead. + */ +public final class KnownTagCodec { + // Plain (non-volatile) fast-path flag: false until a resolver is ever registered. A plain read is + // free and hoistable, unlike a volatile read of `resolver` (costly on weak memory models such as + // ARM). A stale `false` is benign — callers treat the tag as unknown and use the hash buckets, + // which is correct, just unoptimized; the next read after publication takes the slot path. + private static boolean active; + + private static volatile Resolver resolver; + + /** Fast-path gate: true once a resolver has been registered. */ + public static boolean isActive() { + return active; + } + + /* + * tagId bit layout: [63 intercepted] [62-48 globalSerial (15 bits)] [47-32 fieldPos] + * [31-0 nameHash]. Bit 63 (the sign bit) marks a tag the tag interceptor must see, so the check + * is a single {@code tagId < 0}. globalSerial is globally unique per known tag; fieldPos is its + * slot in the global positional layout (TagMap.knownEntries index); nameHash is + * TagMap.Entry#_hash(name) and is layout-independent. Unknown (string-only) tags have the upper + * 32 bits zero. NOTE: TagMap.Entry decodes nameHash inline as (int) tagId on its hot path, so the + * low-32 encoding here must stay in sync with that. + */ + public static int globalSerial(long tagId) { + return (int) ((tagId >>> 48) & 0x7FFF); + } + + /** + * Flag bit (the sign bit) marking a tag the tag interceptor must process — reserved/"virtual" + * tags AND intercepted-but-stored tags (e.g. http.method, which the interceptor side-effects and + * also stores). Encoded in the id so {@code DDSpanContext.setTag(long)} can route with a single + * sign test ({@link #isIntercepted}) instead of resolving the name. Non-intercepted tags (peer.*, + * base.service, …) leave it clear and take the fast store path. Must agree with the interceptor's + * name-based {@code needsIntercept} for every assigned id. + */ + public static final long INTERCEPTED = Long.MIN_VALUE; // 1L << 63 + + /** True if the tagId is flagged for tag-interceptor processing. */ + public static boolean isIntercepted(long tagId) { + return tagId < 0L; + } + + /** Returns the tagId with the {@link #INTERCEPTED} flag set. */ + public static long intercepted(long tagId) { + return tagId | INTERCEPTED; + } + + public static int fieldPos(long tagId) { + return (int) ((tagId >>> 32) & 0xFFFF); + } + + public static int nameHash(long tagId) { + return (int) tagId; + } + + /** + * globalSerial partition. {@code [1, FIRST_STORED_SERIAL)} is reserved for "virtual" tags that + * are specially handled (redirected to span fields or processed by the tag interceptor) and are + * NOT stored in the TagMap — these are hand-assigned in tracer core. {@code [FIRST_STORED_SERIAL, + * ..]} is for generated convention tags that ARE stored (slotted/bucketed). {@code globalSerial + * == 0} means unknown / string-only. Both core and the code generator must agree on this + * boundary. + */ + public static final int FIRST_STORED_SERIAL = 256; + + /** True if the tagId names a reserved "virtual"/specially-handled tag (not stored in the map). */ + public static boolean isReserved(long tagId) { + int globalSerial = globalSerial(tagId); + return globalSerial > 0 && globalSerial < FIRST_STORED_SERIAL; + } + + /** True if the tagId names a generated, map-stored (slotted/bucketed) tag. */ + public static boolean isStored(long tagId) { + return globalSerial(tagId) >= FIRST_STORED_SERIAL; + } + + /** + * Sentinel {@code fieldPos} meaning "no positional slot". It is the maximum value the 16-bit + * fieldPos field can hold, so it always compares {@code >= slotCount()} and routes to the hash + * buckets rather than the fast positional array. Two kinds of tagId use it: + * + *

+ */ + public static final int NO_SLOT = 0xFFFF; + + /** + * True if the tagId names a stored tag that deliberately has no positional slot (bucket-only). + */ + public static boolean isUnslotted(long tagId) { + return isStored(tagId) && fieldPos(tagId) == NO_SLOT; + } + + /** + * Builds a tagId from its parts: {@code globalSerial} (globally unique per known tag), {@code + * fieldPos} (the tag's slot within its span type's positional table), and the tag {@code name} + * (whose hash is computed via the same function the runtime uses, so the low 32 bits match {@link + * TagMap.Entry#hash()}). Inverse of {@link #globalSerial}/{@link #fieldPos}/{@link #nameHash}. + * Intended for the code generator and tests. + */ + public static long tagId(int globalSerial, int fieldPos, String name) { + long nameHash = TagMap.Entry._hash(name) & 0xFFFFFFFFL; + return ((long) globalSerial << 48) | ((long) (fieldPos & 0xFFFF) << 32) | nameHash; + } + + /** + * Builds a tagId with no positional slot ({@code fieldPos == }{@link #NO_SLOT}). Use for reserved + * "virtual" tags and for "low-priority" stored tags that get a stable id but are intentionally + * kept out of the fast slot array (they route to the hash buckets). See {@link #NO_SLOT}. + */ + public static long tagId(int globalSerial, String name) { + return tagId(globalSerial, NO_SLOT, name); + } + + // Number of positional slots in the global layout = (max stored fieldPos) + 1, declared by the + // registered provider. Captured once at registration and read as a dynamic constant; TagMap sizes + // its knownEntries array to exactly this rather than a hardcoded max. 0 when no resolver. + private static int slotCount; + + /** Slot count of the registered provider (max stored fieldPos + 1); 0 if none. */ + public static int slotCount() { + return slotCount; + } + + public interface Resolver { + String nameOf(long tagId); + + long keyOf(String name); + + /** Number of positional slots this provider uses: (max stored fieldPos) + 1. */ + int slotCount(); + } + + @SuppressFBWarnings( + value = "AT_STALE_THREAD_WRITE_OF_PRIMITIVE", + justification = + "active/slotCount are plain by design: written once at tracer-init registration (before" + + " any span processing) and read plain on the hot path. A stale read is benign — the" + + " tag is treated as unknown and takes the hash-bucket path — so plain reads are" + + " deliberately preferred over a costly volatile read on weak memory models.") + public static void register(Resolver resolver) { + KnownTagCodec.resolver = resolver; // volatile write publishes the resolver + KnownTagCodec.slotCount = (resolver != null) ? resolver.slotCount() : 0; + KnownTagCodec.active = + (resolver != null); // plain write; readers re-read resolver volatile anyway + } + + public static String nameOf(long tagId) { + if (!active) return null; + Resolver r = resolver; + return r != null ? r.nameOf(tagId) : null; + } + + public static long keyOf(String name) { + if (!active) return 0L; + Resolver r = resolver; + return r != null ? r.keyOf(name) : 0L; + } + + private KnownTagCodec() {} +} diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTags.java b/internal-api/src/main/java/datadog/trace/api/KnownTags.java new file mode 100644 index 00000000000..391b000b844 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/KnownTags.java @@ -0,0 +1,328 @@ +package datadog.trace.api; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.util.StringIndex; + +/** + * Hand-assigned tag-id constants for well-known tags, plus the {@link KnownTagCodec.Resolver} that + * resolves them. This is the single registry shared by the tracer core and by instrumentation + * (decorators) — it lives in {@code internal-api} so both layers can reference the ids; the + * eventual code generator will replace the hand assignment here. + * + *

Reserved serials {@code [1, KnownTagCodec.FIRST_STORED_SERIAL)} name "virtual" tags handled by + * the tag interceptor / span fields and are NOT stored in the {@code TagMap}; their {@code + * fieldPos} is the {@link KnownTagCodec#NO_SLOT} sentinel that is out of slot range, so any + * incidental store routes to the hash buckets rather than a positional slot. Serials {@code >= + * FIRST_STORED_SERIAL} name stored tags that slot/bucket normally (or, with {@code NO_SLOT}, are + * stored bucket-only). + * + *

The resolver registers on class initialization, so simply referencing any constant here makes + * tag-id resolution live before the first span is built. + * + *

Slice-1 note (keyOf substrate): the {@code fieldPos} assignments below (and {@link + * #SLOT_COUNT}) describe a single universal positional layout (slots 0..25). That layout is + * currently dormant — no dense store consumes {@code fieldPos} yet — and is provisional: the + * dense-store slice replaces the universal layout with per-role / per-type sizing (see the + * over-provision finding in {@code dense-tagmap-design.md}). {@code keyOf}/{@code nameOf} depend + * only on {@code globalSerial} + name, not {@code fieldPos}, so the ids themselves are stable + * across any layout scheme. + */ +public final class KnownTags { + // slot count = (max stored fieldPos) + 1. Stored tags use fieldPos 0..25. PROVISIONAL universal + // layout — see the slice-1 note above; the dense-store slice supersedes this with role/type + // sizing. + static final int SLOT_COUNT = 26; + + // ---- reserved / virtual (tag-interceptor handled, not stored) ---- + // Reserved tags are always intercepted -> set the INTERCEPTED flag. + public static final int ERROR_SERIAL = 1; + public static final long ERROR_ID = + KnownTagCodec.intercepted(KnownTagCodec.tagId(ERROR_SERIAL, Tags.ERROR)); + + // ---- stored (slotted / bucketed) ---- + public static final int PARENT_ID_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL; + public static final long PARENT_ID = KnownTagCodec.tagId(PARENT_ID_SERIAL, 0, DDTags.PARENT_ID); + + // common (process-constant) tags added by InternalTagsAdder to ~every span + public static final int BASE_SERVICE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 1; + public static final long BASE_SERVICE_ID = + KnownTagCodec.tagId(BASE_SERVICE_SERIAL, 1, DDTags.BASE_SERVICE); + + public static final int VERSION_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 2; + public static final long VERSION_ID = KnownTagCodec.tagId(VERSION_SERIAL, 2, Tags.VERSION); + + // build-time-known constant tags merged into defaultSpanTags (see CoreTracer.withTracerTags). + // "env" is a base-mixin tag; the *_ENABLED flags are product-mixin tags. Hand-assigned for now. + public static final String ENV = "env"; + public static final int ENV_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 3; + public static final long ENV_ID = KnownTagCodec.tagId(ENV_SERIAL, 3, ENV); + + public static final int DJM_ENABLED_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 4; + public static final long DJM_ENABLED_ID = + KnownTagCodec.tagId(DJM_ENABLED_SERIAL, 4, DDTags.DJM_ENABLED); + + public static final int DSM_ENABLED_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 5; + public static final long DSM_ENABLED_ID = + KnownTagCodec.tagId(DSM_ENABLED_SERIAL, 5, DDTags.DSM_ENABLED); + + // common tags added by the tag post-processors (RemoteHostnameAdder / IntegrationAdder / + // ServiceNameSourceAdder). Not intercepted; stored. + public static final int TRACER_HOST_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 6; + public static final long TRACER_HOST_ID = + KnownTagCodec.tagId(TRACER_HOST_SERIAL, 6, DDTags.TRACER_HOST); + + public static final int INTEGRATION_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 7; + public static final long INTEGRATION_ID = + KnownTagCodec.tagId(INTEGRATION_SERIAL, 7, DDTags.DD_INTEGRATION); + + public static final int SVC_SRC_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 8; + public static final long SVC_SRC_ID = KnownTagCodec.tagId(SVC_SRC_SERIAL, 8, DDTags.DD_SVC_SRC); + + // peer.service tags, read/written by PeerServiceCalculator (post-processor; uses Map put/get that + // bypass the interceptor). peer.service is intercepted on the set-path but STORED, so it slots. + public static final int PEER_SERVICE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 9; + public static final long PEER_SERVICE_ID = + KnownTagCodec.intercepted(KnownTagCodec.tagId(PEER_SERVICE_SERIAL, 9, Tags.PEER_SERVICE)); + + public static final int PEER_SERVICE_REMAPPED_FROM_SERIAL = + KnownTagCodec.FIRST_STORED_SERIAL + 10; + public static final long PEER_SERVICE_REMAPPED_FROM_ID = + KnownTagCodec.tagId(PEER_SERVICE_REMAPPED_FROM_SERIAL, 10, DDTags.PEER_SERVICE_REMAPPED_FROM); + + // HTTP tags read by HttpEndpointPostProcessor. http.method/http.url are intercepted-but-stored + // (interceptTag side-effects then returns false → stored); http.route is not intercepted. All + // stored, so the string set-path slots them via keyOf and the id reads here find them. + public static final int HTTP_METHOD_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 11; + public static final long HTTP_METHOD_ID = + KnownTagCodec.intercepted(KnownTagCodec.tagId(HTTP_METHOD_SERIAL, 11, Tags.HTTP_METHOD)); + + public static final int HTTP_ROUTE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 12; + public static final long HTTP_ROUTE_ID = + KnownTagCodec.tagId(HTTP_ROUTE_SERIAL, 12, Tags.HTTP_ROUTE); + + public static final int HTTP_URL_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 13; + public static final long HTTP_URL_ID = + KnownTagCodec.intercepted(KnownTagCodec.tagId(HTTP_URL_SERIAL, 13, Tags.HTTP_URL)); + + // peer connection tags set by BaseDecorator.onPeerConnection on ~every client/producer span. + // Not intercepted; stored. Slotted (common across client instrumentations). + public static final int PEER_HOSTNAME_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 14; + public static final long PEER_HOSTNAME_ID = + KnownTagCodec.tagId(PEER_HOSTNAME_SERIAL, 14, Tags.PEER_HOSTNAME); + + public static final int PEER_HOST_IPV4_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 15; + public static final long PEER_HOST_IPV4_ID = + KnownTagCodec.tagId(PEER_HOST_IPV4_SERIAL, 15, Tags.PEER_HOST_IPV4); + + public static final int PEER_HOST_IPV6_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 16; + public static final long PEER_HOST_IPV6_ID = + KnownTagCodec.tagId(PEER_HOST_IPV6_SERIAL, 16, Tags.PEER_HOST_IPV6); + + public static final int PEER_PORT_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 17; + public static final long PEER_PORT_ID = KnownTagCodec.tagId(PEER_PORT_SERIAL, 17, Tags.PEER_PORT); + + // Universal decorator tags — set on ~every span (component/span.kind via Base/Server/Client + // decorators, language via ServerDecorator). span.kind is intercepted (setSpanKindOrdinal). + public static final int COMPONENT_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 18; + public static final long COMPONENT_ID = KnownTagCodec.tagId(COMPONENT_SERIAL, 18, Tags.COMPONENT); + + public static final int SPAN_KIND_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 19; + public static final long SPAN_KIND_ID = + KnownTagCodec.intercepted(KnownTagCodec.tagId(SPAN_KIND_SERIAL, 19, Tags.SPAN_KIND)); + + public static final int LANGUAGE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 20; + public static final long LANGUAGE_ID = + KnownTagCodec.tagId(LANGUAGE_SERIAL, 20, DDTags.LANGUAGE_TAG_KEY); + + // JDBC / database-client tags — set on every db span (58% of petclinic spans). Not intercepted + // (only db.statement is, and that's handled separately). + public static final int DB_TYPE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 21; + public static final long DB_TYPE_ID = KnownTagCodec.tagId(DB_TYPE_SERIAL, 21, Tags.DB_TYPE); + + public static final int DB_INSTANCE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 22; + public static final long DB_INSTANCE_ID = + KnownTagCodec.tagId(DB_INSTANCE_SERIAL, 22, Tags.DB_INSTANCE); + + public static final int DB_USER_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 23; + public static final long DB_USER_ID = KnownTagCodec.tagId(DB_USER_SERIAL, 23, Tags.DB_USER); + + public static final int DB_OPERATION_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 24; + public static final long DB_OPERATION_ID = + KnownTagCodec.tagId(DB_OPERATION_SERIAL, 24, Tags.DB_OPERATION); + + public static final int DB_POOL_NAME_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 25; + public static final long DB_POOL_NAME_ID = + KnownTagCodec.tagId(DB_POOL_NAME_SERIAL, 25, Tags.DB_POOL_NAME); + + // Open-addressed name -> id table backing keyOf (data, not a switch): scales flat as the known + // set grows, where a generated switch eventually falls off the inline threshold. KEYOF_NAMES and + // KEYOF_VALUES are parallel; the table places names by hash and a parallel ids[] by slot. + private static final String[] KEYOF_NAMES = { + Tags.ERROR, + DDTags.PARENT_ID, + DDTags.BASE_SERVICE, + Tags.VERSION, + ENV, + DDTags.DJM_ENABLED, + DDTags.DSM_ENABLED, + DDTags.TRACER_HOST, + DDTags.DD_INTEGRATION, + DDTags.DD_SVC_SRC, + Tags.PEER_SERVICE, + DDTags.PEER_SERVICE_REMAPPED_FROM, + Tags.HTTP_METHOD, + Tags.HTTP_ROUTE, + Tags.HTTP_URL, + Tags.PEER_HOSTNAME, + Tags.PEER_HOST_IPV4, + Tags.PEER_HOST_IPV6, + Tags.PEER_PORT, + Tags.COMPONENT, + Tags.SPAN_KIND, + DDTags.LANGUAGE_TAG_KEY, + Tags.DB_TYPE, + Tags.DB_INSTANCE, + Tags.DB_USER, + Tags.DB_OPERATION, + Tags.DB_POOL_NAME, + }; + + private static final long[] KEYOF_VALUES = { + ERROR_ID, + PARENT_ID, + BASE_SERVICE_ID, + VERSION_ID, + ENV_ID, + DJM_ENABLED_ID, + DSM_ENABLED_ID, + TRACER_HOST_ID, + INTEGRATION_ID, + SVC_SRC_ID, + PEER_SERVICE_ID, + PEER_SERVICE_REMAPPED_FROM_ID, + HTTP_METHOD_ID, + HTTP_ROUTE_ID, + HTTP_URL_ID, + PEER_HOSTNAME_ID, + PEER_HOST_IPV4_ID, + PEER_HOST_IPV6_ID, + PEER_PORT_ID, + COMPONENT_ID, + SPAN_KIND_ID, + LANGUAGE_ID, + DB_TYPE_ID, + DB_INSTANCE_ID, + DB_USER_ID, + DB_OPERATION_ID, + DB_POOL_NAME_ID, + }; + + // Static-final raw arrays placed by StringIndex.EmbeddingSupport: the JIT folds these refs to + // constants on + // the keyOf hot path (the fastest of StringIndex's three usage modes — no instance dereference). + private static final int[] KEYOF_HASHES; + private static final String[] KEYOF_KEYS; + private static final long[] KEYOF_IDS; + + static { + StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); + long[] ids = new long[data.names.length]; + for (int j = 0; j < KEYOF_NAMES.length; j++) { + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = + KEYOF_VALUES[j]; + } + KEYOF_HASHES = data.hashes; + KEYOF_KEYS = data.names; + KEYOF_IDS = ids; + } + + static final KnownTagCodec.Resolver RESOLVER = + new KnownTagCodec.Resolver() { + @Override + public String nameOf(long tagId) { + switch (KnownTagCodec.globalSerial(tagId)) { + case ERROR_SERIAL: + return Tags.ERROR; + case PARENT_ID_SERIAL: + return DDTags.PARENT_ID; + case BASE_SERVICE_SERIAL: + return DDTags.BASE_SERVICE; + case VERSION_SERIAL: + return Tags.VERSION; + case ENV_SERIAL: + return ENV; + case DJM_ENABLED_SERIAL: + return DDTags.DJM_ENABLED; + case DSM_ENABLED_SERIAL: + return DDTags.DSM_ENABLED; + case TRACER_HOST_SERIAL: + return DDTags.TRACER_HOST; + case INTEGRATION_SERIAL: + return DDTags.DD_INTEGRATION; + case SVC_SRC_SERIAL: + return DDTags.DD_SVC_SRC; + case PEER_SERVICE_SERIAL: + return Tags.PEER_SERVICE; + case PEER_SERVICE_REMAPPED_FROM_SERIAL: + return DDTags.PEER_SERVICE_REMAPPED_FROM; + case HTTP_METHOD_SERIAL: + return Tags.HTTP_METHOD; + case HTTP_ROUTE_SERIAL: + return Tags.HTTP_ROUTE; + case HTTP_URL_SERIAL: + return Tags.HTTP_URL; + case PEER_HOSTNAME_SERIAL: + return Tags.PEER_HOSTNAME; + case PEER_HOST_IPV4_SERIAL: + return Tags.PEER_HOST_IPV4; + case PEER_HOST_IPV6_SERIAL: + return Tags.PEER_HOST_IPV6; + case PEER_PORT_SERIAL: + return Tags.PEER_PORT; + case COMPONENT_SERIAL: + return Tags.COMPONENT; + case SPAN_KIND_SERIAL: + return Tags.SPAN_KIND; + case LANGUAGE_SERIAL: + return DDTags.LANGUAGE_TAG_KEY; + case DB_TYPE_SERIAL: + return Tags.DB_TYPE; + case DB_INSTANCE_SERIAL: + return Tags.DB_INSTANCE; + case DB_USER_SERIAL: + return Tags.DB_USER; + case DB_OPERATION_SERIAL: + return Tags.DB_OPERATION; + case DB_POOL_NAME_SERIAL: + return Tags.DB_POOL_NAME; + default: + return null; + } + } + + @Override + public int slotCount() { + return SLOT_COUNT; + } + + @Override + public long keyOf(String name) { + int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name); + return slot < 0 ? 0L : KEYOF_IDS[slot]; + } + }; + + static { + KnownTagCodec.register(RESOLVER); + } + + /** + * Forces resolver registration. Merely invoking this static method runs {@code } (which + * registers {@link #RESOLVER}), so calling it once at tracer init flips the dense store live; + * idempotent. Until something references this class the registry stays dormant and {@code keyOf} + * returns 0, so tag storage is byte-identical to the bucket-only behavior. + */ + public static void init() {} + + private KnownTags() {} +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index e0a2f2b6f70..7c30f373881 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -47,10 +47,9 @@ */ public final class TagMap implements Map, Iterable { /** Immutable empty TagMap - similar to {@link Collections#emptyMap()} */ - // Frozen view over a length-1 array: bucket masking needs a power-of-two array length (size 0 - // would fail with ArrayIndexOutOfBoundsException, size 1 works), and the private constructor - // reads no statics, so this is safe to build directly during TagMap's . - public static final TagMap EMPTY = new TagMap(new Object[1], 0); + // Frozen view over a power-of-two array; the private constructor reads no statics, so this is + // safe to build directly during TagMap's . + public static final TagMap EMPTY = new TagMap(new Object[1 << 4], 0); /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { @@ -1018,10 +1017,44 @@ public EntryChange next() { * removed from the collision chain. */ - private final Object[] buckets; + // Shared immutable empty buckets (all null, length 16). Every map points here until its first + // custom-tag write copies-on-write to a private array (materializeBuckets), so an all-known / + // known-heavy map (e.g. the trace-tier read-through parent) allocates ZERO buckets. Length is + // always 16, so reads need no null guard and read-through bucket alignment (hash & 15) holds. + private static final Object[] EMPTY_BUCKETS = new Object[1 << 4]; + + private Object[] buckets; private int size; private boolean frozen; + /** + * Dense known-tag store (dense-tagmap-design §5). Values for KNOWN tags (those {@link + * KnownTagCodec#keyOf} resolves to a stored id) live in these INSERTION-ORDERED parallel arrays + * with NO per-tag {@link Entry} object — the allocation win. Lazily allocated on the first + * known-tag write ({@code null} until then, so all-unknown maps pay nothing) and grown x2 from + * {@link #KNOWN_INIT_CAP}. Matched by globalSerial via a linear scan ({@link #knownIndexOf}); + * reads aren't hot, so O(knownCount) is fine and positional indexing is deferred. Dormant until a + * resolver is registered: {@code keyOf} returns 0, so nothing routes here and production is + * byte-identical. + * + *

Disjoint from {@link #buckets} by construction: known-ness is global ({@code keyOf} is + * deterministic), so a known tag is ALWAYS dense and never bucketed, and vice-versa. That + * disjointness keeps read-through shadow checks within-region — an ancestor dense entry can only + * be shadowed by a nearer level's dense entry of the same id, an ancestor bucket entry only by a + * nearer level's bucket entry — so the bucket read-through chain walk is unchanged and the dense + * one mirrors it ({@link #parentDenseVisible}). + * + *

{@link #size} counts bucket entries only; {@link #knownCount} counts dense entries; the + * local total is {@code size + knownCount}. + */ + private long[] knownIds; + + private Object[] knownValues; + private int knownCount; + + private static final int KNOWN_INIT_CAP = + 12; // generous per-type max stopgap; exact per-type sizing comes with the tag registry + /** * Optional frozen parent for read-through. When non-null, reads that miss the local buckets fall * through to the parent chain, nearest-level-wins (a local entry shadows the parent's, a nearer @@ -1053,8 +1086,9 @@ public TagMap() { * optimizations can treat it as fixed. */ private TagMap(TagMap parent) { - // needs to be a power of 2 for bucket masking calculation to work as intended - this.buckets = new Object[1 << 4]; + // Start on the shared empty buckets; materializeBuckets() COWs to a private power-of-two array + // on the first custom-tag write. All-known maps never allocate buckets. + this.buckets = EMPTY_BUCKETS; this.size = 0; this.frozen = false; this.parent = parent; @@ -1075,8 +1109,9 @@ public boolean isOptimized() { @Override public int size() { // Exact (Map contract). Under read-through resolves the union; prefer estimateSize() for hints. + int local = this.size + this.knownCount; // buckets + dense TagMap parent = this.parent; - return parent == null ? this.size : this.size + this.visibleParentCount(); + return parent == null ? local : local + this.visibleParentCount(); } /** @@ -1086,6 +1121,12 @@ public int size() { private int visibleParentCount() { int count = 0; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + // dense entries at this ancestor not shadowed/tombstoned by a nearer level + long[] ancestorIds = ancestor.knownIds; + int ancestorKnownCount = ancestor.knownCount; + for (int i = 0; i < ancestorKnownCount; ++i) { + if (this.parentDenseVisible(ancestorIds[i], ancestor)) count++; + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1109,7 +1150,7 @@ private int visibleParentCount() { @Override public boolean isEmpty() { // Exact (Map contract). Under read-through resolves the parent; prefer isDefinitelyEmpty(). - if (this.size != 0) { + if (this.size != 0 || this.knownCount != 0) { return false; } TagMap parent = this.parent; @@ -1128,7 +1169,7 @@ public boolean isEmpty() { public boolean isDefinitelyEmpty() { // Cheap: empty iff no level in the chain holds a local entry (ignores shadowing/tombstones). for (TagMap level = this; level != null; level = level.parent) { - if (level.size != 0) { + if (level.size != 0 || level.knownCount != 0) { return false; } } @@ -1136,10 +1177,10 @@ public boolean isDefinitelyEmpty() { } public int estimateSize() { - // Upper bound: sum of every level's local size, ignoring read-through shadowing/removals. + // Upper bound: sum of every level's local size (buckets + dense), ignoring shadowing/removals. int total = 0; for (TagMap level = this; level != null; level = level.parent) { - total += level.size; + total += level.size + level.knownCount; } return total; } @@ -1267,8 +1308,15 @@ public Entry getEntry(String tag) { return parent.getEntry(tag); } - /** Looks up an entry in this map's own buckets only — no read-through to the parent. */ + /** Looks up an entry in this map's own storage only (dense then buckets) — no read-through. */ private Entry getLocalEntry(String tag) { + // Known tags live in the dense store; resolve identity and check there first. keyOf is a no-op + // (returns 0 -> isStored false) until a resolver is registered, so this is inert in production. + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + Object known = this.knownRawValue(id); + return known == null ? null : Entry.newAnyEntry(tag, known); + } Object[] thisBuckets = this.buckets; int hash = TagMap.Entry._hash(tag); return findInBucket(thisBuckets[hash & (thisBuckets.length - 1)], hash, tag); @@ -1316,10 +1364,102 @@ private boolean parentEntryVisible(Entry parentEntry, TagMap fromAncestor) { return true; } + // ---- dense known-tag store (see the knownIds field doc) + // ---------------------------------------- + + /** + * Linear scan of the dense store for {@code tagId}, returning its index or -1. Ids are canonical + * (the only way one enters is {@link KnownTagCodec#keyOf} or a {@code KnownTags} constant, both + * canonical), so a full {@code long} compare is exact and cheaper than extracting globalSerial. + */ + private int knownIndexOf(long tagId) { + long[] ids = this.knownIds; + int n = this.knownCount; + for (int i = 0; i < n; ++i) { + if (ids[i] == tagId) return i; + } + return -1; + } + + private void ensureKnownCapacity() { + if (this.knownIds == null) { + this.knownIds = new long[KNOWN_INIT_CAP]; + this.knownValues = new Object[KNOWN_INIT_CAP]; + } else if (this.knownCount == this.knownIds.length) { + int newCap = this.knownIds.length << 1; + this.knownIds = Arrays.copyOf(this.knownIds, newCap); + this.knownValues = Arrays.copyOf(this.knownValues, newCap); + } + } + + /** + * Stores a known tag's value densely (no {@link Entry} alloc). Overwrites in place when present + * (returning the prior value materialized as an Entry, per the {@code Map} contract — usually + * discarded by {@code set}); otherwise appends, growing x2 as needed. + */ + private Entry putKnownValue(long tagId, Object value) { + int i = this.knownIndexOf(tagId); + if (i >= 0) { + Object prior = this.knownValues[i]; + this.knownValues[i] = value; + return materializeKnown(tagId, prior); + } + this.ensureKnownCapacity(); + int slot = this.knownCount++; + this.knownIds[slot] = tagId; + this.knownValues[slot] = value; + return null; + } + + /** Raw dense value for {@code tagId}, or {@code null} when absent (no Entry, no boxing). */ + private Object knownRawValue(long tagId) { + int i = this.knownIndexOf(tagId); + return i < 0 ? null : this.knownValues[i]; + } + + /** + * Removes a known tag from the dense store (swap-with-last), returning the prior Entry or null. + */ + private Entry removeKnown(long tagId) { + int i = this.knownIndexOf(tagId); + if (i < 0) return null; + Object prior = this.knownValues[i]; + int last = --this.knownCount; + this.knownIds[i] = this.knownIds[last]; + this.knownValues[i] = this.knownValues[last]; + this.knownIds[last] = 0L; + this.knownValues[last] = null; + return materializeKnown(tagId, prior); + } + + /** Materializes a transient Entry for a dense (id, value) pair — only on explicit get/iterate. */ + private static Entry materializeKnown(long tagId, Object value) { + return Entry.newAnyEntry(KnownTagCodec.nameOf(tagId), value); + } + + /** + * Whether an ancestor dense entry ({@code tagId}, declared at level {@code fromAncestor}) is + * visible from this leaf under read-through: not shadowed by a nearer level's dense entry of the + * same id and not tombstoned by a nearer level. Chain-aware mirror of {@link #parentEntryVisible} + * for the dense store. (Disjointness: a known tag never buckets, so no bucket shadow check is + * needed.) + */ + private boolean parentDenseVisible(long tagId, TagMap fromAncestor) { + String tag = null; // resolved lazily, only if a nearer level carries tombstones + for (TagMap nearer = this; nearer != fromAncestor; nearer = nearer.parent) { + if (nearer.knownIndexOf(tagId) >= 0) return false; // shadowed by a nearer dense entry + if (nearer.removedFromParent != null) { + if (tag == null) tag = KnownTagCodec.nameOf(tagId); + if (nearer.removedFromParent.contains(tag)) return false; // tombstoned by a nearer level + } + } + return true; + } + @Deprecated @Override public Object put(@Nonnull String tag, Object value) { - TagMap.Entry entry = this.getAndSet(Entry.newAnyEntry(tag, value)); + TagMap.Entry entry = this.getAndSet(tag, value); return entry == null ? null : entry.objectValue(); } @@ -1334,32 +1474,70 @@ public void set(@Nullable TagMap.EntryReader newEntryReader) { } } + // The set(String, ...) family resolves keyOf FIRST: a known tag stores its value densely with no + // Entry (boxing the primitive only on that branch) and no parent-fallback lookup (set discards + // the prior value); a custom tag takes the typed bucket insert (no boxing for primitives). public void set(@Nonnull String tag, @Nonnull Object value) { - this.putEntry(Entry.newAnyEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, value); + } else { + this.putBucketEntry(Entry.newAnyEntry(tag, value)); + } } public void set(@Nonnull String tag, @Nonnull CharSequence value) { - this.putEntry(Entry.newObjectEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, value); + } else { + this.putBucketEntry(Entry.newObjectEntry(tag, value)); + } } public void set(@Nonnull String tag, boolean value) { - this.putEntry(Entry.newBooleanEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Boolean.valueOf(value)); + } else { + this.putBucketEntry(Entry.newBooleanEntry(tag, value)); + } } public void set(@Nonnull String tag, int value) { - this.putEntry(Entry.newIntEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Integer.valueOf(value)); + } else { + this.putBucketEntry(Entry.newIntEntry(tag, value)); + } } public void set(@Nonnull String tag, long value) { - this.putEntry(Entry.newLongEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Long.valueOf(value)); + } else { + this.putBucketEntry(Entry.newLongEntry(tag, value)); + } } public void set(@Nonnull String tag, float value) { - this.putEntry(Entry.newFloatEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Float.valueOf(value)); + } else { + this.putBucketEntry(Entry.newFloatEntry(tag, value)); + } } public void set(@Nonnull String tag, double value) { - this.putEntry(Entry.newDoubleEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Double.valueOf(value)); + } else { + this.putBucketEntry(Entry.newDoubleEntry(tag, value)); + } } /** @@ -1372,6 +1550,17 @@ public Entry getAndSet(@Nullable Entry newEntry) { if (newEntry == null) { return null; } + return this.getAndSetWithFallback(newEntry); + } + + /** + * Local insert (via {@link #putEntry}) plus the read-through parent fallback for the prior + * visible value (Map contract). When no local entry was replaced and the key was not tombstoned, + * the prior visible value is the nearest ancestor's, resolved through {@link #getEntry} (which + * handles both dense known tags and bucketed custom tags). Shared by {@link #getAndSet(Entry)} + * and the {@code getAndSet(String, ...)} overloads. + */ + private Entry getAndSetWithFallback(@Nonnull Entry newEntry) { // Capture whether the key was tombstoned BEFORE putEntry clears it: a tombstoned key had no // visible prior value (it was removed), so getAndSet must report null rather than the parent's. boolean wasTombstoned = @@ -1392,10 +1581,44 @@ public Entry getAndSet(@Nullable Entry newEntry) { /** * Inserts or replaces a local entry, returning the replaced local Entry (or null if none). Does * NOT consult the read-through parent -- the {@code set(...)} methods use this so they never pay - * for a prior-value lookup they discard; {@link #getAndSet(Entry)} layers the parent fallback on - * top. + * for a prior-value lookup they discard; {@link #getAndSetWithFallback} layers the parent + * fallback on top. Routes a known tag to the dense store, a custom tag to the hash buckets. */ private Entry putEntry(@Nonnull Entry newEntry) { + long id = KnownTagCodec.keyOf(newEntry.tag); + if (KnownTagCodec.isStored(id)) { + return this.putKnownLocal(id, newEntry.tag, newEntry.objectValue()); + } + return this.putBucketEntry(newEntry); + } + + /** + * Stores a known tag's value densely with NO Entry retained (the alloc win) and NO parent + * fallback — the local-only counterpart used by {@code set} and {@link #putEntry}. Returns the + * prior LOCAL dense value materialized as an Entry (Map contract); usually discarded. + */ + private Entry putKnownLocal(long id, String tag, Object value) { + this.checkWriteAccess(); + if (this.removedFromParent != null) { + this.removedFromParent.remove(tag); + } + return this.putKnownValue(id, value); + } + + /** Copy-on-write the shared empty buckets to a private array on the first bucket write. */ + private Object[] materializeBuckets() { + Object[] b = this.buckets; + if (b == EMPTY_BUCKETS) { + b = new Object[1 << 4]; + this.buckets = b; + } + return b; + } + + /** + * Stores an entry in the hash buckets — the unknown/custom-tag local path (no parent fallback). + */ + private Entry putBucketEntry(@Nonnull Entry newEntry) { this.checkWriteAccess(); // Re-setting a key clears any read-through tombstone for it (the new value overrides the @@ -1404,7 +1627,7 @@ private Entry putEntry(@Nonnull Entry newEntry) { this.removedFromParent.remove(newEntry.tag); } - Object[] thisBuckets = this.buckets; + Object[] thisBuckets = this.materializeBuckets(); int newHash = newEntry.hash(); int bucketIndex = newHash & (thisBuckets.length - 1); @@ -1449,32 +1672,36 @@ private Entry putEntry(@Nonnull Entry newEntry) { return null; } + // Each getAndSet(String, ...) builds the typed Entry (no boxing for primitives) then funnels + // through getAndSetWithFallback, which routes a known tag to the dense store (dropping the Entry) + // and a custom tag to the buckets, layering the read-through parent fallback on top. The Entry- + // free hot path is set(String, ...), which discards the prior value; getAndSet returns it. public Entry getAndSet(@Nonnull String tag, Object value) { - return this.getAndSet(Entry.newAnyEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newAnyEntry(tag, value)); } public Entry getAndSet(@Nonnull String tag, CharSequence value) { - return this.getAndSet(Entry.newObjectEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newObjectEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, boolean value) { - return this.getAndSet(Entry.newBooleanEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newBooleanEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, int value) { - return this.getAndSet(Entry.newIntEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newIntEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, long value) { - return this.getAndSet(Entry.newLongEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newLongEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, float value) { - return this.getAndSet(Entry.newFloatEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newFloatEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, double value) { - return this.getAndSet(Entry.newDoubleEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newDoubleEntry(tag, value)); } public void putAll(Map map) { @@ -1515,7 +1742,9 @@ private void putAllOptimizedMap(TagMap that) { that.forEach(this, (self, entry) -> self.set(entry)); return; } - if (this.size == 0) { + // "empty" must consider BOTH local regions — a map with only dense entries has size == 0 but is + // not empty, and putAllIntoEmptyMap would clobber its dense store. + if (this.size == 0 && this.knownCount == 0) { this.putAllIntoEmptyMap(that); } else { this.putAllMerge(that); @@ -1523,7 +1752,9 @@ private void putAllOptimizedMap(TagMap that) { } private void putAllMerge(TagMap that) { - Object[] thisBuckets = this.buckets; + // COW our buckets only if the source has bucket entries to merge in; otherwise the loop below + // writes nothing and the shared empty buckets stay shared. + Object[] thisBuckets = (that.size > 0) ? this.materializeBuckets() : this.buckets; Object[] thatBuckets = that.buckets; // Since TagMap-s don't support expansion, buckets are perfectly aligned @@ -1634,33 +1865,49 @@ private void putAllMerge(TagMap that) { } } } + + // merge the source's dense known-tag entries; incoming clobbers existing (same as buckets) + for (int i = 0; i < that.knownCount; ++i) { + this.putKnownValue(that.knownIds[i], that.knownValues[i]); + } } /* * Specially optimized version of putAll for the common case of destination map being empty */ private void putAllIntoEmptyMap(TagMap that) { - Object[] thisBuckets = this.buckets; - Object[] thatBuckets = that.buckets; - - // Check against both thisBuckets.length && thatBuckets.length is to help the JIT do bound check - // elimination - for (int i = 0; i < thisBuckets.length && i < thatBuckets.length; ++i) { - Object thatBucket = thatBuckets[i]; - - // faster to explicitly null check first, then do instanceof - if (thatBucket == null) { - // do nothing - } else if (thatBucket instanceof BucketGroup) { - // if it is a BucketGroup, then need to clone - BucketGroup thatGroup = (BucketGroup) thatBucket; + // Only copy buckets (and COW ours) when the source actually has bucket entries; an all-known + // source leaves us on the shared empty buckets. + if (that.size > 0) { + Object[] thisBuckets = this.materializeBuckets(); + Object[] thatBuckets = that.buckets; + + // Check against both thisBuckets.length && thatBuckets.length is to help the JIT do bound + // check elimination + for (int i = 0; i < thisBuckets.length && i < thatBuckets.length; ++i) { + Object thatBucket = thatBuckets[i]; + + // faster to explicitly null check first, then do instanceof + if (thatBucket == null) { + // do nothing + } else if (thatBucket instanceof BucketGroup) { + // if it is a BucketGroup, then need to clone + BucketGroup thatGroup = (BucketGroup) thatBucket; - thisBuckets[i] = thatGroup.cloneChain(); - } else { // if ( thatBucket instanceof Entry ) - thisBuckets[i] = thatBucket; + thisBuckets[i] = thatGroup.cloneChain(); + } else { // if ( thatBucket instanceof Entry ) + thisBuckets[i] = thatBucket; + } } + this.size = that.size; + } + + // clone the dense known-tag store (values are immutable boxes/objects -> safe to share refs) + if (that.knownCount > 0) { + this.knownIds = Arrays.copyOf(that.knownIds, that.knownIds.length); + this.knownValues = Arrays.copyOf(that.knownValues, that.knownValues.length); + this.knownCount = that.knownCount; } - this.size = that.size; } public void fillMap(Map map) { @@ -1679,6 +1926,9 @@ public void fillMap(Map map) { thisGroup.fillMapFromChain(map); } } + for (int i = 0; i < this.knownCount; ++i) { + map.put(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + } } public void fillStringMap(Map stringMap) { @@ -1697,6 +1947,11 @@ public void fillStringMap(Map stringMap) { thisGroup.fillStringMapFromChain(stringMap); } } + for (int i = 0; i < this.knownCount; ++i) { + stringMap.put( + KnownTagCodec.nameOf(this.knownIds[i]), + TagValueConversions.toString(this.knownValues[i])); + } } @Override @@ -1738,8 +1993,13 @@ public Entry getAndRemove(String tag) { return localRemoved; } - /** Removes an entry from this map's own buckets only — no parent/tombstone handling. */ + /** Removes an entry from this map's own storage only — no parent/tombstone handling. */ private Entry removeLocal(String tag) { + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + return this.removeKnown(id); + } + Object[] thisBuckets = this.buckets; int hash = TagMap.Entry._hash(tag); @@ -1807,6 +2067,15 @@ public Stream stream() { @Override public void forEach(Consumer consumer) { + // local dense known tags via a reused flyweight (no per-entry Entry alloc — the serialize win) + if (this.knownCount > 0) { + EntryReadingHelper reader = new EntryReadingHelper(); + for (int i = 0; i < this.knownCount; ++i) { + reader.set(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + consumer.accept(reader); + } + } + Object[] thisBuckets = this.buckets; for (int i = 0; i < thisBuckets.length; ++i) { @@ -1832,8 +2101,23 @@ public void forEach(Consumer consumer) { private void forEachParent(Consumer consumer) { // Walk the ancestor chain, nearest first. Each entry is emitted once, by the nearest level that - // defines its key, when not shadowed by a nearer level and not tombstoned. + // defines its key, when not shadowed by a nearer level and not tombstoned. Dense known tags are + // emitted via a reused flyweight (no per-entry Entry alloc — the serialize win). + EntryReadingHelper reader = null; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + long[] ancestorIds = ancestor.knownIds; + int ancestorKnownCount = ancestor.knownCount; + if (ancestorKnownCount > 0) { + Object[] ancestorValues = ancestor.knownValues; + if (reader == null) reader = new EntryReadingHelper(); + for (int i = 0; i < ancestorKnownCount; ++i) { + long id = ancestorIds[i]; + if (this.parentDenseVisible(id, ancestor)) { + reader.set(KnownTagCodec.nameOf(id), ancestorValues[i]); + consumer.accept(reader); + } + } + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1856,6 +2140,14 @@ private void forEachParent(Consumer consumer) { } public void forEach(T thisObj, BiConsumer consumer) { + if (this.knownCount > 0) { + EntryReadingHelper reader = new EntryReadingHelper(); + for (int i = 0; i < this.knownCount; ++i) { + reader.set(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + consumer.accept(thisObj, reader); + } + } + Object[] thisBuckets = this.buckets; for (int i = 0; i < thisBuckets.length; ++i) { @@ -1879,7 +2171,21 @@ public void forEach(T thisObj, BiConsumer con } private void forEachParent(T thisObj, BiConsumer consumer) { + EntryReadingHelper reader = null; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + int ancestorKnownCount = ancestor.knownCount; + if (ancestorKnownCount > 0) { + long[] ancestorIds = ancestor.knownIds; + Object[] ancestorValues = ancestor.knownValues; + if (reader == null) reader = new EntryReadingHelper(); + for (int i = 0; i < ancestorKnownCount; ++i) { + long id = ancestorIds[i]; + if (this.parentDenseVisible(id, ancestor)) { + reader.set(KnownTagCodec.nameOf(id), ancestorValues[i]); + consumer.accept(thisObj, reader); + } + } + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1904,6 +2210,14 @@ private void forEachParent(T thisObj, BiConsumer void forEach( T thisObj, U otherObj, TriConsumer consumer) { + if (this.knownCount > 0) { + EntryReadingHelper reader = new EntryReadingHelper(); + for (int i = 0; i < this.knownCount; ++i) { + reader.set(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + consumer.accept(thisObj, otherObj, reader); + } + } + Object[] thisBuckets = this.buckets; for (int i = 0; i < thisBuckets.length; ++i) { @@ -1928,7 +2242,21 @@ public void forEach( private void forEachParent( T thisObj, U otherObj, TriConsumer consumer) { + EntryReadingHelper reader = null; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + int ancestorKnownCount = ancestor.knownCount; + if (ancestorKnownCount > 0) { + long[] ancestorIds = ancestor.knownIds; + Object[] ancestorValues = ancestor.knownValues; + if (reader == null) reader = new EntryReadingHelper(); + for (int i = 0; i < ancestorKnownCount; ++i) { + long id = ancestorIds[i]; + if (this.parentDenseVisible(id, ancestor)) { + reader.set(KnownTagCodec.nameOf(id), ancestorValues[i]); + consumer.accept(thisObj, otherObj, reader); + } + } + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1955,13 +2283,17 @@ private void forEachParent( public void clear() { this.checkWriteAccess(); - Arrays.fill(this.buckets, null); + // Drop the private bucket array back to the shared empty sentinel (also avoids mutating it). + this.buckets = EMPTY_BUCKETS; this.size = 0; // clear() removes ALL mappings, including any inherited through read-through. Detaching the // parent (rather than tombstoning every inherited key) is simpler and cheaper, and leaves an // empty, parent-less map. Detach is one-way -- the parent is never re-pointed. this.parent = null; this.removedFromParent = null; + this.knownIds = null; + this.knownValues = null; + this.knownCount = 0; } public TagMap freeze() { @@ -2016,6 +2348,20 @@ void checkIntegrity() { } } + // dense store: ids must be unique (no tag stored twice) and the count within array bounds. + if (this.knownCount > 0) { + if (this.knownIds == null || this.knownCount > this.knownIds.length) { + throw new IllegalStateException("incorrect known count"); + } + for (int i = 0; i < this.knownCount; ++i) { + for (int j = i + 1; j < this.knownCount; ++j) { + if (this.knownIds[i] == this.knownIds[j]) { + throw new IllegalStateException("duplicate known id"); + } + } + } + } + if (this.size != this.computeSize()) { throw new IllegalStateException("incorrect size"); } @@ -2146,13 +2492,23 @@ abstract static class IteratorBase { private TagMap level; private Object[] buckets; - private Entry nextEntry; + // Currency is EntryReader, not Entry: a BUCKET entry is its own (real, retain-safe) Entry, but + // a + // DENSE entry is emitted via the reused denseReader flyweight (alloc-free, "use now"). This is + // the contract of TagMap.iterator()/keySet()/values(). entrySet() (Iterator) sits on + // top and calls .entry() per next() to get a real retain-safe Entry (see EntriesIterator). + private EntryReader nextEntry; + private EntryReadingHelper denseReader; // lazily created on the first dense emit private int bucketIndex = -1; private BucketGroup group = null; private int groupIndex = 0; + // dense-store cursor for the current level's known tags; advance() resets it when it moves to + // the next ancestor level (read-through union). + private int knownIndex = 0; + IteratorBase(TagMap map) { this.map = map; this.level = map; @@ -2166,9 +2522,9 @@ public final boolean hasNext() { return this.nextEntry != null; } - final Entry nextEntryOrThrowNoSuchElement() { + final EntryReader nextEntryOrThrowNoSuchElement() { if (this.nextEntry != null) { - Entry nextEntry = this.nextEntry; + EntryReader nextEntry = this.nextEntry; this.nextEntry = null; return nextEntry; } @@ -2180,9 +2536,9 @@ final Entry nextEntryOrThrowNoSuchElement() { } } - final Entry nextEntryOrNull() { + final EntryReader nextEntryOrNull() { if (this.nextEntry != null) { - Entry nextEntry = this.nextEntry; + EntryReader nextEntry = this.nextEntry; this.nextEntry = null; return nextEntry; } @@ -2190,8 +2546,22 @@ final Entry nextEntryOrNull() { return this.hasNext() ? this.nextEntry : null; } - private final Entry advance() { + private final EntryReader advance() { while (true) { + // phase 1: drain the current level's dense known tags before its buckets. Leaf dense always + // emits; ancestor dense only if visible from the leaf (not shadowed by a nearer dense entry + // and not tombstoned). Emitted via the reused denseReader flyweight -- NO per-entry Entry + // alloc (the read/serialize alloc win). + if (this.knownIndex < this.level.knownCount) { + int i = this.knownIndex++; + long id = this.level.knownIds[i]; + if (this.level == this.map || this.map.parentDenseVisible(id, this.level)) { + return this.emitDense(id, this.level.knownValues[i]); + } + continue; // ancestor dense entry shadowed/tombstoned -> skip + } + + // phase 2: the current level's buckets. Entry tagEntry = this.rawAdvance(); if (tagEntry != null) { // leaf entries emit as-is; ancestor entries only if visible from the leaf -- not shadowed @@ -2202,9 +2572,12 @@ private final Entry advance() { continue; // ancestor entry shadowed/tombstoned -> skip } - // current level exhausted; advance to the next ancestor's buckets (read-through union) + // current level exhausted; advance to the next ancestor (read-through union), resetting + // both + // the per-level dense cursor and the bucket cursor for the new level. if (this.level.parent != null) { this.level = this.level.parent; + this.knownIndex = 0; this.buckets = this.level.buckets; this.bucketIndex = -1; this.group = null; @@ -2215,6 +2588,16 @@ private final Entry advance() { } } + /** Sets and returns the reused dense flyweight (lazily created); "use now", do not retain. */ + private EntryReader emitDense(long tagId, Object value) { + EntryReadingHelper reader = this.denseReader; + if (reader == null) { + reader = this.denseReader = new EntryReadingHelper(); + } + reader.set(KnownTagCodec.nameOf(tagId), value); + return reader; + } + /** Next raw entry in the current bucket array, ignoring shadowing/tombstones. */ private final Entry rawAdvance() { while (this.bucketIndex < this.buckets.length) { @@ -2744,9 +3127,26 @@ public boolean isEmpty() { @Override public Iterator> iterator() { - @SuppressWarnings({"rawtypes", "unchecked"}) - Iterator> iter = (Iterator) this.map.iterator(); - return iter; + return new EntriesIterator(this.map); + } + } + + /** + * entrySet() yields real, retain-safe {@code Map.Entry} objects. It sits on top of the + * EntryReader iterator and materializes each via {@code .entry()}: a bucket entry's reader IS the + * real stored Entry (returns {@code this}, free); a dense entry's flyweight materializes a fresh + * Entry. Deliberately NOT alloc-optimized for dense — bulk reads use {@code forEach}/EntryReader, + * and manual instrumentation does point get/set, not bulk entrySet iteration. + */ + static final class EntriesIterator extends IteratorBase + implements Iterator> { + EntriesIterator(TagMap map) { + super(map); + } + + @Override + public Map.Entry next() { + return this.nextEntryOrThrowNoSuchElement().entry(); } } diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java new file mode 100644 index 00000000000..11d01bc7ec8 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -0,0 +1,152 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Parity test for the keyOf substrate (slice 1): the {@link KnownTags} registry + the {@link + * KnownTagCodec.Resolver} it registers. Verifies name ↔ id resolution without any dense store + * — {@code keyOf}/{@code nameOf} depend only on globalSerial + name, not on the (dormant) + * positional layout. + */ +class KnownTagsTest { + + /** (name, id) pairs — the full registry. keyOf returns the id verbatim (incl. INTERCEPTED). */ + static Stream knownTags() { + return Stream.of( + Arguments.of(Tags.ERROR, KnownTags.ERROR_ID), + Arguments.of(DDTags.PARENT_ID, KnownTags.PARENT_ID), + Arguments.of(DDTags.BASE_SERVICE, KnownTags.BASE_SERVICE_ID), + Arguments.of(Tags.VERSION, KnownTags.VERSION_ID), + Arguments.of(KnownTags.ENV, KnownTags.ENV_ID), + Arguments.of(DDTags.DJM_ENABLED, KnownTags.DJM_ENABLED_ID), + Arguments.of(DDTags.DSM_ENABLED, KnownTags.DSM_ENABLED_ID), + Arguments.of(DDTags.TRACER_HOST, KnownTags.TRACER_HOST_ID), + Arguments.of(DDTags.DD_INTEGRATION, KnownTags.INTEGRATION_ID), + Arguments.of(DDTags.DD_SVC_SRC, KnownTags.SVC_SRC_ID), + Arguments.of(Tags.PEER_SERVICE, KnownTags.PEER_SERVICE_ID), + Arguments.of(DDTags.PEER_SERVICE_REMAPPED_FROM, KnownTags.PEER_SERVICE_REMAPPED_FROM_ID), + Arguments.of(Tags.HTTP_METHOD, KnownTags.HTTP_METHOD_ID), + Arguments.of(Tags.HTTP_ROUTE, KnownTags.HTTP_ROUTE_ID), + Arguments.of(Tags.HTTP_URL, KnownTags.HTTP_URL_ID), + Arguments.of(Tags.PEER_HOSTNAME, KnownTags.PEER_HOSTNAME_ID), + Arguments.of(Tags.PEER_HOST_IPV4, KnownTags.PEER_HOST_IPV4_ID), + Arguments.of(Tags.PEER_HOST_IPV6, KnownTags.PEER_HOST_IPV6_ID), + Arguments.of(Tags.PEER_PORT, KnownTags.PEER_PORT_ID), + Arguments.of(Tags.COMPONENT, KnownTags.COMPONENT_ID), + Arguments.of(Tags.SPAN_KIND, KnownTags.SPAN_KIND_ID), + Arguments.of(DDTags.LANGUAGE_TAG_KEY, KnownTags.LANGUAGE_ID), + Arguments.of(Tags.DB_TYPE, KnownTags.DB_TYPE_ID), + Arguments.of(Tags.DB_INSTANCE, KnownTags.DB_INSTANCE_ID), + Arguments.of(Tags.DB_USER, KnownTags.DB_USER_ID), + Arguments.of(Tags.DB_OPERATION, KnownTags.DB_OPERATION_ID), + Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); + } + + /** + * The subset flagged INTERCEPTED (sign bit) — must agree with the interceptor's needsIntercept. + */ + static Stream interceptedTags() { + return Stream.of( + Arguments.of(KnownTags.ERROR_ID), + Arguments.of(KnownTags.PEER_SERVICE_ID), + Arguments.of(KnownTags.HTTP_METHOD_ID), + Arguments.of(KnownTags.HTTP_URL_ID), + Arguments.of(KnownTags.SPAN_KIND_ID)); + } + + @Test + void resolverIsActiveOnceReferenced() { + // referencing any constant triggers KnownTags. -> KnownTagCodec.register + assertTrue(KnownTags.ERROR_ID != 0L); + assertTrue(KnownTagCodec.isActive()); + assertEquals(KnownTags.SLOT_COUNT, KnownTagCodec.slotCount()); + } + + @ParameterizedTest + @MethodSource("knownTags") + void keyOfResolvesNameToId(String name, long id) { + assertEquals(id, KnownTagCodec.keyOf(name), "keyOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("knownTags") + void nameOfResolvesIdToName(String name, long id) { + assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("knownTags") + void nameHashMatchesEntryHash(String name, long id) { + assertEquals( + (int) TagMap.Entry._hash(name), KnownTagCodec.nameHash(id), "nameHash(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("interceptedTags") + void interceptedTagsCarryFlag(long id) { + assertTrue(KnownTagCodec.isIntercepted(id), "isIntercepted"); + } + + @Test + void nonInterceptedTagsDoNotCarryFlag() { + Set intercepted = new HashSet<>(); + interceptedTags().forEach(a -> intercepted.add((Long) a.get()[0])); + knownTags() + .forEach( + a -> { + long id = (Long) a.get()[1]; + if (!intercepted.contains(id)) { + assertFalse(KnownTagCodec.isIntercepted(id), "not intercepted: " + a.get()[0]); + } + }); + } + + @Test + void unknownNamesResolveToZero() { + assertEquals(0L, KnownTagCodec.keyOf("definitely.not.a.known.tag")); + assertEquals(0L, KnownTagCodec.keyOf("http.statuscode")); // close-but-not-listed + assertEquals(0L, KnownTagCodec.keyOf("")); + } + + @Test + void unknownIdsResolveToNullName() { + assertNull(KnownTagCodec.nameOf(0L)); + assertNull(KnownTagCodec.nameOf(KnownTagCodec.tagId(9999, "made.up"))); + } + + @Test + void errorIsReservedTheRestAreStored() { + assertTrue(KnownTagCodec.isReserved(KnownTags.ERROR_ID), "ERROR reserved"); + assertFalse(KnownTagCodec.isStored(KnownTags.ERROR_ID), "ERROR not stored"); + knownTags() + .forEach( + a -> { + long id = (Long) a.get()[1]; + if (id != KnownTags.ERROR_ID) { + assertTrue(KnownTagCodec.isStored(id), "stored: " + a.get()[0]); + assertFalse(KnownTagCodec.isReserved(id), "not reserved: " + a.get()[0]); + } + }); + } + + @Test + void globalSerialsAreUnique() { + List serials = new ArrayList<>(); + knownTags().forEach(a -> serials.add((long) KnownTagCodec.globalSerial((Long) a.get()[1]))); + assertEquals(serials.size(), new HashSet<>(serials).size(), "globalSerials must be unique"); + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java new file mode 100644 index 00000000000..a82742c3acd --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java @@ -0,0 +1,273 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Exercises the dense known-tag store with a LIVE resolver. Registration ({@link KnownTagCodec}) is + * a global static with no un-register, so this lives in a {@code ForkedTest} (isolated JVM) to keep + * dense routing from leaking into the bucket-only tests in the shared JVM. The dense store is + * dormant in production (no resolver) — this is where it actually executes. + * + *

Stored tags (globalSerial ≥ {@code FIRST_STORED_SERIAL}) route to the dense store; reserved + * tags (e.g. {@code error}) and arbitrary tags stay in the hash buckets. Behavior must be + * observationally identical to the bucket store. + */ +class TagMapDenseForkedTest { + + // stored (dense-routed) tags + static final String BASE_SERVICE = DDTags.BASE_SERVICE; + static final String COMPONENT = Tags.COMPONENT; + static final String DB_TYPE = Tags.DB_TYPE; + static final String HTTP_METHOD = Tags.HTTP_METHOD; // stored + intercepted + static final String DB_INSTANCE = Tags.DB_INSTANCE; + // arbitrary (bucket-routed) tags + static final String CUSTOM_A = "custom.tag.a"; + static final String CUSTOM_B = "custom.tag.b"; + + @BeforeAll + static void registerResolver() { + // referencing any KnownTags constant triggers its -> KnownTagCodec.register + assertTrue(KnownTags.BASE_SERVICE_ID != 0L); + assertTrue(KnownTagCodec.isActive(), "resolver must be live for the dense store to engage"); + assertTrue( + KnownTagCodec.isStored(KnownTagCodec.keyOf(BASE_SERVICE)), "base_service routes dense"); + assertFalse( + KnownTagCodec.isStored(KnownTagCodec.keyOf(CUSTOM_A)), "custom tag stays in buckets"); + assertFalse( + KnownTagCodec.isStored(KnownTagCodec.keyOf(Tags.ERROR)), "error is reserved, not stored"); + } + + private static TagMap map() { + return (TagMap) TagMap.create(); + } + + @Test + void knownTagRoundTripsThroughDenseStore() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(COMPONENT, "spring-web"); + + assertEquals("billing", map.getObject(BASE_SERVICE)); + assertEquals("spring-web", map.getString(COMPONENT)); + assertEquals("billing", map.getEntry(BASE_SERVICE).objectValue()); + assertTrue(map.containsKey(BASE_SERVICE)); + assertEquals(2, map.size()); + map.checkIntegrity(); + } + + @Test + void typedKnownValuesRoundTrip() { + TagMap map = map(); + map.set(DB_TYPE, "postgresql"); + map.set(HTTP_METHOD, "GET"); + map.set(Tags.PEER_PORT, 5432); + + assertEquals("postgresql", map.getString(DB_TYPE)); + assertEquals("GET", map.getString(HTTP_METHOD)); + assertEquals(5432, map.getInt(Tags.PEER_PORT)); + assertEquals(3, map.size()); + map.checkIntegrity(); + } + + @Test + void knownAndUnknownCoexist() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); // dense + map.set(CUSTOM_A, "alpha"); // bucket + map.set(DB_TYPE, "h2"); // dense + map.set(CUSTOM_B, "beta"); // bucket + + assertEquals("billing", map.getObject(BASE_SERVICE)); + assertEquals("alpha", map.getObject(CUSTOM_A)); + assertEquals("h2", map.getObject(DB_TYPE)); + assertEquals("beta", map.getObject(CUSTOM_B)); + assertEquals(4, map.size()); + assertFalse(map.isEmpty()); + map.checkIntegrity(); + + Map collected = new HashMap<>(); + map.fillMap(collected); + assertEquals(4, collected.size()); + assertEquals("billing", collected.get(BASE_SERVICE)); + assertEquals("alpha", collected.get(CUSTOM_A)); + assertEquals("h2", collected.get(DB_TYPE)); + assertEquals("beta", collected.get(CUSTOM_B)); + } + + @Test + void overwriteKnownReplacesInPlace() { + TagMap map = map(); + map.set(COMPONENT, "first"); + assertEquals("first", map.getObject(COMPONENT)); + map.set(COMPONENT, "second"); + assertEquals("second", map.getObject(COMPONENT)); + assertEquals(1, map.size()); // overwrite, not append + map.checkIntegrity(); + } + + @Test + void removeKnownClearsIt() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(DB_TYPE, "h2"); + map.set(CUSTOM_A, "alpha"); + assertEquals(3, map.size()); + + TagMap.Entry removed = map.getAndRemove(BASE_SERVICE); + assertEquals("billing", removed.objectValue()); + assertNull(map.getObject(BASE_SERVICE)); + assertEquals("h2", map.getObject(DB_TYPE)); // sibling dense entry intact + assertEquals("alpha", map.getObject(CUSTOM_A)); + assertEquals(2, map.size()); + map.checkIntegrity(); + } + + @Test + void forEachAndIteratorEmitDenseAndBucketEntries() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(COMPONENT, "web"); + map.set(CUSTOM_A, "alpha"); + + Map viaForEach = new HashMap<>(); + map.forEach(reader -> viaForEach.put(reader.tag(), reader.objectValue())); + assertEquals(3, viaForEach.size()); + assertEquals("billing", viaForEach.get(BASE_SERVICE)); + assertEquals("web", viaForEach.get(COMPONENT)); + assertEquals("alpha", viaForEach.get(CUSTOM_A)); + + Map viaIterator = new HashMap<>(); + for (TagMap.EntryReader reader : map) { + viaIterator.put(reader.tag(), reader.objectValue()); + } + assertEquals(viaForEach, viaIterator); + } + + @Test + void copyPreservesDenseStore() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(CUSTOM_A, "alpha"); + + TagMap copy = (TagMap) map.copy(); + assertEquals("billing", copy.getObject(BASE_SERVICE)); + assertEquals("alpha", copy.getObject(CUSTOM_A)); + assertEquals(2, copy.size()); + + // independence: mutating the copy doesn't touch the original's dense store + copy.set(BASE_SERVICE, "shipping"); + assertEquals("shipping", copy.getObject(BASE_SERVICE)); + assertEquals("billing", map.getObject(BASE_SERVICE)); + copy.checkIntegrity(); + map.checkIntegrity(); + } + + @Test + void clearEmptiesDenseStore() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(CUSTOM_A, "alpha"); + map.clear(); + assertEquals(0, map.size()); + assertTrue(map.isEmpty()); + assertNull(map.getObject(BASE_SERVICE)); + map.checkIntegrity(); + } + + @Test + void putAllMergesDenseStore() { + TagMap src = map(); + src.set(BASE_SERVICE, "billing"); + src.set(DB_TYPE, "h2"); + src.set(CUSTOM_A, "alpha"); + + TagMap dst = map(); + dst.set(COMPONENT, "web"); // dense, distinct + dst.set(BASE_SERVICE, "old"); // dense, clobbered by src + dst.putAll((TagMap) src); + + assertEquals("billing", dst.getObject(BASE_SERVICE)); // src clobbers + assertEquals("h2", dst.getObject(DB_TYPE)); + assertEquals("web", dst.getObject(COMPONENT)); + assertEquals("alpha", dst.getObject(CUSTOM_A)); + assertEquals(4, dst.size()); + dst.checkIntegrity(); + } + + // ---- read-through union (dense parent + dense child) ---- + + private static TagMap frozenParent() { + TagMap parent = map(); + parent.set(BASE_SERVICE, "billing"); // dense + parent.set(COMPONENT, "web"); // dense + parent.set(CUSTOM_A, "alpha"); // bucket + parent.freeze(); + return parent; + } + + @Test + void childReadsThroughToParentDense() { + TagMap child = TagMap.createFromParent(frozenParent()); + child.set(DB_TYPE, "h2"); // child-only dense + child.set(CUSTOM_B, "beta"); // child-only bucket + + // inherited from parent + assertEquals("billing", child.getObject(BASE_SERVICE)); + assertEquals("web", child.getObject(COMPONENT)); + assertEquals("alpha", child.getObject(CUSTOM_A)); + // own + assertEquals("h2", child.getObject(DB_TYPE)); + assertEquals("beta", child.getObject(CUSTOM_B)); + // union size: 3 parent + 2 child + assertEquals(5, child.size()); + assertFalse(child.isEmpty()); + + Map union = new HashMap<>(); + child.forEach(reader -> union.put(reader.tag(), reader.objectValue())); + assertEquals(5, union.size()); + assertEquals("billing", union.get(BASE_SERVICE)); + assertEquals("h2", union.get(DB_TYPE)); + child.checkIntegrity(); + } + + @Test + void childDenseShadowsParentDense() { + TagMap child = TagMap.createFromParent(frozenParent()); + child.set(BASE_SERVICE, "shipping"); // shadows parent's dense base_service + + assertEquals("shipping", child.getObject(BASE_SERVICE)); // local wins + assertEquals("web", child.getObject(COMPONENT)); // still inherited + assertEquals(3, child.size()); // base_service counted once (shadowed, not doubled) + + Map union = new HashMap<>(); + child.forEach(reader -> union.put(reader.tag(), reader.objectValue())); + assertEquals(3, union.size()); + assertEquals("shipping", union.get(BASE_SERVICE)); // shadow value, parent suppressed + } + + @Test + void removingParentDenseKeyTombstonesIt() { + TagMap child = TagMap.createFromParent(frozenParent()); + + TagMap.Entry removed = child.getAndRemove(BASE_SERVICE); // parent-only dense key + assertEquals("billing", removed.objectValue()); // prior visible value was the parent's + assertNull(child.getObject(BASE_SERVICE)); // tombstoned: no read-through + assertEquals("web", child.getObject(COMPONENT)); // sibling still inherited + assertEquals(2, child.size()); // 3 parent - 1 tombstoned + + Map union = new HashMap<>(); + child.forEach(reader -> union.put(reader.tag(), reader.objectValue())); + assertEquals(2, union.size()); + assertFalse(union.containsKey(BASE_SERVICE)); + child.checkIntegrity(); + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java new file mode 100644 index 00000000000..a29795c542a --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java @@ -0,0 +1,202 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.TagMapFuzzTest.MapAction; +import datadog.trace.api.TagMapFuzzTest.TestCase; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Supplier; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Fuzz test for the dense store under a LIVE resolver, across three key regimes. Reuses {@link + * TagMapFuzzTest}'s oracle machinery ({@code test(TestCase)} replays a random action sequence + * against a {@code HashMap}, verifying each step + {@code checkIntegrity}). + * + *

Uses a synthetic prefix resolver ({@code known-N} -> stored / dense, anything else -> bucket) + * rather than the real {@link KnownTags}: it gives an UNBOUNDED known key space, so the dense array + * actually grows past its initial capacity and the linear scan gets long, and it lets each test pin + * the known/custom ratio. The three regimes exercise paths the mixed run alone would miss: + * + *

+ * + *

Forked (isolated JVM) because resolver registration is a global static with no un-register. + */ +class TagMapDenseFuzzForkedTest { + static final int SINGLE_MAP_CASES = 1500; + static final int MERGE_CASES = 400; + static final int MAX_ACTIONS = 40; + static final int MIN_ACTIONS = 8; + + // unbounded synthetic key spaces — large enough to grow the dense array past cap-8 several times + static final int KNOWN_SPACE = 48; + static final int CUSTOM_SPACE = 48; + + enum Regime { + KNOWN_ONLY, + CUSTOM_ONLY, + MIXED + } + + /** + * Synthetic resolver: {@code known-N} -> stored id (serial = FIRST_STORED_SERIAL + N); else 0. + */ + static final KnownTagCodec.Resolver FUZZ_RESOLVER = + new KnownTagCodec.Resolver() { + @Override + public long keyOf(String name) { + if (name.startsWith("known-")) { + int n = Integer.parseInt(name.substring("known-".length())); + return KnownTagCodec.tagId(KnownTagCodec.FIRST_STORED_SERIAL + n, name); + } + return 0L; + } + + @Override + public String nameOf(long tagId) { + int serial = KnownTagCodec.globalSerial(tagId); + return serial >= KnownTagCodec.FIRST_STORED_SERIAL + ? "known-" + (serial - KnownTagCodec.FIRST_STORED_SERIAL) + : null; + } + + @Override + public int slotCount() { + return 0; // positional unused + } + }; + + @BeforeAll + static void registerResolver() { + KnownTagCodec.register(FUZZ_RESOLVER); + assertTrue(KnownTagCodec.isActive(), "resolver must be live"); + assertTrue(KnownTagCodec.isStored(KnownTagCodec.keyOf("known-0")), "known- routes dense"); + assertFalse( + KnownTagCodec.isStored(KnownTagCodec.keyOf("custom-0")), "custom- stays in buckets"); + // round-trip the synthetic encoding + long id = KnownTagCodec.keyOf("known-7"); + assertTrue("known-7".equals(KnownTagCodec.nameOf(id)), "name<->id round-trips"); + } + + @Test + void knownOnlyFuzz() { + runRegime(Regime.KNOWN_ONLY); + } + + @Test + void customOnlyFuzz() { + runRegime(Regime.CUSTOM_ONLY); + } + + @Test + void mixedFuzz() { + runRegime(Regime.MIXED); + } + + private static void runRegime(Regime regime) { + for (int i = 0; i < SINGLE_MAP_CASES; ++i) { + TagMapFuzzTest.test(generateTest(regime)); + } + for (int i = 0; i < MERGE_CASES; ++i) { + TagMap mapA = TagMapFuzzTest.test(generateTest(regime)); + TagMap mapB = TagMapFuzzTest.test(generateTest(regime)); + + HashMap hashA = new HashMap<>(mapA); + HashMap hashB = new HashMap<>(mapB); + + mapA.putAll(mapB); + hashA.putAll(hashB); + + TagMapFuzzTest.assertMapEquals(hashA, mapA); + } + } + + // --- action generation (mirrors TagMapFuzzTest.randomAction, regime-driven key pool) --- + + private static TestCase generateTest(Regime regime) { + ThreadLocalRandom r = ThreadLocalRandom.current(); + int numActions = r.nextInt(MAX_ACTIONS - MIN_ACTIONS) + MIN_ACTIONS; + List actions = new ArrayList<>(numActions); + for (int i = 0; i < numActions; ++i) { + actions.add(randomAction(regime)); + } + return new TestCase(actions); + } + + private static MapAction randomAction(Regime regime) { + switch (randomChoice(0.02, 0.1, 0.2)) { + case 0: + return TagMapFuzzTest.clear(); + case 1: + return choose( + () -> TagMapFuzzTest.putAll(randomKeysAndValues(regime)), + () -> TagMapFuzzTest.putAllTagMap(randomKeysAndValues(regime)), + () -> TagMapFuzzTest.putAllLedger(randomKeysAndValues(regime))); + case 2: + return choose( + () -> TagMapFuzzTest.remove(randomKey(regime)), + () -> TagMapFuzzTest.removeLight(randomKey(regime)), + () -> TagMapFuzzTest.getAndRemove(randomKey(regime))); + default: + return choose( + () -> TagMapFuzzTest.put(randomKey(regime), randomValue()), + () -> TagMapFuzzTest.set(randomKey(regime), randomValue()), + () -> TagMapFuzzTest.getAndSet(randomKey(regime), randomValue())); + } + } + + private static String randomKey(Regime regime) { + ThreadLocalRandom r = ThreadLocalRandom.current(); + boolean known; + switch (regime) { + case KNOWN_ONLY: + known = true; + break; + case CUSTOM_ONLY: + known = false; + break; + default: + known = r.nextBoolean(); + } + return known ? "known-" + r.nextInt(KNOWN_SPACE) : "custom-" + r.nextInt(CUSTOM_SPACE); + } + + private static String randomValue() { + return "values-" + ThreadLocalRandom.current().nextInt(); + } + + private static String[] randomKeysAndValues(Regime regime) { + int numEntries = ThreadLocalRandom.current().nextInt(KNOWN_SPACE + CUSTOM_SPACE); + String[] keysAndValues = new String[numEntries << 1]; + for (int i = 0; i < keysAndValues.length; i += 2) { + keysAndValues[i] = randomKey(regime); + keysAndValues[i + 1] = randomValue(); + } + return keysAndValues; + } + + private static int randomChoice(double... proportions) { + double selector = ThreadLocalRandom.current().nextDouble(); + for (int i = 0; i < proportions.length; ++i) { + if (selector < proportions[i]) return i; + selector -= proportions[i]; + } + return proportions.length; + } + + @SafeVarargs + private static MapAction choose(Supplier... choices) { + return choices[ThreadLocalRandom.current().nextInt(choices.length)].get(); + } +} From 2bc5ae7f7419fdf2255775dce115cc494ebb3fce Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 17 Aug 2026 19:17:27 -0400 Subject: [PATCH 02/14] Encode tag ids with a colored slot + one-long dense presence Reframe of the dense-store presence layer. Replaces the earlier two-tier (group-decl mask + field-decl bloom) design with a single global colored slot: the tag-id middle 16 bits carry one graph-colored slot coordinate (SLOT_SHIFT=32, SLOT_MASK=0xFFFF), so the dense store tracks presence with one occupancy long instead of a group mask plus a field bloom. Adds the trace-level bit (LEVEL_TRACE) and level-bit read-through in the parent visibility check. KnownTags remains hand-maintained here (src/main); the tag-registry code generator that produces these colored ids lands in the following commit, which relocates the file to src/generated. Co-Authored-By: Claude Opus 4.8 --- .../java/datadog/trace/api/KnownTagCodec.java | 145 ++-- .../java/datadog/trace/api/KnownTags.java | 772 ++++++++++++------ .../main/java/datadog/trace/api/TagMap.java | 94 ++- .../java/datadog/trace/api/KnownTagsTest.java | 119 ++- .../trace/api/TagMapDenseForkedTest.java | 5 +- .../trace/api/TagMapDenseFuzzForkedTest.java | 4 +- 6 files changed, 782 insertions(+), 357 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 7ed9dff633a..031e908e5ca 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -22,23 +22,33 @@ public static boolean isActive() { } /* - * tagId bit layout: [63 intercepted] [62-48 globalSerial (15 bits)] [47-32 fieldPos] - * [31-0 nameHash]. Bit 63 (the sign bit) marks a tag the tag interceptor must see, so the check - * is a single {@code tagId < 0}. globalSerial is globally unique per known tag; fieldPos is its - * slot in the global positional layout (TagMap.knownEntries index); nameHash is - * TagMap.Entry#_hash(name) and is layout-independent. Unknown (string-only) tags have the upper - * 32 bits zero. NOTE: TagMap.Entry decodes nameHash inline as (int) tagId on its hot path, so the - * low-32 encoding here must stay in sync with that. + * tagId bit layout: [63 intercepted] [62-48 globalSerial (15 bits)] [47-32 slot (16 bits)] [31-0 + * reserved, zero]. Bit 63 (the sign bit) marks a tag the tag interceptor must see, so the check is + * a single {@code tagId < 0}. globalSerial is globally unique per known tag. The middle 16 bits + * carry the tag's SLOT: one globally stable coordinate assigned by graph-coloring the tag + * co-occurrence graph (the resolved tag set of each concrete span type, plus the trace-level + * tier, is a clique). Co-occurring tags always get distinct slots; slots are reused only between + * tags that never appear together, so slotCount stays bounded by the largest clique (≤ 64) and + * fits one {@code long} occupancy mask — the dense store's single-tier presence fast path (see + * {@link TagMap}). The low 32 bits are unused for known ids (the whole id is fully determined by + * serial + slot, so the generator can emit a literal). The low 32 bits are being carved for + * cross-cutting flags; bit 2 is the trace/span LEVEL bit (set ⟹ trace-level), and bits 1-0 are + * reserved for the dd/otel applicability flags that land with increment 1. The level bit lets + * read-through skip the shadow check across the trace/span boundary — trace and span tags reuse + * the same slots, so occupancy alone can't tell them apart, but a span map (no trace-level tags) + * can never shadow a trace-level ancestor entry (see {@link TagMap}). Unknown (string-only) custom + * tags are NOT known ids — they key off {@code TagMap.Entry#_hash(name)} in their own bucket path + * and never enter here. */ - public static int globalSerial(long tagId) { + public static int serialNum(long tagId) { return (int) ((tagId >>> 48) & 0x7FFF); } /** - * Flag bit (the sign bit) marking a tag the tag interceptor must process — reserved/"virtual" - * tags AND intercepted-but-stored tags (e.g. http.method, which the interceptor side-effects and - * also stores). Encoded in the id so {@code DDSpanContext.setTag(long)} can route with a single - * sign test ({@link #isIntercepted}) instead of resolving the name. Non-intercepted tags (peer.*, + * Flag bit (the sign bit) marking a tag the tag interceptor must process — reserved tags AND + * intercepted-but-stored tags (e.g. http.method, which the interceptor side-effects and also + * stores). Encoded in the id so {@code DDSpanContext.setTag(long)} can route with a single sign + * test ({@link #isIntercepted}) instead of resolving the name. Non-intercepted tags (peer.*, * base.service, …) leave it clear and take the fast store path. Must agree with the interceptor's * name-based {@code needsIntercept} for every assigned id. */ @@ -54,78 +64,115 @@ public static long intercepted(long tagId) { return tagId | INTERCEPTED; } - public static int fieldPos(long tagId) { - return (int) ((tagId >>> 32) & 0xFFFF); + /** + * Trace/span LEVEL bit (low-32 carve, bit 2). Set marks a trace-level tag (lives on the + * TraceSegment's own TagMap); clear marks a span-level tag. Trace and span tags reuse the same + * coloring slots, so this bit is what lets read-through tell the two levels apart — a span map + * (no trace-level tags) can never shadow a trace-level ancestor entry, so its shadow check is + * skipped (see {@link TagMap#parentDenseVisible}). + */ + public static final long LEVEL_TRACE = 1L << 2; + + /** True if the tagId names a trace-level tag. */ + public static boolean isTraceLevel(long tagId) { + return (tagId & LEVEL_TRACE) != 0L; } - public static int nameHash(long tagId) { - return (int) tagId; + /** Returns the tagId with the {@link #LEVEL_TRACE} flag set. */ + public static long traceLevel(long tagId) { + return tagId | LEVEL_TRACE; } + // The middle 16 bits [47-32] hold the tag's SLOT: one globally stable coordinate from graph + // coloring the co-occurrence graph. Co-occurring tags get distinct slots and slotCount stays + // bounded by the largest clique (<= 64), so the dense store's presence fast path is a single + // occupancy long (1L << slot); a clear bit proves the tag absent and enables an O(1) append. See + // TagMap's dense-store fast path. + static final int SLOT_SHIFT = 32; + static final int SLOT_MASK = 0xFFFF; // 16 bits + /** - * globalSerial partition. {@code [1, FIRST_STORED_SERIAL)} is reserved for "virtual" tags that - * are specially handled (redirected to span fields or processed by the tag interceptor) and are - * NOT stored in the TagMap — these are hand-assigned in tracer core. {@code [FIRST_STORED_SERIAL, - * ..]} is for generated convention tags that ARE stored (slotted/bucketed). {@code globalSerial - * == 0} means unknown / string-only. Both core and the code generator must agree on this - * boundary. + * The tag's slot: its globally stable coloring coordinate, or {@link #NO_SLOT} when it has none + * (reserved or deliberately bucket-only). Drives the dense store's single occupancy mask. + */ + public static int slot(long tagId) { + return (int) ((tagId >>> SLOT_SHIFT) & SLOT_MASK); + } + + /** + * globalSerial partition. {@code [1, FIRST_STORED_SERIAL)} is the RESERVED tier and {@code + * [FIRST_STORED_SERIAL, ..]} is the STORED tier; {@code globalSerial == 0} means unknown / + * string-only. Both core and the code generator must agree on this boundary. + * + *

Reserved is the shared mechanism: the tracer reserves the key and handles it itself + * instead of putting it in the TagMap. It says nothing about whether a value exists — that splits + * into two kinds (the {@code kind:} in the overlay): + * + *

+ * + * "virtual" over-claims non-existence (wrong for structural) and "built-in" over-claims existence + * (wrong for directive), so the tier is named for the mechanism they share: reserved. These are + * hand-assigned in the overlay. Stored tags are the generated convention tags that ARE put + * in the map (slotted/bucketed). */ public static final int FIRST_STORED_SERIAL = 256; - /** True if the tagId names a reserved "virtual"/specially-handled tag (not stored in the map). */ + /** True if the tagId names a reserved (structural/directive) tag — handled, not stored. */ public static boolean isReserved(long tagId) { - int globalSerial = globalSerial(tagId); - return globalSerial > 0 && globalSerial < FIRST_STORED_SERIAL; + int serialNum = serialNum(tagId); + return serialNum > 0 && serialNum < FIRST_STORED_SERIAL; } /** True if the tagId names a generated, map-stored (slotted/bucketed) tag. */ public static boolean isStored(long tagId) { - return globalSerial(tagId) >= FIRST_STORED_SERIAL; + return serialNum(tagId) >= FIRST_STORED_SERIAL; } /** - * Sentinel {@code fieldPos} meaning "no positional slot". It is the maximum value the 16-bit - * fieldPos field can hold, so it always compares {@code >= slotCount()} and routes to the hash - * buckets rather than the fast positional array. Two kinds of tagId use it: + * Sentinel {@code slot} meaning "no positional slot". It is the maximum value the 16-bit slot + * field can hold, so it always compares {@code >= slotCount()} and routes to the hash buckets + * rather than the fast positional array. Two kinds of tagId use it: * * */ - public static final int NO_SLOT = 0xFFFF; + public static final int NO_SLOT = SLOT_MASK; // slot all-ones sentinel (16 bits) /** * True if the tagId names a stored tag that deliberately has no positional slot (bucket-only). */ public static boolean isUnslotted(long tagId) { - return isStored(tagId) && fieldPos(tagId) == NO_SLOT; + return isStored(tagId) && slot(tagId) == NO_SLOT; } /** - * Builds a tagId from its parts: {@code globalSerial} (globally unique per known tag), {@code - * fieldPos} (the tag's slot within its span type's positional table), and the tag {@code name} - * (whose hash is computed via the same function the runtime uses, so the low 32 bits match {@link - * TagMap.Entry#hash()}). Inverse of {@link #globalSerial}/{@link #fieldPos}/{@link #nameHash}. - * Intended for the code generator and tests. + * Builds a tagId from its {@code serialNum} (globally unique per known tag) and {@code slot} (its + * coloring coordinate, or {@link #NO_SLOT}). The low 32 bits are zero, so the id is fully + * determined by these parts — the generator emits it as a literal. Inverse of {@link + * #serialNum}/{@link #slot}. Intended for the code generator and tests. */ - public static long tagId(int globalSerial, int fieldPos, String name) { - long nameHash = TagMap.Entry._hash(name) & 0xFFFFFFFFL; - return ((long) globalSerial << 48) | ((long) (fieldPos & 0xFFFF) << 32) | nameHash; + public static long makeTagId(int serialNum, int slot) { + return ((long) serialNum << 48) | ((long) (slot & SLOT_MASK) << SLOT_SHIFT); } /** - * Builds a tagId with no positional slot ({@code fieldPos == }{@link #NO_SLOT}). Use for reserved - * "virtual" tags and for "low-priority" stored tags that get a stable id but are intentionally - * kept out of the fast slot array (they route to the hash buckets). See {@link #NO_SLOT}. + * Builds a tagId with no positional slot ({@code slot == }{@link #NO_SLOT}). Use for reserved + * tags and for "low-priority" stored tags that get a stable id but are intentionally kept out of + * the fast slot array (they route to the hash buckets). See {@link #NO_SLOT}. */ - public static long tagId(int globalSerial, String name) { - return tagId(globalSerial, NO_SLOT, name); + public static long makeTagId(int serialNum) { + return makeTagId(serialNum, NO_SLOT); } // Number of positional slots in the global layout = (max stored fieldPos) + 1, declared by the diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTags.java b/internal-api/src/main/java/datadog/trace/api/KnownTags.java index 391b000b844..599b34f8a25 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTags.java @@ -1,225 +1,412 @@ package datadog.trace.api; -import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.util.StringIndex; -/** - * Hand-assigned tag-id constants for well-known tags, plus the {@link KnownTagCodec.Resolver} that - * resolves them. This is the single registry shared by the tracer core and by instrumentation - * (decorators) — it lives in {@code internal-api} so both layers can reference the ids; the - * eventual code generator will replace the hand assignment here. - * - *

Reserved serials {@code [1, KnownTagCodec.FIRST_STORED_SERIAL)} name "virtual" tags handled by - * the tag interceptor / span fields and are NOT stored in the {@code TagMap}; their {@code - * fieldPos} is the {@link KnownTagCodec#NO_SLOT} sentinel that is out of slot range, so any - * incidental store routes to the hash buckets rather than a positional slot. Serials {@code >= - * FIRST_STORED_SERIAL} name stored tags that slot/bucket normally (or, with {@code NO_SLOT}, are - * stored bucket-only). - * - *

The resolver registers on class initialization, so simply referencing any constant here makes - * tag-id resolution live before the first span is built. - * - *

Slice-1 note (keyOf substrate): the {@code fieldPos} assignments below (and {@link - * #SLOT_COUNT}) describe a single universal positional layout (slots 0..25). That layout is - * currently dormant — no dense store consumes {@code fieldPos} yet — and is provisional: the - * dense-store slice replaces the universal layout with per-role / per-type sizing (see the - * over-provision finding in {@code dense-tagmap-design.md}). {@code keyOf}/{@code nameOf} depend - * only on {@code globalSerial} + name, not {@code fieldPos}, so the ids themselves are stable - * across any layout scheme. - */ +// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator). +// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml. public final class KnownTags { - // slot count = (max stored fieldPos) + 1. Stored tags use fieldPos 0..25. PROVISIONAL universal - // layout — see the slice-1 note above; the dense-store slice supersedes this with role/type - // sizing. - static final int SLOT_COUNT = 26; - - // ---- reserved / virtual (tag-interceptor handled, not stored) ---- - // Reserved tags are always intercepted -> set the INTERCEPTED flag. - public static final int ERROR_SERIAL = 1; - public static final long ERROR_ID = - KnownTagCodec.intercepted(KnownTagCodec.tagId(ERROR_SERIAL, Tags.ERROR)); - - // ---- stored (slotted / bucketed) ---- - public static final int PARENT_ID_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL; - public static final long PARENT_ID = KnownTagCodec.tagId(PARENT_ID_SERIAL, 0, DDTags.PARENT_ID); - - // common (process-constant) tags added by InternalTagsAdder to ~every span - public static final int BASE_SERVICE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 1; - public static final long BASE_SERVICE_ID = - KnownTagCodec.tagId(BASE_SERVICE_SERIAL, 1, DDTags.BASE_SERVICE); - - public static final int VERSION_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 2; - public static final long VERSION_ID = KnownTagCodec.tagId(VERSION_SERIAL, 2, Tags.VERSION); - - // build-time-known constant tags merged into defaultSpanTags (see CoreTracer.withTracerTags). - // "env" is a base-mixin tag; the *_ENABLED flags are product-mixin tags. Hand-assigned for now. - public static final String ENV = "env"; - public static final int ENV_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 3; - public static final long ENV_ID = KnownTagCodec.tagId(ENV_SERIAL, 3, ENV); - - public static final int DJM_ENABLED_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 4; - public static final long DJM_ENABLED_ID = - KnownTagCodec.tagId(DJM_ENABLED_SERIAL, 4, DDTags.DJM_ENABLED); - - public static final int DSM_ENABLED_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 5; - public static final long DSM_ENABLED_ID = - KnownTagCodec.tagId(DSM_ENABLED_SERIAL, 5, DDTags.DSM_ENABLED); - - // common tags added by the tag post-processors (RemoteHostnameAdder / IntegrationAdder / - // ServiceNameSourceAdder). Not intercepted; stored. - public static final int TRACER_HOST_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 6; - public static final long TRACER_HOST_ID = - KnownTagCodec.tagId(TRACER_HOST_SERIAL, 6, DDTags.TRACER_HOST); - - public static final int INTEGRATION_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 7; - public static final long INTEGRATION_ID = - KnownTagCodec.tagId(INTEGRATION_SERIAL, 7, DDTags.DD_INTEGRATION); - - public static final int SVC_SRC_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 8; - public static final long SVC_SRC_ID = KnownTagCodec.tagId(SVC_SRC_SERIAL, 8, DDTags.DD_SVC_SRC); - - // peer.service tags, read/written by PeerServiceCalculator (post-processor; uses Map put/get that - // bypass the interceptor). peer.service is intercepted on the set-path but STORED, so it slots. - public static final int PEER_SERVICE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 9; - public static final long PEER_SERVICE_ID = - KnownTagCodec.intercepted(KnownTagCodec.tagId(PEER_SERVICE_SERIAL, 9, Tags.PEER_SERVICE)); - - public static final int PEER_SERVICE_REMAPPED_FROM_SERIAL = - KnownTagCodec.FIRST_STORED_SERIAL + 10; - public static final long PEER_SERVICE_REMAPPED_FROM_ID = - KnownTagCodec.tagId(PEER_SERVICE_REMAPPED_FROM_SERIAL, 10, DDTags.PEER_SERVICE_REMAPPED_FROM); - - // HTTP tags read by HttpEndpointPostProcessor. http.method/http.url are intercepted-but-stored - // (interceptTag side-effects then returns false → stored); http.route is not intercepted. All - // stored, so the string set-path slots them via keyOf and the id reads here find them. - public static final int HTTP_METHOD_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 11; - public static final long HTTP_METHOD_ID = - KnownTagCodec.intercepted(KnownTagCodec.tagId(HTTP_METHOD_SERIAL, 11, Tags.HTTP_METHOD)); - - public static final int HTTP_ROUTE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 12; - public static final long HTTP_ROUTE_ID = - KnownTagCodec.tagId(HTTP_ROUTE_SERIAL, 12, Tags.HTTP_ROUTE); - - public static final int HTTP_URL_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 13; - public static final long HTTP_URL_ID = - KnownTagCodec.intercepted(KnownTagCodec.tagId(HTTP_URL_SERIAL, 13, Tags.HTTP_URL)); - - // peer connection tags set by BaseDecorator.onPeerConnection on ~every client/producer span. - // Not intercepted; stored. Slotted (common across client instrumentations). - public static final int PEER_HOSTNAME_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 14; - public static final long PEER_HOSTNAME_ID = - KnownTagCodec.tagId(PEER_HOSTNAME_SERIAL, 14, Tags.PEER_HOSTNAME); - - public static final int PEER_HOST_IPV4_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 15; - public static final long PEER_HOST_IPV4_ID = - KnownTagCodec.tagId(PEER_HOST_IPV4_SERIAL, 15, Tags.PEER_HOST_IPV4); - - public static final int PEER_HOST_IPV6_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 16; - public static final long PEER_HOST_IPV6_ID = - KnownTagCodec.tagId(PEER_HOST_IPV6_SERIAL, 16, Tags.PEER_HOST_IPV6); - - public static final int PEER_PORT_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 17; - public static final long PEER_PORT_ID = KnownTagCodec.tagId(PEER_PORT_SERIAL, 17, Tags.PEER_PORT); - - // Universal decorator tags — set on ~every span (component/span.kind via Base/Server/Client - // decorators, language via ServerDecorator). span.kind is intercepted (setSpanKindOrdinal). - public static final int COMPONENT_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 18; - public static final long COMPONENT_ID = KnownTagCodec.tagId(COMPONENT_SERIAL, 18, Tags.COMPONENT); - - public static final int SPAN_KIND_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 19; - public static final long SPAN_KIND_ID = - KnownTagCodec.intercepted(KnownTagCodec.tagId(SPAN_KIND_SERIAL, 19, Tags.SPAN_KIND)); - - public static final int LANGUAGE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 20; - public static final long LANGUAGE_ID = - KnownTagCodec.tagId(LANGUAGE_SERIAL, 20, DDTags.LANGUAGE_TAG_KEY); - - // JDBC / database-client tags — set on every db span (58% of petclinic spans). Not intercepted - // (only db.statement is, and that's handled separately). - public static final int DB_TYPE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 21; - public static final long DB_TYPE_ID = KnownTagCodec.tagId(DB_TYPE_SERIAL, 21, Tags.DB_TYPE); - - public static final int DB_INSTANCE_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 22; - public static final long DB_INSTANCE_ID = - KnownTagCodec.tagId(DB_INSTANCE_SERIAL, 22, Tags.DB_INSTANCE); - - public static final int DB_USER_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 23; - public static final long DB_USER_ID = KnownTagCodec.tagId(DB_USER_SERIAL, 23, Tags.DB_USER); - - public static final int DB_OPERATION_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 24; - public static final long DB_OPERATION_ID = - KnownTagCodec.tagId(DB_OPERATION_SERIAL, 24, Tags.DB_OPERATION); - - public static final int DB_POOL_NAME_SERIAL = KnownTagCodec.FIRST_STORED_SERIAL + 25; - public static final long DB_POOL_NAME_ID = - KnownTagCodec.tagId(DB_POOL_NAME_SERIAL, 25, Tags.DB_POOL_NAME); - - // Open-addressed name -> id table backing keyOf (data, not a switch): scales flat as the known - // set grows, where a generated switch eventually falls off the inline threshold. KEYOF_NAMES and - // KEYOF_VALUES are parallel; the table places names by hash and a parallel ids[] by slot. + static final int SLOT_COUNT = 16; + + // ---- reserved (routed to span fields or directives; not stored) ---- + public static final String ERROR_NAME = "error"; + public static final long ERROR_ID = 0x8001FFFF00000000L; + // makeTagId(serial=1, slot=NO_SLOT) + intercepted [structural -> error] + + public static final String SERVICE_NAME = "service"; + public static final long SERVICE_ID = 0x8002FFFF00000000L; + // makeTagId(serial=2, slot=NO_SLOT) + intercepted [structural -> service] + + public static final String RESOURCE_NAME = "resource.name"; + public static final long RESOURCE_NAME_ID = 0x8003FFFF00000000L; + // makeTagId(serial=3, slot=NO_SLOT) + intercepted [structural -> resource] + + public static final String SPAN_TYPE_NAME = "span.type"; + public static final long SPAN_TYPE_ID = 0x8004FFFF00000000L; + // makeTagId(serial=4, slot=NO_SLOT) + intercepted [structural -> type] + + public static final String ORIGIN_NAME = "origin"; + public static final long ORIGIN_ID = 0x8005FFFF00000000L; + // makeTagId(serial=5, slot=NO_SLOT) + intercepted [structural -> origin] + + public static final String SAMPLING_PRIORITY_NAME = "sampling.priority"; + public static final long SAMPLING_PRIORITY_ID = 0x8006FFFF00000000L; + // makeTagId(serial=6, slot=NO_SLOT) + intercepted [directive] + + public static final String MANUAL_KEEP_NAME = "manual.keep"; + public static final long MANUAL_KEEP_ID = 0x8007FFFF00000000L; + // makeTagId(serial=7, slot=NO_SLOT) + intercepted [directive] + + public static final String MANUAL_DROP_NAME = "manual.drop"; + public static final long MANUAL_DROP_ID = 0x8008FFFF00000000L; + // makeTagId(serial=8, slot=NO_SLOT) + intercepted [directive] + + public static final String MEASURED_NAME = "measured"; + public static final long MEASURED_ID = 0x8009FFFF00000000L; + // makeTagId(serial=9, slot=NO_SLOT) + intercepted [directive] + + public static final String ANALYTICS_SAMPLE_RATE_NAME = "analytics.sample_rate"; + public static final long ANALYTICS_SAMPLE_RATE_ID = 0x800AFFFF00000000L; + // makeTagId(serial=10, slot=NO_SLOT) + intercepted [directive] + + // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ---- + public static final String DD_APPSEC_ENABLED_NAME = "_dd.appsec.enabled"; + public static final long DD_APPSEC_ENABLED_ID = 0x0100000000000004L; + // makeTagId(serial=256, slot=0) + trace-level + + public static final String DD_BASE_SERVICE_NAME = "_dd.base_service"; + public static final long DD_BASE_SERVICE_ID = 0x0101000100000004L; + // makeTagId(serial=257, slot=1) + trace-level + + public static final String DD_CIVISIBILITY_ENABLED_NAME = "_dd.civisibility.enabled"; + public static final long DD_CIVISIBILITY_ENABLED_ID = 0x0102000200000004L; + // makeTagId(serial=258, slot=2) + trace-level + + public static final String DD_DJM_ENABLED_NAME = "_dd.djm.enabled"; + public static final long DD_DJM_ENABLED_ID = 0x0103000300000004L; + // makeTagId(serial=259, slot=3) + trace-level + + public static final String DD_DSM_ENABLED_NAME = "_dd.dsm.enabled"; + public static final long DD_DSM_ENABLED_ID = 0x0104000400000004L; + // makeTagId(serial=260, slot=4) + trace-level + + public static final String DD_GIT_COMMIT_SHA_NAME = "_dd.git.commit.sha"; + public static final long DD_GIT_COMMIT_SHA_ID = 0x0105000500000004L; + // makeTagId(serial=261, slot=5) + trace-level + + public static final String DD_GIT_REPOSITORY_URL_NAME = "_dd.git.repository_url"; + public static final long DD_GIT_REPOSITORY_URL_ID = 0x0106000600000004L; + // makeTagId(serial=262, slot=6) + trace-level + + public static final String DD_INTEGRATION_NAME = "_dd.integration"; + public static final long DD_INTEGRATION_ID = 0x0107000000000000L; + // makeTagId(serial=263, slot=0) + + public static final String DD_PARENT_ID_NAME = "_dd.parent_id"; + public static final long DD_PARENT_ID = 0x0108000100000000L; + // makeTagId(serial=264, slot=1) + + public static final String DD_PEER_SERVICE_REMAPPED_FROM_NAME = "_dd.peer.service.remapped_from"; + public static final long DD_PEER_SERVICE_REMAPPED_FROM_ID = 0x0109000700000000L; + // makeTagId(serial=265, slot=7) + + public static final String DD_PEER_SERVICE_SOURCE_NAME = "_dd.peer.service.source"; + public static final long DD_PEER_SERVICE_SOURCE_ID = 0x010A000800000000L; + // makeTagId(serial=266, slot=8) + + public static final String DD_PROFILING_ENABLED_NAME = "_dd.profiling.enabled"; + public static final long DD_PROFILING_ENABLED_ID = 0x010B000700000004L; + // makeTagId(serial=267, slot=7) + trace-level + + public static final String DD_SVC_SRC_NAME = "_dd.svc_src"; + public static final long DD_SVC_SRC_ID = 0x010CFFFF00000000L; + // makeTagId(serial=268, slot=NO_SLOT) + + public static final String DD_TRACER_HOST_NAME = "_dd.tracer_host"; + public static final long DD_TRACER_HOST_ID = 0x010D000800000004L; + // makeTagId(serial=269, slot=8) + trace-level + + public static final String COMPONENT_NAME = "component"; + public static final long COMPONENT_ID = 0x010E000200000000L; + // makeTagId(serial=270, slot=2) + + public static final String DB_INSTANCE_NAME = "db.instance"; + public static final long DB_INSTANCE_ID = 0x010F000900000000L; + // makeTagId(serial=271, slot=9) + + public static final String DB_OPERATION_NAME = "db.operation"; + public static final long DB_OPERATION_ID = 0x0110000A00000000L; + // makeTagId(serial=272, slot=10) + + public static final String DB_POOL_NAME = "db.pool.name"; + public static final long DB_POOL_NAME_ID = 0x0111FFFF00000000L; + // makeTagId(serial=273, slot=NO_SLOT) + + public static final String DB_STATEMENT_NAME = "db.statement"; + public static final long DB_STATEMENT_ID = 0x8112000B00000000L; + // makeTagId(serial=274, slot=11) + intercepted + + public static final String DB_TYPE_NAME = "db.type"; + public static final long DB_TYPE_ID = 0x0113000C00000000L; + // makeTagId(serial=275, slot=12) + + public static final String DB_USER_NAME = "db.user"; + public static final long DB_USER_ID = 0x0114000F00000000L; + // makeTagId(serial=276, slot=15) + + public static final String ENV_NAME = "env"; + public static final long ENV_ID = 0x0115000900000004L; + // makeTagId(serial=277, slot=9) + trace-level + + public static final String ERROR_MESSAGE_NAME = "error.message"; + public static final long ERROR_MESSAGE_ID = 0x0116000300000000L; + // makeTagId(serial=278, slot=3) + + public static final String ERROR_STACK_NAME = "error.stack"; + public static final long ERROR_STACK_ID = 0x0117000400000000L; + // makeTagId(serial=279, slot=4) + + public static final String ERROR_TYPE_NAME = "error.type"; + public static final long ERROR_TYPE_ID = 0x0118000500000000L; + // makeTagId(serial=280, slot=5) + + public static final String HTTP_HOSTNAME_NAME = "http.hostname"; + public static final long HTTP_HOSTNAME_ID = 0x0119000700000000L; + // makeTagId(serial=281, slot=7) + + public static final String HTTP_METHOD_NAME = "http.method"; + public static final long HTTP_METHOD_ID = 0x811A000900000000L; + // makeTagId(serial=282, slot=9) + intercepted + + public static final String HTTP_QUERY_STRING_NAME = "http.query.string"; + public static final long HTTP_QUERY_STRING_ID = 0x011B000800000000L; + // makeTagId(serial=283, slot=8) + + public static final String HTTP_RESEND_COUNT_NAME = "http.resend_count"; + public static final long HTTP_RESEND_COUNT_ID = 0x011C000F00000000L; + // makeTagId(serial=284, slot=15) + + public static final String HTTP_ROUTE_NAME = "http.route"; + public static final long HTTP_ROUTE_ID = 0x011D000D00000000L; + // makeTagId(serial=285, slot=13) + + public static final String HTTP_STATUS_CODE_NAME = "http.status_code"; + public static final long HTTP_STATUS_CODE_ID = 0x011E000A00000000L; + // makeTagId(serial=286, slot=10) + + public static final String HTTP_URL_NAME = "http.url"; + public static final long HTTP_URL_ID = 0x811F000B00000000L; + // makeTagId(serial=287, slot=11) + intercepted + + public static final String HTTP_USERAGENT_NAME = "http.useragent"; + public static final long HTTP_USERAGENT_ID = 0x0120000E00000000L; + // makeTagId(serial=288, slot=14) + + public static final String LANGUAGE_NAME = "language"; + public static final long LANGUAGE_ID = 0x0121000A00000004L; + // makeTagId(serial=289, slot=10) + trace-level + + public static final String NETWORK_PROTOCOL_VERSION_NAME = "network.protocol.version"; + public static final long NETWORK_PROTOCOL_VERSION_ID = 0x0122000C00000000L; + // makeTagId(serial=290, slot=12) + + public static final String PEER_HOSTNAME_NAME = "peer.hostname"; + public static final long PEER_HOSTNAME_ID = 0x0123000D00000000L; + // makeTagId(serial=291, slot=13) + + public static final String PEER_IPV4_NAME = "peer.ipv4"; + public static final long PEER_IPV4_ID = 0x0124FFFF00000000L; + // makeTagId(serial=292, slot=NO_SLOT) + + public static final String PEER_IPV6_NAME = "peer.ipv6"; + public static final long PEER_IPV6_ID = 0x0125FFFF00000000L; + // makeTagId(serial=293, slot=NO_SLOT) + + public static final String PEER_PORT_NAME = "peer.port"; + public static final long PEER_PORT_ID = 0x0126FFFF00000000L; + // makeTagId(serial=294, slot=NO_SLOT) + + public static final String PEER_SERVICE_NAME = "peer.service"; + public static final long PEER_SERVICE_ID = 0x8127000E00000000L; + // makeTagId(serial=295, slot=14) + intercepted + + public static final String RUNTIME_ID_NAME = "runtime-id"; + public static final long RUNTIME_ID = 0x0128000B00000004L; + // makeTagId(serial=296, slot=11) + trace-level + + public static final String SERVLET_CONTEXT_NAME = "servlet.context"; + public static final long SERVLET_CONTEXT_ID = 0x8129FFFF00000000L; + // makeTagId(serial=297, slot=NO_SLOT) + intercepted + + public static final String SERVLET_PATH_NAME = "servlet.path"; + public static final long SERVLET_PATH_ID = 0x012AFFFF00000000L; + // makeTagId(serial=298, slot=NO_SLOT) + + public static final String SPAN_KIND_NAME = "span.kind"; + public static final long SPAN_KIND_ID = 0x812B000600000000L; + // makeTagId(serial=299, slot=6) + intercepted + + public static final String VERSION_NAME = "version"; + public static final long VERSION_ID = 0x012C000C00000004L; + // makeTagId(serial=300, slot=12) + trace-level + + public static final String VIEW_NAME = "view.name"; + public static final long VIEW_NAME_ID = 0x012D000700000000L; + // makeTagId(serial=301, slot=7) + + // ---- serial numbers ---- + static final int ERROR_SERIAL_NUM = 1; + static final int SERVICE_SERIAL_NUM = 2; + static final int RESOURCE_NAME_SERIAL_NUM = 3; + static final int SPAN_TYPE_SERIAL_NUM = 4; + static final int ORIGIN_SERIAL_NUM = 5; + static final int SAMPLING_PRIORITY_SERIAL_NUM = 6; + static final int MANUAL_KEEP_SERIAL_NUM = 7; + static final int MANUAL_DROP_SERIAL_NUM = 8; + static final int MEASURED_SERIAL_NUM = 9; + static final int ANALYTICS_SAMPLE_RATE_SERIAL_NUM = 10; + static final int DD_APPSEC_ENABLED_SERIAL_NUM = 256; + static final int DD_BASE_SERVICE_SERIAL_NUM = 257; + static final int DD_CIVISIBILITY_ENABLED_SERIAL_NUM = 258; + static final int DD_DJM_ENABLED_SERIAL_NUM = 259; + static final int DD_DSM_ENABLED_SERIAL_NUM = 260; + static final int DD_GIT_COMMIT_SHA_SERIAL_NUM = 261; + static final int DD_GIT_REPOSITORY_URL_SERIAL_NUM = 262; + static final int DD_INTEGRATION_SERIAL_NUM = 263; + static final int DD_PARENT_ID_SERIAL_NUM = 264; + static final int DD_PEER_SERVICE_REMAPPED_FROM_SERIAL_NUM = 265; + static final int DD_PEER_SERVICE_SOURCE_SERIAL_NUM = 266; + static final int DD_PROFILING_ENABLED_SERIAL_NUM = 267; + static final int DD_SVC_SRC_SERIAL_NUM = 268; + static final int DD_TRACER_HOST_SERIAL_NUM = 269; + static final int COMPONENT_SERIAL_NUM = 270; + static final int DB_INSTANCE_SERIAL_NUM = 271; + static final int DB_OPERATION_SERIAL_NUM = 272; + static final int DB_POOL_NAME_SERIAL_NUM = 273; + static final int DB_STATEMENT_SERIAL_NUM = 274; + static final int DB_TYPE_SERIAL_NUM = 275; + static final int DB_USER_SERIAL_NUM = 276; + static final int ENV_SERIAL_NUM = 277; + static final int ERROR_MESSAGE_SERIAL_NUM = 278; + static final int ERROR_STACK_SERIAL_NUM = 279; + static final int ERROR_TYPE_SERIAL_NUM = 280; + static final int HTTP_HOSTNAME_SERIAL_NUM = 281; + static final int HTTP_METHOD_SERIAL_NUM = 282; + static final int HTTP_QUERY_STRING_SERIAL_NUM = 283; + static final int HTTP_RESEND_COUNT_SERIAL_NUM = 284; + static final int HTTP_ROUTE_SERIAL_NUM = 285; + static final int HTTP_STATUS_CODE_SERIAL_NUM = 286; + static final int HTTP_URL_SERIAL_NUM = 287; + static final int HTTP_USERAGENT_SERIAL_NUM = 288; + static final int LANGUAGE_SERIAL_NUM = 289; + static final int NETWORK_PROTOCOL_VERSION_SERIAL_NUM = 290; + static final int PEER_HOSTNAME_SERIAL_NUM = 291; + static final int PEER_IPV4_SERIAL_NUM = 292; + static final int PEER_IPV6_SERIAL_NUM = 293; + static final int PEER_PORT_SERIAL_NUM = 294; + static final int PEER_SERVICE_SERIAL_NUM = 295; + static final int RUNTIME_ID_SERIAL_NUM = 296; + static final int SERVLET_CONTEXT_SERIAL_NUM = 297; + static final int SERVLET_PATH_SERIAL_NUM = 298; + static final int SPAN_KIND_SERIAL_NUM = 299; + static final int VERSION_SERIAL_NUM = 300; + static final int VIEW_NAME_SERIAL_NUM = 301; + private static final String[] KEYOF_NAMES = { - Tags.ERROR, - DDTags.PARENT_ID, - DDTags.BASE_SERVICE, - Tags.VERSION, - ENV, - DDTags.DJM_ENABLED, - DDTags.DSM_ENABLED, - DDTags.TRACER_HOST, - DDTags.DD_INTEGRATION, - DDTags.DD_SVC_SRC, - Tags.PEER_SERVICE, - DDTags.PEER_SERVICE_REMAPPED_FROM, - Tags.HTTP_METHOD, - Tags.HTTP_ROUTE, - Tags.HTTP_URL, - Tags.PEER_HOSTNAME, - Tags.PEER_HOST_IPV4, - Tags.PEER_HOST_IPV6, - Tags.PEER_PORT, - Tags.COMPONENT, - Tags.SPAN_KIND, - DDTags.LANGUAGE_TAG_KEY, - Tags.DB_TYPE, - Tags.DB_INSTANCE, - Tags.DB_USER, - Tags.DB_OPERATION, - Tags.DB_POOL_NAME, + ERROR_NAME, + SERVICE_NAME, + RESOURCE_NAME, + SPAN_TYPE_NAME, + ORIGIN_NAME, + SAMPLING_PRIORITY_NAME, + MANUAL_KEEP_NAME, + MANUAL_DROP_NAME, + MEASURED_NAME, + ANALYTICS_SAMPLE_RATE_NAME, + DD_APPSEC_ENABLED_NAME, + DD_BASE_SERVICE_NAME, + DD_CIVISIBILITY_ENABLED_NAME, + DD_DJM_ENABLED_NAME, + DD_DSM_ENABLED_NAME, + DD_GIT_COMMIT_SHA_NAME, + DD_GIT_REPOSITORY_URL_NAME, + DD_INTEGRATION_NAME, + DD_PARENT_ID_NAME, + DD_PEER_SERVICE_REMAPPED_FROM_NAME, + DD_PEER_SERVICE_SOURCE_NAME, + DD_PROFILING_ENABLED_NAME, + DD_SVC_SRC_NAME, + DD_TRACER_HOST_NAME, + COMPONENT_NAME, + DB_INSTANCE_NAME, + DB_OPERATION_NAME, + DB_POOL_NAME, + DB_STATEMENT_NAME, + DB_TYPE_NAME, + DB_USER_NAME, + ENV_NAME, + ERROR_MESSAGE_NAME, + ERROR_STACK_NAME, + ERROR_TYPE_NAME, + HTTP_HOSTNAME_NAME, + HTTP_METHOD_NAME, + HTTP_QUERY_STRING_NAME, + HTTP_RESEND_COUNT_NAME, + HTTP_ROUTE_NAME, + HTTP_STATUS_CODE_NAME, + HTTP_URL_NAME, + HTTP_USERAGENT_NAME, + LANGUAGE_NAME, + NETWORK_PROTOCOL_VERSION_NAME, + PEER_HOSTNAME_NAME, + PEER_IPV4_NAME, + PEER_IPV6_NAME, + PEER_PORT_NAME, + PEER_SERVICE_NAME, + RUNTIME_ID_NAME, + SERVLET_CONTEXT_NAME, + SERVLET_PATH_NAME, + SPAN_KIND_NAME, + VERSION_NAME, + VIEW_NAME, }; - private static final long[] KEYOF_VALUES = { ERROR_ID, - PARENT_ID, - BASE_SERVICE_ID, - VERSION_ID, + SERVICE_ID, + RESOURCE_NAME_ID, + SPAN_TYPE_ID, + ORIGIN_ID, + SAMPLING_PRIORITY_ID, + MANUAL_KEEP_ID, + MANUAL_DROP_ID, + MEASURED_ID, + ANALYTICS_SAMPLE_RATE_ID, + DD_APPSEC_ENABLED_ID, + DD_BASE_SERVICE_ID, + DD_CIVISIBILITY_ENABLED_ID, + DD_DJM_ENABLED_ID, + DD_DSM_ENABLED_ID, + DD_GIT_COMMIT_SHA_ID, + DD_GIT_REPOSITORY_URL_ID, + DD_INTEGRATION_ID, + DD_PARENT_ID, + DD_PEER_SERVICE_REMAPPED_FROM_ID, + DD_PEER_SERVICE_SOURCE_ID, + DD_PROFILING_ENABLED_ID, + DD_SVC_SRC_ID, + DD_TRACER_HOST_ID, + COMPONENT_ID, + DB_INSTANCE_ID, + DB_OPERATION_ID, + DB_POOL_NAME_ID, + DB_STATEMENT_ID, + DB_TYPE_ID, + DB_USER_ID, ENV_ID, - DJM_ENABLED_ID, - DSM_ENABLED_ID, - TRACER_HOST_ID, - INTEGRATION_ID, - SVC_SRC_ID, - PEER_SERVICE_ID, - PEER_SERVICE_REMAPPED_FROM_ID, + ERROR_MESSAGE_ID, + ERROR_STACK_ID, + ERROR_TYPE_ID, + HTTP_HOSTNAME_ID, HTTP_METHOD_ID, + HTTP_QUERY_STRING_ID, + HTTP_RESEND_COUNT_ID, HTTP_ROUTE_ID, + HTTP_STATUS_CODE_ID, HTTP_URL_ID, + HTTP_USERAGENT_ID, + LANGUAGE_ID, + NETWORK_PROTOCOL_VERSION_ID, PEER_HOSTNAME_ID, - PEER_HOST_IPV4_ID, - PEER_HOST_IPV6_ID, + PEER_IPV4_ID, + PEER_IPV6_ID, PEER_PORT_ID, - COMPONENT_ID, + PEER_SERVICE_ID, + RUNTIME_ID, + SERVLET_CONTEXT_ID, + SERVLET_PATH_ID, SPAN_KIND_ID, - LANGUAGE_ID, - DB_TYPE_ID, - DB_INSTANCE_ID, - DB_USER_ID, - DB_OPERATION_ID, - DB_POOL_NAME_ID, + VERSION_ID, + VIEW_NAME_ID, }; - - // Static-final raw arrays placed by StringIndex.EmbeddingSupport: the JIT folds these refs to - // constants on - // the keyOf hot path (the fastest of StringIndex's three usage modes — no instance dereference). private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; private static final long[] KEYOF_IDS; @@ -240,61 +427,119 @@ public final class KnownTags { new KnownTagCodec.Resolver() { @Override public String nameOf(long tagId) { - switch (KnownTagCodec.globalSerial(tagId)) { - case ERROR_SERIAL: - return Tags.ERROR; - case PARENT_ID_SERIAL: - return DDTags.PARENT_ID; - case BASE_SERVICE_SERIAL: - return DDTags.BASE_SERVICE; - case VERSION_SERIAL: - return Tags.VERSION; - case ENV_SERIAL: - return ENV; - case DJM_ENABLED_SERIAL: - return DDTags.DJM_ENABLED; - case DSM_ENABLED_SERIAL: - return DDTags.DSM_ENABLED; - case TRACER_HOST_SERIAL: - return DDTags.TRACER_HOST; - case INTEGRATION_SERIAL: - return DDTags.DD_INTEGRATION; - case SVC_SRC_SERIAL: - return DDTags.DD_SVC_SRC; - case PEER_SERVICE_SERIAL: - return Tags.PEER_SERVICE; - case PEER_SERVICE_REMAPPED_FROM_SERIAL: - return DDTags.PEER_SERVICE_REMAPPED_FROM; - case HTTP_METHOD_SERIAL: - return Tags.HTTP_METHOD; - case HTTP_ROUTE_SERIAL: - return Tags.HTTP_ROUTE; - case HTTP_URL_SERIAL: - return Tags.HTTP_URL; - case PEER_HOSTNAME_SERIAL: - return Tags.PEER_HOSTNAME; - case PEER_HOST_IPV4_SERIAL: - return Tags.PEER_HOST_IPV4; - case PEER_HOST_IPV6_SERIAL: - return Tags.PEER_HOST_IPV6; - case PEER_PORT_SERIAL: - return Tags.PEER_PORT; - case COMPONENT_SERIAL: - return Tags.COMPONENT; - case SPAN_KIND_SERIAL: - return Tags.SPAN_KIND; - case LANGUAGE_SERIAL: - return DDTags.LANGUAGE_TAG_KEY; - case DB_TYPE_SERIAL: - return Tags.DB_TYPE; - case DB_INSTANCE_SERIAL: - return Tags.DB_INSTANCE; - case DB_USER_SERIAL: - return Tags.DB_USER; - case DB_OPERATION_SERIAL: - return Tags.DB_OPERATION; - case DB_POOL_NAME_SERIAL: - return Tags.DB_POOL_NAME; + switch (KnownTagCodec.serialNum(tagId)) { + case ERROR_SERIAL_NUM: + return ERROR_NAME; + case SERVICE_SERIAL_NUM: + return SERVICE_NAME; + case RESOURCE_NAME_SERIAL_NUM: + return RESOURCE_NAME; + case SPAN_TYPE_SERIAL_NUM: + return SPAN_TYPE_NAME; + case ORIGIN_SERIAL_NUM: + return ORIGIN_NAME; + case SAMPLING_PRIORITY_SERIAL_NUM: + return SAMPLING_PRIORITY_NAME; + case MANUAL_KEEP_SERIAL_NUM: + return MANUAL_KEEP_NAME; + case MANUAL_DROP_SERIAL_NUM: + return MANUAL_DROP_NAME; + case MEASURED_SERIAL_NUM: + return MEASURED_NAME; + case ANALYTICS_SAMPLE_RATE_SERIAL_NUM: + return ANALYTICS_SAMPLE_RATE_NAME; + case DD_APPSEC_ENABLED_SERIAL_NUM: + return DD_APPSEC_ENABLED_NAME; + case DD_BASE_SERVICE_SERIAL_NUM: + return DD_BASE_SERVICE_NAME; + case DD_CIVISIBILITY_ENABLED_SERIAL_NUM: + return DD_CIVISIBILITY_ENABLED_NAME; + case DD_DJM_ENABLED_SERIAL_NUM: + return DD_DJM_ENABLED_NAME; + case DD_DSM_ENABLED_SERIAL_NUM: + return DD_DSM_ENABLED_NAME; + case DD_GIT_COMMIT_SHA_SERIAL_NUM: + return DD_GIT_COMMIT_SHA_NAME; + case DD_GIT_REPOSITORY_URL_SERIAL_NUM: + return DD_GIT_REPOSITORY_URL_NAME; + case DD_INTEGRATION_SERIAL_NUM: + return DD_INTEGRATION_NAME; + case DD_PARENT_ID_SERIAL_NUM: + return DD_PARENT_ID_NAME; + case DD_PEER_SERVICE_REMAPPED_FROM_SERIAL_NUM: + return DD_PEER_SERVICE_REMAPPED_FROM_NAME; + case DD_PEER_SERVICE_SOURCE_SERIAL_NUM: + return DD_PEER_SERVICE_SOURCE_NAME; + case DD_PROFILING_ENABLED_SERIAL_NUM: + return DD_PROFILING_ENABLED_NAME; + case DD_SVC_SRC_SERIAL_NUM: + return DD_SVC_SRC_NAME; + case DD_TRACER_HOST_SERIAL_NUM: + return DD_TRACER_HOST_NAME; + case COMPONENT_SERIAL_NUM: + return COMPONENT_NAME; + case DB_INSTANCE_SERIAL_NUM: + return DB_INSTANCE_NAME; + case DB_OPERATION_SERIAL_NUM: + return DB_OPERATION_NAME; + case DB_POOL_NAME_SERIAL_NUM: + return DB_POOL_NAME; + case DB_STATEMENT_SERIAL_NUM: + return DB_STATEMENT_NAME; + case DB_TYPE_SERIAL_NUM: + return DB_TYPE_NAME; + case DB_USER_SERIAL_NUM: + return DB_USER_NAME; + case ENV_SERIAL_NUM: + return ENV_NAME; + case ERROR_MESSAGE_SERIAL_NUM: + return ERROR_MESSAGE_NAME; + case ERROR_STACK_SERIAL_NUM: + return ERROR_STACK_NAME; + case ERROR_TYPE_SERIAL_NUM: + return ERROR_TYPE_NAME; + case HTTP_HOSTNAME_SERIAL_NUM: + return HTTP_HOSTNAME_NAME; + case HTTP_METHOD_SERIAL_NUM: + return HTTP_METHOD_NAME; + case HTTP_QUERY_STRING_SERIAL_NUM: + return HTTP_QUERY_STRING_NAME; + case HTTP_RESEND_COUNT_SERIAL_NUM: + return HTTP_RESEND_COUNT_NAME; + case HTTP_ROUTE_SERIAL_NUM: + return HTTP_ROUTE_NAME; + case HTTP_STATUS_CODE_SERIAL_NUM: + return HTTP_STATUS_CODE_NAME; + case HTTP_URL_SERIAL_NUM: + return HTTP_URL_NAME; + case HTTP_USERAGENT_SERIAL_NUM: + return HTTP_USERAGENT_NAME; + case LANGUAGE_SERIAL_NUM: + return LANGUAGE_NAME; + case NETWORK_PROTOCOL_VERSION_SERIAL_NUM: + return NETWORK_PROTOCOL_VERSION_NAME; + case PEER_HOSTNAME_SERIAL_NUM: + return PEER_HOSTNAME_NAME; + case PEER_IPV4_SERIAL_NUM: + return PEER_IPV4_NAME; + case PEER_IPV6_SERIAL_NUM: + return PEER_IPV6_NAME; + case PEER_PORT_SERIAL_NUM: + return PEER_PORT_NAME; + case PEER_SERVICE_SERIAL_NUM: + return PEER_SERVICE_NAME; + case RUNTIME_ID_SERIAL_NUM: + return RUNTIME_ID_NAME; + case SERVLET_CONTEXT_SERIAL_NUM: + return SERVLET_CONTEXT_NAME; + case SERVLET_PATH_SERIAL_NUM: + return SERVLET_PATH_NAME; + case SPAN_KIND_SERIAL_NUM: + return SPAN_KIND_NAME; + case VERSION_SERIAL_NUM: + return VERSION_NAME; + case VIEW_NAME_SERIAL_NUM: + return VIEW_NAME; default: return null; } @@ -316,12 +561,7 @@ public long keyOf(String name) { KnownTagCodec.register(RESOLVER); } - /** - * Forces resolver registration. Merely invoking this static method runs {@code } (which - * registers {@link #RESOLVER}), so calling it once at tracer init flips the dense store live; - * idempotent. Until something references this class the registry stays dormant and {@code keyOf} - * returns 0, so tag storage is byte-identical to the bucket-only behavior. - */ + /** Forces resolver registration by triggering . Idempotent. */ public static void init() {} private KnownTags() {} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 7c30f373881..849c4db9911 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -1052,6 +1052,37 @@ public EntryChange next() { private Object[] knownValues; private int knownCount; + /** + * Single-tier presence filter over the dense store — the fast path that lets a definitely-absent + * known tag append in O(1) instead of paying the {@link #knownIndexOf} scan (the common per-build + * insert). Each tag's id carries a globally stable {@code slot} from graph coloring (see {@link + * KnownTagCodec}); a tag is present ONLY IF its slot bit is set in {@link #knownOccupancy}. A + * clear bit ⟹ definitely absent ⟹ skip the scan. + * + *

Because co-occurring tags always get distinct slots (they form a clique in the coloring), + * slotCount is bounded by the largest clique (≤ 64) and every present tag of a well-formed map + * has its own bit — so one {@code long} is the whole filter, collapsing the earlier two-tier + * (group mask + field bloom) design to a single word. Disjoint occupancy across two maps ({@code + * (a.knownOccupancy & b.knownOccupancy) == 0}) proves nothing shadows across them, which the + * read-through shadow check exploits (see {@link #parentDenseVisible}). + * + *

Superset semantics: bits are set on every add and NEVER cleared on remove (a stale bit only + * costs a scan, never a wrong answer), so correctness never depends on the slot→bit collision + * rate — only the fast-path hit rate does. Unslotted stored tags ({@link KnownTagCodec#NO_SLOT}) + * all fold onto one shared bit ({@code slot & 63}); the scan stays authoritative for them. + */ + private long knownOccupancy; + + /** + * Whether this map holds any trace-level known tag ({@link KnownTagCodec#isTraceLevel}). Trace + * and span tags reuse the same slots, so {@link #knownOccupancy} can't tell the two levels apart; + * this flag can. A span map leaves it {@code false}, which lets {@link #parentDenseVisible} skip + * the shadow check when enumerating a trace-level ancestor entry — a map with no trace-level tags + * cannot shadow one. Superset semantics like the occupancy mask: set on add, never cleared on + * remove (a stale {@code true} only costs a scan, never a wrong answer). + */ + private boolean knownTraceLevel; + private static final int KNOWN_INIT_CAP = 12; // generous per-type max stopgap; exact per-type sizing comes with the tag registry @@ -1392,27 +1423,54 @@ private void ensureKnownCapacity() { } } + /** + * Presence bit for {@code tagId}: {@code 1L << slot}. Colored slots are < 64; unslotted stored + * tags ({@link KnownTagCodec#NO_SLOT}) fold onto one shared bit via {@code slot & 63} — crude for + * them, but the scan stays authoritative. + */ + private static long knownSlotBit(long tagId) { + return 1L << (KnownTagCodec.slot(tagId) & 63); + } + + /** + * Whether {@code tagId} MAY be present in the dense store (its slot bit is set), vs DEFINITELY + * absent (the bit is clear ⟹ skip the scan). + */ + private boolean knownMaybePresent(long tagId) { + return (this.knownOccupancy & knownSlotBit(tagId)) != 0; + } + /** * Stores a known tag's value densely (no {@link Entry} alloc). Overwrites in place when present * (returning the prior value materialized as an Entry, per the {@code Map} contract — usually - * discarded by {@code set}); otherwise appends, growing x2 as needed. + * discarded by {@code set}); otherwise appends, growing x2 as needed. The occupancy presence + * filter skips the {@link #knownIndexOf} scan when the tag is definitely absent (the common + * per-build case), so an append is O(1) instead of O(n). */ private Entry putKnownValue(long tagId, Object value) { - int i = this.knownIndexOf(tagId); - if (i >= 0) { - Object prior = this.knownValues[i]; - this.knownValues[i] = value; - return materializeKnown(tagId, prior); + long slotBit = knownSlotBit(tagId); + // maybe present only if the slot bit is set; a clear bit ⟹ definitely absent ⟹ append + if ((this.knownOccupancy & slotBit) != 0) { + int i = this.knownIndexOf(tagId); + if (i >= 0) { + Object prior = this.knownValues[i]; + this.knownValues[i] = value; + return materializeKnown(tagId, prior); + } + // filter false positive (slot collision) -> fall through to append } this.ensureKnownCapacity(); - int slot = this.knownCount++; - this.knownIds[slot] = tagId; - this.knownValues[slot] = value; + int idx = this.knownCount++; + this.knownIds[idx] = tagId; + this.knownValues[idx] = value; + this.knownOccupancy |= slotBit; + this.knownTraceLevel |= KnownTagCodec.isTraceLevel(tagId); return null; } /** Raw dense value for {@code tagId}, or {@code null} when absent (no Entry, no boxing). */ private Object knownRawValue(long tagId) { + if (!this.knownMaybePresent(tagId)) return null; // definitely absent, no scan int i = this.knownIndexOf(tagId); return i < 0 ? null : this.knownValues[i]; } @@ -1421,6 +1479,7 @@ private Object knownRawValue(long tagId) { * Removes a known tag from the dense store (swap-with-last), returning the prior Entry or null. */ private Entry removeKnown(long tagId) { + if (!this.knownMaybePresent(tagId)) return null; // definitely absent int i = this.knownIndexOf(tagId); if (i < 0) return null; Object prior = this.knownValues[i]; @@ -1429,6 +1488,8 @@ private Entry removeKnown(long tagId) { this.knownValues[i] = this.knownValues[last]; this.knownIds[last] = 0L; this.knownValues[last] = null; + // knownOccupancy intentionally NOT cleared: a stale-set bit only costs a scan; clearing could + // drop a bit still shared (via collision) by a present id -> false negative. return materializeKnown(tagId, prior); } @@ -1445,9 +1506,18 @@ private static Entry materializeKnown(long tagId, Object value) { * needed.) */ private boolean parentDenseVisible(long tagId, TagMap fromAncestor) { + // Trace and span tags reuse the same slots, so a trace-level ancestor entry can only be + // shadowed by a nearer level that ALSO holds trace-level tags. A span map (knownTraceLevel + // false) never shadows a trace tag — skip its occupancy+scan entirely (the level-bit win). A + // span-level ancestor entry keeps the plain occupancy-filtered scan below. + boolean traceLevelTag = KnownTagCodec.isTraceLevel(tagId); String tag = null; // resolved lazily, only if a nearer level carries tombstones for (TagMap nearer = this; nearer != fromAncestor; nearer = nearer.parent) { - if (nearer.knownIndexOf(tagId) >= 0) return false; // shadowed by a nearer dense entry + // shadowed by a nearer dense entry — the occupancy filter prunes the scan when definitely + // absent, so a nearer level with disjoint slots never pays a scan here (read-through win) + if ((!traceLevelTag || nearer.knownTraceLevel) + && nearer.knownMaybePresent(tagId) + && nearer.knownIndexOf(tagId) >= 0) return false; if (nearer.removedFromParent != null) { if (tag == null) tag = KnownTagCodec.nameOf(tagId); if (nearer.removedFromParent.contains(tag)) return false; // tombstoned by a nearer level @@ -1907,6 +1977,8 @@ private void putAllIntoEmptyMap(TagMap that) { this.knownIds = Arrays.copyOf(that.knownIds, that.knownIds.length); this.knownValues = Arrays.copyOf(that.knownValues, that.knownValues.length); this.knownCount = that.knownCount; + this.knownOccupancy = that.knownOccupancy; + this.knownTraceLevel = that.knownTraceLevel; } } @@ -2294,6 +2366,8 @@ public void clear() { this.knownIds = null; this.knownValues = null; this.knownCount = 0; + this.knownOccupancy = 0L; + this.knownTraceLevel = false; } public TagMap freeze() { diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java index 11d01bc7ec8..3e0dccaae56 100644 --- a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -11,40 +11,42 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; /** - * Parity test for the keyOf substrate (slice 1): the {@link KnownTags} registry + the {@link - * KnownTagCodec.Resolver} it registers. Verifies name ↔ id resolution without any dense store - * — {@code keyOf}/{@code nameOf} depend only on globalSerial + name, not on the (dormant) - * positional layout. + * Parity test for the keyOf substrate: the generated {@link KnownTags} registry + the {@link + * KnownTagCodec.Resolver} it registers. Verifies name ↔ id resolution and the intercepted / + * reserved / stored partitioning. {@code keyOf}/{@code nameOf} depend only on globalSerial + name, + * not on the (dormant) positional layout, so this is independent of the colored slot the tag + * registry assigns. Also covers the slot/level-bit encoding the coloring adds to the id. */ class KnownTagsTest { - /** (name, id) pairs — the full registry. keyOf returns the id verbatim (incl. INTERCEPTED). */ + /** (name, id) pairs across the groups — keyOf returns the id verbatim (incl. INTERCEPTED). */ static Stream knownTags() { return Stream.of( Arguments.of(Tags.ERROR, KnownTags.ERROR_ID), - Arguments.of(DDTags.PARENT_ID, KnownTags.PARENT_ID), - Arguments.of(DDTags.BASE_SERVICE, KnownTags.BASE_SERVICE_ID), + Arguments.of(DDTags.PARENT_ID, KnownTags.DD_PARENT_ID), + Arguments.of(DDTags.BASE_SERVICE, KnownTags.DD_BASE_SERVICE_ID), Arguments.of(Tags.VERSION, KnownTags.VERSION_ID), - Arguments.of(KnownTags.ENV, KnownTags.ENV_ID), - Arguments.of(DDTags.DJM_ENABLED, KnownTags.DJM_ENABLED_ID), - Arguments.of(DDTags.DSM_ENABLED, KnownTags.DSM_ENABLED_ID), - Arguments.of(DDTags.TRACER_HOST, KnownTags.TRACER_HOST_ID), - Arguments.of(DDTags.DD_INTEGRATION, KnownTags.INTEGRATION_ID), - Arguments.of(DDTags.DD_SVC_SRC, KnownTags.SVC_SRC_ID), + Arguments.of("env", KnownTags.ENV_ID), + Arguments.of(DDTags.DJM_ENABLED, KnownTags.DD_DJM_ENABLED_ID), + Arguments.of(DDTags.DSM_ENABLED, KnownTags.DD_DSM_ENABLED_ID), + Arguments.of(DDTags.TRACER_HOST, KnownTags.DD_TRACER_HOST_ID), + Arguments.of(DDTags.DD_INTEGRATION, KnownTags.DD_INTEGRATION_ID), + Arguments.of(DDTags.DD_SVC_SRC, KnownTags.DD_SVC_SRC_ID), Arguments.of(Tags.PEER_SERVICE, KnownTags.PEER_SERVICE_ID), - Arguments.of(DDTags.PEER_SERVICE_REMAPPED_FROM, KnownTags.PEER_SERVICE_REMAPPED_FROM_ID), + Arguments.of(DDTags.PEER_SERVICE_REMAPPED_FROM, KnownTags.DD_PEER_SERVICE_REMAPPED_FROM_ID), Arguments.of(Tags.HTTP_METHOD, KnownTags.HTTP_METHOD_ID), Arguments.of(Tags.HTTP_ROUTE, KnownTags.HTTP_ROUTE_ID), Arguments.of(Tags.HTTP_URL, KnownTags.HTTP_URL_ID), Arguments.of(Tags.PEER_HOSTNAME, KnownTags.PEER_HOSTNAME_ID), - Arguments.of(Tags.PEER_HOST_IPV4, KnownTags.PEER_HOST_IPV4_ID), - Arguments.of(Tags.PEER_HOST_IPV6, KnownTags.PEER_HOST_IPV6_ID), + Arguments.of(Tags.PEER_HOST_IPV4, KnownTags.PEER_IPV4_ID), + Arguments.of(Tags.PEER_HOST_IPV6, KnownTags.PEER_IPV6_ID), Arguments.of(Tags.PEER_PORT, KnownTags.PEER_PORT_ID), Arguments.of(Tags.COMPONENT, KnownTags.COMPONENT_ID), Arguments.of(Tags.SPAN_KIND, KnownTags.SPAN_KIND_ID), @@ -68,10 +70,40 @@ static Stream interceptedTags() { Arguments.of(KnownTags.SPAN_KIND_ID)); } + /** + * Trace-level tags (live on the TraceSegment's TagMap) — their id carries the LEVEL_TRACE bit. + */ + static Stream traceLevelTags() { + return Stream.of( + Arguments.of(KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(KnownTags.VERSION_ID), + Arguments.of(KnownTags.ENV_ID), + Arguments.of(KnownTags.LANGUAGE_ID), + Arguments.of(KnownTags.RUNTIME_ID), + Arguments.of(KnownTags.DD_TRACER_HOST_ID), + Arguments.of(KnownTags.DD_DJM_ENABLED_ID)); + } + + /** Span-level tags — their id leaves the LEVEL_TRACE bit clear. */ + static Stream spanLevelTags() { + return Stream.of( + Arguments.of(KnownTags.HTTP_METHOD_ID), + Arguments.of(KnownTags.HTTP_URL_ID), + Arguments.of(KnownTags.DB_TYPE_ID), + Arguments.of(KnownTags.COMPONENT_ID), + Arguments.of(KnownTags.SPAN_KIND_ID), + Arguments.of(KnownTags.PEER_SERVICE_ID)); + } + + @BeforeAll + static void registerResolver() { + // Generated ids are compile-time constants (literal), so a constant reference is inlined and + // never triggers KnownTags.. init() forces class-load -> KnownTagCodec.register. + KnownTags.init(); + } + @Test - void resolverIsActiveOnceReferenced() { - // referencing any constant triggers KnownTags. -> KnownTagCodec.register - assertTrue(KnownTags.ERROR_ID != 0L); + void resolverIsActiveAfterInit() { assertTrue(KnownTagCodec.isActive()); assertEquals(KnownTags.SLOT_COUNT, KnownTagCodec.slotCount()); } @@ -88,13 +120,6 @@ void nameOfResolvesIdToName(String name, long id) { assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); } - @ParameterizedTest - @MethodSource("knownTags") - void nameHashMatchesEntryHash(String name, long id) { - assertEquals( - (int) TagMap.Entry._hash(name), KnownTagCodec.nameHash(id), "nameHash(" + name + ")"); - } - @ParameterizedTest @MethodSource("interceptedTags") void interceptedTagsCarryFlag(long id) { @@ -125,7 +150,7 @@ void unknownNamesResolveToZero() { @Test void unknownIdsResolveToNullName() { assertNull(KnownTagCodec.nameOf(0L)); - assertNull(KnownTagCodec.nameOf(KnownTagCodec.tagId(9999, "made.up"))); + assertNull(KnownTagCodec.nameOf(KnownTagCodec.makeTagId(9999))); // serial with no assigned tag } @Test @@ -146,7 +171,45 @@ void errorIsReservedTheRestAreStored() { @Test void globalSerialsAreUnique() { List serials = new ArrayList<>(); - knownTags().forEach(a -> serials.add((long) KnownTagCodec.globalSerial((Long) a.get()[1]))); + knownTags().forEach(a -> serials.add((long) KnownTagCodec.serialNum((Long) a.get()[1]))); assertEquals(serials.size(), new HashSet<>(serials).size(), "globalSerials must be unique"); } + + @ParameterizedTest + @MethodSource("traceLevelTags") + void traceLevelTagsCarryLevelBit(long id) { + assertTrue(KnownTagCodec.isTraceLevel(id), "isTraceLevel"); + } + + @ParameterizedTest + @MethodSource("spanLevelTags") + void spanLevelTagsClearLevelBit(long id) { + assertFalse(KnownTagCodec.isTraceLevel(id), "not trace-level"); + } + + @Test + void levelBitCompositionRoundTrips() { + long spanId = KnownTagCodec.makeTagId(300, 5); // no level bit + assertFalse(KnownTagCodec.isTraceLevel(spanId)); + long traceId = KnownTagCodec.traceLevel(spanId); + assertTrue(KnownTagCodec.isTraceLevel(traceId)); + // level bit is orthogonal to serial/slot — both survive setting it + assertEquals(KnownTagCodec.serialNum(spanId), KnownTagCodec.serialNum(traceId)); + assertEquals(KnownTagCodec.slot(spanId), KnownTagCodec.slot(traceId)); + assertEquals(traceId, KnownTagCodec.traceLevel(traceId), "traceLevel is idempotent"); + } + + @Test + void slotEncodingRoundTrips() { + long id = KnownTagCodec.makeTagId(FIRST_STORED_SERIAL_PLUS_7, 7); + assertEquals(FIRST_STORED_SERIAL_PLUS_7, KnownTagCodec.serialNum(id)); + assertEquals(7, KnownTagCodec.slot(id)); + assertFalse(KnownTagCodec.isUnslotted(id)); + + long unslotted = KnownTagCodec.makeTagId(FIRST_STORED_SERIAL_PLUS_7); // NO_SLOT + assertEquals(KnownTagCodec.NO_SLOT, KnownTagCodec.slot(unslotted)); + assertTrue(KnownTagCodec.isUnslotted(unslotted)); + } + + private static final int FIRST_STORED_SERIAL_PLUS_7 = KnownTagCodec.FIRST_STORED_SERIAL + 7; } diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java index a82742c3acd..70e41c47798 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java @@ -35,8 +35,9 @@ class TagMapDenseForkedTest { @BeforeAll static void registerResolver() { - // referencing any KnownTags constant triggers its -> KnownTagCodec.register - assertTrue(KnownTags.BASE_SERVICE_ID != 0L); + // Generated ids are compile-time constants (literal), so a constant reference is inlined and + // never triggers KnownTags.. init() forces class-load -> KnownTagCodec.register. + KnownTags.init(); assertTrue(KnownTagCodec.isActive(), "resolver must be live for the dense store to engage"); assertTrue( KnownTagCodec.isStored(KnownTagCodec.keyOf(BASE_SERVICE)), "base_service routes dense"); diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java index a29795c542a..f1c8a744d4b 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java @@ -58,14 +58,14 @@ enum Regime { public long keyOf(String name) { if (name.startsWith("known-")) { int n = Integer.parseInt(name.substring("known-".length())); - return KnownTagCodec.tagId(KnownTagCodec.FIRST_STORED_SERIAL + n, name); + return KnownTagCodec.makeTagId(KnownTagCodec.FIRST_STORED_SERIAL + n, n); } return 0L; } @Override public String nameOf(long tagId) { - int serial = KnownTagCodec.globalSerial(tagId); + int serial = KnownTagCodec.serialNum(tagId); return serial >= KnownTagCodec.FIRST_STORED_SERIAL ? "known-" + (serial - KnownTagCodec.FIRST_STORED_SERIAL) : null; From 9b7c3d159e2b5c5820bc768e50919bb6c9d931af Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 17 Aug 2026 19:17:53 -0400 Subject: [PATCH 03/14] Generate KnownTags from tag-conventions via graph coloring Introduce the tag-registry code generator (buildSrc plugin + tag-conventions YAML) and use it to assign the colored slots from the previous commit by graph-coloring the tag co-occurrence graph (each concrete span type's resolved set plus the clique), packing all stored tags into SLOT_COUNT=16 slots. Relocates the hand-maintained KnownTags from src/main to the generated src/generated source set and renames the codec accessors to their generated form (serialNum/makeTagId). Adds the generator verify task so CI fails if the checked-in KnownTags drifts from the conventions. Co-Authored-By: Claude Opus 4.8 --- buildSrc/build.gradle.kts | 6 + .../plugin/tags/GenerateKnownTagsTask.kt | 38 ++++ .../gradle/plugin/tags/KnownTagsEmitter.kt | 142 ++++++++++++++ .../gradle/plugin/tags/TagConventions.kt | 173 ++++++++++++++++++ .../datadog/gradle/plugin/tags/TagRegistry.kt | 153 ++++++++++++++++ .../plugin/tags/TagRegistryGenerator.kt | 172 +++++++++++++++++ .../plugin/tags/TagRegistryGeneratorPlugin.kt | 41 +++++ .../gradle/plugin/tags/VerifyKnownTagsTask.kt | 67 +++++++ gradle/spotless.gradle | 3 +- internal-api/build.gradle.kts | 16 ++ internal-api/src/generated/folded-types.txt | 93 ++++++++++ .../java/datadog/trace/api/KnownTags.java | 4 +- internal-api/src/generated/layout-by-type.txt | 91 +++++++++ internal-api/src/generated/resolved-tags.txt | 77 ++++++++ internal-api/src/generated/tag-assignment.txt | 69 +++++++ tag-conventions.java.yaml | 36 ++++ tag-conventions.yaml | 134 ++++++++++++++ 17 files changed, 1311 insertions(+), 4 deletions(-) create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt create mode 100644 internal-api/src/generated/folded-types.txt rename internal-api/src/{main => generated}/java/datadog/trace/api/KnownTags.java (99%) create mode 100644 internal-api/src/generated/layout-by-type.txt create mode 100644 internal-api/src/generated/resolved-tags.txt create mode 100644 internal-api/src/generated/tag-assignment.txt create mode 100644 tag-conventions.java.yaml create mode 100644 tag-conventions.yaml diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 991ca899596..b7eacb3cd95 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -59,6 +59,11 @@ gradlePlugin { implementationClass = "datadog.gradle.plugin.config.SupportedConfigPlugin" } + create("tag-registry-generator") { + id = "dd-trace-java.tag-registry-generator" + implementationClass = "datadog.gradle.plugin.tags.TagRegistryGeneratorPlugin" + } + create("supported-config-linter") { id = "dd-trace-java.config-inversion-linter" implementationClass = "datadog.gradle.plugin.config.ConfigInversionLinter" @@ -107,6 +112,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.core:jackson-annotations") implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") compileOnly(libs.develocity) } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt new file mode 100644 index 00000000000..ab1c2631c60 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt @@ -0,0 +1,38 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Generates the committed tag registry (KnownTags.java + layout reports) from the language-agnostic + * {@code tag-conventions.yaml} + the Java overlay. The actual emit lives in [TagRegistryGenerator]; + * this task just wires the inputs/outputs so Gradle can cache and up-to-date-check it. + */ +@CacheableTask +abstract class GenerateKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:OutputDirectory val destinationDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun generate() { + val outDir = destinationDirectory.get().asFile + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, outDir) + logger.lifecycle("tag-registry: generated -> $outDir") + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt new file mode 100644 index 00000000000..82c04562f62 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -0,0 +1,142 @@ +package datadog.gradle.plugin.tags + +/** + * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag + * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` + * derivation comment — then the package-private `_SERIAL_NUM` constants, the + * `StringIndex.EmbeddingSupport` keyOf table, the `serialNum` switch `nameOf`, and resolver + * registration. + */ +object KnownTagsEmitter { + + fun emit(reg: TagRegistry, pkg: String, className: String): String { + // Sanitize tag names into unique Java constant identifiers. + val used = HashSet() + val cname = HashMap() + fun mk(name: String): String { + var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_') + if (c.isEmpty() || c[0].isDigit()) c = "T_$c" + var u = c + var n = 2 + while (u in used) { + u = "${c}_$n"; n++ + } + used.add(u) + cname[name] = u + return u + } + reg.reserved.forEach { mk(it.name) } + reg.stored.forEach { mk(it.name) } + + // Constant names. Collapse a duplicated trailing token so e.g. "resource.name" yields NAME + // (not NAME_NAME) and "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs + // (ID + _NAME -> ID_NAME, NAME + _ID -> NAME_ID) are kept as-is. + fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix" + fun nameC(name: String) = withSuffix(cname[name]!!, "_NAME") + fun idC(name: String) = withSuffix(cname[name]!!, "_ID") + fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM") + + val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order + val b = StringBuilder() + b.appendLine("package $pkg;") + b.appendLine() + b.appendLine("import datadog.trace.util.StringIndex;") + b.appendLine() + b.appendLine("// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator).") + b.appendLine("// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml.") + b.appendLine("public final class $className {") + b.appendLine(" static final int SLOT_COUNT = ${reg.slotCount};") + b.appendLine() + + // Public API first (name + encoded id couplets), so readers see the useful parts up top; the + // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment. + b.appendLine(" // ---- reserved (routed to span fields or directives; not stored) ----") + for (v in reg.reserved) { + b.appendLine(" public static final String ${nameC(v.name)} = \"${v.name}\";") + b.appendLine(" public static final long ${idC(v.name)} = ${hex(v.id)};") + b.appendLine(" // makeTagId(serial=${v.serial}, slot=NO_SLOT) + intercepted [${v.kind}${v.field?.let { " -> $it" } ?: ""}]") + b.appendLine() + } + + b.appendLine(" // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ----") + for (t in reg.stored) { + val slot = if (t.slotted) t.slot.toString() else "NO_SLOT" + b.appendLine(" public static final String ${nameC(t.name)} = \"${t.name}\";") + b.appendLine(" public static final long ${idC(t.name)} = ${hex(t.id)};") + b.appendLine(" // makeTagId(serial=${t.serial}, slot=$slot)${if (t.intercepted) " + intercepted" else ""}${if (t.traceLevel) " + trace-level" else ""} <${t.required}>") + b.appendLine() + } + + // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch. + b.appendLine(" // ---- serial numbers ----") + for (v in reg.reserved) { + b.appendLine(" static final int ${serialC(v.name)} = ${v.serial};") + } + for (t in reg.stored) { + b.appendLine(" static final int ${serialC(t.name)} = ${t.serial};") + } + b.appendLine() + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). + b.appendLine(" private static final String[] KEYOF_NAMES = {") + order.forEach { b.appendLine(" ${nameC(it)},") } + b.appendLine(" };") + b.appendLine(" private static final long[] KEYOF_VALUES = {") + order.forEach { b.appendLine(" ${idC(it)},") } + b.appendLine(" };") + b.appendLine(" private static final int[] KEYOF_HASHES;") + b.appendLine(" private static final String[] KEYOF_KEYS;") + b.appendLine(" private static final long[] KEYOF_IDS;") + b.appendLine(" static {") + b.appendLine(" StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);") + b.appendLine(" long[] ids = new long[data.names.length];") + b.appendLine(" for (int j = 0; j < KEYOF_NAMES.length; j++) {") + b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j];") + b.appendLine(" }") + b.appendLine(" KEYOF_HASHES = data.hashes;") + b.appendLine(" KEYOF_KEYS = data.names;") + b.appendLine(" KEYOF_IDS = ids;") + b.appendLine(" }") + b.appendLine() + + // Resolver. + b.appendLine(" static final KnownTagCodec.Resolver RESOLVER =") + b.appendLine(" new KnownTagCodec.Resolver() {") + b.appendLine(" @Override") + b.appendLine(" public String nameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return ${nameC(name)};") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public int slotCount() {") + b.appendLine(" return SLOT_COUNT;") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public long keyOf(String name) {") + b.appendLine(" int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name);") + b.appendLine(" return slot < 0 ? 0L : KEYOF_IDS[slot];") + b.appendLine(" }") + b.appendLine(" };") + b.appendLine() + b.appendLine(" static {") + b.appendLine(" KnownTagCodec.register(RESOLVER);") + b.appendLine(" }") + b.appendLine() + b.appendLine(" /** Forces resolver registration by triggering . Idempotent. */") + b.appendLine(" public static void init() {}") + b.appendLine() + b.appendLine(" private $className() {}") + b.appendLine("}") + return b.toString() + } + + private fun hex(id: Long): String = "0x%016XL".format(id) +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt new file mode 100644 index 00000000000..a01da7bc75c --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt @@ -0,0 +1,173 @@ +package datadog.gradle.plugin.tags + +/** + * Parsed tag-conventions domain model + the per-type tag-set resolver. Language-agnostic: it knows + * only structure (extends / include / applies) and per-tag semantics (name / type / required / + * source). Id assignment and emission are layered on top of the resolved sets. + */ +class TagConventions +private constructor( + private val spanTypes: Map, + private val mixins: Map, + private val traceLevel: List, +) { + /** A tag declaration (domain semantics only). */ + data class Tag( + val name: String, + val type: String, + val required: String, + ) + + data class SpanType( + val name: String, + val abstract: Boolean, + val extends: String?, + val include: List, + val tags: List, + ) + + data class Mixin( + val name: String, + val appliesAll: Boolean, + val appliesTo: Set, + val tags: List, + ) + + /** Concrete (instantiable) span types — the ones a layout is computed for. */ + fun concreteTypes(): List = + spanTypes.values.filter { !it.abstract }.map { it.name }.sorted() + + /** + * resolved(type) = own tags + tags up the `extends` chain (incl. base) + tags of every mixin the + * type or an ancestor `include`s + tags of every mixin whose `applies` matches. De-duped by tag + * name (first occurrence wins). Base-first order, so it is stable across runs. + */ + fun resolve(typeName: String): List { + val result = LinkedHashMap() + fun add(t: Tag) = result.putIfAbsent(t.name, t) + + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { add(it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { add(it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) mx.tags.forEach { add(it) } + } + return result.values.toList() + } + + /** The explicit trace-level tier tags (their own TagMap "type" on the TraceSegment). */ + fun traceLevelTags(): List = traceLevel + + /** A declaration group: the source that *declares* a set of tags (its own `tags:` list). */ + data class Group(val name: String, val kind: String, val tags: List) + + /** + * The declaration groups, in a stable order: the trace-level tier first, then every span type + * (abstract included — `base`/`http` declare real tags) sorted by name, then every mixin sorted by + * name. Each maps to one `group-decl`. A tag is *declared* once (in its own container's `tags:`); + * the same tag reached via extends/include/applies is not re-declared, so first-declaration (in + * this order) is its home group. Groups with no declared tags are omitted. + */ + fun declarationGroups(): List { + val groups = ArrayList() + if (traceLevel.isNotEmpty()) groups.add(Group(TRACE_LAYER, "trace", traceLevel)) + for (name in spanTypes.keys.sorted()) { + val st = spanTypes.getValue(name) + if (st.tags.isNotEmpty()) groups.add(Group(name, "span_type", st.tags)) + } + for (name in mixins.keys.sorted()) { + val mx = mixins.getValue(name) + if (mx.tags.isNotEmpty()) groups.add(Group(name, "mixin", mx.tags)) + } + return groups + } + + /** Full stored-tag universe (concrete span types' resolves + trace-level), de-duped by name. */ + fun allStoredTags(): List { + val union = LinkedHashMap() + for (type in concreteTypes()) for (t in resolve(type)) union.putIfAbsent(t.name, t) + for (t in traceLevel) union.putIfAbsent(t.name, t) + return union.values.toList() + } + + /** + * Full composition for a type as (origin, tag) pairs, in composition order and NOT de-duped, so a + * tag contributed by more than one source shows up more than once. Origin is the contributing + * span type (via extends), `incl:` (via include), or `appl:` (via applies). + */ + fun compose(typeName: String): List> { + val out = ArrayList>() + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { out.add(st.name to it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { out.add("incl:$mixinName" to it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) { + mx.tags.forEach { out.add("appl:${mx.name}" to it) } + } + } + return out + } + + companion object { + /** Group name of the trace-level tier (its own TagMap layer on the TraceSegment). */ + const val TRACE_LAYER = "" + + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): TagConventions { + val spanTypesRaw = (root["span_types"] as? Map) ?: emptyMap() + val spanTypes = + spanTypesRaw.mapValues { (name, v) -> + val m = v as Map + SpanType( + name = name, + abstract = (m["abstract"] as? Boolean) ?: false, + extends = m["extends"] as? String, + include = (m["include"] as? List) ?: emptyList(), + tags = tagList(m["tags"]), + ) + } + + val mixinsRaw = (root["mixins"] as? Map) ?: emptyMap() + val mixins = + mixinsRaw.mapValues { (name, v) -> + val m = v as Map + val applies = m["applies"] + Mixin( + name = name, + appliesAll = applies == "all", + appliesTo = if (applies is List<*>) applies.map { it.toString() }.toSet() else emptySet(), + tags = tagList(m["tags"]), + ) + } + + val traceLevel = tagList((root["trace_level"] as? Map)?.get("tags")) + return TagConventions(spanTypes, mixins, traceLevel) + } + + @Suppress("UNCHECKED_CAST") + private fun tagList(tags: Any?): List = + (tags as? List>)?.map { m -> + Tag( + name = m["tag"].toString(), + type = (m["type"] as? String) ?: "string", + required = (m["required"] as? String) ?: "optional", + ) + } ?: emptyList() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt new file mode 100644 index 00000000000..0eebcbda629 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt @@ -0,0 +1,153 @@ +package datadog.gradle.plugin.tags + +/** + * Assigns tag ids from a parsed [TagConventions] plus the Java overlay (intercepted set + reserved + * registry). The id encoding mirrors KnownTagCodec: [63 intercepted][62-48 serial][47-32 slot][31-0 + * zero] (known ids carry no nameHash — they are dense-store addressed). + * + *

The `slot` is a single globally stable coordinate from GRAPH COLORING the tag co-occurrence + * graph. Each concrete span type's resolved tag set (see [TagConventions.resolve]) is a clique — its + * tags all appear together on one span, so they must get distinct slots. The trace-level tier is its + * own clique (a separate TagMap on the TraceSegment), so it may reuse slot numbers freely with the + * span layers. Slots are shared only between tags that never co-occur, so slotCount stays bounded by + * the largest clique (≤ 64) — small enough that the dense store's presence fast path is a single + * occupancy `long` (`1L << slot`), which is exactly why the earlier two-tier (group + field bloom) + * scheme could collapse to one word. Correctness never depends on the coloring (the dense scan is + * authoritative); only the fast-path hit rate does. + * + *

Slotting (does a tag get a slot / dense presence bit) is derived from the domain `required` + * level: required/conditional/recommended tags are colored (slotted), the rest are NO_SLOT + * (bucketed) and carry no slot bit. + */ +class TagRegistry +private constructor( + val stored: List, + val reserved: List, + val slotCount: Int, +) { + data class StoredTag( + val name: String, + val type: String, + val required: String, + val serial: Int, + val intercepted: Boolean, + val slot: Int, + val traceLevel: Boolean, + val id: Long, + ) { + val slotted: Boolean + get() = slot != NO_SLOT + } + + data class ReservedTag( + val name: String, + val kind: String, + val field: String?, + val serial: Int, + val id: Long, + ) + + /** Java overlay: intercepted tag names + the reserved/special-key registry. */ + class Overlay(val intercepted: Set, val reserved: List) { + data class ReservedDef(val name: String, val kind: String, val field: String?) + + companion object { + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): Overlay { + val intercepted = (root["intercepted"] as? List)?.toSet() ?: emptySet() + val reserved = + (root["reserved"] as? List>)?.map { m -> + ReservedDef(m["tag"].toString(), (m["kind"] as? String) ?: "directive", m["field"] as? String) + } ?: emptyList() + return Overlay(intercepted, reserved) + } + } + } + + companion object { + const val FIRST_STORED_SERIAL = 256 + const val NO_SLOT = 0xFFFF // slot all-ones sentinel (16 bits); mirrors KnownTagCodec.NO_SLOT + const val MAX_SLOT = 63 // one occupancy long: colored slots must fit in [0, 63] + const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + const val TRACE_LAYER = "" + + // Domain `required` levels that get a colored slot (the rest are bucketed with NO_SLOT). + val COLORABLE = setOf("required", "conditional", "recommended") + + /** + * Mirrors KnownTagCodec.makeTagId(serial, slot) + intercepted()/traceLevel() — must stay in + * sync. slot [47-32], LEVEL_TRACE at bit 2, other low bits zero. + */ + fun encode(serial: Int, intercepted: Boolean, slot: Int, traceLevel: Boolean): Long { + var id = (serial.toLong() shl 48) or ((slot.toLong() and 0xFFFF) shl 32) + if (traceLevel) id = id or LEVEL_TRACE + if (intercepted) id = id or Long.MIN_VALUE + return id + } + + fun build(conv: TagConventions, overlay: Overlay): TagRegistry { + val all = conv.allStoredTags() + val traceNames = conv.traceLevelTags().map { it.name }.toSet() + val colorable = all.filter { it.required in COLORABLE }.map { it.name }.toSet() + + // Co-occurrence cliques: each concrete type's resolved colorable tags, plus the trace-level + // tier as its own clique (a separate TagMap -> free to reuse span slot numbers). Tags in the + // same clique must get distinct colors; tags never sharing a clique may share a color. + val cliques = ArrayList>() + for (type in conv.concreteTypes()) { + cliques.add(conv.resolve(type).map { it.name }.filter { it in colorable }.toSet()) + } + cliques.add(traceNames.filter { it in colorable }.toSet()) + + // Adjacency: an edge between every pair of tags that co-occur in some clique. + val adj = HashMap>() + colorable.forEach { adj[it] = HashSet() } + for (clique in cliques) { + val members = clique.toList() + for (i in members.indices) for (j in i + 1 until members.size) { + adj.getValue(members[i]).add(members[j]) + adj.getValue(members[j]).add(members[i]) + } + } + + // Greedy coloring, most-constrained-first (by clique membership count, then name for a stable + // tie-break). Each tag takes the smallest color not used by an already-colored neighbor. + val cliqueCount = colorable.associateWith { n -> cliques.count { n in it } } + val order = colorable.sortedWith(compareByDescending { cliqueCount.getValue(it) }.thenBy { it }) + val color = HashMap() + for (n in order) { + val used = adj.getValue(n).mapNotNull { color[it] }.toSet() + var c = 0 + while (c in used) c++ + color[n] = c + } + val slotCount = (color.values.maxOrNull() ?: -1) + 1 + require(slotCount <= MAX_SLOT + 1) { + "coloring produced $slotCount slots; the single occupancy long holds at most ${MAX_SLOT + 1}" + } + + val reserved = + overlay.reserved.mapIndexed { i, v -> + val serial = 1 + i + ReservedTag( + v.name, v.kind, v.field, serial, + encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false)) + } + + // Stored tags in a stable order (by name); serials are a dense global counter from + // FIRST_STORED_SERIAL. slot comes from the coloring (NO_SLOT for non-colorable/bucketed tags). + val stored = + all.sortedBy { it.name }.mapIndexed { i, t -> + val serial = FIRST_STORED_SERIAL + i + val intercepted = t.name in overlay.intercepted + val slot = color[t.name] ?: NO_SLOT + val traceLevel = t.name in traceNames + StoredTag( + t.name, t.type, t.required, serial, intercepted, slot, traceLevel, + id = encode(serial, intercepted, slot, traceLevel)) + } + + return TagRegistry(stored, reserved, slotCount) + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt new file mode 100644 index 00000000000..c57a055c405 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -0,0 +1,172 @@ +package datadog.gradle.plugin.tags + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import java.io.File + +/** + * Turns the language-agnostic {@code tag-conventions.yaml} + the Java overlay into the generated tag + * registry: {@code KnownTags.java} (under {@code java/}) plus verification report dumps + * (resolved-tags / tag-assignment / layout-by-type / folded-types) at the destination root. + * + * Pure function of its inputs (deterministic ordering throughout), so the same inputs always produce + * byte-identical output -- which is what the {@code verifyKnownTags} freshness gate relies on. + */ +object TagRegistryGenerator { + /** Parses the two YAML files and writes the full generated tree under [outDir]. */ + fun generate(domainYaml: File, overlayYaml: File, outDir: File) { + val mapper = ObjectMapper(YAMLFactory()) + val domain: Map = + domainYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + val overlayMap: Map = + overlayYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + + outDir.mkdirs() + // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. + val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } + + val conv = TagConventions.parse(domain) + val overlay = TagRegistry.Overlay.parse(overlayMap) + val reg = TagRegistry.build(conv, overlay) + + File(outDir, "resolved-tags.txt").writeText(resolvedReport(conv)) + File(outDir, "tag-assignment.txt").writeText(assignmentReport(conv, reg)) + File(outDir, "layout-by-type.txt").writeText(layoutByTypeReport(conv, reg)) + File(outDir, "folded-types.txt").writeText(foldedTypesReport(conv, reg)) + File(javaPkg, "KnownTags.java") + .writeText(KnownTagsEmitter.emit(reg, "datadog.trace.api", "KnownTags")) + } + + /** resolved-tags.txt — the per-type resolved sets (composition check). */ + private fun resolvedReport(conv: TagConventions): String { + val resolved = StringBuilder() + resolved.appendLine("# Resolved per-type tag sets (concrete span types).") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + resolved.appendLine() + resolved.appendLine("$type (${tags.size} tags):") + for (t in tags) resolved.appendLine(" - ${t.name}") + } + return resolved.toString() + } + + /** tag-assignment.txt — serials, colored slots, ids, per-type slot sets (coloring check). */ + private fun assignmentReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val a = StringBuilder() + a.appendLine( + "# Tag id assignment. slotCount=${reg.slotCount} stored=${reg.stored.size} reserved=${reg.reserved.size}") + a.appendLine() + a.appendLine("# STORED serial slot int lvl id required name") + for (t in reg.stored) { + a.appendLine( + " %6d %5s %s %s %-18s %-12s %s".format( + t.serial, + if (t.slotted) t.slot.toString() else "-", + if (t.intercepted) "I" else "-", + if (t.traceLevel) "T" else "-", + "0x%016X".format(t.id), + t.required, + t.name)) + } + a.appendLine() + a.appendLine("# RESERVED serial id kind name") + for (v in reg.reserved) { + a.appendLine( + " %6d %-18s %-12s %s%s".format( + v.serial, "0x%016X".format(v.id), v.kind, v.name, v.field?.let { " -> $it" } ?: "")) + } + a.appendLine() + a.appendLine("# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the") + a.appendLine("# co-occurrence clique); is its own clique and freely reuses span slot numbers.") + for (type in conv.concreteTypes()) { + val slots = + conv.resolve(type).mapNotNull { byName[it.name] } + .filter { it.slotted && !it.traceLevel } + .map { it.slot } + .sorted() + a.appendLine(" %-14s count=%-3d slots=%s".format(type, slots.size, slots)) + } + val traceSlots = + reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() + a.appendLine(" %-14s count=%-3d slots=%s".format("", traceSlots.size, traceSlots)) + return a.toString() + } + + /** + * layout-by-type.txt — full composition per type (origins shown, NOT de-duped), each tag annotated + * with its slot/tier: s = colored slot, trace s = trace-level layer, bkt = bucket. + */ + private fun layoutByTypeReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val lay = StringBuilder() + lay.appendLine("# Full tag composition per concrete span type (after extends/include/applies).") + lay.appendLine("# Not de-duped: a tag from >1 source appears >1 time.") + lay.appendLine("# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted") + for (type in conv.concreteTypes()) { + val comp = conv.compose(type) + val distinct = comp.map { it.second.name }.distinct().size + lay.appendLine() + lay.appendLine("$type (${comp.size} contributions, $distinct distinct):") + val byOrigin = LinkedHashMap>() + for ((origin, tag) in comp) byOrigin.getOrPut(origin) { ArrayList() }.add(tag) + for ((origin, tags) in byOrigin) { + lay.appendLine(" [$origin]") + for (t in tags) { + val st = byName[t.name] + val field = + when { + st == null -> "?" + st.traceLevel && st.slotted -> "trace s${st.slot}" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + lay.appendLine( + " %-26s %-12s %-12s %s".format(t.name, field, t.required, if (st?.intercepted == true) "I" else "")) + } + } + } + return lay.toString() + } + + /** + * folded-types.txt — each type's full resolved set (extends + include + applies, DE-DUPED) with its + * slot; plus the type. This is the "type with everything folded in" view. + */ + private fun foldedTypesReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + fun tierField(st: TagRegistry.StoredTag?): String = + when { + st == null -> "?" + st.traceLevel -> if (st.slotted) "trace s${st.slot}" else "trace-bkt" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + val f = StringBuilder() + f.appendLine("# Folded tag set per type (extends + include + applies, de-duped), with colored slots.") + f.appendLine( + "# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + f.appendLine() + f.appendLine("$type (${tags.size} tags):") + for (t in tags) { + val st = byName[t.name] + f.appendLine(" %-12s %-26s %s".format(tierField(st), t.name, if (st?.intercepted == true) "I" else "")) + } + } + val traceTags = + reg.stored.filter { it.traceLevel }.sortedWith(compareBy({ !it.slotted }, { it.slot }, { it.name })) + f.appendLine() + f.appendLine(" (${traceTags.size} tags):") + for (st in traceTags) { + f.appendLine(" %-12s %-26s %s".format(tierField(st), st.name, if (st.intercepted) "I" else "")) + } + return f.toString() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt new file mode 100644 index 00000000000..313bd859068 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt @@ -0,0 +1,41 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory + +/** Extension configuring the tag-registry generator inputs/outputs. */ +abstract class TagRegistryExtension @Inject constructor(objects: ObjectFactory) { + val domainYaml: RegularFileProperty = objects.fileProperty() + val overlayYaml: RegularFileProperty = objects.fileProperty() + val destinationDirectory: DirectoryProperty = objects.directoryProperty() +} + +/** + * Registers {@code generateKnownTags} (emits the committed tag registry) and {@code verifyKnownTags} + * (a freshness gate that regenerates and byte-compares against the committed output). The verify task + * is wired into {@code check} so stale generated sources fail CI. + */ +class TagRegistryGeneratorPlugin : Plugin { + override fun apply(project: Project) { + val ext = project.extensions.create("tagRegistry", TagRegistryExtension::class.java) + project.tasks.register("generateKnownTags", GenerateKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + destinationDirectory.set(ext.destinationDirectory) + } + val verify = + project.tasks.register("verifyKnownTags", VerifyKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + committedDirectory.set(ext.destinationDirectory) + } + // `check` is contributed by lifecycle-base (via java-library); wait for it before wiring. + project.pluginManager.withPlugin("lifecycle-base") { + project.tasks.named("check").configure { dependsOn(verify) } + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt new file mode 100644 index 00000000000..e990e15e957 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt @@ -0,0 +1,67 @@ +package datadog.gradle.plugin.tags + +import java.io.File +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Freshness gate: regenerates the tag registry into a scratch dir and byte-compares it against the + * committed [committedDirectory]. Fails (pointing at {@code generateKnownTags}) if they differ, so a + * stale commit of the generated sources can't slip through CI. Not cacheable -- it must actually run + * the generator to catch drift, and it is cheap. + */ +abstract class VerifyKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + val committedDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun verify() { + val committed = committedDirectory.get().asFile + val scratch = File(temporaryDir, "generated") + scratch.deleteRecursively() + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, scratch) + + val diffs = ArrayList() + val freshFiles = scratch.walkTopDown().filter { it.isFile }.toList() + for (fresh in freshFiles) { + val rel = fresh.relativeTo(scratch).path + val committedFile = File(committed, rel) + when { + !committedFile.exists() -> diffs.add("missing (not committed): $rel") + committedFile.readText() != fresh.readText() -> diffs.add("out of date: $rel") + } + } + val freshRel = freshFiles.map { it.relativeTo(scratch).path }.toSet() + for (committedFile in committed.walkTopDown().filter { it.isFile }) { + val rel = committedFile.relativeTo(committed).path + if (rel !in freshRel) diffs.add("stale (no longer generated): $rel") + } + + if (diffs.isNotEmpty()) { + throw GradleException( + buildString { + appendLine("Generated tag registry is out of date with tag-conventions.yaml:") + diffs.forEach { appendLine(" - $it") } + append("Run `./gradlew :internal-api:generateKnownTags` and commit the result.") + }) + } + } +} diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle index 93a817e6452..769e153ddd9 100644 --- a/gradle/spotless.gradle +++ b/gradle/spotless.gradle @@ -21,7 +21,8 @@ spotless { // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 target 'src/**/*.java' // ignore embedded test projects and everything in build dir, e.g. generated sources - targetExclude('src/test/resources/**', buildDirectoryFiles) + // src/generated/** is emitted by code generators (e.g. the tag registry) — verified by their own freshness gate + targetExclude('src/test/resources/**', 'src/generated/**', buildDirectoryFiles) tableTestFormatter('1.1.1') googleJavaFormat('1.35.0') } diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 4d48a434c19..f1ab6486aa2 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -5,6 +5,7 @@ import groovy.lang.Closure plugins { `java-library` id("me.champeau.jmh") + id("dd-trace-java.tag-registry-generator") } apply(from = "$rootDir/gradle/java.gradle") @@ -32,6 +33,9 @@ extra["minimumBranchCoverage"] = 0.7 extra["minimumInstructionCoverage"] = 0.8 extra["excludedClassesCoverage"] = listOf( + // Generated by the tag-registry code generator (verified fresh via verifyKnownTags). + "datadog.trace.api.KnownTags", + "datadog.trace.api.KnownTags.*", "datadog.trace.api.ClassloaderConfigurationOverrides", "datadog.trace.api.ClassloaderConfigurationOverrides.Lazy", // Interface @@ -261,6 +265,18 @@ extra["excludedClassesBranchCoverage"] = listOf( extra["excludedClassesInstructionCoverage"] = listOf("datadog.trace.util.stacktrace.StackWalkerFactory") +// Tag registry: generated KnownTags is committed under src/generated (audited via git); the srcDir +// puts it on the main compile path and `verifyKnownTags` (wired into `check`) fails CI if it drifts +// from tag-conventions.yaml. Generation is run on demand (`./gradlew :internal-api:generateKnownTags`), +// not on every build, so the committed source stays the source of truth for the compiler. +tagRegistry { + domainYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.yaml")) + overlayYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.java.yaml")) + destinationDirectory.set(layout.projectDirectory.dir("src/generated")) +} + +sourceSets["main"].java.srcDir("src/generated/java") + dependencies { // references TraceScope and Continuation from public api api(project(":dd-trace-api")) diff --git a/internal-api/src/generated/folded-types.txt b/internal-api/src/generated/folded-types.txt new file mode 100644 index 00000000000..48b631bf5a5 --- /dev/null +++ b/internal-api/src/generated/folded-types.txt @@ -0,0 +1,93 @@ +# Folded tag set per type (extends + include + applies, de-duped), with colored slots. +# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted + +db.client (21 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s12 db.type + s9 db.instance + s10 db.operation + s15 db.user + bkt db.pool.name + s11 db.statement I + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.client (20 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s15 http.resend_count + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.server (18 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s13 http.route + s7 http.hostname + s14 http.useragent + s8 http.query.string + bkt servlet.path + bkt servlet.context I + +view.render (9 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s7 view.name + + (13 tags): + trace s0 _dd.appsec.enabled + trace s1 _dd.base_service + trace s2 _dd.civisibility.enabled + trace s3 _dd.djm.enabled + trace s4 _dd.dsm.enabled + trace s5 _dd.git.commit.sha + trace s6 _dd.git.repository_url + trace s7 _dd.profiling.enabled + trace s8 _dd.tracer_host + trace s9 env + trace s10 language + trace s11 runtime-id + trace s12 version diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java similarity index 99% rename from internal-api/src/main/java/datadog/trace/api/KnownTags.java rename to internal-api/src/generated/java/datadog/trace/api/KnownTags.java index 599b34f8a25..57b703ea697 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -410,13 +410,11 @@ public final class KnownTags { private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; private static final long[] KEYOF_IDS; - static { StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); long[] ids = new long[data.names.length]; for (int j = 0; j < KEYOF_NAMES.length; j++) { - ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = - KEYOF_VALUES[j]; + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j]; } KEYOF_HASHES = data.hashes; KEYOF_KEYS = data.names; diff --git a/internal-api/src/generated/layout-by-type.txt b/internal-api/src/generated/layout-by-type.txt new file mode 100644 index 00000000000..309bdfcf8e3 --- /dev/null +++ b/internal-api/src/generated/layout-by-type.txt @@ -0,0 +1,91 @@ +# Full tag composition per concrete span type (after extends/include/applies). +# Not de-duped: a tag from >1 source appears >1 time. +# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted + +db.client (21 contributions, 21 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [db.client] + db.type s12 required + db.instance s9 recommended + db.operation s10 recommended + db.user s15 recommended + db.pool.name bkt optional + db.statement s11 recommended I + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.client (20 contributions, 20 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.client] + http.url s11 required I + http.resend_count s15 recommended + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.server (18 contributions, 18 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.server] + http.url s11 required I + http.route s13 conditional + http.hostname s7 required + http.useragent s14 recommended + http.query.string s8 recommended + servlet.path bkt optional + servlet.context bkt optional I + +view.render (9 contributions, 9 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [view.render] + view.name s7 recommended diff --git a/internal-api/src/generated/resolved-tags.txt b/internal-api/src/generated/resolved-tags.txt new file mode 100644 index 00000000000..0edb3479608 --- /dev/null +++ b/internal-api/src/generated/resolved-tags.txt @@ -0,0 +1,77 @@ +# Resolved per-type tag sets (concrete span types). + +db.client (21 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - db.type + - db.instance + - db.operation + - db.user + - db.pool.name + - db.statement + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.client (20 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.resend_count + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.server (18 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.route + - http.hostname + - http.useragent + - http.query.string + - servlet.path + - servlet.context + +view.render (9 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - view.name diff --git a/internal-api/src/generated/tag-assignment.txt b/internal-api/src/generated/tag-assignment.txt new file mode 100644 index 00000000000..c1516085a3b --- /dev/null +++ b/internal-api/src/generated/tag-assignment.txt @@ -0,0 +1,69 @@ +# Tag id assignment. slotCount=16 stored=46 reserved=10 + +# STORED serial slot int lvl id required name + 256 0 - T 0x0100000000000004 recommended _dd.appsec.enabled + 257 1 - T 0x0101000100000004 required _dd.base_service + 258 2 - T 0x0102000200000004 recommended _dd.civisibility.enabled + 259 3 - T 0x0103000300000004 recommended _dd.djm.enabled + 260 4 - T 0x0104000400000004 recommended _dd.dsm.enabled + 261 5 - T 0x0105000500000004 recommended _dd.git.commit.sha + 262 6 - T 0x0106000600000004 recommended _dd.git.repository_url + 263 0 - - 0x0107000000000000 recommended _dd.integration + 264 1 - - 0x0108000100000000 required _dd.parent_id + 265 7 - - 0x0109000700000000 recommended _dd.peer.service.remapped_from + 266 8 - - 0x010A000800000000 recommended _dd.peer.service.source + 267 7 - T 0x010B000700000004 recommended _dd.profiling.enabled + 268 - - - 0x010CFFFF00000000 optional _dd.svc_src + 269 8 - T 0x010D000800000004 recommended _dd.tracer_host + 270 2 - - 0x010E000200000000 required component + 271 9 - - 0x010F000900000000 recommended db.instance + 272 10 - - 0x0110000A00000000 recommended db.operation + 273 - - - 0x0111FFFF00000000 optional db.pool.name + 274 11 I - 0x8112000B00000000 recommended db.statement + 275 12 - - 0x0113000C00000000 required db.type + 276 15 - - 0x0114000F00000000 recommended db.user + 277 9 - T 0x0115000900000004 recommended env + 278 3 - - 0x0116000300000000 recommended error.message + 279 4 - - 0x0117000400000000 recommended error.stack + 280 5 - - 0x0118000500000000 recommended error.type + 281 7 - - 0x0119000700000000 required http.hostname + 282 9 I - 0x811A000900000000 required http.method + 283 8 - - 0x011B000800000000 recommended http.query.string + 284 15 - - 0x011C000F00000000 recommended http.resend_count + 285 13 - - 0x011D000D00000000 conditional http.route + 286 10 - - 0x011E000A00000000 conditional http.status_code + 287 11 I - 0x811F000B00000000 required http.url + 288 14 - - 0x0120000E00000000 recommended http.useragent + 289 10 - T 0x0121000A00000004 required language + 290 12 - - 0x0122000C00000000 recommended network.protocol.version + 291 13 - - 0x0123000D00000000 recommended peer.hostname + 292 - - - 0x0124FFFF00000000 optional peer.ipv4 + 293 - - - 0x0125FFFF00000000 optional peer.ipv6 + 294 - - - 0x0126FFFF00000000 optional peer.port + 295 14 I - 0x8127000E00000000 recommended peer.service + 296 11 - T 0x0128000B00000004 required runtime-id + 297 - I - 0x8129FFFF00000000 optional servlet.context + 298 - - - 0x012AFFFF00000000 optional servlet.path + 299 6 I - 0x812B000600000000 required span.kind + 300 12 - T 0x012C000C00000004 recommended version + 301 7 - - 0x012D000700000000 recommended view.name + +# RESERVED serial id kind name + 1 0x8001FFFF00000000 structural error -> error + 2 0x8002FFFF00000000 structural service -> service + 3 0x8003FFFF00000000 structural resource.name -> resource + 4 0x8004FFFF00000000 structural span.type -> type + 5 0x8005FFFF00000000 structural origin -> origin + 6 0x8006FFFF00000000 directive sampling.priority + 7 0x8007FFFF00000000 directive manual.keep + 8 0x8008FFFF00000000 directive manual.drop + 9 0x8009FFFF00000000 directive measured + 10 0x800AFFFF00000000 directive analytics.sample_rate + +# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the +# co-occurrence clique); is its own clique and freely reuses span slot numbers. + db.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.server count=15 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + view.render count=8 slots=[0, 1, 2, 3, 4, 5, 6, 7] + count=13 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] diff --git a/tag-conventions.java.yaml b/tag-conventions.java.yaml new file mode 100644 index 00000000000..ab28b64545a --- /dev/null +++ b/tag-conventions.java.yaml @@ -0,0 +1,36 @@ +# dd-trace-java overlay — impl hints + special-tag registry. +# Consumed alongside the language-agnostic tag-conventions.yaml. Keyed by tag name +# (these are tag-intrinsic, not per-type). Only exceptions are listed. +# --------------------------------------------------------------------------- +# NOTE: the id coordinate (group-decl / field-decl) is NOT here — the generator assigns it from the +# declaration groups; the dense-vs-bucket split derives from the domain `required` level. + +# Tags whose set-path is handled by the Java TagInterceptor (side-effecting, but still stored). +# Generated ids carry the intercepted flag (bit 63). +intercepted: + - span.kind + - http.method + - http.url + - servlet.context + - db.statement + - peer.service + +# Reserved / special keys: accepted by the public setTag(...) API but ROUTED to a span field or a +# trace directive instead of tag storage. Reserved-tier ids (serial < FIRST_STORED_SERIAL, no slot). +# "Reserved" names the shared mechanism (the tracer reserves the key and handles it); the two kinds +# split on whether a value exists: structural has one (in a span/trace field), directive has none. +# The generated id->handler dispatch table is the data-driven replacement for the imperative +# TagInterceptor chain. +# kind: structural -> sets a span/trace field (`field:` names it) +# kind: directive -> triggers sampling/trace behavior +reserved: + - { tag: error, kind: structural, field: error } + - { tag: service, kind: structural, field: service, aliases: [service.name] } + - { tag: resource.name, kind: structural, field: resource } + - { tag: span.type, kind: structural, field: type } + - { tag: origin, kind: structural, field: origin } # trace-level field + - { tag: sampling.priority, kind: directive } + - { tag: manual.keep, kind: directive } + - { tag: manual.drop, kind: directive } + - { tag: measured, kind: directive } + - { tag: analytics.sample_rate, kind: directive } # legacy diff --git a/tag-conventions.yaml b/tag-conventions.yaml new file mode 100644 index 00000000000..2eff24f1784 --- /dev/null +++ b/tag-conventions.yaml @@ -0,0 +1,134 @@ +# Tag conventions — LANGUAGE-AGNOSTIC domain spec (structure + semantics only) +# --------------------------------------------------------------------------- +# The code generator consumes THIS file (domain) plus a per-language overlay +# (tag-conventions..yaml: impl hints like `intercepted`, and the reserved/ +# special-tag registry) to emit each language's tag-id constants, id<->name +# resolver, and slot (bitmask-bit) assignment. +# +# TRACE-LEVEL is its own thing (its own TagMap "type" on the TraceSegment) — the process/trace +# constants + product flags that are set once per trace, NOT per span. Declared explicitly in the +# `trace_level` section below (a distinct tier), never inferred from `source`. +# +# SPAN TYPES compose three ways: +# extends — structural is-a inheritance (http.server is-a http is-a base). `base` is implicitly +# in every span; abstract layers exist only to be extended. +# include — a span type PULLS in a mixin it intrinsically has (has-a; core-owned). +# applies — a mixin PUSHES itself onto span types, gated by `enabled_by`. +# resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). +# +# tag fields (DOMAIN only): tag | type (string|int|long|boolean|double) +# | required (required|conditional|recommended|optional|opt_in) | aliases. +# The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each +# declaration source (the trace-level tier, each span type, each mixin) is a group, and within a +# group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are +# bucketed. See the design doc. +# --------------------------------------------------------------------------- + +# Trace-level tier: its own TagMap on the TraceSegment. Set once per trace, not per span. +trace_level: + tags: + - { tag: _dd.base_service, type: string, required: required } + - { tag: version, type: string, required: recommended } + - { tag: env, type: string, required: recommended } + - { tag: language, type: string, required: required } + - { tag: runtime-id, type: string, required: required } + - { tag: _dd.tracer_host, type: string, required: recommended } + - { tag: _dd.git.commit.sha, type: string, required: recommended } + - { tag: _dd.git.repository_url, type: string, required: recommended } + # product .enabled flags — process-constant; present on the trace segment regardless of whether + # the product is enabled (the flag carries the state), so always-present => recommended. + - { tag: _dd.profiling.enabled, type: boolean, required: recommended } + - { tag: _dd.dsm.enabled, type: boolean, required: recommended } + - { tag: _dd.appsec.enabled, type: boolean, required: recommended } + - { tag: _dd.djm.enabled, type: boolean, required: recommended } + - { tag: _dd.civisibility.enabled, type: boolean, required: recommended } + +span_types: + # root: per-span tags every span has (incl. the per-span core tags parent_id / integration / svc_src + # — core-set but per-span, so NOT trace-level). + base: + abstract: true + tags: + - { tag: _dd.parent_id, type: string, required: required } + - { tag: component, type: string, required: required } + - { tag: span.kind, type: string, required: required } + - { tag: _dd.integration, type: string, required: recommended } + - { tag: _dd.svc_src, type: string, required: optional } + - { tag: error.type, type: string, required: recommended } + - { tag: error.message, type: string, required: recommended } + - { tag: error.stack, type: string, required: recommended } + + http: + abstract: true + extends: base + tags: + - { tag: http.method, type: string, required: required, aliases: [http.request.method] } + - { tag: http.status_code, type: int, required: conditional, aliases: [http.response.status_code] } + - { tag: network.protocol.version, type: string, required: recommended } + + http.server: + extends: http + tags: + - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.route, type: string, required: conditional } + - { tag: http.hostname, type: string, required: required, aliases: [server.address] } + - { tag: http.useragent, type: string, required: recommended } + - { tag: http.query.string, type: string, required: recommended, aliases: [url.query] } + - { tag: servlet.path, type: string, required: optional } + - { tag: servlet.context, type: string, required: optional } + + http.client: + extends: http + include: [ peer ] + tags: + - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.resend_count, type: int, required: recommended } + + db.client: + extends: base + include: [ peer ] + tags: + - { tag: db.type, type: string, required: required, aliases: [db.system] } + - { tag: db.instance, type: string, required: recommended } + - { tag: db.operation, type: string, required: recommended, aliases: [db.operation.name] } + - { tag: db.user, type: string, required: recommended } + - { tag: db.pool.name, type: string, required: optional } + - { tag: db.statement, type: string, required: recommended, aliases: [db.query.text] } + + view.render: + extends: base + tags: + - { tag: view.name, type: string, required: recommended } + +mixins: + # peer — outbound/remote-peer capability, PULLED via `include` by client span types. + peer: + tags: + - { tag: peer.service, type: string, required: recommended } + - { tag: _dd.peer.service.source, type: string, required: recommended } + - { tag: _dd.peer.service.remapped_from, type: string, required: recommended } + - { tag: peer.hostname, type: string, required: recommended } + - { tag: peer.ipv4, type: string } + - { tag: peer.ipv6, type: string } + - { tag: peer.port, type: int } + + # ci_visibility — per-span test tags. Its capability flag (_dd.civisibility.enabled) lives in + # trace_level, outside this mixin (general rule: capability flags are trace-level, mixins hold the + # per-span tags). Applies to the `test` span type (not modeled here yet). + ci_visibility: + enabled_by: dd.civisibility.enabled + applies: [ test ] + tags: + - { tag: test.name, type: string, required: recommended } + - { tag: test.suite, type: string, required: recommended } + - { tag: test.status, type: string, required: recommended } + - { tag: test.framework, type: string, required: recommended } + +# --------------------------------------------------------------------------- +# Notes +# - Product .enabled flags moved to `trace_level` (process-constant) — the old product mixins held +# only those flags, so they dissolved. `enabled_by`/attachment gating is a runtime concern. +# - span.kind enumerates: server | client | producer | consumer | internal | broker. +# - Reserved/special keys (service, resource.name, error, sampling.priority, ...) route to span +# fields/directives, not tag storage — they live in the per-language overlay, not here. +# --------------------------------------------------------------------------- From 098070fd3e0120af409b27f622204e30a99e9941 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 09:41:59 -0400 Subject: [PATCH 04/14] Promote dense-tags gate to trace.dense.tags.enabled Config flag Replaces the raw Boolean.getBoolean("dd.trace.dense.tags.enabled") system-property read with a proper trace.dense.tags.enabled Config flag, captured once into the DENSE_TAGS_ENABLED static constant so it constant-propagates on the span-creation path. Co-Authored-By: Claude Opus 4.8 --- .../datadog/trace/api/config/TracerConfig.java | 7 +++++++ .../main/java/datadog/trace/core/CoreTracer.java | 15 ++++++++++----- .../src/main/java/datadog/trace/api/Config.java | 7 +++++++ metadata/supported-configurations.json | 8 ++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java index 9faf4f4ea8e..49640f0ef6b 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java @@ -173,5 +173,12 @@ public final class TracerConfig { public static final String TRACE_ORG_GUARD_STRICT = "trace.org.guard.strict"; public static final String TRACE_ORG_GUARD_TRUSTED_OPMS = "trace.org.guard.trusted.opms"; + /** + * Routes known tags through the dense (id-keyed) tag store instead of per-tag entries. + * Experimental, OFF by default. The {@code KnownTagCodec} is registered regardless; this flag + * only selects whether tags take the dense storage path. + */ + public static final String TRACE_DENSE_TAGS_ENABLED = "trace.experimental.dense.tags.enabled"; + private TracerConfig() {} } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 4e32410d76c..a13ae3e5863 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -247,6 +247,12 @@ public static CoreTracerBuilder builder() { private static final boolean SPAN_BUILDER_REUSE_ENABLED = Config.get().isSpanBuilderReuseEnabled(); + // Dense known-tag store gate (experimental, OFF by default). Read once into a static constant so + // it constant-propagates and dead-code-eliminates the disabled branches on the span-creation + // path. The KnownTagCodec is registered regardless (ids route both ways: name->id when dense is + // on, id->name when off); this flag only selects whether tags actually take the dense store. + private static final boolean DENSE_TAGS_ENABLED = Config.get().isTraceDenseTagsEnabled(); + // Cache used by buildSpan - instance so it can capture the CoreTracer private final ReusableSingleSpanBuilderThreadLocalCache spanBuilderThreadLocalCache = SPAN_BUILDER_REUSE_ENABLED ? new ReusableSingleSpanBuilderThreadLocalCache(this) : null; @@ -657,11 +663,10 @@ private CoreTracer( // preload this enum to avoid triggering classloading on the hot path TraceCollector.PublishState.values(); - // Dense known-tag store (experimental, OFF by default): registering the KnownTagCodec resolver - // flips the dense store live so known tags store without a per-tag Entry. Gated by a system - // property for A/B benchmarking; when off, keyOf stays a no-op and tag storage is byte-identical - // to today. Promote to a Config flag if this becomes a permanent rollout. - if (Boolean.getBoolean("dd.trace.dense.tags.enabled")) { + // Dense known-tag store (experimental, OFF by default, see DENSE_TAGS_ENABLED): initializing + // KnownTags routes known tags into the dense store so they store without a per-tag Entry. When + // off, tags take the same path as today. Gated by the trace.dense.tags.enabled Config flag. + if (DENSE_TAGS_ENABLED) { KnownTags.init(); } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index bc7b0b904af..75e618e514b 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -1413,6 +1413,7 @@ public static String getHostName() { private final boolean jdkSocketEnabled; private final boolean spanBuilderReuseEnabled; + private final boolean traceDenseTagsEnabled; private final int tagNameUtf8CacheSize; private final int tagValueUtf8CacheSize; private final int stackTraceLengthLimit; @@ -3301,6 +3302,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.spanBuilderReuseEnabled = configProvider.getBoolean(GeneralConfig.SPAN_BUILDER_REUSE_ENABLED, true); + this.traceDenseTagsEnabled = + configProvider.getBoolean(TracerConfig.TRACE_DENSE_TAGS_ENABLED, false); this.tagNameUtf8CacheSize = Math.max(configProvider.getInteger(GeneralConfig.TAG_NAME_UTF8_CACHE_SIZE, 128), 0); this.tagValueUtf8CacheSize = @@ -5153,6 +5156,10 @@ public boolean isSpanBuilderReuseEnabled() { return spanBuilderReuseEnabled; } + public boolean isTraceDenseTagsEnabled() { + return traceDenseTagsEnabled; + } + public int getTagNameUtf8CacheSize() { return tagNameUtf8CacheSize; } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 850c6055d58..4c5bf4c8b0a 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -5641,6 +5641,14 @@ "aliases": [] } ], + "DD_TRACE_EXPERIMENTAL_DENSE_TAGS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_TRACE_EXPERIMENTAL_FEATURES_ENABLED": [ { "version": "A", From a3579bc8145e62ec58ccc22e5cc02c3d73ac95c7 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 10:43:35 -0400 Subject: [PATCH 05/14] Add lazy tagId() to TagMap.Entry and EntryReader Expose the known-tag id on EntryReader: TagMap.Entry resolves it lazily via KnownTagCodec.keyOf (same memoized-field idiom as lazyTagHash; 0L means unknown tag / inactive codec, so the sentinel is Long.MIN_VALUE). The dense reader flyweight already has the id in hand at emit time and records it directly, skipping the keyOf resolve. Co-Authored-By: Claude Opus 4.8 --- .../main/java/datadog/trace/api/TagMap.java | 51 ++++++++++++++++++- .../trace/api/TagMapDenseForkedTest.java | 26 ++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 849c4db9911..45d2984b426 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -51,6 +51,10 @@ public final class TagMap implements Map, Iterable. public static final TagMap EMPTY = new TagMap(new Object[1 << 4], 0); + // Sentinel for a not-yet-resolved lazy tag id. Cannot be 0L: 0L is a valid keyOf result (the tag + // is not a known tag, or the codec is inactive). Shared by Entry and EntryReadingHelper. + static final long TAG_ID_NOT_COMPUTED = Long.MIN_VALUE; + /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { TagMap tagMap = TagMap.create(map.size()); @@ -165,6 +169,12 @@ public interface EntryReader { String tag(); + /** + * The known-tag id for this entry's tag, or {@code 0L} when the tag is not a known tag (or the + * {@link KnownTagCodec} is inactive). Resolved via {@link KnownTagCodec#keyOf(String)}. + */ + long tagId(); + byte type(); boolean is(byte type); @@ -313,6 +323,13 @@ static Entry newDoubleEntry(String tag, Double box) { */ int lazyTagHash; + /* + * Known-tag id, lazily resolved using the same trick as lazyTagHash. TAG_ID_NOT_COMPUTED marks + * "not yet resolved" (0L is a valid result -- unknown tag / inactive codec -- so it cannot be + * the sentinel). Only pays off on the dense-OFF path; dense-ON known tags never become Entry-s. + */ + long lazyTagId = TAG_ID_NOT_COMPUTED; + // To optimize construction of Entry around boxed primitives and Object entries, // no type checks are done during construction. // Any Object entries are initially marked as type ANY, prim set to 0, and the Object put into @@ -353,6 +370,17 @@ int hash() { return hash; } + @Override + public long tagId() { + // Same lazy idiom as hash(): a benign race just recomputes keyOf, which is deterministic. + long id = this.lazyTagId; + if (id != TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.lazyTagId = id; + return id; + } + @Override public Entry entry() { return this; @@ -2668,7 +2696,7 @@ private EntryReader emitDense(long tagId, Object value) { if (reader == null) { reader = this.denseReader = new EntryReadingHelper(); } - reader.set(KnownTagCodec.nameOf(tagId), value); + reader.set(KnownTagCodec.nameOf(tagId), value, tagId); return reader; } @@ -3307,17 +3335,28 @@ final class EntryReadingHelper implements TagMap.EntryReader { private Map.Entry mapEntry; private String tag; private Object value; + private long tagId; void set(String tag, Object value) { this.mapEntry = null; this.tag = tag; this.value = value; + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access + } + + /** Dense emit: the id is known directly, so record it and skip the keyOf resolve. */ + void set(String tag, Object value, long tagId) { + this.mapEntry = null; + this.tag = tag; + this.value = value; + this.tagId = tagId; } void set(Map.Entry mapEntry) { this.mapEntry = mapEntry; this.tag = mapEntry.getKey(); this.value = mapEntry.getValue(); + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } @Override @@ -3325,6 +3364,16 @@ public String tag() { return this.tag; } + @Override + public long tagId() { + long id = this.tagId; + if (id != TagMap.TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.tagId = id; + return id; + } + @Override public byte type() { return TagValueConversions.typeOf(this.value); diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java index 70e41c47798..c9c2b4ff1c0 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java @@ -271,4 +271,30 @@ void removingParentDenseKeyTombstonesIt() { assertFalse(union.containsKey(BASE_SERVICE)); child.checkIntegrity(); } + + @Test + void denseReaderExposesTagId() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); // dense-routed + map.set(CUSTOM_A, "alpha"); // bucket-routed + + Map idsByTag = new HashMap<>(); + map.forEach(reader -> idsByTag.put(reader.tag(), reader.tagId())); + + // dense entry: the reader carries the real known-tag id directly + assertEquals(KnownTagCodec.keyOf(BASE_SERVICE), idsByTag.get(BASE_SERVICE).longValue()); + assertTrue(idsByTag.get(BASE_SERVICE) != 0L, "known tag has a non-zero id"); + // bucket entry for a custom tag: unknown -> 0L + assertEquals(0L, idsByTag.get(CUSTOM_A).longValue()); + } + + @Test + void bucketEntryResolvesTagIdLazily() { + TagMap map = map(); + map.set(CUSTOM_A, "alpha"); // custom tag stays a bucket Entry + + TagMap.Entry entry = map.getEntry(CUSTOM_A); + assertEquals(0L, entry.tagId()); // unknown tag -> 0L + assertEquals(0L, entry.tagId()); // second read hits the memoized field + } } From fd42e978aa63dbd5c2307351243f34766102a534 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:03:11 -0400 Subject: [PATCH 06/14] Harden tag-registry generator: clear output tree + Locale.ROOT formatting Address two Codex robustness findings on the generator: - generateKnownTags now clears its owned destination tree before regenerating, so a report/source file retired by a later revision cannot linger and fail verifyKnownTags with no way for the fix task to remove it. - All String.format calls in the report/emitter use Locale.ROOT, so %d output no longer localizes on machines whose default locale uses non-ASCII digits, preserving the byte-identical-output invariant verifyKnownTags relies on. Co-Authored-By: Claude Opus 4.8 --- .../gradle/plugin/tags/KnownTagsEmitter.kt | 4 ++- .../plugin/tags/TagRegistryGenerator.kt | 31 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt index 82c04562f62..1c35d62c43b 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -1,5 +1,7 @@ package datadog.gradle.plugin.tags +import java.util.Locale + /** * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` @@ -138,5 +140,5 @@ object KnownTagsEmitter { return b.toString() } - private fun hex(id: Long): String = "0x%016XL".format(id) + private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id) } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt index c57a055c405..9cfda68246e 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import java.io.File +import java.util.Locale /** * Turns the language-agnostic {@code tag-conventions.yaml} + the Java overlay into the generated tag @@ -26,6 +27,10 @@ object TagRegistryGenerator { mapper.readValue(it, object : TypeReference>() {}) } + // Clear the owned destination tree first, so a report/source file retired by a later generator + // revision doesn't linger: otherwise verifyKnownTags flags it as stale while telling developers + // to rerun generateKnownTags, which (without this) can't actually remove it. + outDir.deleteRecursively() outDir.mkdirs() // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } @@ -66,11 +71,12 @@ object TagRegistryGenerator { for (t in reg.stored) { a.appendLine( " %6d %5s %s %s %-18s %-12s %s".format( + Locale.ROOT, t.serial, if (t.slotted) t.slot.toString() else "-", if (t.intercepted) "I" else "-", if (t.traceLevel) "T" else "-", - "0x%016X".format(t.id), + "0x%016X".format(Locale.ROOT, t.id), t.required, t.name)) } @@ -79,7 +85,12 @@ object TagRegistryGenerator { for (v in reg.reserved) { a.appendLine( " %6d %-18s %-12s %s%s".format( - v.serial, "0x%016X".format(v.id), v.kind, v.name, v.field?.let { " -> $it" } ?: "")) + Locale.ROOT, + v.serial, + "0x%016X".format(Locale.ROOT, v.id), + v.kind, + v.name, + v.field?.let { " -> $it" } ?: "")) } a.appendLine() a.appendLine("# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the") @@ -90,11 +101,12 @@ object TagRegistryGenerator { .filter { it.slotted && !it.traceLevel } .map { it.slot } .sorted() - a.appendLine(" %-14s count=%-3d slots=%s".format(type, slots.size, slots)) + a.appendLine(" %-14s count=%-3d slots=%s".format(Locale.ROOT, type, slots.size, slots)) } val traceSlots = reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() - a.appendLine(" %-14s count=%-3d slots=%s".format("", traceSlots.size, traceSlots)) + a.appendLine( + " %-14s count=%-3d slots=%s".format(Locale.ROOT, "", traceSlots.size, traceSlots)) return a.toString() } @@ -127,7 +139,8 @@ object TagRegistryGenerator { else -> "bkt" } lay.appendLine( - " %-26s %-12s %-12s %s".format(t.name, field, t.required, if (st?.intercepted == true) "I" else "")) + " %-26s %-12s %-12s %s".format( + Locale.ROOT, t.name, field, t.required, if (st?.intercepted == true) "I" else "")) } } } @@ -157,7 +170,9 @@ object TagRegistryGenerator { f.appendLine("$type (${tags.size} tags):") for (t in tags) { val st = byName[t.name] - f.appendLine(" %-12s %-26s %s".format(tierField(st), t.name, if (st?.intercepted == true) "I" else "")) + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), t.name, if (st?.intercepted == true) "I" else "")) } } val traceTags = @@ -165,7 +180,9 @@ object TagRegistryGenerator { f.appendLine() f.appendLine(" (${traceTags.size} tags):") for (st in traceTags) { - f.appendLine(" %-12s %-26s %s".format(tierField(st), st.name, if (st.intercepted) "I" else "")) + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), st.name, if (st.intercepted) "I" else "")) } return f.toString() } From a2c456ab5a7cdef25798a4883d3bf14bdb139241 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:15:15 -0400 Subject: [PATCH 07/14] Log dense-tags flag in Config.toString when set away from default Address the Codex finding that the new experimental dense-tags flag was stored but never surfaced in Config.toString (repo convention requires new configs appear there for diagnostics). Surface it only when it diverges from the default, so normal config dumps stay uncluttered, and compare against a new DEFAULT_TRACE_DENSE_TAGS_ENABLED constant (also now the getBoolean default) so the condition stays correct if the default ever changes. Co-Authored-By: Claude Opus 4.8 --- .../src/main/java/datadog/trace/api/ConfigDefaults.java | 1 + internal-api/src/main/java/datadog/trace/api/Config.java | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index a57e5d37882..49c9377dff9 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -305,6 +305,7 @@ public final class ConfigDefaults { public static final int DEFAULT_TRACE_X_DATADOG_TAGS_MAX_LENGTH = 512; static final boolean DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH = false; + static final boolean DEFAULT_TRACE_DENSE_TAGS_ENABLED = false; static final boolean DEFAULT_TRACE_LONG_RUNNING_ENABLED = false; static final long DEFAULT_TRACE_LONG_RUNNING_INITIAL_FLUSH_INTERVAL = 20; // seconds static final long DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL = 120; // seconds -> 2 minutes diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 75e618e514b..6e7fbe2d6c0 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -175,6 +175,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_MAX_ITEMS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_TAG_KEYS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES; +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_DENSE_TAGS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_EXPERIMENTAL_FEATURES_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_KEEP_LATENCY_THRESHOLD_MS; @@ -3303,7 +3304,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.spanBuilderReuseEnabled = configProvider.getBoolean(GeneralConfig.SPAN_BUILDER_REUSE_ENABLED, true); this.traceDenseTagsEnabled = - configProvider.getBoolean(TracerConfig.TRACE_DENSE_TAGS_ENABLED, false); + configProvider.getBoolean( + TracerConfig.TRACE_DENSE_TAGS_ENABLED, DEFAULT_TRACE_DENSE_TAGS_ENABLED); this.tagNameUtf8CacheSize = Math.max(configProvider.getInteger(GeneralConfig.TAG_NAME_UTF8_CACHE_SIZE, 128), 0); this.tagValueUtf8CacheSize = @@ -6828,6 +6830,11 @@ public String toString() { + sqsInjectDatadogAttributeEnabled + ", snsInjectDatadogAttributeEnabled=" + snsInjectDatadogAttributeEnabled + // Experimental: surfaced only when set away from the default, keeping normal dumps clean. + // Compared to the default constant (not a literal) so it survives a default change. + + (traceDenseTagsEnabled != DEFAULT_TRACE_DENSE_TAGS_ENABLED + ? ", traceDenseTagsEnabled=" + traceDenseTagsEnabled + : "") + '}'; } } From c4f9645784060606599dbd870cebdd9393b6e1cf Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:21:46 -0400 Subject: [PATCH 08/14] Hold generated tag registry to the formatting standard Address the Codex finding that gradle/spotless.gradle excluded src/generated/**, bypassing google-java-format on committed generated code. Drop the exclude and make KnownTagsEmitter emit google-java-format-clean output (blank line before the keyOf static initializer; wrap the long id-assignment), so spotlessCheck and the byte-identical verifyKnownTags gate both pass on the emitted file. internal-api is the only module with a committed src/generated Java tree, so the exclude removal is otherwise a no-op. Co-Authored-By: Claude Opus 4.8 --- .../kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt | 4 +++- gradle/spotless.gradle | 8 +++++--- .../src/generated/java/datadog/trace/api/KnownTags.java | 4 +++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt index 1c35d62c43b..ff71f4b8ab3 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -89,11 +89,13 @@ object KnownTagsEmitter { b.appendLine(" private static final int[] KEYOF_HASHES;") b.appendLine(" private static final String[] KEYOF_KEYS;") b.appendLine(" private static final long[] KEYOF_IDS;") + b.appendLine() b.appendLine(" static {") b.appendLine(" StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);") b.appendLine(" long[] ids = new long[data.names.length];") b.appendLine(" for (int j = 0; j < KEYOF_NAMES.length; j++) {") - b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j];") + b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] =") + b.appendLine(" KEYOF_VALUES[j];") b.appendLine(" }") b.appendLine(" KEYOF_HASHES = data.hashes;") b.appendLine(" KEYOF_KEYS = data.names;") diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle index 769e153ddd9..f27408c7cec 100644 --- a/gradle/spotless.gradle +++ b/gradle/spotless.gradle @@ -20,9 +20,11 @@ spotless { toggleOffOn() // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 target 'src/**/*.java' - // ignore embedded test projects and everything in build dir, e.g. generated sources - // src/generated/** is emitted by code generators (e.g. the tag registry) — verified by their own freshness gate - targetExclude('src/test/resources/**', 'src/generated/**', buildDirectoryFiles) + // ignore embedded test projects and everything in build dir, e.g. generated sources. + // src/generated/** is committed generated code (e.g. the tag registry) and IS held to the + // formatting standard: emitters must produce google-java-format-clean output, and their + // freshness gate byte-compares against these formatted files. + targetExclude('src/test/resources/**', buildDirectoryFiles) tableTestFormatter('1.1.1') googleJavaFormat('1.35.0') } diff --git a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java index 57b703ea697..599b34f8a25 100644 --- a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -410,11 +410,13 @@ public final class KnownTags { private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; private static final long[] KEYOF_IDS; + static { StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); long[] ids = new long[data.names.length]; for (int j = 0; j < KEYOF_NAMES.length; j++) { - ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = KEYOF_VALUES[j]; + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = + KEYOF_VALUES[j]; } KEYOF_HASHES = data.hashes; KEYOF_KEYS = data.names; From 19222e1a85d946343f870eb69bc0d3e38f88d94e Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 11:56:48 -0400 Subject: [PATCH 09/14] Resolve OpenTelemetry tag names through the registry Adds a per-namespace name to the tag registry: each tag may declare an `open-telemetry-name` (replacing the vestigial `aliases` list). The generator now: - parses it off both YAMLs into the model, - validates it (an OTel name may not collide with a canonical tag name nor be claimed by two tags -> build fails loudly rather than silently picking one), - emits it into the keyOf table so keyOf(otelName) resolves to the canonical tag's id (inbound, many->one), and - emits a reverse switch so openTelemetryNameOf(tagId) recovers it (outbound). KnownTagCodec gains datadogTagOf(tagId) (== nameOf, the canonical name) and openTelemetryTagOf(tagId) (the OTel name, or null). nameOf is unchanged and still returns the Datadog name -- outbound is namespace-specific, not normalized. Serializer applicability (when a span renders under OTel names, fallback policy) and additional namespaces are a follow-on concern. Co-Authored-By: Claude Opus 4.8 --- .../gradle/plugin/tags/KnownTagsEmitter.kt | 33 +++++++++++++- .../gradle/plugin/tags/TagConventions.kt | 7 +++ .../datadog/gradle/plugin/tags/TagRegistry.kt | 43 ++++++++++++++++-- .../plugin/tags/TagRegistryGenerator.kt | 10 +++++ .../java/datadog/trace/api/KnownTags.java | 44 +++++++++++++++++++ internal-api/src/generated/tag-assignment.txt | 12 +++++ .../java/datadog/trace/api/KnownTagCodec.java | 18 ++++++++ .../java/datadog/trace/api/KnownTagsTest.java | 41 +++++++++++++++++ .../trace/api/TagMapDenseFuzzForkedTest.java | 5 +++ tag-conventions.java.yaml | 2 +- tag-conventions.yaml | 20 ++++----- 11 files changed, 219 insertions(+), 16 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt index ff71f4b8ab3..fd0a65eb5a2 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -39,6 +39,11 @@ object KnownTagsEmitter { fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM") val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order + // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch. + val otelName = + (reg.reserved.mapNotNull { v -> v.otelName?.let { v.name to it } } + + reg.stored.mapNotNull { t -> t.otelName?.let { t.name to it } }) + .toMap() val b = StringBuilder() b.appendLine("package $pkg;") b.appendLine() @@ -79,12 +84,23 @@ object KnownTagsEmitter { } b.appendLine() - // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). + // OpenTelemetry name -> canonical tag name, for the tags that declare one. Deterministic order + // (by OTel name) so output stays byte-identical. + val otelByCanonical = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then + // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id + // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name. b.appendLine(" private static final String[] KEYOF_NAMES = {") order.forEach { b.appendLine(" ${nameC(it)},") } + otelByCanonical.forEach { (otel, _) -> b.appendLine(" \"$otel\",") } b.appendLine(" };") b.appendLine(" private static final long[] KEYOF_VALUES = {") order.forEach { b.appendLine(" ${idC(it)},") } + otelByCanonical.forEach { (_, canonical) -> b.appendLine(" ${idC(canonical)},") } b.appendLine(" };") b.appendLine(" private static final int[] KEYOF_HASHES;") b.appendLine(" private static final String[] KEYOF_KEYS;") @@ -118,6 +134,21 @@ object KnownTagsEmitter { b.appendLine(" }") b.appendLine(" }") b.appendLine() + // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The + // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup. + b.appendLine(" @Override") + b.appendLine(" public String openTelemetryNameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + val otel = otelName[name] ?: continue + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return \"$otel\";") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() b.appendLine(" @Override") b.appendLine(" public int slotCount() {") b.appendLine(" return SLOT_COUNT;") diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt index a01da7bc75c..180d2b63a08 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt @@ -16,6 +16,12 @@ private constructor( val name: String, val type: String, val required: String, + /** + * The tag's OpenTelemetry-namespace name, if it has one. keyOf resolves it to this tag's + * canonical id (inbound, many->one); openTelemetryNameOf recovers it (outbound). Further + * namespaces and serializer applicability are a follow-on concern. + */ + val otelName: String? = null, ) data class SpanType( @@ -167,6 +173,7 @@ private constructor( name = m["tag"].toString(), type = (m["type"] as? String) ?: "string", required = (m["required"] as? String) ?: "optional", + otelName = m["open-telemetry-name"] as? String, ) } ?: emptyList() } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt index 0eebcbda629..4c256e1221e 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt @@ -34,6 +34,7 @@ private constructor( val slot: Int, val traceLevel: Boolean, val id: Long, + val otelName: String? = null, ) { val slotted: Boolean get() = slot != NO_SLOT @@ -45,11 +46,17 @@ private constructor( val field: String?, val serial: Int, val id: Long, + val otelName: String? = null, ) /** Java overlay: intercepted tag names + the reserved/special-key registry. */ class Overlay(val intercepted: Set, val reserved: List) { - data class ReservedDef(val name: String, val kind: String, val field: String?) + data class ReservedDef( + val name: String, + val kind: String, + val field: String?, + val otelName: String? = null, + ) companion object { @Suppress("UNCHECKED_CAST") @@ -57,7 +64,11 @@ private constructor( val intercepted = (root["intercepted"] as? List)?.toSet() ?: emptySet() val reserved = (root["reserved"] as? List>)?.map { m -> - ReservedDef(m["tag"].toString(), (m["kind"] as? String) ?: "directive", m["field"] as? String) + ReservedDef( + m["tag"].toString(), + (m["kind"] as? String) ?: "directive", + m["field"] as? String, + m["open-telemetry-name"] as? String) } ?: emptyList() return Overlay(intercepted, reserved) } @@ -131,7 +142,8 @@ private constructor( val serial = 1 + i ReservedTag( v.name, v.kind, v.field, serial, - encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false)) + encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false), + v.otelName) } // Stored tags in a stable order (by name); serials are a dense global counter from @@ -144,10 +156,33 @@ private constructor( val traceLevel = t.name in traceNames StoredTag( t.name, t.type, t.required, serial, intercepted, slot, traceLevel, - id = encode(serial, intercepted, slot, traceLevel)) + id = encode(serial, intercepted, slot, traceLevel), + otelName = t.otelName) } + validateOtelNames(stored, reserved) return TagRegistry(stored, reserved, slotCount) } + + /** + * An OpenTelemetry name must be unambiguous: it may not collide with any canonical tag name, nor + * be claimed by two different tags. Otherwise keyOf(otelName) would have no single right answer. + * Fail the build loudly rather than silently pick a winner. + */ + private fun validateOtelNames(stored: List, reserved: List) { + val canonical = (stored.map { it.name } + reserved.map { it.name }).toSet() + val owner = HashMap() + val check = { name: String, otel: String? -> + if (otel != null) { + require(otel !in canonical) { + "OpenTelemetry name '$otel' (of '$name') collides with canonical tag name '$otel'" + } + val prev = owner.put(otel, name) + require(prev == null) { "OpenTelemetry name '$otel' is claimed by both '$prev' and '$name'" } + } + } + stored.forEach { check(it.name, it.otelName) } + reserved.forEach { check(it.name, it.otelName) } + } } } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt index 9cfda68246e..477e2e9d245 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -107,6 +107,16 @@ object TagRegistryGenerator { reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() a.appendLine( " %-14s count=%-3d slots=%s".format(Locale.ROOT, "", traceSlots.size, traceSlots)) + a.appendLine() + a.appendLine("# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still") + a.appendLine("# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.)") + val otelPairs = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + for ((otel, canonical) in otelPairs) { + a.appendLine(" %-30s -> %s".format(Locale.ROOT, otel, canonical)) + } return a.toString() } diff --git a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java index 599b34f8a25..e0c04dc9142 100644 --- a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -348,6 +348,15 @@ public final class KnownTags { SPAN_KIND_NAME, VERSION_NAME, VIEW_NAME, + "db.operation.name", + "db.query.text", + "db.system", + "http.request.method", + "http.response.status_code", + "server.address", + "service.name", + "url.full", + "url.query", }; private static final long[] KEYOF_VALUES = { ERROR_ID, @@ -406,6 +415,15 @@ public final class KnownTags { SPAN_KIND_ID, VERSION_ID, VIEW_NAME_ID, + DB_OPERATION_ID, + DB_STATEMENT_ID, + DB_TYPE_ID, + HTTP_METHOD_ID, + HTTP_STATUS_CODE_ID, + HTTP_HOSTNAME_ID, + SERVICE_ID, + HTTP_URL_ID, + HTTP_QUERY_STRING_ID, }; private static final int[] KEYOF_HASHES; private static final String[] KEYOF_KEYS; @@ -545,6 +563,32 @@ public String nameOf(long tagId) { } } + @Override + public String openTelemetryNameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + case SERVICE_SERIAL_NUM: + return "service.name"; + case DB_OPERATION_SERIAL_NUM: + return "db.operation.name"; + case DB_STATEMENT_SERIAL_NUM: + return "db.query.text"; + case DB_TYPE_SERIAL_NUM: + return "db.system"; + case HTTP_HOSTNAME_SERIAL_NUM: + return "server.address"; + case HTTP_METHOD_SERIAL_NUM: + return "http.request.method"; + case HTTP_QUERY_STRING_SERIAL_NUM: + return "url.query"; + case HTTP_STATUS_CODE_SERIAL_NUM: + return "http.response.status_code"; + case HTTP_URL_SERIAL_NUM: + return "url.full"; + default: + return null; + } + } + @Override public int slotCount() { return SLOT_COUNT; diff --git a/internal-api/src/generated/tag-assignment.txt b/internal-api/src/generated/tag-assignment.txt index c1516085a3b..9045eb8cde9 100644 --- a/internal-api/src/generated/tag-assignment.txt +++ b/internal-api/src/generated/tag-assignment.txt @@ -67,3 +67,15 @@ http.server count=15 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] view.render count=8 slots=[0, 1, 2, 3, 4, 5, 6, 7] count=13 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + +# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still +# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.) + db.operation.name -> db.operation + db.query.text -> db.statement + db.system -> db.type + http.request.method -> http.method + http.response.status_code -> http.status_code + server.address -> http.hostname + service.name -> service + url.full -> http.url + url.query -> http.query.string diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java index 031e908e5ca..f5203dca92b 100644 --- a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -188,6 +188,9 @@ public static int slotCount() { public interface Resolver { String nameOf(long tagId); + /** The tag's OpenTelemetry-namespace name, or {@code null} when it declares none. */ + String openTelemetryNameOf(long tagId); + long keyOf(String name); /** Number of positional slots this provider uses: (max stored fieldPos) + 1. */ @@ -214,6 +217,21 @@ public static String nameOf(long tagId) { return r != null ? r.nameOf(tagId) : null; } + /** The tag's Datadog-namespace (canonical) name — the same value as {@link #nameOf}. */ + public static String datadogNameOf(long tagId) { + return nameOf(tagId); + } + + /** + * The tag's OpenTelemetry-namespace name, or {@code null} when it declares none (or no resolver + * is registered). A serializer owns any fall-back-to-Datadog-name policy; this is a pure lookup. + */ + public static String openTelemetryNameOf(long tagId) { + if (!active) return null; + Resolver r = resolver; + return r != null ? r.openTelemetryNameOf(tagId) : null; + } + public static long keyOf(String name) { if (!active) return 0L; Resolver r = resolver; diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java index 3e0dccaae56..c70e5ff65fa 100644 --- a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -58,6 +58,24 @@ static Stream knownTags() { Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); } + /** + * (otelName, canonicalId, datadogName) — the OpenTelemetry name resolves (keyOf) to the canonical + * tag's id; datadogNameOf returns the Datadog name and openTelemetryNameOf returns the OTel name. + */ + static Stream otelNamedTags() { + return Stream.of( + Arguments.of("http.request.method", KnownTags.HTTP_METHOD_ID, "http.method"), + Arguments.of( + "http.response.status_code", KnownTags.HTTP_STATUS_CODE_ID, "http.status_code"), + Arguments.of("url.full", KnownTags.HTTP_URL_ID, "http.url"), + Arguments.of("server.address", KnownTags.HTTP_HOSTNAME_ID, "http.hostname"), + Arguments.of("url.query", KnownTags.HTTP_QUERY_STRING_ID, "http.query.string"), + Arguments.of("db.system", KnownTags.DB_TYPE_ID, "db.type"), + Arguments.of("db.operation.name", KnownTags.DB_OPERATION_ID, "db.operation"), + Arguments.of("db.query.text", KnownTags.DB_STATEMENT_ID, "db.statement"), + Arguments.of("service.name", KnownTags.SERVICE_ID, "service")); + } + /** * The subset flagged INTERCEPTED (sign bit) — must agree with the interceptor's needsIntercept. */ @@ -120,6 +138,29 @@ void nameOfResolvesIdToName(String name, long id) { assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); } + @ParameterizedTest + @MethodSource("otelNamedTags") + void otelNameResolvesToCanonicalId(String otelName, long id, String datadogName) { + // Inbound (keyOf) is many->one: both names land on the same canonical id. + assertEquals(id, KnownTagCodec.keyOf(otelName), "keyOf(" + otelName + ")"); + assertEquals(id, KnownTagCodec.keyOf(datadogName), "keyOf(" + datadogName + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void namespaceAccessorsReturnPerNamespaceName(String otelName, long id, String datadogName) { + assertEquals(datadogName, KnownTagCodec.datadogNameOf(id), "datadogNameOf"); + assertEquals(otelName, KnownTagCodec.openTelemetryNameOf(id), "openTelemetryNameOf"); + // nameOf stays the Datadog name -- outbound is namespace-specific, not normalized to OTel. + assertEquals(datadogName, KnownTagCodec.nameOf(id), "nameOf stays Datadog"); + } + + @Test + void tagsWithoutOtelNameReturnNull() { + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID)); // no OTel name declared + assertNull(KnownTagCodec.openTelemetryNameOf(0L)); // unknown id + } + @ParameterizedTest @MethodSource("interceptedTags") void interceptedTagsCarryFlag(long id) { diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java index f1c8a744d4b..6010499d7ea 100644 --- a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java @@ -71,6 +71,11 @@ public String nameOf(long tagId) { : null; } + @Override + public String openTelemetryNameOf(long tagId) { + return null; // synthetic resolver declares no OpenTelemetry names + } + @Override public int slotCount() { return 0; // positional unused diff --git a/tag-conventions.java.yaml b/tag-conventions.java.yaml index ab28b64545a..fcb91ac04f0 100644 --- a/tag-conventions.java.yaml +++ b/tag-conventions.java.yaml @@ -25,7 +25,7 @@ intercepted: # kind: directive -> triggers sampling/trace behavior reserved: - { tag: error, kind: structural, field: error } - - { tag: service, kind: structural, field: service, aliases: [service.name] } + - { tag: service, kind: structural, field: service, open-telemetry-name: service.name } - { tag: resource.name, kind: structural, field: resource } - { tag: span.type, kind: structural, field: type } - { tag: origin, kind: structural, field: origin } # trace-level field diff --git a/tag-conventions.yaml b/tag-conventions.yaml index 2eff24f1784..1639e0e8afc 100644 --- a/tag-conventions.yaml +++ b/tag-conventions.yaml @@ -17,7 +17,7 @@ # resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). # # tag fields (DOMAIN only): tag | type (string|int|long|boolean|double) -# | required (required|conditional|recommended|optional|opt_in) | aliases. +# | required (required|conditional|recommended|optional|opt_in) | open-telemetry-name. # The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each # declaration source (the trace-level tier, each span type, each mixin) is a group, and within a # group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are @@ -62,18 +62,18 @@ span_types: abstract: true extends: base tags: - - { tag: http.method, type: string, required: required, aliases: [http.request.method] } - - { tag: http.status_code, type: int, required: conditional, aliases: [http.response.status_code] } + - { tag: http.method, type: string, required: required, open-telemetry-name: http.request.method } + - { tag: http.status_code, type: int, required: conditional, open-telemetry-name: http.response.status_code } - { tag: network.protocol.version, type: string, required: recommended } http.server: extends: http tags: - - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } - { tag: http.route, type: string, required: conditional } - - { tag: http.hostname, type: string, required: required, aliases: [server.address] } + - { tag: http.hostname, type: string, required: required, open-telemetry-name: server.address } - { tag: http.useragent, type: string, required: recommended } - - { tag: http.query.string, type: string, required: recommended, aliases: [url.query] } + - { tag: http.query.string, type: string, required: recommended, open-telemetry-name: url.query } - { tag: servlet.path, type: string, required: optional } - { tag: servlet.context, type: string, required: optional } @@ -81,19 +81,19 @@ span_types: extends: http include: [ peer ] tags: - - { tag: http.url, type: string, required: required, aliases: [url.full] } + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } - { tag: http.resend_count, type: int, required: recommended } db.client: extends: base include: [ peer ] tags: - - { tag: db.type, type: string, required: required, aliases: [db.system] } + - { tag: db.type, type: string, required: required, open-telemetry-name: db.system } - { tag: db.instance, type: string, required: recommended } - - { tag: db.operation, type: string, required: recommended, aliases: [db.operation.name] } + - { tag: db.operation, type: string, required: recommended, open-telemetry-name: db.operation.name } - { tag: db.user, type: string, required: recommended } - { tag: db.pool.name, type: string, required: optional } - - { tag: db.statement, type: string, required: recommended, aliases: [db.query.text] } + - { tag: db.statement, type: string, required: recommended, open-telemetry-name: db.query.text } view.render: extends: base From 79ff278e1156901b6f31cc00ad15dbc05dd3c95a Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 17 Aug 2026 21:07:44 -0400 Subject: [PATCH 10/14] Add per-operation dense-store sizing (SizingHint) on the colored base Reconciles #11901's SizingHint half onto the colored generator-v2 base as its own layer. A per-operation SizingHint (kept in a bounded, self-tuning SizingHintTable keyed by operation name, two lanes for entry vs child spans) sizes a span's dense TagMap at create and records the observed known-tag high-water mark back on finish, so the reused hint converges to the operation's real size. Erases the fixed KNOWN_INIT_CAP dense-array floor tax that regressed bare/small spans in the #12047-vs-1.65 A/B; the id-keyed write API stacks on top of this as a separate layer. New: SizingHint, SizingHelper, SizingHintTable, FlatHashtable (+ tests, jmh). TagMap: create(SizingHint), recordSize, denseCapHint. DDSpanContext threads a sizingHint through the primary ctor + recordDenseSize on finish; DDSpan hooks it at finishAndAddToTrace; CoreTracer resolves the per-operation hint/lane. Co-Authored-By: Claude Opus 4.8 --- .../java/datadog/trace/core/CoreTracer.java | 12 +- .../main/java/datadog/trace/core/DDSpan.java | 1 + .../datadog/trace/core/DDSpanContext.java | 30 +++- .../trace/util/FlatHashtableBenchmark.java | 97 +++++++++++ .../java/datadog/trace/api/SizingHelper.java | 27 +++ .../java/datadog/trace/api/SizingHint.java | 49 ++++++ .../datadog/trace/api/SizingHintTable.java | 98 +++++++++++ .../main/java/datadog/trace/api/TagMap.java | 31 +++- .../datadog/trace/util/FlatHashtable.java | 152 +++++++++++++++++ .../trace/api/SizingHintTableTest.java | 105 ++++++++++++ .../datadog/trace/util/FlatHashtableTest.java | 158 ++++++++++++++++++ 11 files changed, 754 insertions(+), 6 deletions(-) create mode 100644 internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java create mode 100644 internal-api/src/main/java/datadog/trace/api/SizingHelper.java create mode 100644 internal-api/src/main/java/datadog/trace/api/SizingHint.java create mode 100644 internal-api/src/main/java/datadog/trace/api/SizingHintTable.java create mode 100644 internal-api/src/main/java/datadog/trace/util/FlatHashtable.java create mode 100644 internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java create mode 100644 internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index a13ae3e5863..4b0e5fa61f7 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -42,6 +42,8 @@ import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.KnownTags; import datadog.trace.api.Pair; +import datadog.trace.api.SizingHint; +import datadog.trace.api.SizingHintTable; import datadog.trace.api.TagMap; import datadog.trace.api.TraceConfig; import datadog.trace.api.civisibility.config.BazelMode; @@ -2194,6 +2196,13 @@ protected static final DDSpanContext buildSpanContext( requestContextDataIast = builderRequestContextDataIast; } + // Per-operation dense-store sizing: an entry (local-root) span carries the trace-metadata / + // enriching tags a child doesn't, so pick the lane by whether we have a local parent. A + // resolved hint sizes the span's TagMap and self-tunes on finish; null (no/unkeyable + // operation name) falls back to the generic default capacity. + final boolean entrySpan = !(resolvedParentSpanContext instanceof DDSpanContext); + final SizingHint sizingHint = SizingHintTable.hintFor(operationName, entrySpan); + // some attributes are inherited from the parent context = new DDSpanContext( @@ -2222,7 +2231,8 @@ protected static final DDSpanContext buildSpanContext( tracer.profilingContextIntegration, tracer.injectBaggageAsTags, tracer.injectLinksAsTags, - mergedTracerTagsNeedsIntercept ? null : mergedTracerTags); + mergedTracerTagsNeedsIntercept ? null : mergedTracerTags, + sizingHint); // By setting the tags on the context we apply decorators to any tags that have been set via // the builder. This is the order that the tags were added previously, but maybe the `tags` diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index a288c405e6f..841a1f7a08f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -157,6 +157,7 @@ public boolean isFinished() { private void finishAndAddToTrace(final long durationNano) { // ensure a min duration of 1 if (DURATION_NANO_UPDATER.compareAndSet(this, 0, Math.max(1, durationNano))) { + context.recordDenseSize(); setLongRunningVersion(-this.longRunningVersion); SpanWrapper wrapper = getWrapper(); if (wrapper != null) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index a2d87e2c18b..100b9229145 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -12,6 +12,7 @@ import datadog.trace.api.DDTraceId; import datadog.trace.api.Functions; import datadog.trace.api.ProcessTags; +import datadog.trace.api.SizingHint; import datadog.trace.api.TagMap; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; @@ -138,6 +139,12 @@ public class DDSpanContext */ private final TagMap unsafeTags; + // Per-operation sizing hint (from SizingHintTable, keyed by operation name) this span was sized + // from, if any. Held so the span can feed its final dense-store size back on finish + // (recordDenseSize) -- the self-tuning loop that lets the reused hint converge to the operation's + // real known-tag high-water mark. Null when unsized. + private final SizingHint sizingHint; + /** The service name is required, otherwise the span are dropped by the agent */ private volatile String serviceName; @@ -244,6 +251,7 @@ public DDSpanContext( ProfilingContextIntegration.NoOp.INSTANCE, true, true, + null, null); } @@ -295,6 +303,7 @@ public DDSpanContext( ProfilingContextIntegration.NoOp.INSTANCE, injectBaggageAsTags, injectLinksAsTags, + null, null); } @@ -351,6 +360,7 @@ public DDSpanContext( profilingContextIntegration, injectBaggageAsTags, injectLinksAsTags, + null, null); } @@ -380,7 +390,8 @@ public DDSpanContext( final ProfilingContextIntegration profilingContextIntegration, final boolean injectBaggageAsTags, final boolean injectLinksAsTags, - final TagMap readThroughParent) { + final TagMap readThroughParent, + final SizingHint sizingHint) { assert traceCollector != null; this.traceCollector = traceCollector; @@ -412,7 +423,8 @@ public DDSpanContext( this.unsafeTags = readThroughParent != null ? TagMap.createFromParent(readThroughParent) - : TagMap.create(capacity); + : sizingHint != null ? TagMap.create(sizingHint) : TagMap.create(capacity); + this.sizingHint = sizingHint; // must set this before setting the service and resource names below this.profilingContextIntegration = profilingContextIntegration; @@ -963,6 +975,20 @@ public void setTag(final String tag, final String value) { } } + /** + * Feeds this span's final dense-store size back into the sizingHint it was created from (if any). + * Called once at span finish; lets a reused hint self-tune to the operation's observed known-tag + * high-water mark, so subsequent spans of the operation are sized correctly. + */ + void recordDenseSize() { + if (sizingHint != null) { + // Intentionally unsynchronized: recordSize is benign-racy by design (monotonic-max on a plain + // int), and a stale knownCount read only ever costs one extra growth on a later span. The + // lock would add nothing but contention on the finish path. + unsafeTags.recordSize(sizingHint); + } + } + public void setTag(TagMap.EntryReader entry) { if (entry == null) { return; diff --git a/internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java new file mode 100644 index 00000000000..91052ee6a4b --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java @@ -0,0 +1,97 @@ +package datadog.trace.util; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Directional: is the {@link FlatHashtable} hit-path lookup (what a span pays per create, all hits + * after warmup) cheap vs a {@link HashMap}? Concrete-typed {@code static final} helper so the + * static-poly specialization is in play. One op = one pass over the op-name set. Single-threaded, + * short — a rough signal, not a verdict (real numbers on the box). + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +@Threads(1) +public class FlatHashtableBenchmark { + + static final class StrHelper extends FlatHashtable.StringHelper { + @Override + public boolean matches(String key, String value) { + return key == value || key.equals(value); + } + + @Override + public String create(String key) { + return key; // store the key itself as the (self-identifying) entry + } + } + + private static final StrHelper HELPER = new StrHelper(); + + private static final String[] KEYS = { + "servlet.request", + "database.query", + "http.request", + "grpc.client", + "kafka.produce", + "kafka.consume", + "jdbc.query", + "spring.handler", + "servlet.forward", + "okhttp.request" + }; + + private String[] table; + private Map map; + + @Setup + public void setup() { + table = FlatHashtable.create(String.class, KEYS.length); + map = new HashMap<>(KEYS.length * 2); + for (String k : KEYS) { + FlatHashtable.getOrCreate(table, k, HELPER); + map.put(k, k); + } + } + + /** FlatHashtable all-hit lookups (concrete helper → specialized). */ + @Benchmark + public void flatGet(Blackhole bh) { + for (String k : KEYS) { + bh.consume(FlatHashtable.get(table, k, HELPER)); + } + } + + /** Steady-state span-create shape: get-then-getOrCreate, all hits. */ + @Benchmark + public void flatGetOrCreate(Blackhole bh) { + for (String k : KEYS) { + bh.consume(FlatHashtable.getOrCreate(table, k, HELPER)); + } + } + + /** Baseline: HashMap lookups over the same keys. */ + @Benchmark + public void hashMapGet(Blackhole bh) { + for (String k : KEYS) { + bh.consume(map.get(k)); + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHelper.java b/internal-api/src/main/java/datadog/trace/api/SizingHelper.java new file mode 100644 index 00000000000..282be66079e --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/SizingHelper.java @@ -0,0 +1,27 @@ +package datadog.trace.api; + +import datadog.trace.util.FlatHashtable; + +/** + * {@link FlatHashtable} policy for the per-operation {@link SizingHint} table: keys by operation + * name, entries are {@code SizingHint}s carrying that name plus its cached spread hash. Stateless — + * held by {@link SizingHintTable} as a concrete-typed {@code static final} singleton so {@code + * FlatHashtable.get}/{@code getOrCreate} specialize (devirtualize + inline) at the call site. + * + *

Extends {@link FlatHashtable.StringHelper}, which seals the spread {@code hash}; this class + * only supplies {@code matches} and {@code create}. Both use the inherited {@code hash} so the + * cached {@link SizingHint#labelHash} is always the same spread the probe used. + */ +final class SizingHelper extends FlatHashtable.StringHelper { + @Override + public boolean matches(String key, SizingHint value) { + // int gate on the cached hash before equals; op-names are usually interned literals, so `==` is + // the common hit. + return value.labelHash == hash(key) && (key == value.label || key.equals(value.label)); + } + + @Override + public SizingHint create(String key) { + return new SizingHint(key, hash(key), SizingHintTable.SEED_SIZE); + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHint.java b/internal-api/src/main/java/datadog/trace/api/SizingHint.java new file mode 100644 index 00000000000..498d79f3433 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/SizingHint.java @@ -0,0 +1,49 @@ +package datadog.trace.api; + +/** + * Opaque per-operation dense-store sizing hint, and a self-contained {@link + * datadog.trace.util.FlatHashtable} slot: it carries everything the probe compares ({@link #label} + * + cached {@link #labelHash}) plus the tuned payload ({@link #size}). Holding key, hash, and value + * in ONE object behind ONE array slot is deliberate — entry publication is a single reference + * store, so a reader sees {@code null} or a complete entry (never a torn one), and the {@code + * final} identity fields are visible even under racy publication (JMM final-field guarantee). That + * sidesteps the memory-ordering / visibility problems parallel key/hash/value arrays would create, + * no volatile or atomics. + * + *

Opaque to everything outside {@code datadog.trace.api}: no public members. {@link TagMap} + * reads {@link #size} to size a fresh dense store and writes it back (monotonic-max) at a terminal + * point; {@code SizingHelper} mints and compares by {@link #label}/{@link #labelHash}. Callers only + * ever hold the reference. + * + *

{@link #labelHash} is supplied by the helper (a single spread source — {@code + * FlatHashtable.StringHelper.hash}) so the cached gate always matches the probe hash. + */ +public final class SizingHint { + // Identity: final => safely published under a racy single-reference store. `label` is the + // operation + // name (typically an interned literal, so the `==` fast-path usually hits). `labelHash` is the + // helper's spread hash, cached to gate `equals` with an int compare during probing. + final String label; + final int labelHash; + + // Payload: the tuned dense-store size. Plain racy int, updated monotonic-max — a stale/lost read + // only mis-sizes an array (over/under-provision), never corrupts tag data, so no synchronization. + int size; + + // When true, {@code size} is fixed and recordSize won't grow it. For the shared default / + // overflow + // hint (a HETEROGENEOUS catch-all for operation-less / over-budget spans): self-tuning it via + // monotonic-max would converge to the max across unlike sharers and over-provision the lean ones. + final boolean capped; + + SizingHint(String label, int labelHash, int seedSize) { + this(label, labelHash, seedSize, false); + } + + SizingHint(String label, int labelHash, int seedSize, boolean capped) { + this.label = label; + this.labelHash = labelHash; + this.size = seedSize; + this.capped = capped; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java b/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java new file mode 100644 index 00000000000..c01e63fb4dc --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java @@ -0,0 +1,98 @@ +package datadog.trace.api; + +import datadog.trace.util.FlatHashtable; + +/** + * Process-wide, self-tuning registry of per-operation {@link SizingHint}s, keyed by operation name. + * Pure static: the tracer resolves a hint here at span build and hands it to the span, which sizes + * its dense {@link TagMap} from it and records the actual size back on finish — the hint converges + * to the operation's real known-tag high-water mark, so later spans of that operation size + * correctly. + * + *

Two lanes, because a span's dense size depends systematically on whether it is an entry + * (local-root) span or not: entry spans carry the trace-metadata / enriching tags and children + * don't, so the same operation name has two different steady-state sizes. {@link #hintFor} picks + * the lane. + * + *

Bounded + racy by design. Each lane is a fixed-capacity {@link FlatHashtable} (never + * resized) so memory is bounded even under unbounded/dynamic operation names; once a lane's + * cardinality budget is spent, further operations share a capped default hint. Construction, + * insertion, and the monotonic-max size update are all lock-free and deliberately racy — a lost + * update or a double-mint only mis-sizes an array (over/under-provision) for a span or two, never + * corrupts tag data (see {@link FlatHashtable} and {@link SizingHint} for the rationale). + * + *

Keyed by the operation name's {@code String} form. In practice every operation name we see is + * a {@code String} or a {@link datadog.trace.bootstrap.instrumentation.api.UTF8BytesString}, whose + * {@code toString()} just returns its backing field — so the {@code toString()} here is O(1) and + * allocation-free on the hot path. A {@code null} operation name gets no hint (the span falls back + * to the generic default capacity); a missed hint is benign, never wrong. + */ +public final class SizingHintTable { + private SizingHintTable() {} + + // Seed for a fresh per-operation hint: a floor that self-tunes up via monotonic-max recordSize. + static final int SEED_SIZE = 1; + // Fixed size for the shared over-budget hint: a small lean default. Capped (never grown by + // recordSize) because it's a heterogeneous catch-all -- growing it would over-provision lean + // sharers + // to the max of an unlike cohort. + static final int OVERFLOW_SEED = 8; + // Max distinct operation names per lane that get their own hint before collapsing to the capped + // default. Backing capacity is the next power of two >= 2 * this (load factor <= 0.5). + private static final int CARDINALITY_LIMIT = 512; + + // Concrete-typed static-final singleton => FlatHashtable calls specialize at this call site. + private static final SizingHelper HELPER = new SizingHelper(); + + // Entry (local-root, enriched) lane and non-entry (child) lane, keyed by operation name. + private static final SizingHint[] ENTRY_SLOTS = + FlatHashtable.create(SizingHint.class, CARDINALITY_LIMIT); + private static final SizingHint[] CHILD_SLOTS = + FlatHashtable.create(SizingHint.class, CARDINALITY_LIMIT); + + // Shared capped hint each lane returns once its budget is exhausted. + private static final SizingHint ENTRY_OVERFLOW = + new SizingHint(null, 0, OVERFLOW_SEED, /* capped */ true); + private static final SizingHint CHILD_OVERFLOW = + new SizingHint(null, 0, OVERFLOW_SEED, /* capped */ true); + + // Approximate live counts gating each lane's budget. Plain racy ints -- a few over/under the cap + // under contention is harmless (the cap is a safety bound, not an exact quota). + private static int entrySize; + private static int childSize; + + /** + * The sizing hint for {@code operationName} in the given lane: the existing one, a freshly-minted + * (seeded) one if the lane has budget, or the shared capped default if it's full. Returns {@code + * null} for a {@code null} operation name — the span then uses the generic default capacity + * (operation-less spans aren't reliably similar, so they get no hint). Hits are a single probe; + * the create path is warmup-rare. + */ + public static SizingHint hintFor(CharSequence operationName, boolean entrySpan) { + if (operationName == null) { + return null; + } + final String key = + operationName.toString(); // O(1) for String / UTF8BytesString (see class doc) + final SizingHint[] slots = entrySpan ? ENTRY_SLOTS : CHILD_SLOTS; + final SizingHint overflow = entrySpan ? ENTRY_OVERFLOW : CHILD_OVERFLOW; + + final SizingHint existing = FlatHashtable.get(slots, key, HELPER); + if (existing != null) { + return existing; + } + if ((entrySpan ? entrySize : childSize) >= CARDINALITY_LIMIT) { + return overflow; + } + final SizingHint created = FlatHashtable.getOrCreate(slots, key, HELPER); + if (created == null) { + return overflow; // physically full -- shouldn't happen under the cap, but stay safe + } + if (entrySpan) { + entrySize++; // racy approximate count + } else { + childSize++; + } + return created; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 45d2984b426..9bff9fe1575 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -79,6 +79,14 @@ public static final TagMap create(int size) { return new TagMap(); } + /** Creates a mutable TagMap whose dense store is sized per-operation from {@code hint}. */ + public static final TagMap create(SizingHint hint) { + TagMap tagMap = new TagMap(); + int n = hint.size; + tagMap.denseCapHint = n > 0 ? n : KNOWN_INIT_CAP; + return tagMap; + } + /** * Creates a fresh, mutable TagMap that reads through to {@code parent} on local misses. The * parent must be frozen and is fixed for the life of the returned map (no re-parenting), so @@ -1112,7 +1120,12 @@ public EntryChange next() { private boolean knownTraceLevel; private static final int KNOWN_INIT_CAP = - 12; // generous per-type max stopgap; exact per-type sizing comes with the tag registry + 12; // generous default when no SizingHint; per-operation sizing comes via create(SizingHint) + + // Initial dense-array capacity, set once from a SizingHint at create(SizingHint) (per-operation + // sizing). Just the size (an int), NOT a hint reference -- the hint stays external (recordSize is + // explicit). + private int denseCapHint = KNOWN_INIT_CAP; /** * Optional frozen parent for read-through. When non-null, reads that miss the local buckets fall @@ -1442,8 +1455,8 @@ private int knownIndexOf(long tagId) { private void ensureKnownCapacity() { if (this.knownIds == null) { - this.knownIds = new long[KNOWN_INIT_CAP]; - this.knownValues = new Object[KNOWN_INIT_CAP]; + this.knownIds = new long[this.denseCapHint]; + this.knownValues = new Object[this.denseCapHint]; } else if (this.knownCount == this.knownIds.length) { int newCap = this.knownIds.length << 1; this.knownIds = Arrays.copyOf(this.knownIds, newCap); @@ -1451,6 +1464,18 @@ private void ensureKnownCapacity() { } } + /** + * Feeds this map's final dense-entry count back to {@code hint} (call at a terminal point: freeze + * / span-finish). The hint self-tunes so future maps of the same operation size correctly. + */ + public void recordSize(SizingHint hint) { + // Capped hints (the heterogeneous shared default) are fixed -- don't let unlike sharers grow + // them. + if (!hint.capped && this.knownCount > hint.size) { + hint.size = this.knownCount; // monotonic-max; benign racy plain-int write + } + } + /** * Presence bit for {@code tagId}: {@code 1L << slot}. Colored slots are < 64; unslotted stored * tags ({@link KnownTagCodec#NO_SLOT}) fold onto one shared bit via {@code slot & 63} — crude for diff --git a/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java b/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java new file mode 100644 index 00000000000..b5254271779 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java @@ -0,0 +1,152 @@ +package datadog.trace.util; + +import java.lang.reflect.Array; + +/** + * Open-addressed, single-array find-or-create over self-contained entries — each slot is one + * reference to an entry that carries its own key (and, typically, a cached hash). One array, one + * reference per slot: entry publication is a single reference store, so a reader sees {@code null} + * or a complete entry (never a torn one), and {@code final} identity fields on the entry are + * visible under racy publication. That sidesteps the memory-ordering / visibility problems parallel + * key/hash/value arrays would create — no {@code volatile}, no atomics — as long as the payload is + * one where a stale/lost read is benign (miss → recreate; clobber → one wins). + * + *

Static polymorphism (C++-template-style). The per-use policy is a {@link Helper} — a + * stateless subclass held by each caller as a {@code static final} field declared with + * the concrete helper type (not the {@code Helper} base): + * + * 

{@code
+ * private static final MyHelper HELPER = new MyHelper();  // concrete type => exact type pinned
+ * ...
+ * V v = FlatHashtable.getOrCreate(table, key, HELPER);
+ * }
+ * + * Because {@code HELPER} is a compile-time-constant of an exact type at the call site, once these + * small {@code Support} methods inline the JIT devirtualizes and inlines {@code hash}/{@code + * matches}/{@code create} — each call site specializes to straight-line code, one instantiation per + * helper, with no CHA/type-profiling dependence. Keep the methods small so they inline; verify with + * {@code -XX:+PrintInlining} (the failure mode is silent: it compiles and runs, just stays + * megamorphic and slow). {@code Helper} is an abstract class, so a distinct final subclass is + * required anyway — an exact type gives the inliner an unambiguous receiver. + * + *

Contract: {@code table.length} must be a power of two ({@link #capacityFor}). {@code + * helper.hash} should be well-distributed (this class masks it directly). Cardinality cap / + * overflow / a live-size counter are caller policy (this class is pure mechanism): a capped + * caller does {@link #get} first, and only on a miss checks its budget before {@link #getOrCreate} + * (so hits stay a single probe and the create path is warmup-rare). + */ +public final class FlatHashtable { + private FlatHashtable() {} + + /** + * Per-use policy. Extend as a stateless final class and hold a {@code static final} + * singleton of the concrete type (see class doc) so the JIT can specialize each call site. + * + *

An abstract class (not an interface) on purpose: it forces a named helper type (no + * lambdas, which can blur the receiver the inliner needs), and if specialization ever misses, the + * fallback dispatches via {@code invokevirtual} rather than the costlier megamorphic {@code + * invokeinterface}. On the specialized (inlined) path the choice is a wash — this just hedges the + * fallback and lets shared bits be {@code final}-sealed later. + * + * @param lookup key + * @param stored entry — self-contained (carries its own key, ideally a cached hash) + */ + public abstract static class Helper { + /** Hash of {@code key}; should be well-distributed (this table masks it directly). */ + public abstract int hash(K key); + + /** Whether the stored {@code value} entry is the one for {@code key}. */ + public abstract boolean matches(K key, V value); + + /** Mint a new entry for {@code key} (called once, on insert). */ + public abstract V create(K key); + } + + /** + * {@link Helper} specialized for {@code String} keys: seals a spread {@link #hash} so String-key + * callers write only {@link #matches} and {@link #create}. Extend as a stateless final class held + * in a concrete-typed {@code static final} singleton, exactly like {@link Helper} — the {@code + * final} hash resolves directly and the concrete subclass still specializes the same at each call + * site, so there's no cost to the extra layer. + * + * @param stored entry — self-contained (carries its own key, ideally a cached hash) + */ + public abstract static class StringHelper extends Helper { + @Override + public final int hash(String key) { + final int h = key.hashCode(); + return h ^ (h >>> 16); // spread; FlatHashtable masks this directly + } + } + + /** Power-of-two capacity for a cardinality budget: {@code >= 2 * limit} (load factor <= 0.5). */ + public static int capacityFor(int cardinalityLimit) { + if (cardinalityLimit <= 0) { + throw new IllegalArgumentException("cardinalityLimit must be positive: " + cardinalityLimit); + } + return Integer.highestOneBit(cardinalityLimit * 2 - 1) << 1; + } + + /** + * Allocates a correctly-typed table for a cardinality budget ({@link #capacityFor} slots). + * Passing {@code type} makes the array's runtime component type {@code T} rather than {@code + * Object[]} — typed reads, real array-store checks, and a monomorphic element type for the JIT. + * Callers can't {@code new T[]} themselves under erasure; this does the one reflective allocation + * at construction (off any hot path). Note: this {@code create} mints the backing array; {@link + * Helper#create} mints an entry — different types, no ambiguity at the call site. + */ + @SuppressWarnings("unchecked") + public static T[] create(Class type, int cardinalityLimit) { + return (T[]) Array.newInstance(type, capacityFor(cardinalityLimit)); + } + + /** + * Existing entry for {@code key}, or {@code null}. Read-only — never creates. Single probe on a + * hit; walks to the first empty slot (or all the way around) on a miss. + */ + public static V get(V[] table, K key, Helper helper) { + final int mask = table.length - 1; + final int start = helper.hash(key) & mask; + int i = start; + for (; ; ) { + final V e = table[i]; + if (e == null) { + return null; // empty slot terminates the probe (no tombstones) + } + if (helper.matches(key, e)) { + return e; + } + i = (i + 1) & mask; + if (i == start) { + return null; // wrapped ⇒ full, absent + } + } + } + + /** + * Existing entry for {@code key}, or a freshly {@link Helper#create created} + inserted one. + * Returns {@code null} only if the table is full (no empty slot) — the caller supplies its + * overflow default. The insert is a single plain reference store: a concurrent clobber / + * double-create is acceptable only when the payload makes it benign (see class doc). + */ + public static V getOrCreate(V[] table, K key, Helper helper) { + final int mask = table.length - 1; + final int start = helper.hash(key) & mask; + int i = start; + for (; ; ) { + final V e = table[i]; + if (e == null) { + final V created = helper.create(key); + table[i] = created; // single-reference publish; benign clobber (see class doc) + return created; + } + if (helper.matches(key, e)) { + return e; + } + i = (i + 1) & mask; + if (i == start) { + return null; // wrapped ⇒ full + } + } + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java b/internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java new file mode 100644 index 00000000000..59b826a5019 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java @@ -0,0 +1,105 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import org.junit.jupiter.api.Test; + +/** + * Registry semantics for {@link SizingHintTable}. The table is process-wide static state; each test + * uses distinct operation names so tests don't interfere (the only shared, cumulative state is the + * per-lane cardinality counter, which the overflow test drives past its bound with its own names). + */ +class SizingHintTableTest { + + @Test + void nullOperationNameGetsNoHint() { + assertNull(SizingHintTable.hintFor(null, true)); + assertNull(SizingHintTable.hintFor(null, false)); + } + + @Test + void nonStringCharSequenceIsKeyedByContent() { + // Operation names are commonly UTF8BytesString, not String; a content-equal name must resolve + // to the same hint as its String form (keyed by toString(), which is O(1) for these types). + SizingHint viaString = SizingHintTable.hintFor("registry.utf8", true); + CharSequence utf8 = UTF8BytesString.create("registry.utf8"); + assertNotNull(viaString); + assertSame(viaString, SizingHintTable.hintFor(utf8, true)); + } + + @Test + void freshHintIsSeededAndUncapped() { + SizingHint hint = SizingHintTable.hintFor("registry.fresh", true); + assertNotNull(hint); + assertEquals(SizingHintTable.SEED_SIZE, hint.size); + assertFalse(hint.capped); + assertEquals("registry.fresh", hint.label); + } + + @Test + void sameOperationAndLaneReturnsSameInstance() { + SizingHint a = SizingHintTable.hintFor("registry.stable", true); + SizingHint b = SizingHintTable.hintFor("registry.stable", true); + assertSame(a, b); + } + + @Test + void entryAndChildLanesAreIndependent() { + SizingHint entry = SizingHintTable.hintFor("registry.twolane", true); + SizingHint child = SizingHintTable.hintFor("registry.twolane", false); + assertNotNull(entry); + assertNotNull(child); + assertNotSame(entry, child, "each lane holds its own hint for the same operation name"); + // ...and each lane is internally stable. + assertSame(entry, SizingHintTable.hintFor("registry.twolane", true)); + assertSame(child, SizingHintTable.hintFor("registry.twolane", false)); + } + + @Test + void beyondCardinalityBudgetSharesACappedOverflowHint() { + // Push far more distinct names than any lane's budget; a capped shared hint must appear. + SizingHint firstOverflow = null; + for (int i = 0; i < 4096 && firstOverflow == null; i++) { + SizingHint hint = SizingHintTable.hintFor("registry.flood." + i, true); + assertNotNull(hint); + if (hint.capped) { + firstOverflow = hint; + } + } + assertNotNull(firstOverflow, "lane eventually collapses to a capped overflow hint"); + assertEquals(SizingHintTable.OVERFLOW_SEED, firstOverflow.size); + + // Every further over-budget name shares that same capped instance. + SizingHint another = SizingHintTable.hintFor("registry.flood.after", true); + assertTrue(another.capped); + assertSame(firstOverflow, another); + } + + @Test + void sizingHintFeedsAndTunesTheDenseStore() { + KnownTags.init(); // register the real allocation-free resolver so known tags route dense + SizingHint hint = SizingHintTable.hintFor("registry.tuning", true); + assertEquals(SizingHintTable.SEED_SIZE, hint.size); + + TagMap map = TagMap.create(hint); + map.set(DDTags.BASE_SERVICE, "svc"); + map.set(datadog.trace.bootstrap.instrumentation.api.Tags.COMPONENT, "comp"); + map.set(datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND, "server"); + + map.recordSize(hint); + assertEquals(3, hint.size, "hint self-tunes up to the observed known-tag high-water mark"); + + // Monotonic-max: a smaller later observation does not shrink the hint. + TagMap smaller = TagMap.create(hint); + smaller.set(DDTags.BASE_SERVICE, "svc"); + smaller.recordSize(hint); + assertEquals(3, hint.size, "recordSize never shrinks the hint"); + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java b/internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java new file mode 100644 index 00000000000..c02ca085d53 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java @@ -0,0 +1,158 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class FlatHashtableTest { + + /** + * Self-contained entry: carries its own key + cached spread hash (the FlatHashtable contract). + */ + static final class Entry { + final String key; + final int hash; + + Entry(String key, int hash) { + this.key = key; + this.hash = hash; + } + } + + /** Counts create() calls so tests can prove getOrCreate mints exactly once per key. */ + static final class CountingHelper extends FlatHashtable.StringHelper { + int creates; + + @Override + public boolean matches(String key, Entry value) { + return this.hash(key) == value.hash && key.equals(value.key); + } + + @Override + public Entry create(String key) { + this.creates++; + return new Entry(key, this.hash(key)); + } + } + + /** A helper whose keys all collide to slot 0, to exercise linear probing + fill-to-full. */ + static final class CollidingHelper extends FlatHashtable.Helper { + @Override + public int hash(String key) { + return 0; + } + + @Override + public boolean matches(String key, Entry value) { + return key.equals(value.key); + } + + @Override + public Entry create(String key) { + return new Entry(key, 0); + } + } + + @Test + void capacityForIsPowerOfTwoAtLeastTwiceLimit() { + for (int limit = 1; limit <= 1024; limit++) { + int cap = FlatHashtable.capacityFor(limit); + assertTrue(cap >= 2 * limit, "cap " + cap + " >= 2*" + limit); + assertEquals(0, cap & (cap - 1), "cap " + cap + " is a power of two"); + } + assertEquals(2, FlatHashtable.capacityFor(1)); + assertEquals(4, FlatHashtable.capacityFor(2)); + assertEquals(8, FlatHashtable.capacityFor(3)); + assertEquals(8, FlatHashtable.capacityFor(4)); + assertEquals(1024, FlatHashtable.capacityFor(512)); + } + + @Test + void capacityForRejectsNonPositive() { + assertThrows(IllegalArgumentException.class, () -> FlatHashtable.capacityFor(0)); + assertThrows(IllegalArgumentException.class, () -> FlatHashtable.capacityFor(-1)); + } + + @Test + void createAllocatesTypedArrayOfCapacity() { + Entry[] table = FlatHashtable.create(Entry.class, 512); + assertEquals(1024, table.length); + assertEquals(Entry.class, table.getClass().getComponentType()); + } + + @Test + void getReturnsNullOnEmptyTable() { + Entry[] table = FlatHashtable.create(Entry.class, 8); + assertNull(FlatHashtable.get(table, "absent", new CountingHelper())); + } + + @Test + void getOrCreateMintsOnceThenReturnsSameInstance() { + Entry[] table = FlatHashtable.create(Entry.class, 8); + CountingHelper helper = new CountingHelper(); + + Entry first = FlatHashtable.getOrCreate(table, "op", helper); + assertNotNull(first); + assertEquals(1, helper.creates); + + Entry again = FlatHashtable.getOrCreate(table, "op", helper); + assertSame(first, again, "second getOrCreate returns the existing entry"); + assertEquals(1, helper.creates, "no re-mint on hit"); + + assertSame(first, FlatHashtable.get(table, "op", helper), "get sees the inserted entry"); + } + + @Test + void storesManyDistinctKeysWithinBudget() { + int limit = 200; + Entry[] table = FlatHashtable.create(Entry.class, limit); + CountingHelper helper = new CountingHelper(); + + Set seen = new HashSet<>(); + for (int i = 0; i < limit; i++) { + Entry e = FlatHashtable.getOrCreate(table, "op-" + i, helper); + assertNotNull(e); + seen.add(e); + } + assertEquals(limit, seen.size()); + assertEquals(limit, helper.creates); + + // All still retrievable (probing across collisions works). + for (int i = 0; i < limit; i++) { + assertNotNull(FlatHashtable.get(table, "op-" + i, helper)); + } + } + + @Test + void getOrCreateReturnsNullWhenPhysicallyFull() { + // capacityFor(1) == 2 slots; all keys collide to slot 0 so 2 fills the table. + Entry[] table = FlatHashtable.create(Entry.class, 1); + assertEquals(2, table.length); + CollidingHelper helper = new CollidingHelper(); + + assertNotNull(FlatHashtable.getOrCreate(table, "a", helper)); + assertNotNull(FlatHashtable.getOrCreate(table, "b", helper)); + // Table is now full; a third distinct key has no empty slot. + assertNull(FlatHashtable.getOrCreate(table, "c", helper)); + // But existing keys are still found via the wrapped probe. + assertNotNull(FlatHashtable.get(table, "a", helper)); + assertNotNull(FlatHashtable.get(table, "b", helper)); + assertNull(FlatHashtable.get(table, "c", helper)); + } + + @Test + void stringHelperHashIsSpreadAndStable() { + CountingHelper helper = new CountingHelper(); + int h = helper.hash("component"); + assertEquals(h, helper.hash("component"), "hash is deterministic"); + int raw = "component".hashCode(); + assertEquals(raw ^ (raw >>> 16), h, "hash is the spread of String.hashCode()"); + } +} From dd548217acd95c6ea66b64da0c51c92a70490612 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Mon, 17 Aug 2026 21:20:46 -0400 Subject: [PATCH 11/14] Add span-creation + trace-creation JMH harness Near-no-op DropWriter (blackhole-consuming Writer) plus two front-half benchmarks used to measure the dense-store / per-operation-sizing (SizingHint) allocation behavior with the dense flag on: - SpanCreationBenchmark: single span create -> (set tags) -> finish, bare/web/jdbc arms; the drift-stable old-API shape for version A/Bs. - TraceCreationBenchmark: a whole trace (local root + children) so the entry-vs-child sizing win, which a single-span bench can't reach, surfaces and scales with childCount. Read gc.alloc.rate.norm (B/op, deterministic); throughput is directional. Co-Authored-By: Claude Opus 4.8 --- .../java/datadog/trace/core/DropWriter.java | 42 ++++ .../trace/core/SpanCreationBenchmark.java | 146 +++++++++++ .../trace/core/TraceCreationBenchmark.java | 227 ++++++++++++++++++ 3 files changed, 415 insertions(+) create mode 100644 dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java create mode 100644 dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java create mode 100644 dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java b/dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java new file mode 100644 index 00000000000..6e375ee828b --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java @@ -0,0 +1,42 @@ +package datadog.trace.core; + +import datadog.trace.common.writer.Writer; +import java.util.List; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Near-no-op {@link Writer}: drops finished traces (no serialization, no agent I/O, no {@link + * TraceCounters} bookkeeping) so span-creation benchmarks measure only the application-thread + * (front-half) allocation — create, tag, finish, PendingTrace completion. {@link #write} still + * hands the trace to a {@link Blackhole} rather than truly doing nothing with it, so the JIT can't + * treat the finish()-triggered write as dead code and eliminate work the real path performs. + * + *

Drift-stable: implements only the five-method {@link Writer} interface, unchanged + * v1.53→master. + */ +final class DropWriter implements Writer { + private final Blackhole blackhole; + + DropWriter(Blackhole blackhole) { + this.blackhole = blackhole; + } + + @Override + public void write(List trace) { + blackhole.consume(trace); + } + + @Override + public void start() {} + + @Override + public boolean flush() { + return true; + } + + @Override + public void close() {} + + @Override + public void incrementDropCounts(int spanCount) {} +} diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java new file mode 100644 index 00000000000..7dff7d3cf84 --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java @@ -0,0 +1,146 @@ +package datadog.trace.core; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Cross-version portable span-creation benchmark: create -> (set tags) -> finish. Only the + * drift-stable old-API arms (buildSpan/startSpan + setTag/withTag + Tags constants + the + * five-method Writer) so it compiles byte-identically on v1.53..master and can be grafted onto any + * release tag. + * + *

Used for the #12047 (dense tag store + graph-colored slots) vs v1.65.0 A/B. The dense store is + * activated on the #12047 build by {@code -Ddd.trace.dense.tags.enabled=true} in the {@code @Fork} + * args below; the same flag is an unknown/no-op property on v1.65.0, so the ONLY difference between + * the two runs is the tracer version. Read {@code gc.alloc.rate.norm} (B/op, deterministic) as the + * primary signal; throughput is directional-only (per-fork JIT bimodality at @Threads(8)). + */ +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.Throughput) +@Threads(8) +@OutputTimeUnit(MICROSECONDS) +@Fork( + value = 3, + jvmArgsAppend = { + "-DTEST_LOG_LEVEL=warn", + // Activates the dense known-tag store on #12047; unknown/no-op property on v1.65.0. + "-Ddd.trace.dense.tags.enabled=true", + // Production-shaped tracer config so mergedTracerTags is a realistically-sized shared bundle. + "-Ddd.service=petclinic", + "-Ddd.env=staging", + "-Ddd.version=1.2.3", + "-Ddd.tags=team:apm,dc:us1,cluster:prod-1,owner:tracing,tier:backend,region:us-east-1" + }) +public class SpanCreationBenchmark { + private static final String INSTRUMENTATION_NAME = "bench"; + private static final String SERVER_OPERATION_NAME = "servlet.request"; + private static final String JDBC_OPERATION_NAME = "database.query"; + + private static final String COMPONENT_VALUE = "tomcat-server"; + private static final String HTTP_METHOD_VALUE = "GET"; + private static final String HTTP_ROUTE_VALUE = "/owners/{ownerId}"; + private static final String HTTP_URL_VALUE = "http://localhost:8080/owners/42"; + private static final int HTTP_STATUS_VALUE = 100; // in-cache; value itself is immaterial here + private static final int PEER_PORT_VALUE = 80; + + private static final String DB_COMPONENT_VALUE = "java-jdbc-statement"; + private static final String DB_TYPE_VALUE = "postgresql"; + private static final String DB_INSTANCE_VALUE = "petclinic"; + private static final String DB_USER_VALUE = "app"; + private static final String DB_OPERATION_VALUE = "SELECT"; + private static final String DB_STATEMENT_VALUE = "SELECT * FROM owners WHERE id = ?"; + private static final String DB_PEER_HOSTNAME_VALUE = "db.internal"; + private static final int DB_PEER_PORT_VALUE = 90; // in-cache; value itself is immaterial here + + CoreTracer tracer; + + @Setup + public void setup(Blackhole blackhole) { + this.tracer = CoreTracer.builder().writer(new DropWriter(blackhole)).build(); + } + + @TearDown + public void tearDown() { + this.tracer.close(); + } + + /** Baseline: create + finish a bare span via startSpan, no tags. */ + @Benchmark + public void bareStartSpan() { + AgentSpan span = tracer.startSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME); + span.finish(); + } + + /** Baseline: create + finish a bare span via the builder path, no tags. */ + @Benchmark + public void bareBuildSpan() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + span.finish(); + } + + /** Web-server-shaped span: create -> set the typical known tags (7) -> finish. */ + @Benchmark + public void webServerSpan() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + span.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + span.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + span.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + span.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + span.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + span.finish(); + } + + /** + * Web-server-shaped span via the builder tag path (withTag before start, the OTel-bridge shape). + */ + @Benchmark + public void webServerSpanViaBuilder() { + AgentSpan span = + tracer + .buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME) + .withTag(Tags.COMPONENT, COMPONENT_VALUE) + .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER) + .withTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE) + .withTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE) + .withTag(Tags.HTTP_URL, HTTP_URL_VALUE) + .withTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE) + .withTag(Tags.PEER_PORT, PEER_PORT_VALUE) + .start(); + span.finish(); + } + + /** JDBC/DB-client-shaped span: create -> set the typical DB known tags (9) -> finish. */ + @Benchmark + public void jdbcClientSpan() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, JDBC_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, DB_COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + span.setTag(Tags.DB_TYPE, DB_TYPE_VALUE); + span.setTag(Tags.DB_INSTANCE, DB_INSTANCE_VALUE); + span.setTag(Tags.DB_USER, DB_USER_VALUE); + span.setTag(Tags.DB_OPERATION, DB_OPERATION_VALUE); + span.setTag(Tags.DB_STATEMENT, DB_STATEMENT_VALUE); + span.setTag(Tags.PEER_HOSTNAME, DB_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); + span.finish(); + } +} diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java new file mode 100644 index 00000000000..0eeffc43a48 --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java @@ -0,0 +1,227 @@ +package datadog.trace.core; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Front-half allocation/throughput for creating a whole trace (a local-root span plus its + * children), as opposed to the single-span {@link SpanCreationBenchmark}. One op = one complete + * trace: root started, activated, children created-and-finished under it, then root finished. + * + *

Purpose — a reference others can key off when implementing core optimizations: + * + *

+ * + *

Isolation: children are created under an active root scope, so this exercises the real + * parent-context propagation and trace-tag inheritance a single-span bench cannot reach. A no-op + * {@link DropWriter} drops finished traces (handing them to a {@link Blackhole} so the JIT can't + * dead-code the finish()-triggered write) so only application-thread (front-half) allocation lands + * in the {@code -prof gc} number, with no serialization or agent I/O. + * + *

The tracer is production-shaped via {@code @Fork} jvmArgs (service/env/version + global {@code + * dd.tags}) so {@code mergedTracerTags} is a realistically-sized shared bundle — the same config as + * {@link SpanCreationBenchmark}, keeping numbers comparable across the two. + * + *

Read {@code gc.alloc.rate.norm} (B/op, deterministic) as the primary signal; throughput is + * directional-only (laptop thermals + per-fork inlining bimodality). + * + *

Historical results (populate from a committed run on the branch this file lives on; + * matches the {@link SpanCreationBenchmark} header convention): + * + * 

+ *   date        commit           arm                        alloc B/op    thrpt ops/us
+ *   ----        ------           ---                        ----------    ------------
+ *   2026-08-18  sizing-hint-v2   webRequestTrace            4424.1        1.21
+ *   2026-08-18  sizing-hint-v2   fanoutTrace   (child=1)    2632.0        2.42
+ *   2026-08-18  sizing-hint-v2   fanoutTrace   (child=5)    6867.6        0.91
+ *   2026-08-18  sizing-hint-v2   fanoutTrace   (child=10)  11167.7        0.50
+ *
+ *   @Fork(3) @Threads(8), dense on. webRequestTrace is flat across childCount
+ *   (param ignored) — read any row.
+ *
+ *   Sizing A/B vs base #12047 (no SizingHint): fanoutTrace alloc win grows with
+ *   childCount — ~0 @1, -6.3% (-756 B/op) @10 — from child-lane resize-avoidance
+ *   on repeated heavy children; webRequestTrace (one-off mixed ops) shows ~0.
+ *   Throughput held flat ON=OFF. Mean B/op under-sells this feature; the value is
+ *   evolvability + worst-case (tail under fanout x tight heap), not the mean.
+ *
+ */ +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.Throughput) +@Threads(8) +@OutputTimeUnit(MICROSECONDS) +@Fork( + value = 3, + jvmArgsAppend = { + "-DTEST_LOG_LEVEL=warn", + // Production-shaped tracer config so mergedTracerTags is a realistically-sized shared bundle + // (env + 6 global DD_TAGS + runtime-id/language) — the trace-level tags every span merges. + // Same config as SpanCreationBenchmark so the two benchmarks' numbers stay comparable. + "-Ddd.service=petclinic", + "-Ddd.env=staging", + "-Ddd.version=1.2.3", + "-Ddd.tags=team:apm,dc:us1,cluster:prod-1,owner:tracing,tier:backend,region:us-east-1" + }) +public class TraceCreationBenchmark { + private static final String INSTRUMENTATION_NAME = "bench"; + private static final String SERVER_OPERATION_NAME = "servlet.request"; + private static final String JDBC_OPERATION_NAME = "database.query"; + private static final String HTTP_CLIENT_OPERATION_NAME = "http.request"; + private static final String INTERNAL_OPERATION_NAME = "internal.work"; + + // Web-server (local root) tag shape — mirrors SpanCreationBenchmark.webServerSpan. + private static final String COMPONENT_VALUE = "tomcat-server"; + private static final String HTTP_METHOD_VALUE = "GET"; + private static final String HTTP_ROUTE_VALUE = "/owners/{ownerId}"; + private static final String HTTP_URL_VALUE = "http://localhost:8080/owners/42"; + private static final int HTTP_STATUS_VALUE = 100; // in Integer cache; boxing does not allocate + private static final int PEER_PORT_VALUE = 80; + + // JDBC-client child tag shape — mirrors SpanCreationBenchmark.jdbcClientSpan. + private static final String DB_COMPONENT_VALUE = "java-jdbc-statement"; + private static final String DB_TYPE_VALUE = "postgresql"; + private static final String DB_INSTANCE_VALUE = "petclinic"; + private static final String DB_USER_VALUE = "app"; + private static final String DB_OPERATION_VALUE = "SELECT"; + private static final String DB_STATEMENT_VALUE = "SELECT * FROM owners WHERE id = ?"; + private static final String DB_PEER_HOSTNAME_VALUE = "db.internal"; + private static final int DB_PEER_PORT_VALUE = 90; // in Integer cache; boxing does not allocate + + // HTTP-client child tag shape. + private static final String HTTP_CLIENT_COMPONENT_VALUE = "apache-httpclient"; + private static final String HTTP_CLIENT_URL_VALUE = "http://billing.internal/charge"; + private static final String HTTP_CLIENT_PEER_HOSTNAME_VALUE = "billing.internal"; + private static final int HTTP_CLIENT_PEER_PORT_VALUE = 90; // in Integer cache; no alloc + + // Internal child tag shape (a light span, few tags). + private static final String INTERNAL_COMPONENT_VALUE = "spring-scheduler"; + + /** Fan-out width for {@link #fanoutTrace()} — root + this many jdbc-shaped children. */ + @Param({"1", "5", "10"}) + int childCount; + + CoreTracer tracer; + + @Setup + public void setup(Blackhole blackhole) { + this.tracer = CoreTracer.builder().writer(new DropWriter(blackhole)).build(); + } + + @TearDown + public void tearDown() { + this.tracer.close(); + } + + /** + * Fixed, realistic web request: a server local root with a JDBC-client, an HTTP-client, and an + * internal child. One headline number that mirrors the shape of a real request. Not affected by + * {@link #childCount} (JMH still runs it once per param value — read any single row). + */ + @Benchmark + public void webRequestTrace() { + AgentSpan root = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + root.setTag(Tags.COMPONENT, COMPONENT_VALUE); + root.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + root.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + root.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + root.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + root.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + root.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + + AgentScope scope = tracer.activateSpan(root); + try { + jdbcChild(); + httpClientChild(); + internalChild(); + } finally { + scope.close(); + } + root.finish(); + } + + /** + * Fan-out trace: a server local root with {@link #childCount} JDBC-shaped children. The scaling + * axis — per-trace cost as span count grows exposes how per-span optimizations compound. + */ + @Benchmark + public void fanoutTrace() { + AgentSpan root = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + root.setTag(Tags.COMPONENT, COMPONENT_VALUE); + root.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + root.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + root.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + root.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + root.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + root.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + + AgentScope scope = tracer.activateSpan(root); + try { + for (int i = 0; i < childCount; i++) { + jdbcChild(); + } + } finally { + scope.close(); + } + root.finish(); + } + + /** A JDBC-client child of the currently-active span. */ + private void jdbcChild() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, JDBC_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, DB_COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + span.setTag(Tags.DB_TYPE, DB_TYPE_VALUE); + span.setTag(Tags.DB_INSTANCE, DB_INSTANCE_VALUE); + span.setTag(Tags.DB_USER, DB_USER_VALUE); + span.setTag(Tags.DB_OPERATION, DB_OPERATION_VALUE); + span.setTag(Tags.DB_STATEMENT, DB_STATEMENT_VALUE); + span.setTag(Tags.PEER_HOSTNAME, DB_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); + span.finish(); + } + + /** An HTTP-client child of the currently-active span. */ + private void httpClientChild() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, HTTP_CLIENT_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, HTTP_CLIENT_COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + span.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + span.setTag(Tags.HTTP_URL, HTTP_CLIENT_URL_VALUE); + span.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + span.setTag(Tags.PEER_HOSTNAME, HTTP_CLIENT_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, HTTP_CLIENT_PEER_PORT_VALUE); + span.finish(); + } + + /** A light internal child of the currently-active span. */ + private void internalChild() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, INTERNAL_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, INTERNAL_COMPONENT_VALUE); + span.finish(); + } +} From 01e3a71a9be09981cd50eedae261c3dd309820b6 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 09:19:08 -0400 Subject: [PATCH 12/14] Address Codex review on #12056: gate sizing off default, fix record point, lane count, read-through sizing - Do no harm: resolve a SizingHint only when KnownTagCodec.isActive() (the dense store is live). With dd.trace.dense.tags.enabled off (the default), skip the operationName.toString() + global-table probe entirely -- a hint buys nothing when known tags don't take the dense path. - Record point: move recordDenseSize() from DDSpan.finishAndAddToTrace to DDSpanContext.processTagsAndBaggage (at serialization). That is the single terminal point that runs for BOTH finish modes -- plain finish() and phasedFinish()+publish() (gRPC/Netty/WebFlux) -- and runs AFTER the lazy tag post-processors append serialization-time tags (_dd.integration, host, ...), so the recorded high-water mark is the span's true final footprint. - Read-through sizing: add TagMap.createFromParent(parent, hint) so the LOCAL dense store of a read-through map is sized from the hint too; route it in DDSpanContext. Sizing now fires on the common child / shared-parent path (previously the hint was discarded whenever mergedTracerTags was non-null). - Lane count: FlatHashtable.getOrCreate reports genuine inserts; SizingHintTable bumps the per-lane cardinality count only on a real insert, so a cold-start burst of one operation can't spend the budget on a single name. - Docs: correct the 'monotonic-max' over-claim to best-effort racy max (a concurrent finish can transiently lower size; benign -- only mis-sizes an array). Co-Authored-By: Claude Opus 4.8 --- .../java/datadog/trace/core/CoreTracer.java | 16 ++++++++-- .../main/java/datadog/trace/core/DDSpan.java | 1 - .../datadog/trace/core/DDSpanContext.java | 24 ++++++++++---- .../java/datadog/trace/api/SizingHint.java | 11 ++++--- .../datadog/trace/api/SizingHintTable.java | 27 ++++++++++------ .../main/java/datadog/trace/api/TagMap.java | 32 +++++++++++++++++-- .../datadog/trace/util/FlatHashtable.java | 17 ++++++++++ 7 files changed, 102 insertions(+), 26 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 4b0e5fa61f7..be9fd036207 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -40,6 +40,7 @@ import datadog.trace.api.EndpointTracker; import datadog.trace.api.IdGenerationStrategy; import datadog.trace.api.InstrumenterConfig; +import datadog.trace.api.KnownTagCodec; import datadog.trace.api.KnownTags; import datadog.trace.api.Pair; import datadog.trace.api.SizingHint; @@ -2200,8 +2201,19 @@ protected static final DDSpanContext buildSpanContext( // enriching tags a child doesn't, so pick the lane by whether we have a local parent. A // resolved hint sizes the span's TagMap and self-tunes on finish; null (no/unkeyable // operation name) falls back to the generic default capacity. - final boolean entrySpan = !(resolvedParentSpanContext instanceof DDSpanContext); - final SizingHint sizingHint = SizingHintTable.hintFor(operationName, entrySpan); + // + // Gated on KnownTagCodec.isActive(): sizing only helps when the dense known-tag store is live + // (the experimental dd.trace.dense.tags.enabled path). With it off -- the default -- known + // tags don't take the dense path, so a hint buys nothing; skipping resolution here keeps the + // operationName.toString() and the global-table probe off the default span-creation path + // entirely (do no harm). + final SizingHint sizingHint; + if (KnownTagCodec.isActive()) { + final boolean entrySpan = !(resolvedParentSpanContext instanceof DDSpanContext); + sizingHint = SizingHintTable.hintFor(operationName, entrySpan); + } else { + sizingHint = null; + } // some attributes are inherited from the parent context = diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index 841a1f7a08f..a288c405e6f 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -157,7 +157,6 @@ public boolean isFinished() { private void finishAndAddToTrace(final long durationNano) { // ensure a min duration of 1 if (DURATION_NANO_UPDATER.compareAndSet(this, 0, Math.max(1, durationNano))) { - context.recordDenseSize(); setLongRunningVersion(-this.longRunningVersion); SpanWrapper wrapper = getWrapper(); if (wrapper != null) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 100b9229145..f3b76e3dff5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -422,7 +422,7 @@ public DDSpanContext( final int capacity = Math.max((tagsSize <= 0 ? 3 : (tagsSize + 1)) * 4 / 3, 8); this.unsafeTags = readThroughParent != null - ? TagMap.createFromParent(readThroughParent) + ? TagMap.createFromParent(readThroughParent, sizingHint) : sizingHint != null ? TagMap.create(sizingHint) : TagMap.create(capacity); this.sizingHint = sizingHint; @@ -976,15 +976,21 @@ public void setTag(final String tag, final String value) { } /** - * Feeds this span's final dense-store size back into the sizingHint it was created from (if any). - * Called once at span finish; lets a reused hint self-tune to the operation's observed known-tag - * high-water mark, so subsequent spans of the operation are sized correctly. + * Feeds this span's final dense-store size back into the sizingHint it was created from (if any), + * so a reused hint self-tunes to the operation's observed known-tag high-water mark and later + * spans of the operation are sized correctly. + * + *

Called from {@link #processTagsAndBaggage} at serialization — the one terminal point that + * (a) runs for every serialized span regardless of finish mode (plain {@code finish()} AND {@code + * phasedFinish()}+{@code publish()}, which the async gRPC/Netty/WebFlux paths use and which never + * enter {@code finishAndAddToTrace}), and (b) runs AFTER the lazy tag post-processors have + * appended their serialization-time tags ({@code _dd.integration}, host, ...), so the recorded + * count is the span's true final footprint rather than an underestimate. The caller already holds + * {@code synchronized (unsafeTags)}, so the {@code knownCount} read is consistent; the hint write + * itself is best-effort racy by design (see {@link TagMap#recordSize}). */ void recordDenseSize() { if (sizingHint != null) { - // Intentionally unsynchronized: recordSize is benign-racy by design (monotonic-max on a plain - // int), and a stale knownCount read only ever costs one extra growth on a later span. The - // lock would add nothing but contention on the finish path. unsafeTags.recordSize(sizingHint); } } @@ -1316,6 +1322,10 @@ void processTagsAndBaggage( // Tags TagsPostProcessorFactory.lazyProcessor().processTags(unsafeTags, this, restrictedSpan); + // Self-tune the per-operation sizing hint now that every tag (incl. the just-appended + // serialization-time tags) is present -- the terminal point shared by all finish modes. + recordDenseSize(); + // Links if (injectLinksAsTags) { String linksTag = DDSpanLink.toTag(restrictedSpan.getLinks()); diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHint.java b/internal-api/src/main/java/datadog/trace/api/SizingHint.java index 498d79f3433..78bc2de63ff 100644 --- a/internal-api/src/main/java/datadog/trace/api/SizingHint.java +++ b/internal-api/src/main/java/datadog/trace/api/SizingHint.java @@ -26,14 +26,17 @@ public final class SizingHint { final String label; final int labelHash; - // Payload: the tuned dense-store size. Plain racy int, updated monotonic-max — a stale/lost read - // only mis-sizes an array (over/under-provision), never corrupts tag data, so no synchronization. + // Payload: the tuned dense-store size. Plain racy int, updated best-effort max (see + // TagMap#recordSize -- not strictly monotonic under concurrent finishes) — a stale/lost/lowered + // read only mis-sizes an array (over/under-provision), never corrupts tag data, so no + // synchronization. int size; // When true, {@code size} is fixed and recordSize won't grow it. For the shared default / // overflow - // hint (a HETEROGENEOUS catch-all for operation-less / over-budget spans): self-tuning it via - // monotonic-max would converge to the max across unlike sharers and over-provision the lean ones. + // hint (a HETEROGENEOUS catch-all for operation-less / over-budget spans): self-tuning it toward + // the observed max would converge to the max across unlike sharers and over-provision the lean + // ones. final boolean capped; SizingHint(String label, int labelHash, int seedSize) { diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java b/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java index c01e63fb4dc..d77b02546db 100644 --- a/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java +++ b/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java @@ -17,7 +17,7 @@ *

Bounded + racy by design. Each lane is a fixed-capacity {@link FlatHashtable} (never * resized) so memory is bounded even under unbounded/dynamic operation names; once a lane's * cardinality budget is spent, further operations share a capped default hint. Construction, - * insertion, and the monotonic-max size update are all lock-free and deliberately racy — a lost + * insertion, and the best-effort-max size update are all lock-free and deliberately racy — a lost * update or a double-mint only mis-sizes an array (over/under-provision) for a span or two, never * corrupts tag data (see {@link FlatHashtable} and {@link SizingHint} for the rationale). * @@ -30,7 +30,7 @@ public final class SizingHintTable { private SizingHintTable() {} - // Seed for a fresh per-operation hint: a floor that self-tunes up via monotonic-max recordSize. + // Seed for a fresh per-operation hint: a floor that self-tunes up via best-effort-max recordSize. static final int SEED_SIZE = 1; // Fixed size for the shared over-budget hint: a small lean default. Capped (never grown by // recordSize) because it's a heterogeneous catch-all -- growing it would over-provision lean @@ -84,15 +84,24 @@ public static SizingHint hintFor(CharSequence operationName, boolean entrySpan) if ((entrySpan ? entrySize : childSize) >= CARDINALITY_LIMIT) { return overflow; } - final SizingHint created = FlatHashtable.getOrCreate(slots, key, HELPER); - if (created == null) { + // Count only a genuine insert. Between the get() miss above and this getOrCreate another thread + // may have inserted the same operation's hint; getOrCreate then returns that existing entry to + // us -- without `created` we'd still bump the lane count, so a cold-start burst of many spans + // of + // one operation could spend the budget on a single distinct name. `created[0]` is set only on + // the actual store branch. Allocated here on the create path only, which is warmup-rare. + final boolean[] created = new boolean[1]; + final SizingHint hint = FlatHashtable.getOrCreate(slots, key, HELPER, created); + if (hint == null) { return overflow; // physically full -- shouldn't happen under the cap, but stay safe } - if (entrySpan) { - entrySize++; // racy approximate count - } else { - childSize++; + if (created[0]) { + if (entrySpan) { + entrySize++; // racy approximate count; only a real insert bumps it + } else { + childSize++; + } } - return created; + return hint; } } diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 9bff9fe1575..5de1b26c995 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -102,6 +102,19 @@ public static final TagMap create(SizingHint hint) { * local miss/removal for no benefit. */ public static final TagMap createFromParent(TagMap parent) { + return createFromParent(parent, null); + } + + /** + * Read-through variant of {@link #create(SizingHint)}: reads through to {@code parent} on local + * misses AND sizes the LOCAL dense store from {@code hint}. The two are orthogonal — the parent + * supplies inherited reads, while locally-set known tags live in this map's own dense array, + * whose initial capacity the hint tunes. So the common child / shared-parent path (a non-null + * read-through parent) still gets per-operation sizing; without this the local store would fall + * back to the fixed default. A {@code null} hint behaves exactly like {@link + * #createFromParent(TagMap)}. + */ + public static final TagMap createFromParent(TagMap parent, SizingHint hint) { if (parent != null) { if (!parent.frozen) { throw new IllegalStateException("read-through parent must be frozen"); @@ -110,7 +123,11 @@ public static final TagMap createFromParent(TagMap parent) { parent = null; } } - return new TagMap(parent); + TagMap tagMap = new TagMap(parent); + if (hint != null && hint.size > 0) { + tagMap.denseCapHint = hint.size; + } + return tagMap; } /** Creates a new TagMap.Ledger */ @@ -1466,13 +1483,22 @@ private void ensureKnownCapacity() { /** * Feeds this map's final dense-entry count back to {@code hint} (call at a terminal point: freeze - * / span-finish). The hint self-tunes so future maps of the same operation size correctly. + * / serialization). The hint self-tunes so future maps of the same operation size correctly. + * + *

Best-effort max, not a strict monotonic-max: the read-compare-write is unsynchronized, so + * two spans of the same operation finishing concurrently can both read the same old {@code size} + * and the later write can lower it (a 10-tag span then a 5-tag span settles at 5). And {@code + * size} is a plain field, so a fresh update isn't promptly visible to span-creation threads. Both + * are deliberately tolerated: a stale or lowered hint only mis-sizes a later span's dense array + * (one extra grow-copy), never corrupts tag data — so it stays lock- and atomic-free, and the + * next heavy span of the operation nudges it back up. A CAS/atomic-max would buy exactness at the + * cost of contention on this terminal path, which isn't worth it for a sizing hint. */ public void recordSize(SizingHint hint) { // Capped hints (the heterogeneous shared default) are fixed -- don't let unlike sharers grow // them. if (!hint.capped && this.knownCount > hint.size) { - hint.size = this.knownCount; // monotonic-max; benign racy plain-int write + hint.size = this.knownCount; // best-effort racy max (see javadoc); benign plain-int write } } diff --git a/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java b/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java index b5254271779..d803e22f2c2 100644 --- a/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java @@ -130,6 +130,20 @@ public static V get(V[] table, K key, Helper helper) { * double-create is acceptable only when the payload makes it benign (see class doc). */ public static V getOrCreate(V[] table, K key, Helper helper) { + return getOrCreate(table, key, helper, null); + } + + /** + * Like {@link #getOrCreate(Object[], Object, Helper)}, but reports whether this call actually + * minted-and-stored a new entry: when {@code createdOut} is non-null, {@code createdOut[0]} is + * set to {@code true} only on the store branch, and left untouched when an existing entry + * matched. Callers that keep an approximate live-size counter (for a cardinality budget) + * increment it only when {@code createdOut[0]} — so racing callers that get an already-inserted + * entry back don't each over-count the same operation. Still racy by design: a benign double-mint + * (two threads store into the same slot) reports {@code true} on both, an over-count of at most + * the store-race width, not of every caller (see class doc). + */ + public static V getOrCreate(V[] table, K key, Helper helper, boolean[] createdOut) { final int mask = table.length - 1; final int start = helper.hash(key) & mask; int i = start; @@ -138,6 +152,9 @@ public static V getOrCreate(V[] table, K key, Helper helper) { if (e == null) { final V created = helper.create(key); table[i] = created; // single-reference publish; benign clobber (see class doc) + if (createdOut != null) { + createdOut[0] = true; + } return created; } if (helper.matches(key, e)) { From e0674dcff0b5f46b680a34c4115a475171aa9098 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 09:24:15 -0400 Subject: [PATCH 13/14] Fix leftover monotonic-max wording in SizingHint class javadoc --- .../src/main/java/datadog/trace/api/SizingHint.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHint.java b/internal-api/src/main/java/datadog/trace/api/SizingHint.java index 78bc2de63ff..b4fdd3a4cfe 100644 --- a/internal-api/src/main/java/datadog/trace/api/SizingHint.java +++ b/internal-api/src/main/java/datadog/trace/api/SizingHint.java @@ -11,9 +11,9 @@ * no volatile or atomics. * *

Opaque to everything outside {@code datadog.trace.api}: no public members. {@link TagMap} - * reads {@link #size} to size a fresh dense store and writes it back (monotonic-max) at a terminal - * point; {@code SizingHelper} mints and compares by {@link #label}/{@link #labelHash}. Callers only - * ever hold the reference. + * reads {@link #size} to size a fresh dense store and writes it back (best-effort max, see {@link + * TagMap#recordSize}) at a terminal point; {@code SizingHelper} mints and compares by {@link + * #label}/{@link #labelHash}. Callers only ever hold the reference. * *

{@link #labelHash} is supplied by the helper (a single spread source — {@code * FlatHashtable.StringHelper.hash}) so the cached gate always matches the probe hash. From b4242147bea4cef08ab1c25060ffe86d69427b55 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Tue, 18 Aug 2026 09:46:31 -0400 Subject: [PATCH 14/14] Gate dense sizing on DENSE_TAGS_ENABLED (inherited Config flag) not KnownTagCodec.isActive --- .../src/main/java/datadog/trace/core/CoreTracer.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index be9fd036207..98b645cf219 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -40,7 +40,6 @@ import datadog.trace.api.EndpointTracker; import datadog.trace.api.IdGenerationStrategy; import datadog.trace.api.InstrumenterConfig; -import datadog.trace.api.KnownTagCodec; import datadog.trace.api.KnownTags; import datadog.trace.api.Pair; import datadog.trace.api.SizingHint; @@ -2202,13 +2201,13 @@ protected static final DDSpanContext buildSpanContext( // resolved hint sizes the span's TagMap and self-tunes on finish; null (no/unkeyable // operation name) falls back to the generic default capacity. // - // Gated on KnownTagCodec.isActive(): sizing only helps when the dense known-tag store is live - // (the experimental dd.trace.dense.tags.enabled path). With it off -- the default -- known - // tags don't take the dense path, so a hint buys nothing; skipping resolution here keeps the + // Gated on DENSE_TAGS_ENABLED: sizing only helps when tags take the dense store (the + // experimental trace.dense.tags.enabled path). With it off -- the default -- known tags don't + // take the dense path, so a hint buys nothing; skipping resolution here keeps the // operationName.toString() and the global-table probe off the default span-creation path - // entirely (do no harm). + // entirely (do no harm). Not KnownTagCodec.isActive(): the codec is always registered. final SizingHint sizingHint; - if (KnownTagCodec.isActive()) { + if (DENSE_TAGS_ENABLED) { final boolean entrySpan = !(resolvedParentSpanContext instanceof DDSpanContext); sizingHint = SizingHintTable.hintFor(operationName, entrySpan); } else {