diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 991ca899596..b7eacb3cd95 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -59,6 +59,11 @@ gradlePlugin { implementationClass = "datadog.gradle.plugin.config.SupportedConfigPlugin" } + create("tag-registry-generator") { + id = "dd-trace-java.tag-registry-generator" + implementationClass = "datadog.gradle.plugin.tags.TagRegistryGeneratorPlugin" + } + create("supported-config-linter") { id = "dd-trace-java.config-inversion-linter" implementationClass = "datadog.gradle.plugin.config.ConfigInversionLinter" @@ -107,6 +112,7 @@ dependencies { implementation("com.fasterxml.jackson.core:jackson-databind") implementation("com.fasterxml.jackson.core:jackson-annotations") implementation("com.fasterxml.jackson.core:jackson-core") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") compileOnly(libs.develocity) } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt new file mode 100644 index 00000000000..ab1c2631c60 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt @@ -0,0 +1,38 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Generates the committed tag registry (KnownTags.java + layout reports) from the language-agnostic + * {@code tag-conventions.yaml} + the Java overlay. The actual emit lives in [TagRegistryGenerator]; + * this task just wires the inputs/outputs so Gradle can cache and up-to-date-check it. + */ +@CacheableTask +abstract class GenerateKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:OutputDirectory val destinationDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun generate() { + val outDir = destinationDirectory.get().asFile + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, outDir) + logger.lifecycle("tag-registry: generated -> $outDir") + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt new file mode 100644 index 00000000000..fd0a65eb5a2 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt @@ -0,0 +1,177 @@ +package datadog.gradle.plugin.tags + +import java.util.Locale + +/** + * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag + * `_NAME` (string) + `_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)` + * derivation comment — then the package-private `_SERIAL_NUM` constants, the + * `StringIndex.EmbeddingSupport` keyOf table, the `serialNum` switch `nameOf`, and resolver + * registration. + */ +object KnownTagsEmitter { + + fun emit(reg: TagRegistry, pkg: String, className: String): String { + // Sanitize tag names into unique Java constant identifiers. + val used = HashSet() + val cname = HashMap() + fun mk(name: String): String { + var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_') + if (c.isEmpty() || c[0].isDigit()) c = "T_$c" + var u = c + var n = 2 + while (u in used) { + u = "${c}_$n"; n++ + } + used.add(u) + cname[name] = u + return u + } + reg.reserved.forEach { mk(it.name) } + reg.stored.forEach { mk(it.name) } + + // Constant names. Collapse a duplicated trailing token so e.g. "resource.name" yields NAME + // (not NAME_NAME) and "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs + // (ID + _NAME -> ID_NAME, NAME + _ID -> NAME_ID) are kept as-is. + fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix" + fun nameC(name: String) = withSuffix(cname[name]!!, "_NAME") + fun idC(name: String) = withSuffix(cname[name]!!, "_ID") + fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM") + + val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order + // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch. + val otelName = + (reg.reserved.mapNotNull { v -> v.otelName?.let { v.name to it } } + + reg.stored.mapNotNull { t -> t.otelName?.let { t.name to it } }) + .toMap() + val b = StringBuilder() + b.appendLine("package $pkg;") + b.appendLine() + b.appendLine("import datadog.trace.util.StringIndex;") + b.appendLine() + b.appendLine("// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator).") + b.appendLine("// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml.") + b.appendLine("public final class $className {") + b.appendLine(" static final int SLOT_COUNT = ${reg.slotCount};") + b.appendLine() + + // Public API first (name + encoded id couplets), so readers see the useful parts up top; the + // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment. + b.appendLine(" // ---- reserved (routed to span fields or directives; not stored) ----") + for (v in reg.reserved) { + b.appendLine(" public static final String ${nameC(v.name)} = \"${v.name}\";") + b.appendLine(" public static final long ${idC(v.name)} = ${hex(v.id)};") + b.appendLine(" // makeTagId(serial=${v.serial}, slot=NO_SLOT) + intercepted [${v.kind}${v.field?.let { " -> $it" } ?: ""}]") + b.appendLine() + } + + b.appendLine(" // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ----") + for (t in reg.stored) { + val slot = if (t.slotted) t.slot.toString() else "NO_SLOT" + b.appendLine(" public static final String ${nameC(t.name)} = \"${t.name}\";") + b.appendLine(" public static final long ${idC(t.name)} = ${hex(t.id)};") + b.appendLine(" // makeTagId(serial=${t.serial}, slot=$slot)${if (t.intercepted) " + intercepted" else ""}${if (t.traceLevel) " + trace-level" else ""} <${t.required}>") + b.appendLine() + } + + // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch. + b.appendLine(" // ---- serial numbers ----") + for (v in reg.reserved) { + b.appendLine(" static final int ${serialC(v.name)} = ${v.serial};") + } + for (t in reg.stored) { + b.appendLine(" static final int ${serialC(t.name)} = ${t.serial};") + } + b.appendLine() + + // OpenTelemetry name -> canonical tag name, for the tags that declare one. Deterministic order + // (by OTel name) so output stays byte-identical. + val otelByCanonical = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + + // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then + // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id + // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name. + b.appendLine(" private static final String[] KEYOF_NAMES = {") + order.forEach { b.appendLine(" ${nameC(it)},") } + otelByCanonical.forEach { (otel, _) -> b.appendLine(" \"$otel\",") } + b.appendLine(" };") + b.appendLine(" private static final long[] KEYOF_VALUES = {") + order.forEach { b.appendLine(" ${idC(it)},") } + otelByCanonical.forEach { (_, canonical) -> b.appendLine(" ${idC(canonical)},") } + b.appendLine(" };") + b.appendLine(" private static final int[] KEYOF_HASHES;") + b.appendLine(" private static final String[] KEYOF_KEYS;") + b.appendLine(" private static final long[] KEYOF_IDS;") + b.appendLine() + b.appendLine(" static {") + b.appendLine(" StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);") + b.appendLine(" long[] ids = new long[data.names.length];") + b.appendLine(" for (int j = 0; j < KEYOF_NAMES.length; j++) {") + b.appendLine(" ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] =") + b.appendLine(" KEYOF_VALUES[j];") + b.appendLine(" }") + b.appendLine(" KEYOF_HASHES = data.hashes;") + b.appendLine(" KEYOF_KEYS = data.names;") + b.appendLine(" KEYOF_IDS = ids;") + b.appendLine(" }") + b.appendLine() + + // Resolver. + b.appendLine(" static final KnownTagCodec.Resolver RESOLVER =") + b.appendLine(" new KnownTagCodec.Resolver() {") + b.appendLine(" @Override") + b.appendLine(" public String nameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return ${nameC(name)};") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() + // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The + // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup. + b.appendLine(" @Override") + b.appendLine(" public String openTelemetryNameOf(long tagId) {") + b.appendLine(" switch (KnownTagCodec.serialNum(tagId)) {") + for (name in order) { + val otel = otelName[name] ?: continue + b.appendLine(" case ${serialC(name)}:") + b.appendLine(" return \"$otel\";") + } + b.appendLine(" default:") + b.appendLine(" return null;") + b.appendLine(" }") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public int slotCount() {") + b.appendLine(" return SLOT_COUNT;") + b.appendLine(" }") + b.appendLine() + b.appendLine(" @Override") + b.appendLine(" public long keyOf(String name) {") + b.appendLine(" int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name);") + b.appendLine(" return slot < 0 ? 0L : KEYOF_IDS[slot];") + b.appendLine(" }") + b.appendLine(" };") + b.appendLine() + b.appendLine(" static {") + b.appendLine(" KnownTagCodec.register(RESOLVER);") + b.appendLine(" }") + b.appendLine() + b.appendLine(" /** Forces resolver registration by triggering . Idempotent. */") + b.appendLine(" public static void init() {}") + b.appendLine() + b.appendLine(" private $className() {}") + b.appendLine("}") + return b.toString() + } + + private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id) +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt new file mode 100644 index 00000000000..180d2b63a08 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagConventions.kt @@ -0,0 +1,180 @@ +package datadog.gradle.plugin.tags + +/** + * Parsed tag-conventions domain model + the per-type tag-set resolver. Language-agnostic: it knows + * only structure (extends / include / applies) and per-tag semantics (name / type / required / + * source). Id assignment and emission are layered on top of the resolved sets. + */ +class TagConventions +private constructor( + private val spanTypes: Map, + private val mixins: Map, + private val traceLevel: List, +) { + /** A tag declaration (domain semantics only). */ + data class Tag( + val name: String, + val type: String, + val required: String, + /** + * The tag's OpenTelemetry-namespace name, if it has one. keyOf resolves it to this tag's + * canonical id (inbound, many->one); openTelemetryNameOf recovers it (outbound). Further + * namespaces and serializer applicability are a follow-on concern. + */ + val otelName: String? = null, + ) + + data class SpanType( + val name: String, + val abstract: Boolean, + val extends: String?, + val include: List, + val tags: List, + ) + + data class Mixin( + val name: String, + val appliesAll: Boolean, + val appliesTo: Set, + val tags: List, + ) + + /** Concrete (instantiable) span types — the ones a layout is computed for. */ + fun concreteTypes(): List = + spanTypes.values.filter { !it.abstract }.map { it.name }.sorted() + + /** + * resolved(type) = own tags + tags up the `extends` chain (incl. base) + tags of every mixin the + * type or an ancestor `include`s + tags of every mixin whose `applies` matches. De-duped by tag + * name (first occurrence wins). Base-first order, so it is stable across runs. + */ + fun resolve(typeName: String): List { + val result = LinkedHashMap() + fun add(t: Tag) = result.putIfAbsent(t.name, t) + + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { add(it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { add(it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) mx.tags.forEach { add(it) } + } + return result.values.toList() + } + + /** The explicit trace-level tier tags (their own TagMap "type" on the TraceSegment). */ + fun traceLevelTags(): List = traceLevel + + /** A declaration group: the source that *declares* a set of tags (its own `tags:` list). */ + data class Group(val name: String, val kind: String, val tags: List) + + /** + * The declaration groups, in a stable order: the trace-level tier first, then every span type + * (abstract included — `base`/`http` declare real tags) sorted by name, then every mixin sorted by + * name. Each maps to one `group-decl`. A tag is *declared* once (in its own container's `tags:`); + * the same tag reached via extends/include/applies is not re-declared, so first-declaration (in + * this order) is its home group. Groups with no declared tags are omitted. + */ + fun declarationGroups(): List { + val groups = ArrayList() + if (traceLevel.isNotEmpty()) groups.add(Group(TRACE_LAYER, "trace", traceLevel)) + for (name in spanTypes.keys.sorted()) { + val st = spanTypes.getValue(name) + if (st.tags.isNotEmpty()) groups.add(Group(name, "span_type", st.tags)) + } + for (name in mixins.keys.sorted()) { + val mx = mixins.getValue(name) + if (mx.tags.isNotEmpty()) groups.add(Group(name, "mixin", mx.tags)) + } + return groups + } + + /** Full stored-tag universe (concrete span types' resolves + trace-level), de-duped by name. */ + fun allStoredTags(): List { + val union = LinkedHashMap() + for (type in concreteTypes()) for (t in resolve(type)) union.putIfAbsent(t.name, t) + for (t in traceLevel) union.putIfAbsent(t.name, t) + return union.values.toList() + } + + /** + * Full composition for a type as (origin, tag) pairs, in composition order and NOT de-duped, so a + * tag contributed by more than one source shows up more than once. Origin is the contributing + * span type (via extends), `incl:` (via include), or `appl:` (via applies). + */ + fun compose(typeName: String): List> { + val out = ArrayList>() + val chain = ArrayList() + var cur: SpanType? = spanTypes[typeName] + while (cur != null) { + chain.add(cur) + cur = cur.extends?.let { spanTypes[it] } + } + for (st in chain.asReversed()) { + st.tags.forEach { out.add(st.name to it) } + for (mixinName in st.include) mixins[mixinName]?.tags?.forEach { out.add("incl:$mixinName" to it) } + } + val chainNames = chain.map { it.name }.toSet() + for (mx in mixins.values) { + if (mx.appliesAll || mx.appliesTo.any { it in chainNames }) { + mx.tags.forEach { out.add("appl:${mx.name}" to it) } + } + } + return out + } + + companion object { + /** Group name of the trace-level tier (its own TagMap layer on the TraceSegment). */ + const val TRACE_LAYER = "" + + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): TagConventions { + val spanTypesRaw = (root["span_types"] as? Map) ?: emptyMap() + val spanTypes = + spanTypesRaw.mapValues { (name, v) -> + val m = v as Map + SpanType( + name = name, + abstract = (m["abstract"] as? Boolean) ?: false, + extends = m["extends"] as? String, + include = (m["include"] as? List) ?: emptyList(), + tags = tagList(m["tags"]), + ) + } + + val mixinsRaw = (root["mixins"] as? Map) ?: emptyMap() + val mixins = + mixinsRaw.mapValues { (name, v) -> + val m = v as Map + val applies = m["applies"] + Mixin( + name = name, + appliesAll = applies == "all", + appliesTo = if (applies is List<*>) applies.map { it.toString() }.toSet() else emptySet(), + tags = tagList(m["tags"]), + ) + } + + val traceLevel = tagList((root["trace_level"] as? Map)?.get("tags")) + return TagConventions(spanTypes, mixins, traceLevel) + } + + @Suppress("UNCHECKED_CAST") + private fun tagList(tags: Any?): List = + (tags as? List>)?.map { m -> + Tag( + name = m["tag"].toString(), + type = (m["type"] as? String) ?: "string", + required = (m["required"] as? String) ?: "optional", + otelName = m["open-telemetry-name"] as? String, + ) + } ?: emptyList() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt new file mode 100644 index 00000000000..4c256e1221e --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistry.kt @@ -0,0 +1,188 @@ +package datadog.gradle.plugin.tags + +/** + * Assigns tag ids from a parsed [TagConventions] plus the Java overlay (intercepted set + reserved + * registry). The id encoding mirrors KnownTagCodec: [63 intercepted][62-48 serial][47-32 slot][31-0 + * zero] (known ids carry no nameHash — they are dense-store addressed). + * + *

The `slot` is a single globally stable coordinate from GRAPH COLORING the tag co-occurrence + * graph. Each concrete span type's resolved tag set (see [TagConventions.resolve]) is a clique — its + * tags all appear together on one span, so they must get distinct slots. The trace-level tier is its + * own clique (a separate TagMap on the TraceSegment), so it may reuse slot numbers freely with the + * span layers. Slots are shared only between tags that never co-occur, so slotCount stays bounded by + * the largest clique (≤ 64) — small enough that the dense store's presence fast path is a single + * occupancy `long` (`1L << slot`), which is exactly why the earlier two-tier (group + field bloom) + * scheme could collapse to one word. Correctness never depends on the coloring (the dense scan is + * authoritative); only the fast-path hit rate does. + * + *

Slotting (does a tag get a slot / dense presence bit) is derived from the domain `required` + * level: required/conditional/recommended tags are colored (slotted), the rest are NO_SLOT + * (bucketed) and carry no slot bit. + */ +class TagRegistry +private constructor( + val stored: List, + val reserved: List, + val slotCount: Int, +) { + data class StoredTag( + val name: String, + val type: String, + val required: String, + val serial: Int, + val intercepted: Boolean, + val slot: Int, + val traceLevel: Boolean, + val id: Long, + val otelName: String? = null, + ) { + val slotted: Boolean + get() = slot != NO_SLOT + } + + data class ReservedTag( + val name: String, + val kind: String, + val field: String?, + val serial: Int, + val id: Long, + val otelName: String? = null, + ) + + /** Java overlay: intercepted tag names + the reserved/special-key registry. */ + class Overlay(val intercepted: Set, val reserved: List) { + data class ReservedDef( + val name: String, + val kind: String, + val field: String?, + val otelName: String? = null, + ) + + companion object { + @Suppress("UNCHECKED_CAST") + fun parse(root: Map): Overlay { + val intercepted = (root["intercepted"] as? List)?.toSet() ?: emptySet() + val reserved = + (root["reserved"] as? List>)?.map { m -> + ReservedDef( + m["tag"].toString(), + (m["kind"] as? String) ?: "directive", + m["field"] as? String, + m["open-telemetry-name"] as? String) + } ?: emptyList() + return Overlay(intercepted, reserved) + } + } + } + + companion object { + const val FIRST_STORED_SERIAL = 256 + const val NO_SLOT = 0xFFFF // slot all-ones sentinel (16 bits); mirrors KnownTagCodec.NO_SLOT + const val MAX_SLOT = 63 // one occupancy long: colored slots must fit in [0, 63] + const val LEVEL_TRACE = 1L shl 2 // low-32 carve bit 2; mirrors KnownTagCodec.LEVEL_TRACE + const val TRACE_LAYER = "" + + // Domain `required` levels that get a colored slot (the rest are bucketed with NO_SLOT). + val COLORABLE = setOf("required", "conditional", "recommended") + + /** + * Mirrors KnownTagCodec.makeTagId(serial, slot) + intercepted()/traceLevel() — must stay in + * sync. slot [47-32], LEVEL_TRACE at bit 2, other low bits zero. + */ + fun encode(serial: Int, intercepted: Boolean, slot: Int, traceLevel: Boolean): Long { + var id = (serial.toLong() shl 48) or ((slot.toLong() and 0xFFFF) shl 32) + if (traceLevel) id = id or LEVEL_TRACE + if (intercepted) id = id or Long.MIN_VALUE + return id + } + + fun build(conv: TagConventions, overlay: Overlay): TagRegistry { + val all = conv.allStoredTags() + val traceNames = conv.traceLevelTags().map { it.name }.toSet() + val colorable = all.filter { it.required in COLORABLE }.map { it.name }.toSet() + + // Co-occurrence cliques: each concrete type's resolved colorable tags, plus the trace-level + // tier as its own clique (a separate TagMap -> free to reuse span slot numbers). Tags in the + // same clique must get distinct colors; tags never sharing a clique may share a color. + val cliques = ArrayList>() + for (type in conv.concreteTypes()) { + cliques.add(conv.resolve(type).map { it.name }.filter { it in colorable }.toSet()) + } + cliques.add(traceNames.filter { it in colorable }.toSet()) + + // Adjacency: an edge between every pair of tags that co-occur in some clique. + val adj = HashMap>() + colorable.forEach { adj[it] = HashSet() } + for (clique in cliques) { + val members = clique.toList() + for (i in members.indices) for (j in i + 1 until members.size) { + adj.getValue(members[i]).add(members[j]) + adj.getValue(members[j]).add(members[i]) + } + } + + // Greedy coloring, most-constrained-first (by clique membership count, then name for a stable + // tie-break). Each tag takes the smallest color not used by an already-colored neighbor. + val cliqueCount = colorable.associateWith { n -> cliques.count { n in it } } + val order = colorable.sortedWith(compareByDescending { cliqueCount.getValue(it) }.thenBy { it }) + val color = HashMap() + for (n in order) { + val used = adj.getValue(n).mapNotNull { color[it] }.toSet() + var c = 0 + while (c in used) c++ + color[n] = c + } + val slotCount = (color.values.maxOrNull() ?: -1) + 1 + require(slotCount <= MAX_SLOT + 1) { + "coloring produced $slotCount slots; the single occupancy long holds at most ${MAX_SLOT + 1}" + } + + val reserved = + overlay.reserved.mapIndexed { i, v -> + val serial = 1 + i + ReservedTag( + v.name, v.kind, v.field, serial, + encode(serial, intercepted = true, slot = NO_SLOT, traceLevel = false), + v.otelName) + } + + // Stored tags in a stable order (by name); serials are a dense global counter from + // FIRST_STORED_SERIAL. slot comes from the coloring (NO_SLOT for non-colorable/bucketed tags). + val stored = + all.sortedBy { it.name }.mapIndexed { i, t -> + val serial = FIRST_STORED_SERIAL + i + val intercepted = t.name in overlay.intercepted + val slot = color[t.name] ?: NO_SLOT + val traceLevel = t.name in traceNames + StoredTag( + t.name, t.type, t.required, serial, intercepted, slot, traceLevel, + id = encode(serial, intercepted, slot, traceLevel), + otelName = t.otelName) + } + + validateOtelNames(stored, reserved) + return TagRegistry(stored, reserved, slotCount) + } + + /** + * An OpenTelemetry name must be unambiguous: it may not collide with any canonical tag name, nor + * be claimed by two different tags. Otherwise keyOf(otelName) would have no single right answer. + * Fail the build loudly rather than silently pick a winner. + */ + private fun validateOtelNames(stored: List, reserved: List) { + val canonical = (stored.map { it.name } + reserved.map { it.name }).toSet() + val owner = HashMap() + val check = { name: String, otel: String? -> + if (otel != null) { + require(otel !in canonical) { + "OpenTelemetry name '$otel' (of '$name') collides with canonical tag name '$otel'" + } + val prev = owner.put(otel, name) + require(prev == null) { "OpenTelemetry name '$otel' is claimed by both '$prev' and '$name'" } + } + } + stored.forEach { check(it.name, it.otelName) } + reserved.forEach { check(it.name, it.otelName) } + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt new file mode 100644 index 00000000000..477e2e9d245 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGenerator.kt @@ -0,0 +1,199 @@ +package datadog.gradle.plugin.tags + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import java.io.File +import java.util.Locale + +/** + * Turns the language-agnostic {@code tag-conventions.yaml} + the Java overlay into the generated tag + * registry: {@code KnownTags.java} (under {@code java/}) plus verification report dumps + * (resolved-tags / tag-assignment / layout-by-type / folded-types) at the destination root. + * + * Pure function of its inputs (deterministic ordering throughout), so the same inputs always produce + * byte-identical output -- which is what the {@code verifyKnownTags} freshness gate relies on. + */ +object TagRegistryGenerator { + /** Parses the two YAML files and writes the full generated tree under [outDir]. */ + fun generate(domainYaml: File, overlayYaml: File, outDir: File) { + val mapper = ObjectMapper(YAMLFactory()) + val domain: Map = + domainYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + val overlayMap: Map = + overlayYaml.inputStream().use { + mapper.readValue(it, object : TypeReference>() {}) + } + + // Clear the owned destination tree first, so a report/source file retired by a later generator + // revision doesn't linger: otherwise verifyKnownTags flags it as stale while telling developers + // to rerun generateKnownTags, which (without this) can't actually remove it. + outDir.deleteRecursively() + outDir.mkdirs() + // KnownTags.java goes under java/ (added as a srcDir); the .txt reports sit at the root. + val javaPkg = File(outDir, "java/datadog/trace/api").apply { mkdirs() } + + val conv = TagConventions.parse(domain) + val overlay = TagRegistry.Overlay.parse(overlayMap) + val reg = TagRegistry.build(conv, overlay) + + File(outDir, "resolved-tags.txt").writeText(resolvedReport(conv)) + File(outDir, "tag-assignment.txt").writeText(assignmentReport(conv, reg)) + File(outDir, "layout-by-type.txt").writeText(layoutByTypeReport(conv, reg)) + File(outDir, "folded-types.txt").writeText(foldedTypesReport(conv, reg)) + File(javaPkg, "KnownTags.java") + .writeText(KnownTagsEmitter.emit(reg, "datadog.trace.api", "KnownTags")) + } + + /** resolved-tags.txt — the per-type resolved sets (composition check). */ + private fun resolvedReport(conv: TagConventions): String { + val resolved = StringBuilder() + resolved.appendLine("# Resolved per-type tag sets (concrete span types).") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + resolved.appendLine() + resolved.appendLine("$type (${tags.size} tags):") + for (t in tags) resolved.appendLine(" - ${t.name}") + } + return resolved.toString() + } + + /** tag-assignment.txt — serials, colored slots, ids, per-type slot sets (coloring check). */ + private fun assignmentReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val a = StringBuilder() + a.appendLine( + "# Tag id assignment. slotCount=${reg.slotCount} stored=${reg.stored.size} reserved=${reg.reserved.size}") + a.appendLine() + a.appendLine("# STORED serial slot int lvl id required name") + for (t in reg.stored) { + a.appendLine( + " %6d %5s %s %s %-18s %-12s %s".format( + Locale.ROOT, + t.serial, + if (t.slotted) t.slot.toString() else "-", + if (t.intercepted) "I" else "-", + if (t.traceLevel) "T" else "-", + "0x%016X".format(Locale.ROOT, t.id), + t.required, + t.name)) + } + a.appendLine() + a.appendLine("# RESERVED serial id kind name") + for (v in reg.reserved) { + a.appendLine( + " %6d %-18s %-12s %s%s".format( + Locale.ROOT, + v.serial, + "0x%016X".format(Locale.ROOT, v.id), + v.kind, + v.name, + v.field?.let { " -> $it" } ?: "")) + } + a.appendLine() + a.appendLine("# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the") + a.appendLine("# co-occurrence clique); is its own clique and freely reuses span slot numbers.") + for (type in conv.concreteTypes()) { + val slots = + conv.resolve(type).mapNotNull { byName[it.name] } + .filter { it.slotted && !it.traceLevel } + .map { it.slot } + .sorted() + a.appendLine(" %-14s count=%-3d slots=%s".format(Locale.ROOT, type, slots.size, slots)) + } + val traceSlots = + reg.stored.filter { it.traceLevel && it.slotted }.map { it.slot }.sorted() + a.appendLine( + " %-14s count=%-3d slots=%s".format(Locale.ROOT, "", traceSlots.size, traceSlots)) + a.appendLine() + a.appendLine("# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still") + a.appendLine("# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.)") + val otelPairs = + (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } + + reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } }) + .sortedBy { it.first } + for ((otel, canonical) in otelPairs) { + a.appendLine(" %-30s -> %s".format(Locale.ROOT, otel, canonical)) + } + return a.toString() + } + + /** + * layout-by-type.txt — full composition per type (origins shown, NOT de-duped), each tag annotated + * with its slot/tier: s = colored slot, trace s = trace-level layer, bkt = bucket. + */ + private fun layoutByTypeReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + val lay = StringBuilder() + lay.appendLine("# Full tag composition per concrete span type (after extends/include/applies).") + lay.appendLine("# Not de-duped: a tag from >1 source appears >1 time.") + lay.appendLine("# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted") + for (type in conv.concreteTypes()) { + val comp = conv.compose(type) + val distinct = comp.map { it.second.name }.distinct().size + lay.appendLine() + lay.appendLine("$type (${comp.size} contributions, $distinct distinct):") + val byOrigin = LinkedHashMap>() + for ((origin, tag) in comp) byOrigin.getOrPut(origin) { ArrayList() }.add(tag) + for ((origin, tags) in byOrigin) { + lay.appendLine(" [$origin]") + for (t in tags) { + val st = byName[t.name] + val field = + when { + st == null -> "?" + st.traceLevel && st.slotted -> "trace s${st.slot}" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + lay.appendLine( + " %-26s %-12s %-12s %s".format( + Locale.ROOT, t.name, field, t.required, if (st?.intercepted == true) "I" else "")) + } + } + } + return lay.toString() + } + + /** + * folded-types.txt — each type's full resolved set (extends + include + applies, DE-DUPED) with its + * slot; plus the type. This is the "type with everything folded in" view. + */ + private fun foldedTypesReport(conv: TagConventions, reg: TagRegistry): String { + val byName = reg.stored.associateBy { it.name } + fun tierField(st: TagRegistry.StoredTag?): String = + when { + st == null -> "?" + st.traceLevel -> if (st.slotted) "trace s${st.slot}" else "trace-bkt" + st.slotted -> "s${st.slot}" + else -> "bkt" + } + val f = StringBuilder() + f.appendLine("# Folded tag set per type (extends + include + applies, de-duped), with colored slots.") + f.appendLine( + "# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted") + for (type in conv.concreteTypes()) { + val tags = conv.resolve(type) + f.appendLine() + f.appendLine("$type (${tags.size} tags):") + for (t in tags) { + val st = byName[t.name] + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), t.name, if (st?.intercepted == true) "I" else "")) + } + } + val traceTags = + reg.stored.filter { it.traceLevel }.sortedWith(compareBy({ !it.slotted }, { it.slot }, { it.name })) + f.appendLine() + f.appendLine(" (${traceTags.size} tags):") + for (st in traceTags) { + f.appendLine( + " %-12s %-26s %s".format( + Locale.ROOT, tierField(st), st.name, if (st.intercepted) "I" else "")) + } + return f.toString() + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt new file mode 100644 index 00000000000..313bd859068 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/TagRegistryGeneratorPlugin.kt @@ -0,0 +1,41 @@ +package datadog.gradle.plugin.tags + +import javax.inject.Inject +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory + +/** Extension configuring the tag-registry generator inputs/outputs. */ +abstract class TagRegistryExtension @Inject constructor(objects: ObjectFactory) { + val domainYaml: RegularFileProperty = objects.fileProperty() + val overlayYaml: RegularFileProperty = objects.fileProperty() + val destinationDirectory: DirectoryProperty = objects.directoryProperty() +} + +/** + * Registers {@code generateKnownTags} (emits the committed tag registry) and {@code verifyKnownTags} + * (a freshness gate that regenerates and byte-compares against the committed output). The verify task + * is wired into {@code check} so stale generated sources fail CI. + */ +class TagRegistryGeneratorPlugin : Plugin { + override fun apply(project: Project) { + val ext = project.extensions.create("tagRegistry", TagRegistryExtension::class.java) + project.tasks.register("generateKnownTags", GenerateKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + destinationDirectory.set(ext.destinationDirectory) + } + val verify = + project.tasks.register("verifyKnownTags", VerifyKnownTagsTask::class.java) { + domainYaml.set(ext.domainYaml) + overlayYaml.set(ext.overlayYaml) + committedDirectory.set(ext.destinationDirectory) + } + // `check` is contributed by lifecycle-base (via java-library); wait for it before wiring. + project.pluginManager.withPlugin("lifecycle-base") { + project.tasks.named("check").configure { dependsOn(verify) } + } + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt new file mode 100644 index 00000000000..e990e15e957 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/VerifyKnownTagsTask.kt @@ -0,0 +1,67 @@ +package datadog.gradle.plugin.tags + +import java.io.File +import javax.inject.Inject +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * Freshness gate: regenerates the tag registry into a scratch dir and byte-compares it against the + * committed [committedDirectory]. Fails (pointing at {@code generateKnownTags}) if they differ, so a + * stale commit of the generated sources can't slip through CI. Not cacheable -- it must actually run + * the generator to catch drift, and it is cheap. + */ +abstract class VerifyKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() { + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val domainYaml: RegularFileProperty = objects.fileProperty() + + @get:InputFile + @get:PathSensitive(PathSensitivity.NONE) + val overlayYaml: RegularFileProperty = objects.fileProperty() + + @get:InputDirectory + @get:PathSensitive(PathSensitivity.RELATIVE) + val committedDirectory: DirectoryProperty = objects.directoryProperty() + + @TaskAction + fun verify() { + val committed = committedDirectory.get().asFile + val scratch = File(temporaryDir, "generated") + scratch.deleteRecursively() + TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, scratch) + + val diffs = ArrayList() + val freshFiles = scratch.walkTopDown().filter { it.isFile }.toList() + for (fresh in freshFiles) { + val rel = fresh.relativeTo(scratch).path + val committedFile = File(committed, rel) + when { + !committedFile.exists() -> diffs.add("missing (not committed): $rel") + committedFile.readText() != fresh.readText() -> diffs.add("out of date: $rel") + } + } + val freshRel = freshFiles.map { it.relativeTo(scratch).path }.toSet() + for (committedFile in committed.walkTopDown().filter { it.isFile }) { + val rel = committedFile.relativeTo(committed).path + if (rel !in freshRel) diffs.add("stale (no longer generated): $rel") + } + + if (diffs.isNotEmpty()) { + throw GradleException( + buildString { + appendLine("Generated tag registry is out of date with tag-conventions.yaml:") + diffs.forEach { appendLine(" - $it") } + append("Run `./gradlew :internal-api:generateKnownTags` and commit the result.") + }) + } + } +} diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index a57e5d37882..49c9377dff9 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -305,6 +305,7 @@ public final class ConfigDefaults { public static final int DEFAULT_TRACE_X_DATADOG_TAGS_MAX_LENGTH = 512; static final boolean DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH = false; + static final boolean DEFAULT_TRACE_DENSE_TAGS_ENABLED = false; static final boolean DEFAULT_TRACE_LONG_RUNNING_ENABLED = false; static final long DEFAULT_TRACE_LONG_RUNNING_INITIAL_FLUSH_INTERVAL = 20; // seconds static final long DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL = 120; // seconds -> 2 minutes diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java index 9faf4f4ea8e..49640f0ef6b 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/TracerConfig.java @@ -173,5 +173,12 @@ public final class TracerConfig { public static final String TRACE_ORG_GUARD_STRICT = "trace.org.guard.strict"; public static final String TRACE_ORG_GUARD_TRUSTED_OPMS = "trace.org.guard.trusted.opms"; + /** + * Routes known tags through the dense (id-keyed) tag store instead of per-tag entries. + * Experimental, OFF by default. The {@code KnownTagCodec} is registered regardless; this flag + * only selects whether tags take the dense storage path. + */ + public static final String TRACE_DENSE_TAGS_ENABLED = "trace.experimental.dense.tags.enabled"; + private TracerConfig() {} } diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java b/dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java new file mode 100644 index 00000000000..6e375ee828b --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/DropWriter.java @@ -0,0 +1,42 @@ +package datadog.trace.core; + +import datadog.trace.common.writer.Writer; +import java.util.List; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Near-no-op {@link Writer}: drops finished traces (no serialization, no agent I/O, no {@link + * TraceCounters} bookkeeping) so span-creation benchmarks measure only the application-thread + * (front-half) allocation — create, tag, finish, PendingTrace completion. {@link #write} still + * hands the trace to a {@link Blackhole} rather than truly doing nothing with it, so the JIT can't + * treat the finish()-triggered write as dead code and eliminate work the real path performs. + * + *

Drift-stable: implements only the five-method {@link Writer} interface, unchanged + * v1.53→master. + */ +final class DropWriter implements Writer { + private final Blackhole blackhole; + + DropWriter(Blackhole blackhole) { + this.blackhole = blackhole; + } + + @Override + public void write(List trace) { + blackhole.consume(trace); + } + + @Override + public void start() {} + + @Override + public boolean flush() { + return true; + } + + @Override + public void close() {} + + @Override + public void incrementDropCounts(int spanCount) {} +} diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java new file mode 100644 index 00000000000..7dff7d3cf84 --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java @@ -0,0 +1,146 @@ +package datadog.trace.core; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Cross-version portable span-creation benchmark: create -> (set tags) -> finish. Only the + * drift-stable old-API arms (buildSpan/startSpan + setTag/withTag + Tags constants + the + * five-method Writer) so it compiles byte-identically on v1.53..master and can be grafted onto any + * release tag. + * + *

Used for the #12047 (dense tag store + graph-colored slots) vs v1.65.0 A/B. The dense store is + * activated on the #12047 build by {@code -Ddd.trace.dense.tags.enabled=true} in the {@code @Fork} + * args below; the same flag is an unknown/no-op property on v1.65.0, so the ONLY difference between + * the two runs is the tracer version. Read {@code gc.alloc.rate.norm} (B/op, deterministic) as the + * primary signal; throughput is directional-only (per-fork JIT bimodality at @Threads(8)). + */ +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.Throughput) +@Threads(8) +@OutputTimeUnit(MICROSECONDS) +@Fork( + value = 3, + jvmArgsAppend = { + "-DTEST_LOG_LEVEL=warn", + // Activates the dense known-tag store on #12047; unknown/no-op property on v1.65.0. + "-Ddd.trace.dense.tags.enabled=true", + // Production-shaped tracer config so mergedTracerTags is a realistically-sized shared bundle. + "-Ddd.service=petclinic", + "-Ddd.env=staging", + "-Ddd.version=1.2.3", + "-Ddd.tags=team:apm,dc:us1,cluster:prod-1,owner:tracing,tier:backend,region:us-east-1" + }) +public class SpanCreationBenchmark { + private static final String INSTRUMENTATION_NAME = "bench"; + private static final String SERVER_OPERATION_NAME = "servlet.request"; + private static final String JDBC_OPERATION_NAME = "database.query"; + + private static final String COMPONENT_VALUE = "tomcat-server"; + private static final String HTTP_METHOD_VALUE = "GET"; + private static final String HTTP_ROUTE_VALUE = "/owners/{ownerId}"; + private static final String HTTP_URL_VALUE = "http://localhost:8080/owners/42"; + private static final int HTTP_STATUS_VALUE = 100; // in-cache; value itself is immaterial here + private static final int PEER_PORT_VALUE = 80; + + private static final String DB_COMPONENT_VALUE = "java-jdbc-statement"; + private static final String DB_TYPE_VALUE = "postgresql"; + private static final String DB_INSTANCE_VALUE = "petclinic"; + private static final String DB_USER_VALUE = "app"; + private static final String DB_OPERATION_VALUE = "SELECT"; + private static final String DB_STATEMENT_VALUE = "SELECT * FROM owners WHERE id = ?"; + private static final String DB_PEER_HOSTNAME_VALUE = "db.internal"; + private static final int DB_PEER_PORT_VALUE = 90; // in-cache; value itself is immaterial here + + CoreTracer tracer; + + @Setup + public void setup(Blackhole blackhole) { + this.tracer = CoreTracer.builder().writer(new DropWriter(blackhole)).build(); + } + + @TearDown + public void tearDown() { + this.tracer.close(); + } + + /** Baseline: create + finish a bare span via startSpan, no tags. */ + @Benchmark + public void bareStartSpan() { + AgentSpan span = tracer.startSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME); + span.finish(); + } + + /** Baseline: create + finish a bare span via the builder path, no tags. */ + @Benchmark + public void bareBuildSpan() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + span.finish(); + } + + /** Web-server-shaped span: create -> set the typical known tags (7) -> finish. */ + @Benchmark + public void webServerSpan() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + span.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + span.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + span.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + span.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + span.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + span.finish(); + } + + /** + * Web-server-shaped span via the builder tag path (withTag before start, the OTel-bridge shape). + */ + @Benchmark + public void webServerSpanViaBuilder() { + AgentSpan span = + tracer + .buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME) + .withTag(Tags.COMPONENT, COMPONENT_VALUE) + .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER) + .withTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE) + .withTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE) + .withTag(Tags.HTTP_URL, HTTP_URL_VALUE) + .withTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE) + .withTag(Tags.PEER_PORT, PEER_PORT_VALUE) + .start(); + span.finish(); + } + + /** JDBC/DB-client-shaped span: create -> set the typical DB known tags (9) -> finish. */ + @Benchmark + public void jdbcClientSpan() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, JDBC_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, DB_COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + span.setTag(Tags.DB_TYPE, DB_TYPE_VALUE); + span.setTag(Tags.DB_INSTANCE, DB_INSTANCE_VALUE); + span.setTag(Tags.DB_USER, DB_USER_VALUE); + span.setTag(Tags.DB_OPERATION, DB_OPERATION_VALUE); + span.setTag(Tags.DB_STATEMENT, DB_STATEMENT_VALUE); + span.setTag(Tags.PEER_HOSTNAME, DB_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); + span.finish(); + } +} diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java new file mode 100644 index 00000000000..0eeffc43a48 --- /dev/null +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/TraceCreationBenchmark.java @@ -0,0 +1,227 @@ +package datadog.trace.core; + +import static java.util.concurrent.TimeUnit.MICROSECONDS; + +import datadog.trace.bootstrap.instrumentation.api.AgentScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Front-half allocation/throughput for creating a whole trace (a local-root span plus its + * children), as opposed to the single-span {@link SpanCreationBenchmark}. One op = one complete + * trace: root started, activated, children created-and-finished under it, then root finished. + * + *

Purpose — a reference others can key off when implementing core optimizations: + * + *

    + *
  • Win visibility. Optimizations that touch per-span cost (trace-level tag sharing, + * context propagation, pending-trace bookkeeping) accrue per span, so a trace of + * {@code 1+childCount} spans surfaces the per-trace payoff a single-span bench cannot. + *
  • Regression guard. A documented baseline (below) lets a later change to core show up + * as a delta — "did this addition move overhead?" — rather than passing silently. + *
+ * + *

Isolation: children are created under an active root scope, so this exercises the real + * parent-context propagation and trace-tag inheritance a single-span bench cannot reach. A no-op + * {@link DropWriter} drops finished traces (handing them to a {@link Blackhole} so the JIT can't + * dead-code the finish()-triggered write) so only application-thread (front-half) allocation lands + * in the {@code -prof gc} number, with no serialization or agent I/O. + * + *

The tracer is production-shaped via {@code @Fork} jvmArgs (service/env/version + global {@code + * dd.tags}) so {@code mergedTracerTags} is a realistically-sized shared bundle — the same config as + * {@link SpanCreationBenchmark}, keeping numbers comparable across the two. + * + *

Read {@code gc.alloc.rate.norm} (B/op, deterministic) as the primary signal; throughput is + * directional-only (laptop thermals + per-fork inlining bimodality). + * + *

Historical results (populate from a committed run on the branch this file lives on; + * matches the {@link SpanCreationBenchmark} header convention): + * + * 

+ *   date        commit           arm                        alloc B/op    thrpt ops/us
+ *   ----        ------           ---                        ----------    ------------
+ *   2026-08-18  sizing-hint-v2   webRequestTrace            4424.1        1.21
+ *   2026-08-18  sizing-hint-v2   fanoutTrace   (child=1)    2632.0        2.42
+ *   2026-08-18  sizing-hint-v2   fanoutTrace   (child=5)    6867.6        0.91
+ *   2026-08-18  sizing-hint-v2   fanoutTrace   (child=10)  11167.7        0.50
+ *
+ *   @Fork(3) @Threads(8), dense on. webRequestTrace is flat across childCount
+ *   (param ignored) — read any row.
+ *
+ *   Sizing A/B vs base #12047 (no SizingHint): fanoutTrace alloc win grows with
+ *   childCount — ~0 @1, -6.3% (-756 B/op) @10 — from child-lane resize-avoidance
+ *   on repeated heavy children; webRequestTrace (one-off mixed ops) shows ~0.
+ *   Throughput held flat ON=OFF. Mean B/op under-sells this feature; the value is
+ *   evolvability + worst-case (tail under fanout x tight heap), not the mean.
+ *
+ */ +@State(Scope.Benchmark) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +@BenchmarkMode(Mode.Throughput) +@Threads(8) +@OutputTimeUnit(MICROSECONDS) +@Fork( + value = 3, + jvmArgsAppend = { + "-DTEST_LOG_LEVEL=warn", + // Production-shaped tracer config so mergedTracerTags is a realistically-sized shared bundle + // (env + 6 global DD_TAGS + runtime-id/language) — the trace-level tags every span merges. + // Same config as SpanCreationBenchmark so the two benchmarks' numbers stay comparable. + "-Ddd.service=petclinic", + "-Ddd.env=staging", + "-Ddd.version=1.2.3", + "-Ddd.tags=team:apm,dc:us1,cluster:prod-1,owner:tracing,tier:backend,region:us-east-1" + }) +public class TraceCreationBenchmark { + private static final String INSTRUMENTATION_NAME = "bench"; + private static final String SERVER_OPERATION_NAME = "servlet.request"; + private static final String JDBC_OPERATION_NAME = "database.query"; + private static final String HTTP_CLIENT_OPERATION_NAME = "http.request"; + private static final String INTERNAL_OPERATION_NAME = "internal.work"; + + // Web-server (local root) tag shape — mirrors SpanCreationBenchmark.webServerSpan. + private static final String COMPONENT_VALUE = "tomcat-server"; + private static final String HTTP_METHOD_VALUE = "GET"; + private static final String HTTP_ROUTE_VALUE = "/owners/{ownerId}"; + private static final String HTTP_URL_VALUE = "http://localhost:8080/owners/42"; + private static final int HTTP_STATUS_VALUE = 100; // in Integer cache; boxing does not allocate + private static final int PEER_PORT_VALUE = 80; + + // JDBC-client child tag shape — mirrors SpanCreationBenchmark.jdbcClientSpan. + private static final String DB_COMPONENT_VALUE = "java-jdbc-statement"; + private static final String DB_TYPE_VALUE = "postgresql"; + private static final String DB_INSTANCE_VALUE = "petclinic"; + private static final String DB_USER_VALUE = "app"; + private static final String DB_OPERATION_VALUE = "SELECT"; + private static final String DB_STATEMENT_VALUE = "SELECT * FROM owners WHERE id = ?"; + private static final String DB_PEER_HOSTNAME_VALUE = "db.internal"; + private static final int DB_PEER_PORT_VALUE = 90; // in Integer cache; boxing does not allocate + + // HTTP-client child tag shape. + private static final String HTTP_CLIENT_COMPONENT_VALUE = "apache-httpclient"; + private static final String HTTP_CLIENT_URL_VALUE = "http://billing.internal/charge"; + private static final String HTTP_CLIENT_PEER_HOSTNAME_VALUE = "billing.internal"; + private static final int HTTP_CLIENT_PEER_PORT_VALUE = 90; // in Integer cache; no alloc + + // Internal child tag shape (a light span, few tags). + private static final String INTERNAL_COMPONENT_VALUE = "spring-scheduler"; + + /** Fan-out width for {@link #fanoutTrace()} — root + this many jdbc-shaped children. */ + @Param({"1", "5", "10"}) + int childCount; + + CoreTracer tracer; + + @Setup + public void setup(Blackhole blackhole) { + this.tracer = CoreTracer.builder().writer(new DropWriter(blackhole)).build(); + } + + @TearDown + public void tearDown() { + this.tracer.close(); + } + + /** + * Fixed, realistic web request: a server local root with a JDBC-client, an HTTP-client, and an + * internal child. One headline number that mirrors the shape of a real request. Not affected by + * {@link #childCount} (JMH still runs it once per param value — read any single row). + */ + @Benchmark + public void webRequestTrace() { + AgentSpan root = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + root.setTag(Tags.COMPONENT, COMPONENT_VALUE); + root.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + root.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + root.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + root.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + root.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + root.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + + AgentScope scope = tracer.activateSpan(root); + try { + jdbcChild(); + httpClientChild(); + internalChild(); + } finally { + scope.close(); + } + root.finish(); + } + + /** + * Fan-out trace: a server local root with {@link #childCount} JDBC-shaped children. The scaling + * axis — per-trace cost as span count grows exposes how per-span optimizations compound. + */ + @Benchmark + public void fanoutTrace() { + AgentSpan root = tracer.buildSpan(INSTRUMENTATION_NAME, SERVER_OPERATION_NAME).start(); + root.setTag(Tags.COMPONENT, COMPONENT_VALUE); + root.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + root.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + root.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + root.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + root.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + root.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + + AgentScope scope = tracer.activateSpan(root); + try { + for (int i = 0; i < childCount; i++) { + jdbcChild(); + } + } finally { + scope.close(); + } + root.finish(); + } + + /** A JDBC-client child of the currently-active span. */ + private void jdbcChild() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, JDBC_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, DB_COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + span.setTag(Tags.DB_TYPE, DB_TYPE_VALUE); + span.setTag(Tags.DB_INSTANCE, DB_INSTANCE_VALUE); + span.setTag(Tags.DB_USER, DB_USER_VALUE); + span.setTag(Tags.DB_OPERATION, DB_OPERATION_VALUE); + span.setTag(Tags.DB_STATEMENT, DB_STATEMENT_VALUE); + span.setTag(Tags.PEER_HOSTNAME, DB_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); + span.finish(); + } + + /** An HTTP-client child of the currently-active span. */ + private void httpClientChild() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, HTTP_CLIENT_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, HTTP_CLIENT_COMPONENT_VALUE); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_CLIENT); + span.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + span.setTag(Tags.HTTP_URL, HTTP_CLIENT_URL_VALUE); + span.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + span.setTag(Tags.PEER_HOSTNAME, HTTP_CLIENT_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, HTTP_CLIENT_PEER_PORT_VALUE); + span.finish(); + } + + /** A light internal child of the currently-active span. */ + private void internalChild() { + AgentSpan span = tracer.buildSpan(INSTRUMENTATION_NAME, INTERNAL_OPERATION_NAME).start(); + span.setTag(Tags.COMPONENT, INTERNAL_COMPONENT_VALUE); + span.finish(); + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index 120ba6e8df8..98b645cf219 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -40,7 +40,10 @@ import datadog.trace.api.EndpointTracker; import datadog.trace.api.IdGenerationStrategy; import datadog.trace.api.InstrumenterConfig; +import datadog.trace.api.KnownTags; import datadog.trace.api.Pair; +import datadog.trace.api.SizingHint; +import datadog.trace.api.SizingHintTable; import datadog.trace.api.TagMap; import datadog.trace.api.TraceConfig; import datadog.trace.api.civisibility.config.BazelMode; @@ -246,6 +249,12 @@ public static CoreTracerBuilder builder() { private static final boolean SPAN_BUILDER_REUSE_ENABLED = Config.get().isSpanBuilderReuseEnabled(); + // Dense known-tag store gate (experimental, OFF by default). Read once into a static constant so + // it constant-propagates and dead-code-eliminates the disabled branches on the span-creation + // path. The KnownTagCodec is registered regardless (ids route both ways: name->id when dense is + // on, id->name when off); this flag only selects whether tags actually take the dense store. + private static final boolean DENSE_TAGS_ENABLED = Config.get().isTraceDenseTagsEnabled(); + // Cache used by buildSpan - instance so it can capture the CoreTracer private final ReusableSingleSpanBuilderThreadLocalCache spanBuilderThreadLocalCache = SPAN_BUILDER_REUSE_ENABLED ? new ReusableSingleSpanBuilderThreadLocalCache(this) : null; @@ -656,6 +665,13 @@ private CoreTracer( // preload this enum to avoid triggering classloading on the hot path TraceCollector.PublishState.values(); + // Dense known-tag store (experimental, OFF by default, see DENSE_TAGS_ENABLED): initializing + // KnownTags routes known tags into the dense store so they store without a per-tag Entry. When + // off, tags take the same path as today. Gated by the trace.dense.tags.enabled Config flag. + if (DENSE_TAGS_ENABLED) { + KnownTags.init(); + } + if (reportInTracerFlare) { TracerFlare.addReporter(this); } @@ -2180,6 +2196,24 @@ protected static final DDSpanContext buildSpanContext( requestContextDataIast = builderRequestContextDataIast; } + // Per-operation dense-store sizing: an entry (local-root) span carries the trace-metadata / + // enriching tags a child doesn't, so pick the lane by whether we have a local parent. A + // resolved hint sizes the span's TagMap and self-tunes on finish; null (no/unkeyable + // operation name) falls back to the generic default capacity. + // + // Gated on DENSE_TAGS_ENABLED: sizing only helps when tags take the dense store (the + // experimental trace.dense.tags.enabled path). With it off -- the default -- known tags don't + // take the dense path, so a hint buys nothing; skipping resolution here keeps the + // operationName.toString() and the global-table probe off the default span-creation path + // entirely (do no harm). Not KnownTagCodec.isActive(): the codec is always registered. + final SizingHint sizingHint; + if (DENSE_TAGS_ENABLED) { + final boolean entrySpan = !(resolvedParentSpanContext instanceof DDSpanContext); + sizingHint = SizingHintTable.hintFor(operationName, entrySpan); + } else { + sizingHint = null; + } + // some attributes are inherited from the parent context = new DDSpanContext( @@ -2208,7 +2242,8 @@ protected static final DDSpanContext buildSpanContext( tracer.profilingContextIntegration, tracer.injectBaggageAsTags, tracer.injectLinksAsTags, - mergedTracerTagsNeedsIntercept ? null : mergedTracerTags); + mergedTracerTagsNeedsIntercept ? null : mergedTracerTags, + sizingHint); // By setting the tags on the context we apply decorators to any tags that have been set via // the builder. This is the order that the tags were added previously, but maybe the `tags` diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index a2d87e2c18b..f3b76e3dff5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -12,6 +12,7 @@ import datadog.trace.api.DDTraceId; import datadog.trace.api.Functions; import datadog.trace.api.ProcessTags; +import datadog.trace.api.SizingHint; import datadog.trace.api.TagMap; import datadog.trace.api.cache.DDCache; import datadog.trace.api.cache.DDCaches; @@ -138,6 +139,12 @@ public class DDSpanContext */ private final TagMap unsafeTags; + // Per-operation sizing hint (from SizingHintTable, keyed by operation name) this span was sized + // from, if any. Held so the span can feed its final dense-store size back on finish + // (recordDenseSize) -- the self-tuning loop that lets the reused hint converge to the operation's + // real known-tag high-water mark. Null when unsized. + private final SizingHint sizingHint; + /** The service name is required, otherwise the span are dropped by the agent */ private volatile String serviceName; @@ -244,6 +251,7 @@ public DDSpanContext( ProfilingContextIntegration.NoOp.INSTANCE, true, true, + null, null); } @@ -295,6 +303,7 @@ public DDSpanContext( ProfilingContextIntegration.NoOp.INSTANCE, injectBaggageAsTags, injectLinksAsTags, + null, null); } @@ -351,6 +360,7 @@ public DDSpanContext( profilingContextIntegration, injectBaggageAsTags, injectLinksAsTags, + null, null); } @@ -380,7 +390,8 @@ public DDSpanContext( final ProfilingContextIntegration profilingContextIntegration, final boolean injectBaggageAsTags, final boolean injectLinksAsTags, - final TagMap readThroughParent) { + final TagMap readThroughParent, + final SizingHint sizingHint) { assert traceCollector != null; this.traceCollector = traceCollector; @@ -411,8 +422,9 @@ public DDSpanContext( final int capacity = Math.max((tagsSize <= 0 ? 3 : (tagsSize + 1)) * 4 / 3, 8); this.unsafeTags = readThroughParent != null - ? TagMap.createFromParent(readThroughParent) - : TagMap.create(capacity); + ? TagMap.createFromParent(readThroughParent, sizingHint) + : sizingHint != null ? TagMap.create(sizingHint) : TagMap.create(capacity); + this.sizingHint = sizingHint; // must set this before setting the service and resource names below this.profilingContextIntegration = profilingContextIntegration; @@ -963,6 +975,26 @@ public void setTag(final String tag, final String value) { } } + /** + * Feeds this span's final dense-store size back into the sizingHint it was created from (if any), + * so a reused hint self-tunes to the operation's observed known-tag high-water mark and later + * spans of the operation are sized correctly. + * + *

Called from {@link #processTagsAndBaggage} at serialization — the one terminal point that + * (a) runs for every serialized span regardless of finish mode (plain {@code finish()} AND {@code + * phasedFinish()}+{@code publish()}, which the async gRPC/Netty/WebFlux paths use and which never + * enter {@code finishAndAddToTrace}), and (b) runs AFTER the lazy tag post-processors have + * appended their serialization-time tags ({@code _dd.integration}, host, ...), so the recorded + * count is the span's true final footprint rather than an underestimate. The caller already holds + * {@code synchronized (unsafeTags)}, so the {@code knownCount} read is consistent; the hint write + * itself is best-effort racy by design (see {@link TagMap#recordSize}). + */ + void recordDenseSize() { + if (sizingHint != null) { + unsafeTags.recordSize(sizingHint); + } + } + public void setTag(TagMap.EntryReader entry) { if (entry == null) { return; @@ -1290,6 +1322,10 @@ void processTagsAndBaggage( // Tags TagsPostProcessorFactory.lazyProcessor().processTags(unsafeTags, this, restrictedSpan); + // Self-tune the per-operation sizing hint now that every tag (incl. the just-appended + // serialization-time tags) is present -- the terminal point shared by all finish modes. + recordDenseSize(); + // Links if (injectLinksAsTags) { String linksTag = DDSpanLink.toTag(restrictedSpan.getLinks()); diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle index 93a817e6452..f27408c7cec 100644 --- a/gradle/spotless.gradle +++ b/gradle/spotless.gradle @@ -20,7 +20,10 @@ spotless { toggleOffOn() // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 target 'src/**/*.java' - // ignore embedded test projects and everything in build dir, e.g. generated sources + // ignore embedded test projects and everything in build dir, e.g. generated sources. + // src/generated/** is committed generated code (e.g. the tag registry) and IS held to the + // formatting standard: emitters must produce google-java-format-clean output, and their + // freshness gate byte-compares against these formatted files. targetExclude('src/test/resources/**', buildDirectoryFiles) tableTestFormatter('1.1.1') googleJavaFormat('1.35.0') diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 4d48a434c19..f1ab6486aa2 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -5,6 +5,7 @@ import groovy.lang.Closure plugins { `java-library` id("me.champeau.jmh") + id("dd-trace-java.tag-registry-generator") } apply(from = "$rootDir/gradle/java.gradle") @@ -32,6 +33,9 @@ extra["minimumBranchCoverage"] = 0.7 extra["minimumInstructionCoverage"] = 0.8 extra["excludedClassesCoverage"] = listOf( + // Generated by the tag-registry code generator (verified fresh via verifyKnownTags). + "datadog.trace.api.KnownTags", + "datadog.trace.api.KnownTags.*", "datadog.trace.api.ClassloaderConfigurationOverrides", "datadog.trace.api.ClassloaderConfigurationOverrides.Lazy", // Interface @@ -261,6 +265,18 @@ extra["excludedClassesBranchCoverage"] = listOf( extra["excludedClassesInstructionCoverage"] = listOf("datadog.trace.util.stacktrace.StackWalkerFactory") +// Tag registry: generated KnownTags is committed under src/generated (audited via git); the srcDir +// puts it on the main compile path and `verifyKnownTags` (wired into `check`) fails CI if it drifts +// from tag-conventions.yaml. Generation is run on demand (`./gradlew :internal-api:generateKnownTags`), +// not on every build, so the committed source stays the source of truth for the compiler. +tagRegistry { + domainYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.yaml")) + overlayYaml.set(rootProject.layout.projectDirectory.file("tag-conventions.java.yaml")) + destinationDirectory.set(layout.projectDirectory.dir("src/generated")) +} + +sourceSets["main"].java.srcDir("src/generated/java") + dependencies { // references TraceScope and Continuation from public api api(project(":dd-trace-api")) diff --git a/internal-api/src/generated/folded-types.txt b/internal-api/src/generated/folded-types.txt new file mode 100644 index 00000000000..48b631bf5a5 --- /dev/null +++ b/internal-api/src/generated/folded-types.txt @@ -0,0 +1,93 @@ +# Folded tag set per type (extends + include + applies, de-duped), with colored slots. +# field: s=colored slot trace s=trace-level layer bkt=bucketed trace-bkt=trace-level bucketed I=intercepted + +db.client (21 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s12 db.type + s9 db.instance + s10 db.operation + s15 db.user + bkt db.pool.name + s11 db.statement I + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.client (20 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s15 http.resend_count + s14 peer.service I + s8 _dd.peer.service.source + s7 _dd.peer.service.remapped_from + s13 peer.hostname + bkt peer.ipv4 + bkt peer.ipv6 + bkt peer.port + +http.server (18 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s9 http.method I + s10 http.status_code + s12 network.protocol.version + s11 http.url I + s13 http.route + s7 http.hostname + s14 http.useragent + s8 http.query.string + bkt servlet.path + bkt servlet.context I + +view.render (9 tags): + s1 _dd.parent_id + s2 component + s6 span.kind I + s0 _dd.integration + bkt _dd.svc_src + s5 error.type + s3 error.message + s4 error.stack + s7 view.name + + (13 tags): + trace s0 _dd.appsec.enabled + trace s1 _dd.base_service + trace s2 _dd.civisibility.enabled + trace s3 _dd.djm.enabled + trace s4 _dd.dsm.enabled + trace s5 _dd.git.commit.sha + trace s6 _dd.git.repository_url + trace s7 _dd.profiling.enabled + trace s8 _dd.tracer_host + trace s9 env + trace s10 language + trace s11 runtime-id + trace s12 version diff --git a/internal-api/src/generated/java/datadog/trace/api/KnownTags.java b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java new file mode 100644 index 00000000000..e0c04dc9142 --- /dev/null +++ b/internal-api/src/generated/java/datadog/trace/api/KnownTags.java @@ -0,0 +1,612 @@ +package datadog.trace.api; + +import datadog.trace.util.StringIndex; + +// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator). +// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml. +public final class KnownTags { + static final int SLOT_COUNT = 16; + + // ---- reserved (routed to span fields or directives; not stored) ---- + public static final String ERROR_NAME = "error"; + public static final long ERROR_ID = 0x8001FFFF00000000L; + // makeTagId(serial=1, slot=NO_SLOT) + intercepted [structural -> error] + + public static final String SERVICE_NAME = "service"; + public static final long SERVICE_ID = 0x8002FFFF00000000L; + // makeTagId(serial=2, slot=NO_SLOT) + intercepted [structural -> service] + + public static final String RESOURCE_NAME = "resource.name"; + public static final long RESOURCE_NAME_ID = 0x8003FFFF00000000L; + // makeTagId(serial=3, slot=NO_SLOT) + intercepted [structural -> resource] + + public static final String SPAN_TYPE_NAME = "span.type"; + public static final long SPAN_TYPE_ID = 0x8004FFFF00000000L; + // makeTagId(serial=4, slot=NO_SLOT) + intercepted [structural -> type] + + public static final String ORIGIN_NAME = "origin"; + public static final long ORIGIN_ID = 0x8005FFFF00000000L; + // makeTagId(serial=5, slot=NO_SLOT) + intercepted [structural -> origin] + + public static final String SAMPLING_PRIORITY_NAME = "sampling.priority"; + public static final long SAMPLING_PRIORITY_ID = 0x8006FFFF00000000L; + // makeTagId(serial=6, slot=NO_SLOT) + intercepted [directive] + + public static final String MANUAL_KEEP_NAME = "manual.keep"; + public static final long MANUAL_KEEP_ID = 0x8007FFFF00000000L; + // makeTagId(serial=7, slot=NO_SLOT) + intercepted [directive] + + public static final String MANUAL_DROP_NAME = "manual.drop"; + public static final long MANUAL_DROP_ID = 0x8008FFFF00000000L; + // makeTagId(serial=8, slot=NO_SLOT) + intercepted [directive] + + public static final String MEASURED_NAME = "measured"; + public static final long MEASURED_ID = 0x8009FFFF00000000L; + // makeTagId(serial=9, slot=NO_SLOT) + intercepted [directive] + + public static final String ANALYTICS_SAMPLE_RATE_NAME = "analytics.sample_rate"; + public static final long ANALYTICS_SAMPLE_RATE_ID = 0x800AFFFF00000000L; + // makeTagId(serial=10, slot=NO_SLOT) + intercepted [directive] + + // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ---- + public static final String DD_APPSEC_ENABLED_NAME = "_dd.appsec.enabled"; + public static final long DD_APPSEC_ENABLED_ID = 0x0100000000000004L; + // makeTagId(serial=256, slot=0) + trace-level + + public static final String DD_BASE_SERVICE_NAME = "_dd.base_service"; + public static final long DD_BASE_SERVICE_ID = 0x0101000100000004L; + // makeTagId(serial=257, slot=1) + trace-level + + public static final String DD_CIVISIBILITY_ENABLED_NAME = "_dd.civisibility.enabled"; + public static final long DD_CIVISIBILITY_ENABLED_ID = 0x0102000200000004L; + // makeTagId(serial=258, slot=2) + trace-level + + public static final String DD_DJM_ENABLED_NAME = "_dd.djm.enabled"; + public static final long DD_DJM_ENABLED_ID = 0x0103000300000004L; + // makeTagId(serial=259, slot=3) + trace-level + + public static final String DD_DSM_ENABLED_NAME = "_dd.dsm.enabled"; + public static final long DD_DSM_ENABLED_ID = 0x0104000400000004L; + // makeTagId(serial=260, slot=4) + trace-level + + public static final String DD_GIT_COMMIT_SHA_NAME = "_dd.git.commit.sha"; + public static final long DD_GIT_COMMIT_SHA_ID = 0x0105000500000004L; + // makeTagId(serial=261, slot=5) + trace-level + + public static final String DD_GIT_REPOSITORY_URL_NAME = "_dd.git.repository_url"; + public static final long DD_GIT_REPOSITORY_URL_ID = 0x0106000600000004L; + // makeTagId(serial=262, slot=6) + trace-level + + public static final String DD_INTEGRATION_NAME = "_dd.integration"; + public static final long DD_INTEGRATION_ID = 0x0107000000000000L; + // makeTagId(serial=263, slot=0) + + public static final String DD_PARENT_ID_NAME = "_dd.parent_id"; + public static final long DD_PARENT_ID = 0x0108000100000000L; + // makeTagId(serial=264, slot=1) + + public static final String DD_PEER_SERVICE_REMAPPED_FROM_NAME = "_dd.peer.service.remapped_from"; + public static final long DD_PEER_SERVICE_REMAPPED_FROM_ID = 0x0109000700000000L; + // makeTagId(serial=265, slot=7) + + public static final String DD_PEER_SERVICE_SOURCE_NAME = "_dd.peer.service.source"; + public static final long DD_PEER_SERVICE_SOURCE_ID = 0x010A000800000000L; + // makeTagId(serial=266, slot=8) + + public static final String DD_PROFILING_ENABLED_NAME = "_dd.profiling.enabled"; + public static final long DD_PROFILING_ENABLED_ID = 0x010B000700000004L; + // makeTagId(serial=267, slot=7) + trace-level + + public static final String DD_SVC_SRC_NAME = "_dd.svc_src"; + public static final long DD_SVC_SRC_ID = 0x010CFFFF00000000L; + // makeTagId(serial=268, slot=NO_SLOT) + + public static final String DD_TRACER_HOST_NAME = "_dd.tracer_host"; + public static final long DD_TRACER_HOST_ID = 0x010D000800000004L; + // makeTagId(serial=269, slot=8) + trace-level + + public static final String COMPONENT_NAME = "component"; + public static final long COMPONENT_ID = 0x010E000200000000L; + // makeTagId(serial=270, slot=2) + + public static final String DB_INSTANCE_NAME = "db.instance"; + public static final long DB_INSTANCE_ID = 0x010F000900000000L; + // makeTagId(serial=271, slot=9) + + public static final String DB_OPERATION_NAME = "db.operation"; + public static final long DB_OPERATION_ID = 0x0110000A00000000L; + // makeTagId(serial=272, slot=10) + + public static final String DB_POOL_NAME = "db.pool.name"; + public static final long DB_POOL_NAME_ID = 0x0111FFFF00000000L; + // makeTagId(serial=273, slot=NO_SLOT) + + public static final String DB_STATEMENT_NAME = "db.statement"; + public static final long DB_STATEMENT_ID = 0x8112000B00000000L; + // makeTagId(serial=274, slot=11) + intercepted + + public static final String DB_TYPE_NAME = "db.type"; + public static final long DB_TYPE_ID = 0x0113000C00000000L; + // makeTagId(serial=275, slot=12) + + public static final String DB_USER_NAME = "db.user"; + public static final long DB_USER_ID = 0x0114000F00000000L; + // makeTagId(serial=276, slot=15) + + public static final String ENV_NAME = "env"; + public static final long ENV_ID = 0x0115000900000004L; + // makeTagId(serial=277, slot=9) + trace-level + + public static final String ERROR_MESSAGE_NAME = "error.message"; + public static final long ERROR_MESSAGE_ID = 0x0116000300000000L; + // makeTagId(serial=278, slot=3) + + public static final String ERROR_STACK_NAME = "error.stack"; + public static final long ERROR_STACK_ID = 0x0117000400000000L; + // makeTagId(serial=279, slot=4) + + public static final String ERROR_TYPE_NAME = "error.type"; + public static final long ERROR_TYPE_ID = 0x0118000500000000L; + // makeTagId(serial=280, slot=5) + + public static final String HTTP_HOSTNAME_NAME = "http.hostname"; + public static final long HTTP_HOSTNAME_ID = 0x0119000700000000L; + // makeTagId(serial=281, slot=7) + + public static final String HTTP_METHOD_NAME = "http.method"; + public static final long HTTP_METHOD_ID = 0x811A000900000000L; + // makeTagId(serial=282, slot=9) + intercepted + + public static final String HTTP_QUERY_STRING_NAME = "http.query.string"; + public static final long HTTP_QUERY_STRING_ID = 0x011B000800000000L; + // makeTagId(serial=283, slot=8) + + public static final String HTTP_RESEND_COUNT_NAME = "http.resend_count"; + public static final long HTTP_RESEND_COUNT_ID = 0x011C000F00000000L; + // makeTagId(serial=284, slot=15) + + public static final String HTTP_ROUTE_NAME = "http.route"; + public static final long HTTP_ROUTE_ID = 0x011D000D00000000L; + // makeTagId(serial=285, slot=13) + + public static final String HTTP_STATUS_CODE_NAME = "http.status_code"; + public static final long HTTP_STATUS_CODE_ID = 0x011E000A00000000L; + // makeTagId(serial=286, slot=10) + + public static final String HTTP_URL_NAME = "http.url"; + public static final long HTTP_URL_ID = 0x811F000B00000000L; + // makeTagId(serial=287, slot=11) + intercepted + + public static final String HTTP_USERAGENT_NAME = "http.useragent"; + public static final long HTTP_USERAGENT_ID = 0x0120000E00000000L; + // makeTagId(serial=288, slot=14) + + public static final String LANGUAGE_NAME = "language"; + public static final long LANGUAGE_ID = 0x0121000A00000004L; + // makeTagId(serial=289, slot=10) + trace-level + + public static final String NETWORK_PROTOCOL_VERSION_NAME = "network.protocol.version"; + public static final long NETWORK_PROTOCOL_VERSION_ID = 0x0122000C00000000L; + // makeTagId(serial=290, slot=12) + + public static final String PEER_HOSTNAME_NAME = "peer.hostname"; + public static final long PEER_HOSTNAME_ID = 0x0123000D00000000L; + // makeTagId(serial=291, slot=13) + + public static final String PEER_IPV4_NAME = "peer.ipv4"; + public static final long PEER_IPV4_ID = 0x0124FFFF00000000L; + // makeTagId(serial=292, slot=NO_SLOT) + + public static final String PEER_IPV6_NAME = "peer.ipv6"; + public static final long PEER_IPV6_ID = 0x0125FFFF00000000L; + // makeTagId(serial=293, slot=NO_SLOT) + + public static final String PEER_PORT_NAME = "peer.port"; + public static final long PEER_PORT_ID = 0x0126FFFF00000000L; + // makeTagId(serial=294, slot=NO_SLOT) + + public static final String PEER_SERVICE_NAME = "peer.service"; + public static final long PEER_SERVICE_ID = 0x8127000E00000000L; + // makeTagId(serial=295, slot=14) + intercepted + + public static final String RUNTIME_ID_NAME = "runtime-id"; + public static final long RUNTIME_ID = 0x0128000B00000004L; + // makeTagId(serial=296, slot=11) + trace-level + + public static final String SERVLET_CONTEXT_NAME = "servlet.context"; + public static final long SERVLET_CONTEXT_ID = 0x8129FFFF00000000L; + // makeTagId(serial=297, slot=NO_SLOT) + intercepted + + public static final String SERVLET_PATH_NAME = "servlet.path"; + public static final long SERVLET_PATH_ID = 0x012AFFFF00000000L; + // makeTagId(serial=298, slot=NO_SLOT) + + public static final String SPAN_KIND_NAME = "span.kind"; + public static final long SPAN_KIND_ID = 0x812B000600000000L; + // makeTagId(serial=299, slot=6) + intercepted + + public static final String VERSION_NAME = "version"; + public static final long VERSION_ID = 0x012C000C00000004L; + // makeTagId(serial=300, slot=12) + trace-level + + public static final String VIEW_NAME = "view.name"; + public static final long VIEW_NAME_ID = 0x012D000700000000L; + // makeTagId(serial=301, slot=7) + + // ---- serial numbers ---- + static final int ERROR_SERIAL_NUM = 1; + static final int SERVICE_SERIAL_NUM = 2; + static final int RESOURCE_NAME_SERIAL_NUM = 3; + static final int SPAN_TYPE_SERIAL_NUM = 4; + static final int ORIGIN_SERIAL_NUM = 5; + static final int SAMPLING_PRIORITY_SERIAL_NUM = 6; + static final int MANUAL_KEEP_SERIAL_NUM = 7; + static final int MANUAL_DROP_SERIAL_NUM = 8; + static final int MEASURED_SERIAL_NUM = 9; + static final int ANALYTICS_SAMPLE_RATE_SERIAL_NUM = 10; + static final int DD_APPSEC_ENABLED_SERIAL_NUM = 256; + static final int DD_BASE_SERVICE_SERIAL_NUM = 257; + static final int DD_CIVISIBILITY_ENABLED_SERIAL_NUM = 258; + static final int DD_DJM_ENABLED_SERIAL_NUM = 259; + static final int DD_DSM_ENABLED_SERIAL_NUM = 260; + static final int DD_GIT_COMMIT_SHA_SERIAL_NUM = 261; + static final int DD_GIT_REPOSITORY_URL_SERIAL_NUM = 262; + static final int DD_INTEGRATION_SERIAL_NUM = 263; + static final int DD_PARENT_ID_SERIAL_NUM = 264; + static final int DD_PEER_SERVICE_REMAPPED_FROM_SERIAL_NUM = 265; + static final int DD_PEER_SERVICE_SOURCE_SERIAL_NUM = 266; + static final int DD_PROFILING_ENABLED_SERIAL_NUM = 267; + static final int DD_SVC_SRC_SERIAL_NUM = 268; + static final int DD_TRACER_HOST_SERIAL_NUM = 269; + static final int COMPONENT_SERIAL_NUM = 270; + static final int DB_INSTANCE_SERIAL_NUM = 271; + static final int DB_OPERATION_SERIAL_NUM = 272; + static final int DB_POOL_NAME_SERIAL_NUM = 273; + static final int DB_STATEMENT_SERIAL_NUM = 274; + static final int DB_TYPE_SERIAL_NUM = 275; + static final int DB_USER_SERIAL_NUM = 276; + static final int ENV_SERIAL_NUM = 277; + static final int ERROR_MESSAGE_SERIAL_NUM = 278; + static final int ERROR_STACK_SERIAL_NUM = 279; + static final int ERROR_TYPE_SERIAL_NUM = 280; + static final int HTTP_HOSTNAME_SERIAL_NUM = 281; + static final int HTTP_METHOD_SERIAL_NUM = 282; + static final int HTTP_QUERY_STRING_SERIAL_NUM = 283; + static final int HTTP_RESEND_COUNT_SERIAL_NUM = 284; + static final int HTTP_ROUTE_SERIAL_NUM = 285; + static final int HTTP_STATUS_CODE_SERIAL_NUM = 286; + static final int HTTP_URL_SERIAL_NUM = 287; + static final int HTTP_USERAGENT_SERIAL_NUM = 288; + static final int LANGUAGE_SERIAL_NUM = 289; + static final int NETWORK_PROTOCOL_VERSION_SERIAL_NUM = 290; + static final int PEER_HOSTNAME_SERIAL_NUM = 291; + static final int PEER_IPV4_SERIAL_NUM = 292; + static final int PEER_IPV6_SERIAL_NUM = 293; + static final int PEER_PORT_SERIAL_NUM = 294; + static final int PEER_SERVICE_SERIAL_NUM = 295; + static final int RUNTIME_ID_SERIAL_NUM = 296; + static final int SERVLET_CONTEXT_SERIAL_NUM = 297; + static final int SERVLET_PATH_SERIAL_NUM = 298; + static final int SPAN_KIND_SERIAL_NUM = 299; + static final int VERSION_SERIAL_NUM = 300; + static final int VIEW_NAME_SERIAL_NUM = 301; + + private static final String[] KEYOF_NAMES = { + ERROR_NAME, + SERVICE_NAME, + RESOURCE_NAME, + SPAN_TYPE_NAME, + ORIGIN_NAME, + SAMPLING_PRIORITY_NAME, + MANUAL_KEEP_NAME, + MANUAL_DROP_NAME, + MEASURED_NAME, + ANALYTICS_SAMPLE_RATE_NAME, + DD_APPSEC_ENABLED_NAME, + DD_BASE_SERVICE_NAME, + DD_CIVISIBILITY_ENABLED_NAME, + DD_DJM_ENABLED_NAME, + DD_DSM_ENABLED_NAME, + DD_GIT_COMMIT_SHA_NAME, + DD_GIT_REPOSITORY_URL_NAME, + DD_INTEGRATION_NAME, + DD_PARENT_ID_NAME, + DD_PEER_SERVICE_REMAPPED_FROM_NAME, + DD_PEER_SERVICE_SOURCE_NAME, + DD_PROFILING_ENABLED_NAME, + DD_SVC_SRC_NAME, + DD_TRACER_HOST_NAME, + COMPONENT_NAME, + DB_INSTANCE_NAME, + DB_OPERATION_NAME, + DB_POOL_NAME, + DB_STATEMENT_NAME, + DB_TYPE_NAME, + DB_USER_NAME, + ENV_NAME, + ERROR_MESSAGE_NAME, + ERROR_STACK_NAME, + ERROR_TYPE_NAME, + HTTP_HOSTNAME_NAME, + HTTP_METHOD_NAME, + HTTP_QUERY_STRING_NAME, + HTTP_RESEND_COUNT_NAME, + HTTP_ROUTE_NAME, + HTTP_STATUS_CODE_NAME, + HTTP_URL_NAME, + HTTP_USERAGENT_NAME, + LANGUAGE_NAME, + NETWORK_PROTOCOL_VERSION_NAME, + PEER_HOSTNAME_NAME, + PEER_IPV4_NAME, + PEER_IPV6_NAME, + PEER_PORT_NAME, + PEER_SERVICE_NAME, + RUNTIME_ID_NAME, + SERVLET_CONTEXT_NAME, + SERVLET_PATH_NAME, + SPAN_KIND_NAME, + VERSION_NAME, + VIEW_NAME, + "db.operation.name", + "db.query.text", + "db.system", + "http.request.method", + "http.response.status_code", + "server.address", + "service.name", + "url.full", + "url.query", + }; + private static final long[] KEYOF_VALUES = { + ERROR_ID, + SERVICE_ID, + RESOURCE_NAME_ID, + SPAN_TYPE_ID, + ORIGIN_ID, + SAMPLING_PRIORITY_ID, + MANUAL_KEEP_ID, + MANUAL_DROP_ID, + MEASURED_ID, + ANALYTICS_SAMPLE_RATE_ID, + DD_APPSEC_ENABLED_ID, + DD_BASE_SERVICE_ID, + DD_CIVISIBILITY_ENABLED_ID, + DD_DJM_ENABLED_ID, + DD_DSM_ENABLED_ID, + DD_GIT_COMMIT_SHA_ID, + DD_GIT_REPOSITORY_URL_ID, + DD_INTEGRATION_ID, + DD_PARENT_ID, + DD_PEER_SERVICE_REMAPPED_FROM_ID, + DD_PEER_SERVICE_SOURCE_ID, + DD_PROFILING_ENABLED_ID, + DD_SVC_SRC_ID, + DD_TRACER_HOST_ID, + COMPONENT_ID, + DB_INSTANCE_ID, + DB_OPERATION_ID, + DB_POOL_NAME_ID, + DB_STATEMENT_ID, + DB_TYPE_ID, + DB_USER_ID, + ENV_ID, + ERROR_MESSAGE_ID, + ERROR_STACK_ID, + ERROR_TYPE_ID, + HTTP_HOSTNAME_ID, + HTTP_METHOD_ID, + HTTP_QUERY_STRING_ID, + HTTP_RESEND_COUNT_ID, + HTTP_ROUTE_ID, + HTTP_STATUS_CODE_ID, + HTTP_URL_ID, + HTTP_USERAGENT_ID, + LANGUAGE_ID, + NETWORK_PROTOCOL_VERSION_ID, + PEER_HOSTNAME_ID, + PEER_IPV4_ID, + PEER_IPV6_ID, + PEER_PORT_ID, + PEER_SERVICE_ID, + RUNTIME_ID, + SERVLET_CONTEXT_ID, + SERVLET_PATH_ID, + SPAN_KIND_ID, + VERSION_ID, + VIEW_NAME_ID, + DB_OPERATION_ID, + DB_STATEMENT_ID, + DB_TYPE_ID, + HTTP_METHOD_ID, + HTTP_STATUS_CODE_ID, + HTTP_HOSTNAME_ID, + SERVICE_ID, + HTTP_URL_ID, + HTTP_QUERY_STRING_ID, + }; + private static final int[] KEYOF_HASHES; + private static final String[] KEYOF_KEYS; + private static final long[] KEYOF_IDS; + + static { + StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES); + long[] ids = new long[data.names.length]; + for (int j = 0; j < KEYOF_NAMES.length; j++) { + ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] = + KEYOF_VALUES[j]; + } + KEYOF_HASHES = data.hashes; + KEYOF_KEYS = data.names; + KEYOF_IDS = ids; + } + + static final KnownTagCodec.Resolver RESOLVER = + new KnownTagCodec.Resolver() { + @Override + public String nameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + case ERROR_SERIAL_NUM: + return ERROR_NAME; + case SERVICE_SERIAL_NUM: + return SERVICE_NAME; + case RESOURCE_NAME_SERIAL_NUM: + return RESOURCE_NAME; + case SPAN_TYPE_SERIAL_NUM: + return SPAN_TYPE_NAME; + case ORIGIN_SERIAL_NUM: + return ORIGIN_NAME; + case SAMPLING_PRIORITY_SERIAL_NUM: + return SAMPLING_PRIORITY_NAME; + case MANUAL_KEEP_SERIAL_NUM: + return MANUAL_KEEP_NAME; + case MANUAL_DROP_SERIAL_NUM: + return MANUAL_DROP_NAME; + case MEASURED_SERIAL_NUM: + return MEASURED_NAME; + case ANALYTICS_SAMPLE_RATE_SERIAL_NUM: + return ANALYTICS_SAMPLE_RATE_NAME; + case DD_APPSEC_ENABLED_SERIAL_NUM: + return DD_APPSEC_ENABLED_NAME; + case DD_BASE_SERVICE_SERIAL_NUM: + return DD_BASE_SERVICE_NAME; + case DD_CIVISIBILITY_ENABLED_SERIAL_NUM: + return DD_CIVISIBILITY_ENABLED_NAME; + case DD_DJM_ENABLED_SERIAL_NUM: + return DD_DJM_ENABLED_NAME; + case DD_DSM_ENABLED_SERIAL_NUM: + return DD_DSM_ENABLED_NAME; + case DD_GIT_COMMIT_SHA_SERIAL_NUM: + return DD_GIT_COMMIT_SHA_NAME; + case DD_GIT_REPOSITORY_URL_SERIAL_NUM: + return DD_GIT_REPOSITORY_URL_NAME; + case DD_INTEGRATION_SERIAL_NUM: + return DD_INTEGRATION_NAME; + case DD_PARENT_ID_SERIAL_NUM: + return DD_PARENT_ID_NAME; + case DD_PEER_SERVICE_REMAPPED_FROM_SERIAL_NUM: + return DD_PEER_SERVICE_REMAPPED_FROM_NAME; + case DD_PEER_SERVICE_SOURCE_SERIAL_NUM: + return DD_PEER_SERVICE_SOURCE_NAME; + case DD_PROFILING_ENABLED_SERIAL_NUM: + return DD_PROFILING_ENABLED_NAME; + case DD_SVC_SRC_SERIAL_NUM: + return DD_SVC_SRC_NAME; + case DD_TRACER_HOST_SERIAL_NUM: + return DD_TRACER_HOST_NAME; + case COMPONENT_SERIAL_NUM: + return COMPONENT_NAME; + case DB_INSTANCE_SERIAL_NUM: + return DB_INSTANCE_NAME; + case DB_OPERATION_SERIAL_NUM: + return DB_OPERATION_NAME; + case DB_POOL_NAME_SERIAL_NUM: + return DB_POOL_NAME; + case DB_STATEMENT_SERIAL_NUM: + return DB_STATEMENT_NAME; + case DB_TYPE_SERIAL_NUM: + return DB_TYPE_NAME; + case DB_USER_SERIAL_NUM: + return DB_USER_NAME; + case ENV_SERIAL_NUM: + return ENV_NAME; + case ERROR_MESSAGE_SERIAL_NUM: + return ERROR_MESSAGE_NAME; + case ERROR_STACK_SERIAL_NUM: + return ERROR_STACK_NAME; + case ERROR_TYPE_SERIAL_NUM: + return ERROR_TYPE_NAME; + case HTTP_HOSTNAME_SERIAL_NUM: + return HTTP_HOSTNAME_NAME; + case HTTP_METHOD_SERIAL_NUM: + return HTTP_METHOD_NAME; + case HTTP_QUERY_STRING_SERIAL_NUM: + return HTTP_QUERY_STRING_NAME; + case HTTP_RESEND_COUNT_SERIAL_NUM: + return HTTP_RESEND_COUNT_NAME; + case HTTP_ROUTE_SERIAL_NUM: + return HTTP_ROUTE_NAME; + case HTTP_STATUS_CODE_SERIAL_NUM: + return HTTP_STATUS_CODE_NAME; + case HTTP_URL_SERIAL_NUM: + return HTTP_URL_NAME; + case HTTP_USERAGENT_SERIAL_NUM: + return HTTP_USERAGENT_NAME; + case LANGUAGE_SERIAL_NUM: + return LANGUAGE_NAME; + case NETWORK_PROTOCOL_VERSION_SERIAL_NUM: + return NETWORK_PROTOCOL_VERSION_NAME; + case PEER_HOSTNAME_SERIAL_NUM: + return PEER_HOSTNAME_NAME; + case PEER_IPV4_SERIAL_NUM: + return PEER_IPV4_NAME; + case PEER_IPV6_SERIAL_NUM: + return PEER_IPV6_NAME; + case PEER_PORT_SERIAL_NUM: + return PEER_PORT_NAME; + case PEER_SERVICE_SERIAL_NUM: + return PEER_SERVICE_NAME; + case RUNTIME_ID_SERIAL_NUM: + return RUNTIME_ID_NAME; + case SERVLET_CONTEXT_SERIAL_NUM: + return SERVLET_CONTEXT_NAME; + case SERVLET_PATH_SERIAL_NUM: + return SERVLET_PATH_NAME; + case SPAN_KIND_SERIAL_NUM: + return SPAN_KIND_NAME; + case VERSION_SERIAL_NUM: + return VERSION_NAME; + case VIEW_NAME_SERIAL_NUM: + return VIEW_NAME; + default: + return null; + } + } + + @Override + public String openTelemetryNameOf(long tagId) { + switch (KnownTagCodec.serialNum(tagId)) { + case SERVICE_SERIAL_NUM: + return "service.name"; + case DB_OPERATION_SERIAL_NUM: + return "db.operation.name"; + case DB_STATEMENT_SERIAL_NUM: + return "db.query.text"; + case DB_TYPE_SERIAL_NUM: + return "db.system"; + case HTTP_HOSTNAME_SERIAL_NUM: + return "server.address"; + case HTTP_METHOD_SERIAL_NUM: + return "http.request.method"; + case HTTP_QUERY_STRING_SERIAL_NUM: + return "url.query"; + case HTTP_STATUS_CODE_SERIAL_NUM: + return "http.response.status_code"; + case HTTP_URL_SERIAL_NUM: + return "url.full"; + default: + return null; + } + } + + @Override + public int slotCount() { + return SLOT_COUNT; + } + + @Override + public long keyOf(String name) { + int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name); + return slot < 0 ? 0L : KEYOF_IDS[slot]; + } + }; + + static { + KnownTagCodec.register(RESOLVER); + } + + /** Forces resolver registration by triggering . Idempotent. */ + public static void init() {} + + private KnownTags() {} +} diff --git a/internal-api/src/generated/layout-by-type.txt b/internal-api/src/generated/layout-by-type.txt new file mode 100644 index 00000000000..309bdfcf8e3 --- /dev/null +++ b/internal-api/src/generated/layout-by-type.txt @@ -0,0 +1,91 @@ +# Full tag composition per concrete span type (after extends/include/applies). +# Not de-duped: a tag from >1 source appears >1 time. +# annotation: [s colored slot | trace s trace layer | bkt bucketed] I=intercepted + +db.client (21 contributions, 21 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [db.client] + db.type s12 required + db.instance s9 recommended + db.operation s10 recommended + db.user s15 recommended + db.pool.name bkt optional + db.statement s11 recommended I + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.client (20 contributions, 20 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.client] + http.url s11 required I + http.resend_count s15 recommended + [incl:peer] + peer.service s14 recommended I + _dd.peer.service.source s8 recommended + _dd.peer.service.remapped_from s7 recommended + peer.hostname s13 recommended + peer.ipv4 bkt optional + peer.ipv6 bkt optional + peer.port bkt optional + +http.server (18 contributions, 18 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [http] + http.method s9 required I + http.status_code s10 conditional + network.protocol.version s12 recommended + [http.server] + http.url s11 required I + http.route s13 conditional + http.hostname s7 required + http.useragent s14 recommended + http.query.string s8 recommended + servlet.path bkt optional + servlet.context bkt optional I + +view.render (9 contributions, 9 distinct): + [base] + _dd.parent_id s1 required + component s2 required + span.kind s6 required I + _dd.integration s0 recommended + _dd.svc_src bkt optional + error.type s5 recommended + error.message s3 recommended + error.stack s4 recommended + [view.render] + view.name s7 recommended diff --git a/internal-api/src/generated/resolved-tags.txt b/internal-api/src/generated/resolved-tags.txt new file mode 100644 index 00000000000..0edb3479608 --- /dev/null +++ b/internal-api/src/generated/resolved-tags.txt @@ -0,0 +1,77 @@ +# Resolved per-type tag sets (concrete span types). + +db.client (21 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - db.type + - db.instance + - db.operation + - db.user + - db.pool.name + - db.statement + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.client (20 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.resend_count + - peer.service + - _dd.peer.service.source + - _dd.peer.service.remapped_from + - peer.hostname + - peer.ipv4 + - peer.ipv6 + - peer.port + +http.server (18 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - http.method + - http.status_code + - network.protocol.version + - http.url + - http.route + - http.hostname + - http.useragent + - http.query.string + - servlet.path + - servlet.context + +view.render (9 tags): + - _dd.parent_id + - component + - span.kind + - _dd.integration + - _dd.svc_src + - error.type + - error.message + - error.stack + - view.name diff --git a/internal-api/src/generated/tag-assignment.txt b/internal-api/src/generated/tag-assignment.txt new file mode 100644 index 00000000000..9045eb8cde9 --- /dev/null +++ b/internal-api/src/generated/tag-assignment.txt @@ -0,0 +1,81 @@ +# Tag id assignment. slotCount=16 stored=46 reserved=10 + +# STORED serial slot int lvl id required name + 256 0 - T 0x0100000000000004 recommended _dd.appsec.enabled + 257 1 - T 0x0101000100000004 required _dd.base_service + 258 2 - T 0x0102000200000004 recommended _dd.civisibility.enabled + 259 3 - T 0x0103000300000004 recommended _dd.djm.enabled + 260 4 - T 0x0104000400000004 recommended _dd.dsm.enabled + 261 5 - T 0x0105000500000004 recommended _dd.git.commit.sha + 262 6 - T 0x0106000600000004 recommended _dd.git.repository_url + 263 0 - - 0x0107000000000000 recommended _dd.integration + 264 1 - - 0x0108000100000000 required _dd.parent_id + 265 7 - - 0x0109000700000000 recommended _dd.peer.service.remapped_from + 266 8 - - 0x010A000800000000 recommended _dd.peer.service.source + 267 7 - T 0x010B000700000004 recommended _dd.profiling.enabled + 268 - - - 0x010CFFFF00000000 optional _dd.svc_src + 269 8 - T 0x010D000800000004 recommended _dd.tracer_host + 270 2 - - 0x010E000200000000 required component + 271 9 - - 0x010F000900000000 recommended db.instance + 272 10 - - 0x0110000A00000000 recommended db.operation + 273 - - - 0x0111FFFF00000000 optional db.pool.name + 274 11 I - 0x8112000B00000000 recommended db.statement + 275 12 - - 0x0113000C00000000 required db.type + 276 15 - - 0x0114000F00000000 recommended db.user + 277 9 - T 0x0115000900000004 recommended env + 278 3 - - 0x0116000300000000 recommended error.message + 279 4 - - 0x0117000400000000 recommended error.stack + 280 5 - - 0x0118000500000000 recommended error.type + 281 7 - - 0x0119000700000000 required http.hostname + 282 9 I - 0x811A000900000000 required http.method + 283 8 - - 0x011B000800000000 recommended http.query.string + 284 15 - - 0x011C000F00000000 recommended http.resend_count + 285 13 - - 0x011D000D00000000 conditional http.route + 286 10 - - 0x011E000A00000000 conditional http.status_code + 287 11 I - 0x811F000B00000000 required http.url + 288 14 - - 0x0120000E00000000 recommended http.useragent + 289 10 - T 0x0121000A00000004 required language + 290 12 - - 0x0122000C00000000 recommended network.protocol.version + 291 13 - - 0x0123000D00000000 recommended peer.hostname + 292 - - - 0x0124FFFF00000000 optional peer.ipv4 + 293 - - - 0x0125FFFF00000000 optional peer.ipv6 + 294 - - - 0x0126FFFF00000000 optional peer.port + 295 14 I - 0x8127000E00000000 recommended peer.service + 296 11 - T 0x0128000B00000004 required runtime-id + 297 - I - 0x8129FFFF00000000 optional servlet.context + 298 - - - 0x012AFFFF00000000 optional servlet.path + 299 6 I - 0x812B000600000000 required span.kind + 300 12 - T 0x012C000C00000004 recommended version + 301 7 - - 0x012D000700000000 recommended view.name + +# RESERVED serial id kind name + 1 0x8001FFFF00000000 structural error -> error + 2 0x8002FFFF00000000 structural service -> service + 3 0x8003FFFF00000000 structural resource.name -> resource + 4 0x8004FFFF00000000 structural span.type -> type + 5 0x8005FFFF00000000 structural origin -> origin + 6 0x8006FFFF00000000 directive sampling.priority + 7 0x8007FFFF00000000 directive manual.keep + 8 0x8008FFFF00000000 directive manual.drop + 9 0x8009FFFF00000000 directive measured + 10 0x800AFFFF00000000 directive analytics.sample_rate + +# PER-TYPE colored slots. Slots within a type must be DISTINCT (a valid coloring of the +# co-occurrence clique); is its own clique and freely reuses span slot numbers. + db.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.client count=16 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + http.server count=15 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + view.render count=8 slots=[0, 1, 2, 3, 4, 5, 6, 7] + count=13 slots=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + +# OPENTELEMETRY NAMES. keyOf(otelName) resolves to the canonical tag's id; nameOf still +# returns the Datadog name, openTelemetryNameOf returns the name below. (No distinct id.) + db.operation.name -> db.operation + db.query.text -> db.statement + db.system -> db.type + http.request.method -> http.method + http.response.status_code -> http.status_code + server.address -> http.hostname + service.name -> service + url.full -> http.url + url.query -> http.query.string diff --git a/internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java new file mode 100644 index 00000000000..6c11f8bc212 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/api/DenseStoreAllocBenchmark.java @@ -0,0 +1,148 @@ +package datadog.trace.api; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Deterministic allocation A/B for the dense known-tag store, using the REAL {@link KnownTags} + * resolver (a {@code StringIndex} probe + a constant-returning {@code switch} — allocation-free, + * exactly like production). An earlier synthetic prefix resolver allocated in {@code keyOf} + * (substring) and {@code nameOf} (concat), contaminating the dense arm; this measures the store, + * not the resolver. + * + *

Models how a real span's tags route: {@code today} = all custom (what ships now — every tag + * buckets, since nothing is registered as known), {@code dense} = the same tag count with a + * realistic fraction routed to the dense store (real known tag names) and the rest custom. Run with + * {@code -prof gc}; the {@code gc.alloc.rate.norm} (B/op) delta at the same {@code tagCount} is + * what enabling the dense store does to a real span's per-build allocation. + * + *

Results — buildMap, JDK 17 (Zulu 17.0.7, Apple Silicon), {@code -prof gc -f 1 -wi 2 -i 3}, + * 2026-07-08. Allocation is deterministic (±0.001 B/op); throughput on this run is NOT + * trustworthy (single fork, short) — read B/op only. + * + * 

{@code
+ * scenario    tagCount=7   tagCount=12
+ * today          408 B/op     704 B/op
+ * dense          376 B/op     416 B/op
+ * allKnown       176 B/op     400 B/op
+ * }
+ * + *

Gate met: {@code dense < today} at both counts (the over-provision artifact is gone). The + * Entry-less win scales with the known-tag fraction — ~8% at 7 tags (~70% known), ~41% at 12; + * {@code allKnown} (the codegen endgame / read-through parent shape) reaches ~57% at 7. + * + *

Serialize paths (same run, B/op). {@code buildAndSerialize} (alloc-free {@code forEach} + * flyweight) adds a flat +16 B/op over {@code buildMap} in every scenario (7: 392, 12: 432 dense). + * {@code buildAndSerializeViaIterator} — the {@code EntryReader} enhanced-for modeling the count + * pre-pass at {@code TraceMapperV0_4:95} — adds a CONSTANT per-call cost (+56 custom / +80 dense, + * identical at 7 and 12 tags): that flat-vs-tagCount signature is the {@code EntryReaderIterator} + * OBJECT, NOT per-tag Entry — the iterator reuses a dense flyweight (TagMap:2182/2652). So the + * dense win SURVIVES serialization; the only nit is {@code iterator()} allocating one Iterator per + * call, which {@code forEach} avoids and which can be recycled away. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 2, time = 2) +@Measurement(iterations = 3, time = 2) +@Fork(1) +@Threads(1) +public class DenseStoreAllocBenchmark { + + // Real stored (dense-routed) tag names — a realistic web/db span's known set. + static final String[] KNOWN = + new String[] { + DDTags.BASE_SERVICE, + Tags.VERSION, + Tags.COMPONENT, + Tags.SPAN_KIND, + Tags.HTTP_METHOD, + Tags.HTTP_ROUTE, + Tags.DB_TYPE, + Tags.DB_INSTANCE, + Tags.PEER_HOSTNAME, + Tags.DB_USER, + DDTags.LANGUAGE_TAG_KEY, + Tags.PEER_PORT, + }; + + // today = all custom (all bucket, what ships now); dense = ~70% known + custom (a real span); + // allKnown = 100% known (the trace-tier read-through parent's shape — exercises lazy buckets). + @Param({"today", "dense", "allKnown"}) + String scenario; + + @Param({"7", "12"}) + int tagCount; + + private String[] keys; + private String[] values; + + @Setup(Level.Trial) + public void setup() { + KnownTags.init(); // registers the real (allocation-free) resolver + int knownCount; + if ("allKnown".equals(scenario)) { + knownCount = tagCount; // 100% known (<= KNOWN.length) + } else if ("dense".equals(scenario)) { + knownCount = (tagCount * 7) / 10; // ~70% known + custom + } else { + knownCount = 0; // today: all custom (all bucket) + } + this.keys = new String[tagCount]; + this.values = new String[tagCount]; + for (int i = 0; i < tagCount; i++) { + this.keys[i] = i < knownCount ? KNOWN[i] : "custom.tag." + i; + this.values[i] = "value-" + i; + } + } + + @Benchmark + public TagMap buildMap() { + TagMap m = TagMap.create(16); + for (int i = 0; i < tagCount; i++) { + m.set(keys[i], values[i]); + } + return m; + } + + @Benchmark + public void buildAndSerialize(Blackhole bh) { + TagMap m = TagMap.create(16); + for (int i = 0; i < tagCount; i++) { + m.set(keys[i], values[i]); + } + // forEach: the alloc-free flyweight emit for dense + m.forEach(reader -> bh.consume(reader.objectValue())); + bh.consume(m); + } + + @Benchmark + public void buildAndSerializeViaIterator(Blackhole bh) { + TagMap m = TagMap.create(16); + for (int i = 0; i < tagCount; i++) { + m.set(keys[i], values[i]); + } + // models the REAL serializer's count pre-pass (TraceMapperV0_4:95). The EntryReader iterator + // uses a reused dense flyweight (NO per-tag Entry alloc — TagMap:2182/2652), so the dense win + // SURVIVES; the only extra cost vs forEach is the EntryReaderIterator object itself (a fixed + // per-call cost, constant across tagCount — not per-tag). forEach avoids even that. + for (TagMap.EntryReader reader : m) { + bh.consume(reader.objectValue()); + } + bh.consume(m); + } +} diff --git a/internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java b/internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java new file mode 100644 index 00000000000..91052ee6a4b --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/util/FlatHashtableBenchmark.java @@ -0,0 +1,97 @@ +package datadog.trace.util; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Directional: is the {@link FlatHashtable} hit-path lookup (what a span pays per create, all hits + * after warmup) cheap vs a {@link HashMap}? Concrete-typed {@code static final} helper so the + * static-poly specialization is in play. One op = one pass over the op-name set. Single-threaded, + * short — a rough signal, not a verdict (real numbers on the box). + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 3, time = 1) +@Fork(1) +@Threads(1) +public class FlatHashtableBenchmark { + + static final class StrHelper extends FlatHashtable.StringHelper { + @Override + public boolean matches(String key, String value) { + return key == value || key.equals(value); + } + + @Override + public String create(String key) { + return key; // store the key itself as the (self-identifying) entry + } + } + + private static final StrHelper HELPER = new StrHelper(); + + private static final String[] KEYS = { + "servlet.request", + "database.query", + "http.request", + "grpc.client", + "kafka.produce", + "kafka.consume", + "jdbc.query", + "spring.handler", + "servlet.forward", + "okhttp.request" + }; + + private String[] table; + private Map map; + + @Setup + public void setup() { + table = FlatHashtable.create(String.class, KEYS.length); + map = new HashMap<>(KEYS.length * 2); + for (String k : KEYS) { + FlatHashtable.getOrCreate(table, k, HELPER); + map.put(k, k); + } + } + + /** FlatHashtable all-hit lookups (concrete helper → specialized). */ + @Benchmark + public void flatGet(Blackhole bh) { + for (String k : KEYS) { + bh.consume(FlatHashtable.get(table, k, HELPER)); + } + } + + /** Steady-state span-create shape: get-then-getOrCreate, all hits. */ + @Benchmark + public void flatGetOrCreate(Blackhole bh) { + for (String k : KEYS) { + bh.consume(FlatHashtable.getOrCreate(table, k, HELPER)); + } + } + + /** Baseline: HashMap lookups over the same keys. */ + @Benchmark + public void hashMapGet(Blackhole bh) { + for (String k : KEYS) { + bh.consume(map.get(k)); + } + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index bc7b0b904af..6e7fbe2d6c0 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -175,6 +175,7 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_MAX_ITEMS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_BAGGAGE_TAG_KEYS; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_CLOUD_PAYLOAD_TAGGING_SERVICES; +import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_DENSE_TAGS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_EXPERIMENTAL_FEATURES_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_HTTP_RESOURCE_REMOVE_TRAILING_SLASH; import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_KEEP_LATENCY_THRESHOLD_MS; @@ -1413,6 +1414,7 @@ public static String getHostName() { private final boolean jdkSocketEnabled; private final boolean spanBuilderReuseEnabled; + private final boolean traceDenseTagsEnabled; private final int tagNameUtf8CacheSize; private final int tagValueUtf8CacheSize; private final int stackTraceLengthLimit; @@ -3301,6 +3303,9 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) this.spanBuilderReuseEnabled = configProvider.getBoolean(GeneralConfig.SPAN_BUILDER_REUSE_ENABLED, true); + this.traceDenseTagsEnabled = + configProvider.getBoolean( + TracerConfig.TRACE_DENSE_TAGS_ENABLED, DEFAULT_TRACE_DENSE_TAGS_ENABLED); this.tagNameUtf8CacheSize = Math.max(configProvider.getInteger(GeneralConfig.TAG_NAME_UTF8_CACHE_SIZE, 128), 0); this.tagValueUtf8CacheSize = @@ -5153,6 +5158,10 @@ public boolean isSpanBuilderReuseEnabled() { return spanBuilderReuseEnabled; } + public boolean isTraceDenseTagsEnabled() { + return traceDenseTagsEnabled; + } + public int getTagNameUtf8CacheSize() { return tagNameUtf8CacheSize; } @@ -6821,6 +6830,11 @@ public String toString() { + sqsInjectDatadogAttributeEnabled + ", snsInjectDatadogAttributeEnabled=" + snsInjectDatadogAttributeEnabled + // Experimental: surfaced only when set away from the default, keeping normal dumps clean. + // Compared to the default constant (not a literal) so it survives a default change. + + (traceDenseTagsEnabled != DEFAULT_TRACE_DENSE_TAGS_ENABLED + ? ", traceDenseTagsEnabled=" + traceDenseTagsEnabled + : "") + '}'; } } diff --git a/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java new file mode 100644 index 00000000000..f5203dca92b --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/KnownTagCodec.java @@ -0,0 +1,242 @@ +package datadog.trace.api; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * Registry for generated tag ID ↔ name resolution. The code generator populates this at tracer init + * via {@link #register(Resolver)}. Once registered, HotSpot CHA devirtualizes and inlines the + * resolver's switch, making {@link #nameOf}/{@link #keyOf} effectively zero-overhead. + */ +public final class KnownTagCodec { + // Plain (non-volatile) fast-path flag: false until a resolver is ever registered. A plain read is + // free and hoistable, unlike a volatile read of `resolver` (costly on weak memory models such as + // ARM). A stale `false` is benign — callers treat the tag as unknown and use the hash buckets, + // which is correct, just unoptimized; the next read after publication takes the slot path. + private static boolean active; + + private static volatile Resolver resolver; + + /** Fast-path gate: true once a resolver has been registered. */ + public static boolean isActive() { + return active; + } + + /* + * tagId bit layout: [63 intercepted] [62-48 globalSerial (15 bits)] [47-32 slot (16 bits)] [31-0 + * reserved, zero]. Bit 63 (the sign bit) marks a tag the tag interceptor must see, so the check is + * a single {@code tagId < 0}. globalSerial is globally unique per known tag. The middle 16 bits + * carry the tag's SLOT: one globally stable coordinate assigned by graph-coloring the tag + * co-occurrence graph (the resolved tag set of each concrete span type, plus the trace-level + * tier, is a clique). Co-occurring tags always get distinct slots; slots are reused only between + * tags that never appear together, so slotCount stays bounded by the largest clique (≤ 64) and + * fits one {@code long} occupancy mask — the dense store's single-tier presence fast path (see + * {@link TagMap}). The low 32 bits are unused for known ids (the whole id is fully determined by + * serial + slot, so the generator can emit a literal). The low 32 bits are being carved for + * cross-cutting flags; bit 2 is the trace/span LEVEL bit (set ⟹ trace-level), and bits 1-0 are + * reserved for the dd/otel applicability flags that land with increment 1. The level bit lets + * read-through skip the shadow check across the trace/span boundary — trace and span tags reuse + * the same slots, so occupancy alone can't tell them apart, but a span map (no trace-level tags) + * can never shadow a trace-level ancestor entry (see {@link TagMap}). Unknown (string-only) custom + * tags are NOT known ids — they key off {@code TagMap.Entry#_hash(name)} in their own bucket path + * and never enter here. + */ + public static int serialNum(long tagId) { + return (int) ((tagId >>> 48) & 0x7FFF); + } + + /** + * Flag bit (the sign bit) marking a tag the tag interceptor must process — reserved tags AND + * intercepted-but-stored tags (e.g. http.method, which the interceptor side-effects and also + * stores). Encoded in the id so {@code DDSpanContext.setTag(long)} can route with a single sign + * test ({@link #isIntercepted}) instead of resolving the name. Non-intercepted tags (peer.*, + * base.service, …) leave it clear and take the fast store path. Must agree with the interceptor's + * name-based {@code needsIntercept} for every assigned id. + */ + public static final long INTERCEPTED = Long.MIN_VALUE; // 1L << 63 + + /** True if the tagId is flagged for tag-interceptor processing. */ + public static boolean isIntercepted(long tagId) { + return tagId < 0L; + } + + /** Returns the tagId with the {@link #INTERCEPTED} flag set. */ + public static long intercepted(long tagId) { + return tagId | INTERCEPTED; + } + + /** + * Trace/span LEVEL bit (low-32 carve, bit 2). Set marks a trace-level tag (lives on the + * TraceSegment's own TagMap); clear marks a span-level tag. Trace and span tags reuse the same + * coloring slots, so this bit is what lets read-through tell the two levels apart — a span map + * (no trace-level tags) can never shadow a trace-level ancestor entry, so its shadow check is + * skipped (see {@link TagMap#parentDenseVisible}). + */ + public static final long LEVEL_TRACE = 1L << 2; + + /** True if the tagId names a trace-level tag. */ + public static boolean isTraceLevel(long tagId) { + return (tagId & LEVEL_TRACE) != 0L; + } + + /** Returns the tagId with the {@link #LEVEL_TRACE} flag set. */ + public static long traceLevel(long tagId) { + return tagId | LEVEL_TRACE; + } + + // The middle 16 bits [47-32] hold the tag's SLOT: one globally stable coordinate from graph + // coloring the co-occurrence graph. Co-occurring tags get distinct slots and slotCount stays + // bounded by the largest clique (<= 64), so the dense store's presence fast path is a single + // occupancy long (1L << slot); a clear bit proves the tag absent and enables an O(1) append. See + // TagMap's dense-store fast path. + static final int SLOT_SHIFT = 32; + static final int SLOT_MASK = 0xFFFF; // 16 bits + + /** + * The tag's slot: its globally stable coloring coordinate, or {@link #NO_SLOT} when it has none + * (reserved or deliberately bucket-only). Drives the dense store's single occupancy mask. + */ + public static int slot(long tagId) { + return (int) ((tagId >>> SLOT_SHIFT) & SLOT_MASK); + } + + /** + * globalSerial partition. {@code [1, FIRST_STORED_SERIAL)} is the RESERVED tier and {@code + * [FIRST_STORED_SERIAL, ..]} is the STORED tier; {@code globalSerial == 0} means unknown / + * string-only. Both core and the code generator must agree on this boundary. + * + *

Reserved is the shared mechanism: the tracer reserves the key and handles it itself + * instead of putting it in the TagMap. It says nothing about whether a value exists — that splits + * into two kinds (the {@code kind:} in the overlay): + * + *

    + *
  • structural — the value does exist, it just lives in a first-class + * span/trace field (service, resource.name, error, span.type, origin), not the tag map. + *
  • directive — there is no stored value; the key is a command that triggers + * trace behavior (sampling.priority, manual.keep, measured). + *
+ * + * "virtual" over-claims non-existence (wrong for structural) and "built-in" over-claims existence + * (wrong for directive), so the tier is named for the mechanism they share: reserved. These are + * hand-assigned in the overlay. Stored tags are the generated convention tags that ARE put + * in the map (slotted/bucketed). + */ + public static final int FIRST_STORED_SERIAL = 256; + + /** True if the tagId names a reserved (structural/directive) tag — handled, not stored. */ + public static boolean isReserved(long tagId) { + int serialNum = serialNum(tagId); + return serialNum > 0 && serialNum < FIRST_STORED_SERIAL; + } + + /** True if the tagId names a generated, map-stored (slotted/bucketed) tag. */ + public static boolean isStored(long tagId) { + return serialNum(tagId) >= FIRST_STORED_SERIAL; + } + + /** + * Sentinel {@code slot} meaning "no positional slot". It is the maximum value the 16-bit slot + * field can hold, so it always compares {@code >= slotCount()} and routes to the hash buckets + * rather than the fast positional array. Two kinds of tagId use it: + * + *
    + *
  • Reserved tags ({@code globalSerial < FIRST_STORED_SERIAL}) — not stored at all; the + * sentinel just guarantees an incidental store never lands in a slot. + *
  • Unslotted stored tags ({@code globalSerial >= FIRST_STORED_SERIAL}) — "low-priority" tags + * that get a stable id (and so {@code keyOf}/{@code nameOf} unification with their string + * form) but are deliberately not given a slot, so they live in the buckets. {@code + * getEntry(long)} for these resolves the name and rehashes — the cost of not owning a slot. + *
+ */ + public static final int NO_SLOT = SLOT_MASK; // slot all-ones sentinel (16 bits) + + /** + * True if the tagId names a stored tag that deliberately has no positional slot (bucket-only). + */ + public static boolean isUnslotted(long tagId) { + return isStored(tagId) && slot(tagId) == NO_SLOT; + } + + /** + * Builds a tagId from its {@code serialNum} (globally unique per known tag) and {@code slot} (its + * coloring coordinate, or {@link #NO_SLOT}). The low 32 bits are zero, so the id is fully + * determined by these parts — the generator emits it as a literal. Inverse of {@link + * #serialNum}/{@link #slot}. Intended for the code generator and tests. + */ + public static long makeTagId(int serialNum, int slot) { + return ((long) serialNum << 48) | ((long) (slot & SLOT_MASK) << SLOT_SHIFT); + } + + /** + * Builds a tagId with no positional slot ({@code slot == }{@link #NO_SLOT}). Use for reserved + * tags and for "low-priority" stored tags that get a stable id but are intentionally kept out of + * the fast slot array (they route to the hash buckets). See {@link #NO_SLOT}. + */ + public static long makeTagId(int serialNum) { + return makeTagId(serialNum, NO_SLOT); + } + + // Number of positional slots in the global layout = (max stored fieldPos) + 1, declared by the + // registered provider. Captured once at registration and read as a dynamic constant; TagMap sizes + // its knownEntries array to exactly this rather than a hardcoded max. 0 when no resolver. + private static int slotCount; + + /** Slot count of the registered provider (max stored fieldPos + 1); 0 if none. */ + public static int slotCount() { + return slotCount; + } + + public interface Resolver { + String nameOf(long tagId); + + /** The tag's OpenTelemetry-namespace name, or {@code null} when it declares none. */ + String openTelemetryNameOf(long tagId); + + long keyOf(String name); + + /** Number of positional slots this provider uses: (max stored fieldPos) + 1. */ + int slotCount(); + } + + @SuppressFBWarnings( + value = "AT_STALE_THREAD_WRITE_OF_PRIMITIVE", + justification = + "active/slotCount are plain by design: written once at tracer-init registration (before" + + " any span processing) and read plain on the hot path. A stale read is benign — the" + + " tag is treated as unknown and takes the hash-bucket path — so plain reads are" + + " deliberately preferred over a costly volatile read on weak memory models.") + public static void register(Resolver resolver) { + KnownTagCodec.resolver = resolver; // volatile write publishes the resolver + KnownTagCodec.slotCount = (resolver != null) ? resolver.slotCount() : 0; + KnownTagCodec.active = + (resolver != null); // plain write; readers re-read resolver volatile anyway + } + + public static String nameOf(long tagId) { + if (!active) return null; + Resolver r = resolver; + return r != null ? r.nameOf(tagId) : null; + } + + /** The tag's Datadog-namespace (canonical) name — the same value as {@link #nameOf}. */ + public static String datadogNameOf(long tagId) { + return nameOf(tagId); + } + + /** + * The tag's OpenTelemetry-namespace name, or {@code null} when it declares none (or no resolver + * is registered). A serializer owns any fall-back-to-Datadog-name policy; this is a pure lookup. + */ + public static String openTelemetryNameOf(long tagId) { + if (!active) return null; + Resolver r = resolver; + return r != null ? r.openTelemetryNameOf(tagId) : null; + } + + public static long keyOf(String name) { + if (!active) return 0L; + Resolver r = resolver; + return r != null ? r.keyOf(name) : 0L; + } + + private KnownTagCodec() {} +} diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHelper.java b/internal-api/src/main/java/datadog/trace/api/SizingHelper.java new file mode 100644 index 00000000000..282be66079e --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/SizingHelper.java @@ -0,0 +1,27 @@ +package datadog.trace.api; + +import datadog.trace.util.FlatHashtable; + +/** + * {@link FlatHashtable} policy for the per-operation {@link SizingHint} table: keys by operation + * name, entries are {@code SizingHint}s carrying that name plus its cached spread hash. Stateless — + * held by {@link SizingHintTable} as a concrete-typed {@code static final} singleton so {@code + * FlatHashtable.get}/{@code getOrCreate} specialize (devirtualize + inline) at the call site. + * + *

Extends {@link FlatHashtable.StringHelper}, which seals the spread {@code hash}; this class + * only supplies {@code matches} and {@code create}. Both use the inherited {@code hash} so the + * cached {@link SizingHint#labelHash} is always the same spread the probe used. + */ +final class SizingHelper extends FlatHashtable.StringHelper { + @Override + public boolean matches(String key, SizingHint value) { + // int gate on the cached hash before equals; op-names are usually interned literals, so `==` is + // the common hit. + return value.labelHash == hash(key) && (key == value.label || key.equals(value.label)); + } + + @Override + public SizingHint create(String key) { + return new SizingHint(key, hash(key), SizingHintTable.SEED_SIZE); + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHint.java b/internal-api/src/main/java/datadog/trace/api/SizingHint.java new file mode 100644 index 00000000000..b4fdd3a4cfe --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/SizingHint.java @@ -0,0 +1,52 @@ +package datadog.trace.api; + +/** + * Opaque per-operation dense-store sizing hint, and a self-contained {@link + * datadog.trace.util.FlatHashtable} slot: it carries everything the probe compares ({@link #label} + * + cached {@link #labelHash}) plus the tuned payload ({@link #size}). Holding key, hash, and value + * in ONE object behind ONE array slot is deliberate — entry publication is a single reference + * store, so a reader sees {@code null} or a complete entry (never a torn one), and the {@code + * final} identity fields are visible even under racy publication (JMM final-field guarantee). That + * sidesteps the memory-ordering / visibility problems parallel key/hash/value arrays would create, + * no volatile or atomics. + * + *

Opaque to everything outside {@code datadog.trace.api}: no public members. {@link TagMap} + * reads {@link #size} to size a fresh dense store and writes it back (best-effort max, see {@link + * TagMap#recordSize}) at a terminal point; {@code SizingHelper} mints and compares by {@link + * #label}/{@link #labelHash}. Callers only ever hold the reference. + * + *

{@link #labelHash} is supplied by the helper (a single spread source — {@code + * FlatHashtable.StringHelper.hash}) so the cached gate always matches the probe hash. + */ +public final class SizingHint { + // Identity: final => safely published under a racy single-reference store. `label` is the + // operation + // name (typically an interned literal, so the `==` fast-path usually hits). `labelHash` is the + // helper's spread hash, cached to gate `equals` with an int compare during probing. + final String label; + final int labelHash; + + // Payload: the tuned dense-store size. Plain racy int, updated best-effort max (see + // TagMap#recordSize -- not strictly monotonic under concurrent finishes) — a stale/lost/lowered + // read only mis-sizes an array (over/under-provision), never corrupts tag data, so no + // synchronization. + int size; + + // When true, {@code size} is fixed and recordSize won't grow it. For the shared default / + // overflow + // hint (a HETEROGENEOUS catch-all for operation-less / over-budget spans): self-tuning it toward + // the observed max would converge to the max across unlike sharers and over-provision the lean + // ones. + final boolean capped; + + SizingHint(String label, int labelHash, int seedSize) { + this(label, labelHash, seedSize, false); + } + + SizingHint(String label, int labelHash, int seedSize, boolean capped) { + this.label = label; + this.labelHash = labelHash; + this.size = seedSize; + this.capped = capped; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java b/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java new file mode 100644 index 00000000000..d77b02546db --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/SizingHintTable.java @@ -0,0 +1,107 @@ +package datadog.trace.api; + +import datadog.trace.util.FlatHashtable; + +/** + * Process-wide, self-tuning registry of per-operation {@link SizingHint}s, keyed by operation name. + * Pure static: the tracer resolves a hint here at span build and hands it to the span, which sizes + * its dense {@link TagMap} from it and records the actual size back on finish — the hint converges + * to the operation's real known-tag high-water mark, so later spans of that operation size + * correctly. + * + *

Two lanes, because a span's dense size depends systematically on whether it is an entry + * (local-root) span or not: entry spans carry the trace-metadata / enriching tags and children + * don't, so the same operation name has two different steady-state sizes. {@link #hintFor} picks + * the lane. + * + *

Bounded + racy by design. Each lane is a fixed-capacity {@link FlatHashtable} (never + * resized) so memory is bounded even under unbounded/dynamic operation names; once a lane's + * cardinality budget is spent, further operations share a capped default hint. Construction, + * insertion, and the best-effort-max size update are all lock-free and deliberately racy — a lost + * update or a double-mint only mis-sizes an array (over/under-provision) for a span or two, never + * corrupts tag data (see {@link FlatHashtable} and {@link SizingHint} for the rationale). + * + *

Keyed by the operation name's {@code String} form. In practice every operation name we see is + * a {@code String} or a {@link datadog.trace.bootstrap.instrumentation.api.UTF8BytesString}, whose + * {@code toString()} just returns its backing field — so the {@code toString()} here is O(1) and + * allocation-free on the hot path. A {@code null} operation name gets no hint (the span falls back + * to the generic default capacity); a missed hint is benign, never wrong. + */ +public final class SizingHintTable { + private SizingHintTable() {} + + // Seed for a fresh per-operation hint: a floor that self-tunes up via best-effort-max recordSize. + static final int SEED_SIZE = 1; + // Fixed size for the shared over-budget hint: a small lean default. Capped (never grown by + // recordSize) because it's a heterogeneous catch-all -- growing it would over-provision lean + // sharers + // to the max of an unlike cohort. + static final int OVERFLOW_SEED = 8; + // Max distinct operation names per lane that get their own hint before collapsing to the capped + // default. Backing capacity is the next power of two >= 2 * this (load factor <= 0.5). + private static final int CARDINALITY_LIMIT = 512; + + // Concrete-typed static-final singleton => FlatHashtable calls specialize at this call site. + private static final SizingHelper HELPER = new SizingHelper(); + + // Entry (local-root, enriched) lane and non-entry (child) lane, keyed by operation name. + private static final SizingHint[] ENTRY_SLOTS = + FlatHashtable.create(SizingHint.class, CARDINALITY_LIMIT); + private static final SizingHint[] CHILD_SLOTS = + FlatHashtable.create(SizingHint.class, CARDINALITY_LIMIT); + + // Shared capped hint each lane returns once its budget is exhausted. + private static final SizingHint ENTRY_OVERFLOW = + new SizingHint(null, 0, OVERFLOW_SEED, /* capped */ true); + private static final SizingHint CHILD_OVERFLOW = + new SizingHint(null, 0, OVERFLOW_SEED, /* capped */ true); + + // Approximate live counts gating each lane's budget. Plain racy ints -- a few over/under the cap + // under contention is harmless (the cap is a safety bound, not an exact quota). + private static int entrySize; + private static int childSize; + + /** + * The sizing hint for {@code operationName} in the given lane: the existing one, a freshly-minted + * (seeded) one if the lane has budget, or the shared capped default if it's full. Returns {@code + * null} for a {@code null} operation name — the span then uses the generic default capacity + * (operation-less spans aren't reliably similar, so they get no hint). Hits are a single probe; + * the create path is warmup-rare. + */ + public static SizingHint hintFor(CharSequence operationName, boolean entrySpan) { + if (operationName == null) { + return null; + } + final String key = + operationName.toString(); // O(1) for String / UTF8BytesString (see class doc) + final SizingHint[] slots = entrySpan ? ENTRY_SLOTS : CHILD_SLOTS; + final SizingHint overflow = entrySpan ? ENTRY_OVERFLOW : CHILD_OVERFLOW; + + final SizingHint existing = FlatHashtable.get(slots, key, HELPER); + if (existing != null) { + return existing; + } + if ((entrySpan ? entrySize : childSize) >= CARDINALITY_LIMIT) { + return overflow; + } + // Count only a genuine insert. Between the get() miss above and this getOrCreate another thread + // may have inserted the same operation's hint; getOrCreate then returns that existing entry to + // us -- without `created` we'd still bump the lane count, so a cold-start burst of many spans + // of + // one operation could spend the budget on a single distinct name. `created[0]` is set only on + // the actual store branch. Allocated here on the create path only, which is warmup-rare. + final boolean[] created = new boolean[1]; + final SizingHint hint = FlatHashtable.getOrCreate(slots, key, HELPER, created); + if (hint == null) { + return overflow; // physically full -- shouldn't happen under the cap, but stay safe + } + if (created[0]) { + if (entrySpan) { + entrySize++; // racy approximate count; only a real insert bumps it + } else { + childSize++; + } + } + return hint; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index e0a2f2b6f70..5de1b26c995 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -47,10 +47,13 @@ */ public final class TagMap implements Map, Iterable { /** Immutable empty TagMap - similar to {@link Collections#emptyMap()} */ - // Frozen view over a length-1 array: bucket masking needs a power-of-two array length (size 0 - // would fail with ArrayIndexOutOfBoundsException, size 1 works), and the private constructor - // reads no statics, so this is safe to build directly during TagMap's . - public static final TagMap EMPTY = new TagMap(new Object[1], 0); + // Frozen view over a power-of-two array; the private constructor reads no statics, so this is + // safe to build directly during TagMap's . + public static final TagMap EMPTY = new TagMap(new Object[1 << 4], 0); + + // Sentinel for a not-yet-resolved lazy tag id. Cannot be 0L: 0L is a valid keyOf result (the tag + // is not a known tag, or the codec is inactive). Shared by Entry and EntryReadingHelper. + static final long TAG_ID_NOT_COMPUTED = Long.MIN_VALUE; /** Creates a new mutable TagMap that contains the contents of map */ public static final TagMap fromMap(@Nonnull Map map) { @@ -76,6 +79,14 @@ public static final TagMap create(int size) { return new TagMap(); } + /** Creates a mutable TagMap whose dense store is sized per-operation from {@code hint}. */ + public static final TagMap create(SizingHint hint) { + TagMap tagMap = new TagMap(); + int n = hint.size; + tagMap.denseCapHint = n > 0 ? n : KNOWN_INIT_CAP; + return tagMap; + } + /** * Creates a fresh, mutable TagMap that reads through to {@code parent} on local misses. The * parent must be frozen and is fixed for the life of the returned map (no re-parenting), so @@ -91,6 +102,19 @@ public static final TagMap create(int size) { * local miss/removal for no benefit. */ public static final TagMap createFromParent(TagMap parent) { + return createFromParent(parent, null); + } + + /** + * Read-through variant of {@link #create(SizingHint)}: reads through to {@code parent} on local + * misses AND sizes the LOCAL dense store from {@code hint}. The two are orthogonal — the parent + * supplies inherited reads, while locally-set known tags live in this map's own dense array, + * whose initial capacity the hint tunes. So the common child / shared-parent path (a non-null + * read-through parent) still gets per-operation sizing; without this the local store would fall + * back to the fixed default. A {@code null} hint behaves exactly like {@link + * #createFromParent(TagMap)}. + */ + public static final TagMap createFromParent(TagMap parent, SizingHint hint) { if (parent != null) { if (!parent.frozen) { throw new IllegalStateException("read-through parent must be frozen"); @@ -99,7 +123,11 @@ public static final TagMap createFromParent(TagMap parent) { parent = null; } } - return new TagMap(parent); + TagMap tagMap = new TagMap(parent); + if (hint != null && hint.size > 0) { + tagMap.denseCapHint = hint.size; + } + return tagMap; } /** Creates a new TagMap.Ledger */ @@ -166,6 +194,12 @@ public interface EntryReader { String tag(); + /** + * The known-tag id for this entry's tag, or {@code 0L} when the tag is not a known tag (or the + * {@link KnownTagCodec} is inactive). Resolved via {@link KnownTagCodec#keyOf(String)}. + */ + long tagId(); + byte type(); boolean is(byte type); @@ -314,6 +348,13 @@ static Entry newDoubleEntry(String tag, Double box) { */ int lazyTagHash; + /* + * Known-tag id, lazily resolved using the same trick as lazyTagHash. TAG_ID_NOT_COMPUTED marks + * "not yet resolved" (0L is a valid result -- unknown tag / inactive codec -- so it cannot be + * the sentinel). Only pays off on the dense-OFF path; dense-ON known tags never become Entry-s. + */ + long lazyTagId = TAG_ID_NOT_COMPUTED; + // To optimize construction of Entry around boxed primitives and Object entries, // no type checks are done during construction. // Any Object entries are initially marked as type ANY, prim set to 0, and the Object put into @@ -354,6 +395,17 @@ int hash() { return hash; } + @Override + public long tagId() { + // Same lazy idiom as hash(): a benign race just recomputes keyOf, which is deterministic. + long id = this.lazyTagId; + if (id != TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.lazyTagId = id; + return id; + } + @Override public Entry entry() { return this; @@ -1018,10 +1070,80 @@ public EntryChange next() { * removed from the collision chain. */ - private final Object[] buckets; + // Shared immutable empty buckets (all null, length 16). Every map points here until its first + // custom-tag write copies-on-write to a private array (materializeBuckets), so an all-known / + // known-heavy map (e.g. the trace-tier read-through parent) allocates ZERO buckets. Length is + // always 16, so reads need no null guard and read-through bucket alignment (hash & 15) holds. + private static final Object[] EMPTY_BUCKETS = new Object[1 << 4]; + + private Object[] buckets; private int size; private boolean frozen; + /** + * Dense known-tag store (dense-tagmap-design §5). Values for KNOWN tags (those {@link + * KnownTagCodec#keyOf} resolves to a stored id) live in these INSERTION-ORDERED parallel arrays + * with NO per-tag {@link Entry} object — the allocation win. Lazily allocated on the first + * known-tag write ({@code null} until then, so all-unknown maps pay nothing) and grown x2 from + * {@link #KNOWN_INIT_CAP}. Matched by globalSerial via a linear scan ({@link #knownIndexOf}); + * reads aren't hot, so O(knownCount) is fine and positional indexing is deferred. Dormant until a + * resolver is registered: {@code keyOf} returns 0, so nothing routes here and production is + * byte-identical. + * + *

Disjoint from {@link #buckets} by construction: known-ness is global ({@code keyOf} is + * deterministic), so a known tag is ALWAYS dense and never bucketed, and vice-versa. That + * disjointness keeps read-through shadow checks within-region — an ancestor dense entry can only + * be shadowed by a nearer level's dense entry of the same id, an ancestor bucket entry only by a + * nearer level's bucket entry — so the bucket read-through chain walk is unchanged and the dense + * one mirrors it ({@link #parentDenseVisible}). + * + *

{@link #size} counts bucket entries only; {@link #knownCount} counts dense entries; the + * local total is {@code size + knownCount}. + */ + private long[] knownIds; + + private Object[] knownValues; + private int knownCount; + + /** + * Single-tier presence filter over the dense store — the fast path that lets a definitely-absent + * known tag append in O(1) instead of paying the {@link #knownIndexOf} scan (the common per-build + * insert). Each tag's id carries a globally stable {@code slot} from graph coloring (see {@link + * KnownTagCodec}); a tag is present ONLY IF its slot bit is set in {@link #knownOccupancy}. A + * clear bit ⟹ definitely absent ⟹ skip the scan. + * + *

Because co-occurring tags always get distinct slots (they form a clique in the coloring), + * slotCount is bounded by the largest clique (≤ 64) and every present tag of a well-formed map + * has its own bit — so one {@code long} is the whole filter, collapsing the earlier two-tier + * (group mask + field bloom) design to a single word. Disjoint occupancy across two maps ({@code + * (a.knownOccupancy & b.knownOccupancy) == 0}) proves nothing shadows across them, which the + * read-through shadow check exploits (see {@link #parentDenseVisible}). + * + *

Superset semantics: bits are set on every add and NEVER cleared on remove (a stale bit only + * costs a scan, never a wrong answer), so correctness never depends on the slot→bit collision + * rate — only the fast-path hit rate does. Unslotted stored tags ({@link KnownTagCodec#NO_SLOT}) + * all fold onto one shared bit ({@code slot & 63}); the scan stays authoritative for them. + */ + private long knownOccupancy; + + /** + * Whether this map holds any trace-level known tag ({@link KnownTagCodec#isTraceLevel}). Trace + * and span tags reuse the same slots, so {@link #knownOccupancy} can't tell the two levels apart; + * this flag can. A span map leaves it {@code false}, which lets {@link #parentDenseVisible} skip + * the shadow check when enumerating a trace-level ancestor entry — a map with no trace-level tags + * cannot shadow one. Superset semantics like the occupancy mask: set on add, never cleared on + * remove (a stale {@code true} only costs a scan, never a wrong answer). + */ + private boolean knownTraceLevel; + + private static final int KNOWN_INIT_CAP = + 12; // generous default when no SizingHint; per-operation sizing comes via create(SizingHint) + + // Initial dense-array capacity, set once from a SizingHint at create(SizingHint) (per-operation + // sizing). Just the size (an int), NOT a hint reference -- the hint stays external (recordSize is + // explicit). + private int denseCapHint = KNOWN_INIT_CAP; + /** * Optional frozen parent for read-through. When non-null, reads that miss the local buckets fall * through to the parent chain, nearest-level-wins (a local entry shadows the parent's, a nearer @@ -1053,8 +1175,9 @@ public TagMap() { * optimizations can treat it as fixed. */ private TagMap(TagMap parent) { - // needs to be a power of 2 for bucket masking calculation to work as intended - this.buckets = new Object[1 << 4]; + // Start on the shared empty buckets; materializeBuckets() COWs to a private power-of-two array + // on the first custom-tag write. All-known maps never allocate buckets. + this.buckets = EMPTY_BUCKETS; this.size = 0; this.frozen = false; this.parent = parent; @@ -1075,8 +1198,9 @@ public boolean isOptimized() { @Override public int size() { // Exact (Map contract). Under read-through resolves the union; prefer estimateSize() for hints. + int local = this.size + this.knownCount; // buckets + dense TagMap parent = this.parent; - return parent == null ? this.size : this.size + this.visibleParentCount(); + return parent == null ? local : local + this.visibleParentCount(); } /** @@ -1086,6 +1210,12 @@ public int size() { private int visibleParentCount() { int count = 0; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + // dense entries at this ancestor not shadowed/tombstoned by a nearer level + long[] ancestorIds = ancestor.knownIds; + int ancestorKnownCount = ancestor.knownCount; + for (int i = 0; i < ancestorKnownCount; ++i) { + if (this.parentDenseVisible(ancestorIds[i], ancestor)) count++; + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1109,7 +1239,7 @@ private int visibleParentCount() { @Override public boolean isEmpty() { // Exact (Map contract). Under read-through resolves the parent; prefer isDefinitelyEmpty(). - if (this.size != 0) { + if (this.size != 0 || this.knownCount != 0) { return false; } TagMap parent = this.parent; @@ -1128,7 +1258,7 @@ public boolean isEmpty() { public boolean isDefinitelyEmpty() { // Cheap: empty iff no level in the chain holds a local entry (ignores shadowing/tombstones). for (TagMap level = this; level != null; level = level.parent) { - if (level.size != 0) { + if (level.size != 0 || level.knownCount != 0) { return false; } } @@ -1136,10 +1266,10 @@ public boolean isDefinitelyEmpty() { } public int estimateSize() { - // Upper bound: sum of every level's local size, ignoring read-through shadowing/removals. + // Upper bound: sum of every level's local size (buckets + dense), ignoring shadowing/removals. int total = 0; for (TagMap level = this; level != null; level = level.parent) { - total += level.size; + total += level.size + level.knownCount; } return total; } @@ -1267,8 +1397,15 @@ public Entry getEntry(String tag) { return parent.getEntry(tag); } - /** Looks up an entry in this map's own buckets only — no read-through to the parent. */ + /** Looks up an entry in this map's own storage only (dense then buckets) — no read-through. */ private Entry getLocalEntry(String tag) { + // Known tags live in the dense store; resolve identity and check there first. keyOf is a no-op + // (returns 0 -> isStored false) until a resolver is registered, so this is inert in production. + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + Object known = this.knownRawValue(id); + return known == null ? null : Entry.newAnyEntry(tag, known); + } Object[] thisBuckets = this.buckets; int hash = TagMap.Entry._hash(tag); return findInBucket(thisBuckets[hash & (thisBuckets.length - 1)], hash, tag); @@ -1316,10 +1453,162 @@ private boolean parentEntryVisible(Entry parentEntry, TagMap fromAncestor) { return true; } + // ---- dense known-tag store (see the knownIds field doc) + // ---------------------------------------- + + /** + * Linear scan of the dense store for {@code tagId}, returning its index or -1. Ids are canonical + * (the only way one enters is {@link KnownTagCodec#keyOf} or a {@code KnownTags} constant, both + * canonical), so a full {@code long} compare is exact and cheaper than extracting globalSerial. + */ + private int knownIndexOf(long tagId) { + long[] ids = this.knownIds; + int n = this.knownCount; + for (int i = 0; i < n; ++i) { + if (ids[i] == tagId) return i; + } + return -1; + } + + private void ensureKnownCapacity() { + if (this.knownIds == null) { + this.knownIds = new long[this.denseCapHint]; + this.knownValues = new Object[this.denseCapHint]; + } else if (this.knownCount == this.knownIds.length) { + int newCap = this.knownIds.length << 1; + this.knownIds = Arrays.copyOf(this.knownIds, newCap); + this.knownValues = Arrays.copyOf(this.knownValues, newCap); + } + } + + /** + * Feeds this map's final dense-entry count back to {@code hint} (call at a terminal point: freeze + * / serialization). The hint self-tunes so future maps of the same operation size correctly. + * + *

Best-effort max, not a strict monotonic-max: the read-compare-write is unsynchronized, so + * two spans of the same operation finishing concurrently can both read the same old {@code size} + * and the later write can lower it (a 10-tag span then a 5-tag span settles at 5). And {@code + * size} is a plain field, so a fresh update isn't promptly visible to span-creation threads. Both + * are deliberately tolerated: a stale or lowered hint only mis-sizes a later span's dense array + * (one extra grow-copy), never corrupts tag data — so it stays lock- and atomic-free, and the + * next heavy span of the operation nudges it back up. A CAS/atomic-max would buy exactness at the + * cost of contention on this terminal path, which isn't worth it for a sizing hint. + */ + public void recordSize(SizingHint hint) { + // Capped hints (the heterogeneous shared default) are fixed -- don't let unlike sharers grow + // them. + if (!hint.capped && this.knownCount > hint.size) { + hint.size = this.knownCount; // best-effort racy max (see javadoc); benign plain-int write + } + } + + /** + * Presence bit for {@code tagId}: {@code 1L << slot}. Colored slots are < 64; unslotted stored + * tags ({@link KnownTagCodec#NO_SLOT}) fold onto one shared bit via {@code slot & 63} — crude for + * them, but the scan stays authoritative. + */ + private static long knownSlotBit(long tagId) { + return 1L << (KnownTagCodec.slot(tagId) & 63); + } + + /** + * Whether {@code tagId} MAY be present in the dense store (its slot bit is set), vs DEFINITELY + * absent (the bit is clear ⟹ skip the scan). + */ + private boolean knownMaybePresent(long tagId) { + return (this.knownOccupancy & knownSlotBit(tagId)) != 0; + } + + /** + * Stores a known tag's value densely (no {@link Entry} alloc). Overwrites in place when present + * (returning the prior value materialized as an Entry, per the {@code Map} contract — usually + * discarded by {@code set}); otherwise appends, growing x2 as needed. The occupancy presence + * filter skips the {@link #knownIndexOf} scan when the tag is definitely absent (the common + * per-build case), so an append is O(1) instead of O(n). + */ + private Entry putKnownValue(long tagId, Object value) { + long slotBit = knownSlotBit(tagId); + // maybe present only if the slot bit is set; a clear bit ⟹ definitely absent ⟹ append + if ((this.knownOccupancy & slotBit) != 0) { + int i = this.knownIndexOf(tagId); + if (i >= 0) { + Object prior = this.knownValues[i]; + this.knownValues[i] = value; + return materializeKnown(tagId, prior); + } + // filter false positive (slot collision) -> fall through to append + } + this.ensureKnownCapacity(); + int idx = this.knownCount++; + this.knownIds[idx] = tagId; + this.knownValues[idx] = value; + this.knownOccupancy |= slotBit; + this.knownTraceLevel |= KnownTagCodec.isTraceLevel(tagId); + return null; + } + + /** Raw dense value for {@code tagId}, or {@code null} when absent (no Entry, no boxing). */ + private Object knownRawValue(long tagId) { + if (!this.knownMaybePresent(tagId)) return null; // definitely absent, no scan + int i = this.knownIndexOf(tagId); + return i < 0 ? null : this.knownValues[i]; + } + + /** + * Removes a known tag from the dense store (swap-with-last), returning the prior Entry or null. + */ + private Entry removeKnown(long tagId) { + if (!this.knownMaybePresent(tagId)) return null; // definitely absent + int i = this.knownIndexOf(tagId); + if (i < 0) return null; + Object prior = this.knownValues[i]; + int last = --this.knownCount; + this.knownIds[i] = this.knownIds[last]; + this.knownValues[i] = this.knownValues[last]; + this.knownIds[last] = 0L; + this.knownValues[last] = null; + // knownOccupancy intentionally NOT cleared: a stale-set bit only costs a scan; clearing could + // drop a bit still shared (via collision) by a present id -> false negative. + return materializeKnown(tagId, prior); + } + + /** Materializes a transient Entry for a dense (id, value) pair — only on explicit get/iterate. */ + private static Entry materializeKnown(long tagId, Object value) { + return Entry.newAnyEntry(KnownTagCodec.nameOf(tagId), value); + } + + /** + * Whether an ancestor dense entry ({@code tagId}, declared at level {@code fromAncestor}) is + * visible from this leaf under read-through: not shadowed by a nearer level's dense entry of the + * same id and not tombstoned by a nearer level. Chain-aware mirror of {@link #parentEntryVisible} + * for the dense store. (Disjointness: a known tag never buckets, so no bucket shadow check is + * needed.) + */ + private boolean parentDenseVisible(long tagId, TagMap fromAncestor) { + // Trace and span tags reuse the same slots, so a trace-level ancestor entry can only be + // shadowed by a nearer level that ALSO holds trace-level tags. A span map (knownTraceLevel + // false) never shadows a trace tag — skip its occupancy+scan entirely (the level-bit win). A + // span-level ancestor entry keeps the plain occupancy-filtered scan below. + boolean traceLevelTag = KnownTagCodec.isTraceLevel(tagId); + String tag = null; // resolved lazily, only if a nearer level carries tombstones + for (TagMap nearer = this; nearer != fromAncestor; nearer = nearer.parent) { + // shadowed by a nearer dense entry — the occupancy filter prunes the scan when definitely + // absent, so a nearer level with disjoint slots never pays a scan here (read-through win) + if ((!traceLevelTag || nearer.knownTraceLevel) + && nearer.knownMaybePresent(tagId) + && nearer.knownIndexOf(tagId) >= 0) return false; + if (nearer.removedFromParent != null) { + if (tag == null) tag = KnownTagCodec.nameOf(tagId); + if (nearer.removedFromParent.contains(tag)) return false; // tombstoned by a nearer level + } + } + return true; + } + @Deprecated @Override public Object put(@Nonnull String tag, Object value) { - TagMap.Entry entry = this.getAndSet(Entry.newAnyEntry(tag, value)); + TagMap.Entry entry = this.getAndSet(tag, value); return entry == null ? null : entry.objectValue(); } @@ -1334,32 +1623,70 @@ public void set(@Nullable TagMap.EntryReader newEntryReader) { } } + // The set(String, ...) family resolves keyOf FIRST: a known tag stores its value densely with no + // Entry (boxing the primitive only on that branch) and no parent-fallback lookup (set discards + // the prior value); a custom tag takes the typed bucket insert (no boxing for primitives). public void set(@Nonnull String tag, @Nonnull Object value) { - this.putEntry(Entry.newAnyEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, value); + } else { + this.putBucketEntry(Entry.newAnyEntry(tag, value)); + } } public void set(@Nonnull String tag, @Nonnull CharSequence value) { - this.putEntry(Entry.newObjectEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, value); + } else { + this.putBucketEntry(Entry.newObjectEntry(tag, value)); + } } public void set(@Nonnull String tag, boolean value) { - this.putEntry(Entry.newBooleanEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Boolean.valueOf(value)); + } else { + this.putBucketEntry(Entry.newBooleanEntry(tag, value)); + } } public void set(@Nonnull String tag, int value) { - this.putEntry(Entry.newIntEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Integer.valueOf(value)); + } else { + this.putBucketEntry(Entry.newIntEntry(tag, value)); + } } public void set(@Nonnull String tag, long value) { - this.putEntry(Entry.newLongEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Long.valueOf(value)); + } else { + this.putBucketEntry(Entry.newLongEntry(tag, value)); + } } public void set(@Nonnull String tag, float value) { - this.putEntry(Entry.newFloatEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Float.valueOf(value)); + } else { + this.putBucketEntry(Entry.newFloatEntry(tag, value)); + } } public void set(@Nonnull String tag, double value) { - this.putEntry(Entry.newDoubleEntry(tag, value)); + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + this.putKnownLocal(id, tag, Double.valueOf(value)); + } else { + this.putBucketEntry(Entry.newDoubleEntry(tag, value)); + } } /** @@ -1372,6 +1699,17 @@ public Entry getAndSet(@Nullable Entry newEntry) { if (newEntry == null) { return null; } + return this.getAndSetWithFallback(newEntry); + } + + /** + * Local insert (via {@link #putEntry}) plus the read-through parent fallback for the prior + * visible value (Map contract). When no local entry was replaced and the key was not tombstoned, + * the prior visible value is the nearest ancestor's, resolved through {@link #getEntry} (which + * handles both dense known tags and bucketed custom tags). Shared by {@link #getAndSet(Entry)} + * and the {@code getAndSet(String, ...)} overloads. + */ + private Entry getAndSetWithFallback(@Nonnull Entry newEntry) { // Capture whether the key was tombstoned BEFORE putEntry clears it: a tombstoned key had no // visible prior value (it was removed), so getAndSet must report null rather than the parent's. boolean wasTombstoned = @@ -1392,10 +1730,44 @@ public Entry getAndSet(@Nullable Entry newEntry) { /** * Inserts or replaces a local entry, returning the replaced local Entry (or null if none). Does * NOT consult the read-through parent -- the {@code set(...)} methods use this so they never pay - * for a prior-value lookup they discard; {@link #getAndSet(Entry)} layers the parent fallback on - * top. + * for a prior-value lookup they discard; {@link #getAndSetWithFallback} layers the parent + * fallback on top. Routes a known tag to the dense store, a custom tag to the hash buckets. */ private Entry putEntry(@Nonnull Entry newEntry) { + long id = KnownTagCodec.keyOf(newEntry.tag); + if (KnownTagCodec.isStored(id)) { + return this.putKnownLocal(id, newEntry.tag, newEntry.objectValue()); + } + return this.putBucketEntry(newEntry); + } + + /** + * Stores a known tag's value densely with NO Entry retained (the alloc win) and NO parent + * fallback — the local-only counterpart used by {@code set} and {@link #putEntry}. Returns the + * prior LOCAL dense value materialized as an Entry (Map contract); usually discarded. + */ + private Entry putKnownLocal(long id, String tag, Object value) { + this.checkWriteAccess(); + if (this.removedFromParent != null) { + this.removedFromParent.remove(tag); + } + return this.putKnownValue(id, value); + } + + /** Copy-on-write the shared empty buckets to a private array on the first bucket write. */ + private Object[] materializeBuckets() { + Object[] b = this.buckets; + if (b == EMPTY_BUCKETS) { + b = new Object[1 << 4]; + this.buckets = b; + } + return b; + } + + /** + * Stores an entry in the hash buckets — the unknown/custom-tag local path (no parent fallback). + */ + private Entry putBucketEntry(@Nonnull Entry newEntry) { this.checkWriteAccess(); // Re-setting a key clears any read-through tombstone for it (the new value overrides the @@ -1404,7 +1776,7 @@ private Entry putEntry(@Nonnull Entry newEntry) { this.removedFromParent.remove(newEntry.tag); } - Object[] thisBuckets = this.buckets; + Object[] thisBuckets = this.materializeBuckets(); int newHash = newEntry.hash(); int bucketIndex = newHash & (thisBuckets.length - 1); @@ -1449,32 +1821,36 @@ private Entry putEntry(@Nonnull Entry newEntry) { return null; } + // Each getAndSet(String, ...) builds the typed Entry (no boxing for primitives) then funnels + // through getAndSetWithFallback, which routes a known tag to the dense store (dropping the Entry) + // and a custom tag to the buckets, layering the read-through parent fallback on top. The Entry- + // free hot path is set(String, ...), which discards the prior value; getAndSet returns it. public Entry getAndSet(@Nonnull String tag, Object value) { - return this.getAndSet(Entry.newAnyEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newAnyEntry(tag, value)); } public Entry getAndSet(@Nonnull String tag, CharSequence value) { - return this.getAndSet(Entry.newObjectEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newObjectEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, boolean value) { - return this.getAndSet(Entry.newBooleanEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newBooleanEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, int value) { - return this.getAndSet(Entry.newIntEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newIntEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, long value) { - return this.getAndSet(Entry.newLongEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newLongEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, float value) { - return this.getAndSet(Entry.newFloatEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newFloatEntry(tag, value)); } public TagMap.Entry getAndSet(@Nonnull String tag, double value) { - return this.getAndSet(Entry.newDoubleEntry(tag, value)); + return this.getAndSetWithFallback(Entry.newDoubleEntry(tag, value)); } public void putAll(Map map) { @@ -1515,7 +1891,9 @@ private void putAllOptimizedMap(TagMap that) { that.forEach(this, (self, entry) -> self.set(entry)); return; } - if (this.size == 0) { + // "empty" must consider BOTH local regions — a map with only dense entries has size == 0 but is + // not empty, and putAllIntoEmptyMap would clobber its dense store. + if (this.size == 0 && this.knownCount == 0) { this.putAllIntoEmptyMap(that); } else { this.putAllMerge(that); @@ -1523,7 +1901,9 @@ private void putAllOptimizedMap(TagMap that) { } private void putAllMerge(TagMap that) { - Object[] thisBuckets = this.buckets; + // COW our buckets only if the source has bucket entries to merge in; otherwise the loop below + // writes nothing and the shared empty buckets stay shared. + Object[] thisBuckets = (that.size > 0) ? this.materializeBuckets() : this.buckets; Object[] thatBuckets = that.buckets; // Since TagMap-s don't support expansion, buckets are perfectly aligned @@ -1634,33 +2014,51 @@ private void putAllMerge(TagMap that) { } } } + + // merge the source's dense known-tag entries; incoming clobbers existing (same as buckets) + for (int i = 0; i < that.knownCount; ++i) { + this.putKnownValue(that.knownIds[i], that.knownValues[i]); + } } /* * Specially optimized version of putAll for the common case of destination map being empty */ private void putAllIntoEmptyMap(TagMap that) { - Object[] thisBuckets = this.buckets; - Object[] thatBuckets = that.buckets; - - // Check against both thisBuckets.length && thatBuckets.length is to help the JIT do bound check - // elimination - for (int i = 0; i < thisBuckets.length && i < thatBuckets.length; ++i) { - Object thatBucket = thatBuckets[i]; - - // faster to explicitly null check first, then do instanceof - if (thatBucket == null) { - // do nothing - } else if (thatBucket instanceof BucketGroup) { - // if it is a BucketGroup, then need to clone - BucketGroup thatGroup = (BucketGroup) thatBucket; + // Only copy buckets (and COW ours) when the source actually has bucket entries; an all-known + // source leaves us on the shared empty buckets. + if (that.size > 0) { + Object[] thisBuckets = this.materializeBuckets(); + Object[] thatBuckets = that.buckets; + + // Check against both thisBuckets.length && thatBuckets.length is to help the JIT do bound + // check elimination + for (int i = 0; i < thisBuckets.length && i < thatBuckets.length; ++i) { + Object thatBucket = thatBuckets[i]; + + // faster to explicitly null check first, then do instanceof + if (thatBucket == null) { + // do nothing + } else if (thatBucket instanceof BucketGroup) { + // if it is a BucketGroup, then need to clone + BucketGroup thatGroup = (BucketGroup) thatBucket; - thisBuckets[i] = thatGroup.cloneChain(); - } else { // if ( thatBucket instanceof Entry ) - thisBuckets[i] = thatBucket; + thisBuckets[i] = thatGroup.cloneChain(); + } else { // if ( thatBucket instanceof Entry ) + thisBuckets[i] = thatBucket; + } } + this.size = that.size; + } + + // clone the dense known-tag store (values are immutable boxes/objects -> safe to share refs) + if (that.knownCount > 0) { + this.knownIds = Arrays.copyOf(that.knownIds, that.knownIds.length); + this.knownValues = Arrays.copyOf(that.knownValues, that.knownValues.length); + this.knownCount = that.knownCount; + this.knownOccupancy = that.knownOccupancy; + this.knownTraceLevel = that.knownTraceLevel; } - this.size = that.size; } public void fillMap(Map map) { @@ -1679,6 +2077,9 @@ public void fillMap(Map map) { thisGroup.fillMapFromChain(map); } } + for (int i = 0; i < this.knownCount; ++i) { + map.put(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + } } public void fillStringMap(Map stringMap) { @@ -1697,6 +2098,11 @@ public void fillStringMap(Map stringMap) { thisGroup.fillStringMapFromChain(stringMap); } } + for (int i = 0; i < this.knownCount; ++i) { + stringMap.put( + KnownTagCodec.nameOf(this.knownIds[i]), + TagValueConversions.toString(this.knownValues[i])); + } } @Override @@ -1738,8 +2144,13 @@ public Entry getAndRemove(String tag) { return localRemoved; } - /** Removes an entry from this map's own buckets only — no parent/tombstone handling. */ + /** Removes an entry from this map's own storage only — no parent/tombstone handling. */ private Entry removeLocal(String tag) { + long id = KnownTagCodec.keyOf(tag); + if (KnownTagCodec.isStored(id)) { + return this.removeKnown(id); + } + Object[] thisBuckets = this.buckets; int hash = TagMap.Entry._hash(tag); @@ -1807,6 +2218,15 @@ public Stream stream() { @Override public void forEach(Consumer consumer) { + // local dense known tags via a reused flyweight (no per-entry Entry alloc — the serialize win) + if (this.knownCount > 0) { + EntryReadingHelper reader = new EntryReadingHelper(); + for (int i = 0; i < this.knownCount; ++i) { + reader.set(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + consumer.accept(reader); + } + } + Object[] thisBuckets = this.buckets; for (int i = 0; i < thisBuckets.length; ++i) { @@ -1832,8 +2252,23 @@ public void forEach(Consumer consumer) { private void forEachParent(Consumer consumer) { // Walk the ancestor chain, nearest first. Each entry is emitted once, by the nearest level that - // defines its key, when not shadowed by a nearer level and not tombstoned. + // defines its key, when not shadowed by a nearer level and not tombstoned. Dense known tags are + // emitted via a reused flyweight (no per-entry Entry alloc — the serialize win). + EntryReadingHelper reader = null; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + long[] ancestorIds = ancestor.knownIds; + int ancestorKnownCount = ancestor.knownCount; + if (ancestorKnownCount > 0) { + Object[] ancestorValues = ancestor.knownValues; + if (reader == null) reader = new EntryReadingHelper(); + for (int i = 0; i < ancestorKnownCount; ++i) { + long id = ancestorIds[i]; + if (this.parentDenseVisible(id, ancestor)) { + reader.set(KnownTagCodec.nameOf(id), ancestorValues[i]); + consumer.accept(reader); + } + } + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1856,6 +2291,14 @@ private void forEachParent(Consumer consumer) { } public void forEach(T thisObj, BiConsumer consumer) { + if (this.knownCount > 0) { + EntryReadingHelper reader = new EntryReadingHelper(); + for (int i = 0; i < this.knownCount; ++i) { + reader.set(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + consumer.accept(thisObj, reader); + } + } + Object[] thisBuckets = this.buckets; for (int i = 0; i < thisBuckets.length; ++i) { @@ -1879,7 +2322,21 @@ public void forEach(T thisObj, BiConsumer con } private void forEachParent(T thisObj, BiConsumer consumer) { + EntryReadingHelper reader = null; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + int ancestorKnownCount = ancestor.knownCount; + if (ancestorKnownCount > 0) { + long[] ancestorIds = ancestor.knownIds; + Object[] ancestorValues = ancestor.knownValues; + if (reader == null) reader = new EntryReadingHelper(); + for (int i = 0; i < ancestorKnownCount; ++i) { + long id = ancestorIds[i]; + if (this.parentDenseVisible(id, ancestor)) { + reader.set(KnownTagCodec.nameOf(id), ancestorValues[i]); + consumer.accept(thisObj, reader); + } + } + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1904,6 +2361,14 @@ private void forEachParent(T thisObj, BiConsumer void forEach( T thisObj, U otherObj, TriConsumer consumer) { + if (this.knownCount > 0) { + EntryReadingHelper reader = new EntryReadingHelper(); + for (int i = 0; i < this.knownCount; ++i) { + reader.set(KnownTagCodec.nameOf(this.knownIds[i]), this.knownValues[i]); + consumer.accept(thisObj, otherObj, reader); + } + } + Object[] thisBuckets = this.buckets; for (int i = 0; i < thisBuckets.length; ++i) { @@ -1928,7 +2393,21 @@ public void forEach( private void forEachParent( T thisObj, U otherObj, TriConsumer consumer) { + EntryReadingHelper reader = null; for (TagMap ancestor = this.parent; ancestor != null; ancestor = ancestor.parent) { + int ancestorKnownCount = ancestor.knownCount; + if (ancestorKnownCount > 0) { + long[] ancestorIds = ancestor.knownIds; + Object[] ancestorValues = ancestor.knownValues; + if (reader == null) reader = new EntryReadingHelper(); + for (int i = 0; i < ancestorKnownCount; ++i) { + long id = ancestorIds[i]; + if (this.parentDenseVisible(id, ancestor)) { + reader.set(KnownTagCodec.nameOf(id), ancestorValues[i]); + consumer.accept(thisObj, otherObj, reader); + } + } + } Object[] parentBuckets = ancestor.buckets; for (int i = 0; i < parentBuckets.length; ++i) { Object parentBucket = parentBuckets[i]; @@ -1955,13 +2434,19 @@ private void forEachParent( public void clear() { this.checkWriteAccess(); - Arrays.fill(this.buckets, null); + // Drop the private bucket array back to the shared empty sentinel (also avoids mutating it). + this.buckets = EMPTY_BUCKETS; this.size = 0; // clear() removes ALL mappings, including any inherited through read-through. Detaching the // parent (rather than tombstoning every inherited key) is simpler and cheaper, and leaves an // empty, parent-less map. Detach is one-way -- the parent is never re-pointed. this.parent = null; this.removedFromParent = null; + this.knownIds = null; + this.knownValues = null; + this.knownCount = 0; + this.knownOccupancy = 0L; + this.knownTraceLevel = false; } public TagMap freeze() { @@ -2016,6 +2501,20 @@ void checkIntegrity() { } } + // dense store: ids must be unique (no tag stored twice) and the count within array bounds. + if (this.knownCount > 0) { + if (this.knownIds == null || this.knownCount > this.knownIds.length) { + throw new IllegalStateException("incorrect known count"); + } + for (int i = 0; i < this.knownCount; ++i) { + for (int j = i + 1; j < this.knownCount; ++j) { + if (this.knownIds[i] == this.knownIds[j]) { + throw new IllegalStateException("duplicate known id"); + } + } + } + } + if (this.size != this.computeSize()) { throw new IllegalStateException("incorrect size"); } @@ -2146,13 +2645,23 @@ abstract static class IteratorBase { private TagMap level; private Object[] buckets; - private Entry nextEntry; + // Currency is EntryReader, not Entry: a BUCKET entry is its own (real, retain-safe) Entry, but + // a + // DENSE entry is emitted via the reused denseReader flyweight (alloc-free, "use now"). This is + // the contract of TagMap.iterator()/keySet()/values(). entrySet() (Iterator) sits on + // top and calls .entry() per next() to get a real retain-safe Entry (see EntriesIterator). + private EntryReader nextEntry; + private EntryReadingHelper denseReader; // lazily created on the first dense emit private int bucketIndex = -1; private BucketGroup group = null; private int groupIndex = 0; + // dense-store cursor for the current level's known tags; advance() resets it when it moves to + // the next ancestor level (read-through union). + private int knownIndex = 0; + IteratorBase(TagMap map) { this.map = map; this.level = map; @@ -2166,9 +2675,9 @@ public final boolean hasNext() { return this.nextEntry != null; } - final Entry nextEntryOrThrowNoSuchElement() { + final EntryReader nextEntryOrThrowNoSuchElement() { if (this.nextEntry != null) { - Entry nextEntry = this.nextEntry; + EntryReader nextEntry = this.nextEntry; this.nextEntry = null; return nextEntry; } @@ -2180,9 +2689,9 @@ final Entry nextEntryOrThrowNoSuchElement() { } } - final Entry nextEntryOrNull() { + final EntryReader nextEntryOrNull() { if (this.nextEntry != null) { - Entry nextEntry = this.nextEntry; + EntryReader nextEntry = this.nextEntry; this.nextEntry = null; return nextEntry; } @@ -2190,8 +2699,22 @@ final Entry nextEntryOrNull() { return this.hasNext() ? this.nextEntry : null; } - private final Entry advance() { + private final EntryReader advance() { while (true) { + // phase 1: drain the current level's dense known tags before its buckets. Leaf dense always + // emits; ancestor dense only if visible from the leaf (not shadowed by a nearer dense entry + // and not tombstoned). Emitted via the reused denseReader flyweight -- NO per-entry Entry + // alloc (the read/serialize alloc win). + if (this.knownIndex < this.level.knownCount) { + int i = this.knownIndex++; + long id = this.level.knownIds[i]; + if (this.level == this.map || this.map.parentDenseVisible(id, this.level)) { + return this.emitDense(id, this.level.knownValues[i]); + } + continue; // ancestor dense entry shadowed/tombstoned -> skip + } + + // phase 2: the current level's buckets. Entry tagEntry = this.rawAdvance(); if (tagEntry != null) { // leaf entries emit as-is; ancestor entries only if visible from the leaf -- not shadowed @@ -2202,9 +2725,12 @@ private final Entry advance() { continue; // ancestor entry shadowed/tombstoned -> skip } - // current level exhausted; advance to the next ancestor's buckets (read-through union) + // current level exhausted; advance to the next ancestor (read-through union), resetting + // both + // the per-level dense cursor and the bucket cursor for the new level. if (this.level.parent != null) { this.level = this.level.parent; + this.knownIndex = 0; this.buckets = this.level.buckets; this.bucketIndex = -1; this.group = null; @@ -2215,6 +2741,16 @@ private final Entry advance() { } } + /** Sets and returns the reused dense flyweight (lazily created); "use now", do not retain. */ + private EntryReader emitDense(long tagId, Object value) { + EntryReadingHelper reader = this.denseReader; + if (reader == null) { + reader = this.denseReader = new EntryReadingHelper(); + } + reader.set(KnownTagCodec.nameOf(tagId), value, tagId); + return reader; + } + /** Next raw entry in the current bucket array, ignoring shadowing/tombstones. */ private final Entry rawAdvance() { while (this.bucketIndex < this.buckets.length) { @@ -2744,9 +3280,26 @@ public boolean isEmpty() { @Override public Iterator> iterator() { - @SuppressWarnings({"rawtypes", "unchecked"}) - Iterator> iter = (Iterator) this.map.iterator(); - return iter; + return new EntriesIterator(this.map); + } + } + + /** + * entrySet() yields real, retain-safe {@code Map.Entry} objects. It sits on top of the + * EntryReader iterator and materializes each via {@code .entry()}: a bucket entry's reader IS the + * real stored Entry (returns {@code this}, free); a dense entry's flyweight materializes a fresh + * Entry. Deliberately NOT alloc-optimized for dense — bulk reads use {@code forEach}/EntryReader, + * and manual instrumentation does point get/set, not bulk entrySet iteration. + */ + static final class EntriesIterator extends IteratorBase + implements Iterator> { + EntriesIterator(TagMap map) { + super(map); + } + + @Override + public Map.Entry next() { + return this.nextEntryOrThrowNoSuchElement().entry(); } } @@ -2833,17 +3386,28 @@ final class EntryReadingHelper implements TagMap.EntryReader { private Map.Entry mapEntry; private String tag; private Object value; + private long tagId; void set(String tag, Object value) { this.mapEntry = null; this.tag = tag; this.value = value; + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access + } + + /** Dense emit: the id is known directly, so record it and skip the keyOf resolve. */ + void set(String tag, Object value, long tagId) { + this.mapEntry = null; + this.tag = tag; + this.value = value; + this.tagId = tagId; } void set(Map.Entry mapEntry) { this.mapEntry = mapEntry; this.tag = mapEntry.getKey(); this.value = mapEntry.getValue(); + this.tagId = TagMap.TAG_ID_NOT_COMPUTED; // resolve lazily via keyOf on first tagId() access } @Override @@ -2851,6 +3415,16 @@ public String tag() { return this.tag; } + @Override + public long tagId() { + long id = this.tagId; + if (id != TagMap.TAG_ID_NOT_COMPUTED) return id; + + id = KnownTagCodec.keyOf(this.tag); + this.tagId = id; + return id; + } + @Override public byte type() { return TagValueConversions.typeOf(this.value); diff --git a/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java b/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java new file mode 100644 index 00000000000..d803e22f2c2 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/util/FlatHashtable.java @@ -0,0 +1,169 @@ +package datadog.trace.util; + +import java.lang.reflect.Array; + +/** + * Open-addressed, single-array find-or-create over self-contained entries — each slot is one + * reference to an entry that carries its own key (and, typically, a cached hash). One array, one + * reference per slot: entry publication is a single reference store, so a reader sees {@code null} + * or a complete entry (never a torn one), and {@code final} identity fields on the entry are + * visible under racy publication. That sidesteps the memory-ordering / visibility problems parallel + * key/hash/value arrays would create — no {@code volatile}, no atomics — as long as the payload is + * one where a stale/lost read is benign (miss → recreate; clobber → one wins). + * + *

Static polymorphism (C++-template-style). The per-use policy is a {@link Helper} — a + * stateless subclass held by each caller as a {@code static final} field declared with + * the concrete helper type (not the {@code Helper} base): + * + * 

{@code
+ * private static final MyHelper HELPER = new MyHelper();  // concrete type => exact type pinned
+ * ...
+ * V v = FlatHashtable.getOrCreate(table, key, HELPER);
+ * }
+ * + * Because {@code HELPER} is a compile-time-constant of an exact type at the call site, once these + * small {@code Support} methods inline the JIT devirtualizes and inlines {@code hash}/{@code + * matches}/{@code create} — each call site specializes to straight-line code, one instantiation per + * helper, with no CHA/type-profiling dependence. Keep the methods small so they inline; verify with + * {@code -XX:+PrintInlining} (the failure mode is silent: it compiles and runs, just stays + * megamorphic and slow). {@code Helper} is an abstract class, so a distinct final subclass is + * required anyway — an exact type gives the inliner an unambiguous receiver. + * + *

Contract: {@code table.length} must be a power of two ({@link #capacityFor}). {@code + * helper.hash} should be well-distributed (this class masks it directly). Cardinality cap / + * overflow / a live-size counter are caller policy (this class is pure mechanism): a capped + * caller does {@link #get} first, and only on a miss checks its budget before {@link #getOrCreate} + * (so hits stay a single probe and the create path is warmup-rare). + */ +public final class FlatHashtable { + private FlatHashtable() {} + + /** + * Per-use policy. Extend as a stateless final class and hold a {@code static final} + * singleton of the concrete type (see class doc) so the JIT can specialize each call site. + * + *

An abstract class (not an interface) on purpose: it forces a named helper type (no + * lambdas, which can blur the receiver the inliner needs), and if specialization ever misses, the + * fallback dispatches via {@code invokevirtual} rather than the costlier megamorphic {@code + * invokeinterface}. On the specialized (inlined) path the choice is a wash — this just hedges the + * fallback and lets shared bits be {@code final}-sealed later. + * + * @param lookup key + * @param stored entry — self-contained (carries its own key, ideally a cached hash) + */ + public abstract static class Helper { + /** Hash of {@code key}; should be well-distributed (this table masks it directly). */ + public abstract int hash(K key); + + /** Whether the stored {@code value} entry is the one for {@code key}. */ + public abstract boolean matches(K key, V value); + + /** Mint a new entry for {@code key} (called once, on insert). */ + public abstract V create(K key); + } + + /** + * {@link Helper} specialized for {@code String} keys: seals a spread {@link #hash} so String-key + * callers write only {@link #matches} and {@link #create}. Extend as a stateless final class held + * in a concrete-typed {@code static final} singleton, exactly like {@link Helper} — the {@code + * final} hash resolves directly and the concrete subclass still specializes the same at each call + * site, so there's no cost to the extra layer. + * + * @param stored entry — self-contained (carries its own key, ideally a cached hash) + */ + public abstract static class StringHelper extends Helper { + @Override + public final int hash(String key) { + final int h = key.hashCode(); + return h ^ (h >>> 16); // spread; FlatHashtable masks this directly + } + } + + /** Power-of-two capacity for a cardinality budget: {@code >= 2 * limit} (load factor <= 0.5). */ + public static int capacityFor(int cardinalityLimit) { + if (cardinalityLimit <= 0) { + throw new IllegalArgumentException("cardinalityLimit must be positive: " + cardinalityLimit); + } + return Integer.highestOneBit(cardinalityLimit * 2 - 1) << 1; + } + + /** + * Allocates a correctly-typed table for a cardinality budget ({@link #capacityFor} slots). + * Passing {@code type} makes the array's runtime component type {@code T} rather than {@code + * Object[]} — typed reads, real array-store checks, and a monomorphic element type for the JIT. + * Callers can't {@code new T[]} themselves under erasure; this does the one reflective allocation + * at construction (off any hot path). Note: this {@code create} mints the backing array; {@link + * Helper#create} mints an entry — different types, no ambiguity at the call site. + */ + @SuppressWarnings("unchecked") + public static T[] create(Class type, int cardinalityLimit) { + return (T[]) Array.newInstance(type, capacityFor(cardinalityLimit)); + } + + /** + * Existing entry for {@code key}, or {@code null}. Read-only — never creates. Single probe on a + * hit; walks to the first empty slot (or all the way around) on a miss. + */ + public static V get(V[] table, K key, Helper helper) { + final int mask = table.length - 1; + final int start = helper.hash(key) & mask; + int i = start; + for (; ; ) { + final V e = table[i]; + if (e == null) { + return null; // empty slot terminates the probe (no tombstones) + } + if (helper.matches(key, e)) { + return e; + } + i = (i + 1) & mask; + if (i == start) { + return null; // wrapped ⇒ full, absent + } + } + } + + /** + * Existing entry for {@code key}, or a freshly {@link Helper#create created} + inserted one. + * Returns {@code null} only if the table is full (no empty slot) — the caller supplies its + * overflow default. The insert is a single plain reference store: a concurrent clobber / + * double-create is acceptable only when the payload makes it benign (see class doc). + */ + public static V getOrCreate(V[] table, K key, Helper helper) { + return getOrCreate(table, key, helper, null); + } + + /** + * Like {@link #getOrCreate(Object[], Object, Helper)}, but reports whether this call actually + * minted-and-stored a new entry: when {@code createdOut} is non-null, {@code createdOut[0]} is + * set to {@code true} only on the store branch, and left untouched when an existing entry + * matched. Callers that keep an approximate live-size counter (for a cardinality budget) + * increment it only when {@code createdOut[0]} — so racing callers that get an already-inserted + * entry back don't each over-count the same operation. Still racy by design: a benign double-mint + * (two threads store into the same slot) reports {@code true} on both, an over-count of at most + * the store-race width, not of every caller (see class doc). + */ + public static V getOrCreate(V[] table, K key, Helper helper, boolean[] createdOut) { + final int mask = table.length - 1; + final int start = helper.hash(key) & mask; + int i = start; + for (; ; ) { + final V e = table[i]; + if (e == null) { + final V created = helper.create(key); + table[i] = created; // single-reference publish; benign clobber (see class doc) + if (createdOut != null) { + createdOut[0] = true; + } + return created; + } + if (helper.matches(key, e)) { + return e; + } + i = (i + 1) & mask; + if (i == start) { + return null; // wrapped ⇒ full + } + } + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java new file mode 100644 index 00000000000..c70e5ff65fa --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/KnownTagsTest.java @@ -0,0 +1,256 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Parity test for the keyOf substrate: the generated {@link KnownTags} registry + the {@link + * KnownTagCodec.Resolver} it registers. Verifies name ↔ id resolution and the intercepted / + * reserved / stored partitioning. {@code keyOf}/{@code nameOf} depend only on globalSerial + name, + * not on the (dormant) positional layout, so this is independent of the colored slot the tag + * registry assigns. Also covers the slot/level-bit encoding the coloring adds to the id. + */ +class KnownTagsTest { + + /** (name, id) pairs across the groups — keyOf returns the id verbatim (incl. INTERCEPTED). */ + static Stream knownTags() { + return Stream.of( + Arguments.of(Tags.ERROR, KnownTags.ERROR_ID), + Arguments.of(DDTags.PARENT_ID, KnownTags.DD_PARENT_ID), + Arguments.of(DDTags.BASE_SERVICE, KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(Tags.VERSION, KnownTags.VERSION_ID), + Arguments.of("env", KnownTags.ENV_ID), + Arguments.of(DDTags.DJM_ENABLED, KnownTags.DD_DJM_ENABLED_ID), + Arguments.of(DDTags.DSM_ENABLED, KnownTags.DD_DSM_ENABLED_ID), + Arguments.of(DDTags.TRACER_HOST, KnownTags.DD_TRACER_HOST_ID), + Arguments.of(DDTags.DD_INTEGRATION, KnownTags.DD_INTEGRATION_ID), + Arguments.of(DDTags.DD_SVC_SRC, KnownTags.DD_SVC_SRC_ID), + Arguments.of(Tags.PEER_SERVICE, KnownTags.PEER_SERVICE_ID), + Arguments.of(DDTags.PEER_SERVICE_REMAPPED_FROM, KnownTags.DD_PEER_SERVICE_REMAPPED_FROM_ID), + Arguments.of(Tags.HTTP_METHOD, KnownTags.HTTP_METHOD_ID), + Arguments.of(Tags.HTTP_ROUTE, KnownTags.HTTP_ROUTE_ID), + Arguments.of(Tags.HTTP_URL, KnownTags.HTTP_URL_ID), + Arguments.of(Tags.PEER_HOSTNAME, KnownTags.PEER_HOSTNAME_ID), + Arguments.of(Tags.PEER_HOST_IPV4, KnownTags.PEER_IPV4_ID), + Arguments.of(Tags.PEER_HOST_IPV6, KnownTags.PEER_IPV6_ID), + Arguments.of(Tags.PEER_PORT, KnownTags.PEER_PORT_ID), + Arguments.of(Tags.COMPONENT, KnownTags.COMPONENT_ID), + Arguments.of(Tags.SPAN_KIND, KnownTags.SPAN_KIND_ID), + Arguments.of(DDTags.LANGUAGE_TAG_KEY, KnownTags.LANGUAGE_ID), + Arguments.of(Tags.DB_TYPE, KnownTags.DB_TYPE_ID), + Arguments.of(Tags.DB_INSTANCE, KnownTags.DB_INSTANCE_ID), + Arguments.of(Tags.DB_USER, KnownTags.DB_USER_ID), + Arguments.of(Tags.DB_OPERATION, KnownTags.DB_OPERATION_ID), + Arguments.of(Tags.DB_POOL_NAME, KnownTags.DB_POOL_NAME_ID)); + } + + /** + * (otelName, canonicalId, datadogName) — the OpenTelemetry name resolves (keyOf) to the canonical + * tag's id; datadogNameOf returns the Datadog name and openTelemetryNameOf returns the OTel name. + */ + static Stream otelNamedTags() { + return Stream.of( + Arguments.of("http.request.method", KnownTags.HTTP_METHOD_ID, "http.method"), + Arguments.of( + "http.response.status_code", KnownTags.HTTP_STATUS_CODE_ID, "http.status_code"), + Arguments.of("url.full", KnownTags.HTTP_URL_ID, "http.url"), + Arguments.of("server.address", KnownTags.HTTP_HOSTNAME_ID, "http.hostname"), + Arguments.of("url.query", KnownTags.HTTP_QUERY_STRING_ID, "http.query.string"), + Arguments.of("db.system", KnownTags.DB_TYPE_ID, "db.type"), + Arguments.of("db.operation.name", KnownTags.DB_OPERATION_ID, "db.operation"), + Arguments.of("db.query.text", KnownTags.DB_STATEMENT_ID, "db.statement"), + Arguments.of("service.name", KnownTags.SERVICE_ID, "service")); + } + + /** + * The subset flagged INTERCEPTED (sign bit) — must agree with the interceptor's needsIntercept. + */ + static Stream interceptedTags() { + return Stream.of( + Arguments.of(KnownTags.ERROR_ID), + Arguments.of(KnownTags.PEER_SERVICE_ID), + Arguments.of(KnownTags.HTTP_METHOD_ID), + Arguments.of(KnownTags.HTTP_URL_ID), + Arguments.of(KnownTags.SPAN_KIND_ID)); + } + + /** + * Trace-level tags (live on the TraceSegment's TagMap) — their id carries the LEVEL_TRACE bit. + */ + static Stream traceLevelTags() { + return Stream.of( + Arguments.of(KnownTags.DD_BASE_SERVICE_ID), + Arguments.of(KnownTags.VERSION_ID), + Arguments.of(KnownTags.ENV_ID), + Arguments.of(KnownTags.LANGUAGE_ID), + Arguments.of(KnownTags.RUNTIME_ID), + Arguments.of(KnownTags.DD_TRACER_HOST_ID), + Arguments.of(KnownTags.DD_DJM_ENABLED_ID)); + } + + /** Span-level tags — their id leaves the LEVEL_TRACE bit clear. */ + static Stream spanLevelTags() { + return Stream.of( + Arguments.of(KnownTags.HTTP_METHOD_ID), + Arguments.of(KnownTags.HTTP_URL_ID), + Arguments.of(KnownTags.DB_TYPE_ID), + Arguments.of(KnownTags.COMPONENT_ID), + Arguments.of(KnownTags.SPAN_KIND_ID), + Arguments.of(KnownTags.PEER_SERVICE_ID)); + } + + @BeforeAll + static void registerResolver() { + // Generated ids are compile-time constants (literal), so a constant reference is inlined and + // never triggers KnownTags.. init() forces class-load -> KnownTagCodec.register. + KnownTags.init(); + } + + @Test + void resolverIsActiveAfterInit() { + assertTrue(KnownTagCodec.isActive()); + assertEquals(KnownTags.SLOT_COUNT, KnownTagCodec.slotCount()); + } + + @ParameterizedTest + @MethodSource("knownTags") + void keyOfResolvesNameToId(String name, long id) { + assertEquals(id, KnownTagCodec.keyOf(name), "keyOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("knownTags") + void nameOfResolvesIdToName(String name, long id) { + assertEquals(name, KnownTagCodec.nameOf(id), "nameOf(" + name + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void otelNameResolvesToCanonicalId(String otelName, long id, String datadogName) { + // Inbound (keyOf) is many->one: both names land on the same canonical id. + assertEquals(id, KnownTagCodec.keyOf(otelName), "keyOf(" + otelName + ")"); + assertEquals(id, KnownTagCodec.keyOf(datadogName), "keyOf(" + datadogName + ")"); + } + + @ParameterizedTest + @MethodSource("otelNamedTags") + void namespaceAccessorsReturnPerNamespaceName(String otelName, long id, String datadogName) { + assertEquals(datadogName, KnownTagCodec.datadogNameOf(id), "datadogNameOf"); + assertEquals(otelName, KnownTagCodec.openTelemetryNameOf(id), "openTelemetryNameOf"); + // nameOf stays the Datadog name -- outbound is namespace-specific, not normalized to OTel. + assertEquals(datadogName, KnownTagCodec.nameOf(id), "nameOf stays Datadog"); + } + + @Test + void tagsWithoutOtelNameReturnNull() { + assertNull(KnownTagCodec.openTelemetryNameOf(KnownTags.HTTP_ROUTE_ID)); // no OTel name declared + assertNull(KnownTagCodec.openTelemetryNameOf(0L)); // unknown id + } + + @ParameterizedTest + @MethodSource("interceptedTags") + void interceptedTagsCarryFlag(long id) { + assertTrue(KnownTagCodec.isIntercepted(id), "isIntercepted"); + } + + @Test + void nonInterceptedTagsDoNotCarryFlag() { + Set intercepted = new HashSet<>(); + interceptedTags().forEach(a -> intercepted.add((Long) a.get()[0])); + knownTags() + .forEach( + a -> { + long id = (Long) a.get()[1]; + if (!intercepted.contains(id)) { + assertFalse(KnownTagCodec.isIntercepted(id), "not intercepted: " + a.get()[0]); + } + }); + } + + @Test + void unknownNamesResolveToZero() { + assertEquals(0L, KnownTagCodec.keyOf("definitely.not.a.known.tag")); + assertEquals(0L, KnownTagCodec.keyOf("http.statuscode")); // close-but-not-listed + assertEquals(0L, KnownTagCodec.keyOf("")); + } + + @Test + void unknownIdsResolveToNullName() { + assertNull(KnownTagCodec.nameOf(0L)); + assertNull(KnownTagCodec.nameOf(KnownTagCodec.makeTagId(9999))); // serial with no assigned tag + } + + @Test + void errorIsReservedTheRestAreStored() { + assertTrue(KnownTagCodec.isReserved(KnownTags.ERROR_ID), "ERROR reserved"); + assertFalse(KnownTagCodec.isStored(KnownTags.ERROR_ID), "ERROR not stored"); + knownTags() + .forEach( + a -> { + long id = (Long) a.get()[1]; + if (id != KnownTags.ERROR_ID) { + assertTrue(KnownTagCodec.isStored(id), "stored: " + a.get()[0]); + assertFalse(KnownTagCodec.isReserved(id), "not reserved: " + a.get()[0]); + } + }); + } + + @Test + void globalSerialsAreUnique() { + List serials = new ArrayList<>(); + knownTags().forEach(a -> serials.add((long) KnownTagCodec.serialNum((Long) a.get()[1]))); + assertEquals(serials.size(), new HashSet<>(serials).size(), "globalSerials must be unique"); + } + + @ParameterizedTest + @MethodSource("traceLevelTags") + void traceLevelTagsCarryLevelBit(long id) { + assertTrue(KnownTagCodec.isTraceLevel(id), "isTraceLevel"); + } + + @ParameterizedTest + @MethodSource("spanLevelTags") + void spanLevelTagsClearLevelBit(long id) { + assertFalse(KnownTagCodec.isTraceLevel(id), "not trace-level"); + } + + @Test + void levelBitCompositionRoundTrips() { + long spanId = KnownTagCodec.makeTagId(300, 5); // no level bit + assertFalse(KnownTagCodec.isTraceLevel(spanId)); + long traceId = KnownTagCodec.traceLevel(spanId); + assertTrue(KnownTagCodec.isTraceLevel(traceId)); + // level bit is orthogonal to serial/slot — both survive setting it + assertEquals(KnownTagCodec.serialNum(spanId), KnownTagCodec.serialNum(traceId)); + assertEquals(KnownTagCodec.slot(spanId), KnownTagCodec.slot(traceId)); + assertEquals(traceId, KnownTagCodec.traceLevel(traceId), "traceLevel is idempotent"); + } + + @Test + void slotEncodingRoundTrips() { + long id = KnownTagCodec.makeTagId(FIRST_STORED_SERIAL_PLUS_7, 7); + assertEquals(FIRST_STORED_SERIAL_PLUS_7, KnownTagCodec.serialNum(id)); + assertEquals(7, KnownTagCodec.slot(id)); + assertFalse(KnownTagCodec.isUnslotted(id)); + + long unslotted = KnownTagCodec.makeTagId(FIRST_STORED_SERIAL_PLUS_7); // NO_SLOT + assertEquals(KnownTagCodec.NO_SLOT, KnownTagCodec.slot(unslotted)); + assertTrue(KnownTagCodec.isUnslotted(unslotted)); + } + + private static final int FIRST_STORED_SERIAL_PLUS_7 = KnownTagCodec.FIRST_STORED_SERIAL + 7; +} diff --git a/internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java b/internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java new file mode 100644 index 00000000000..59b826a5019 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/SizingHintTableTest.java @@ -0,0 +1,105 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import org.junit.jupiter.api.Test; + +/** + * Registry semantics for {@link SizingHintTable}. The table is process-wide static state; each test + * uses distinct operation names so tests don't interfere (the only shared, cumulative state is the + * per-lane cardinality counter, which the overflow test drives past its bound with its own names). + */ +class SizingHintTableTest { + + @Test + void nullOperationNameGetsNoHint() { + assertNull(SizingHintTable.hintFor(null, true)); + assertNull(SizingHintTable.hintFor(null, false)); + } + + @Test + void nonStringCharSequenceIsKeyedByContent() { + // Operation names are commonly UTF8BytesString, not String; a content-equal name must resolve + // to the same hint as its String form (keyed by toString(), which is O(1) for these types). + SizingHint viaString = SizingHintTable.hintFor("registry.utf8", true); + CharSequence utf8 = UTF8BytesString.create("registry.utf8"); + assertNotNull(viaString); + assertSame(viaString, SizingHintTable.hintFor(utf8, true)); + } + + @Test + void freshHintIsSeededAndUncapped() { + SizingHint hint = SizingHintTable.hintFor("registry.fresh", true); + assertNotNull(hint); + assertEquals(SizingHintTable.SEED_SIZE, hint.size); + assertFalse(hint.capped); + assertEquals("registry.fresh", hint.label); + } + + @Test + void sameOperationAndLaneReturnsSameInstance() { + SizingHint a = SizingHintTable.hintFor("registry.stable", true); + SizingHint b = SizingHintTable.hintFor("registry.stable", true); + assertSame(a, b); + } + + @Test + void entryAndChildLanesAreIndependent() { + SizingHint entry = SizingHintTable.hintFor("registry.twolane", true); + SizingHint child = SizingHintTable.hintFor("registry.twolane", false); + assertNotNull(entry); + assertNotNull(child); + assertNotSame(entry, child, "each lane holds its own hint for the same operation name"); + // ...and each lane is internally stable. + assertSame(entry, SizingHintTable.hintFor("registry.twolane", true)); + assertSame(child, SizingHintTable.hintFor("registry.twolane", false)); + } + + @Test + void beyondCardinalityBudgetSharesACappedOverflowHint() { + // Push far more distinct names than any lane's budget; a capped shared hint must appear. + SizingHint firstOverflow = null; + for (int i = 0; i < 4096 && firstOverflow == null; i++) { + SizingHint hint = SizingHintTable.hintFor("registry.flood." + i, true); + assertNotNull(hint); + if (hint.capped) { + firstOverflow = hint; + } + } + assertNotNull(firstOverflow, "lane eventually collapses to a capped overflow hint"); + assertEquals(SizingHintTable.OVERFLOW_SEED, firstOverflow.size); + + // Every further over-budget name shares that same capped instance. + SizingHint another = SizingHintTable.hintFor("registry.flood.after", true); + assertTrue(another.capped); + assertSame(firstOverflow, another); + } + + @Test + void sizingHintFeedsAndTunesTheDenseStore() { + KnownTags.init(); // register the real allocation-free resolver so known tags route dense + SizingHint hint = SizingHintTable.hintFor("registry.tuning", true); + assertEquals(SizingHintTable.SEED_SIZE, hint.size); + + TagMap map = TagMap.create(hint); + map.set(DDTags.BASE_SERVICE, "svc"); + map.set(datadog.trace.bootstrap.instrumentation.api.Tags.COMPONENT, "comp"); + map.set(datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND, "server"); + + map.recordSize(hint); + assertEquals(3, hint.size, "hint self-tunes up to the observed known-tag high-water mark"); + + // Monotonic-max: a smaller later observation does not shrink the hint. + TagMap smaller = TagMap.create(hint); + smaller.set(DDTags.BASE_SERVICE, "svc"); + smaller.recordSize(hint); + assertEquals(3, hint.size, "recordSize never shrinks the hint"); + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java new file mode 100644 index 00000000000..c9c2b4ff1c0 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseForkedTest.java @@ -0,0 +1,300 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Exercises the dense known-tag store with a LIVE resolver. Registration ({@link KnownTagCodec}) is + * a global static with no un-register, so this lives in a {@code ForkedTest} (isolated JVM) to keep + * dense routing from leaking into the bucket-only tests in the shared JVM. The dense store is + * dormant in production (no resolver) — this is where it actually executes. + * + *

Stored tags (globalSerial ≥ {@code FIRST_STORED_SERIAL}) route to the dense store; reserved + * tags (e.g. {@code error}) and arbitrary tags stay in the hash buckets. Behavior must be + * observationally identical to the bucket store. + */ +class TagMapDenseForkedTest { + + // stored (dense-routed) tags + static final String BASE_SERVICE = DDTags.BASE_SERVICE; + static final String COMPONENT = Tags.COMPONENT; + static final String DB_TYPE = Tags.DB_TYPE; + static final String HTTP_METHOD = Tags.HTTP_METHOD; // stored + intercepted + static final String DB_INSTANCE = Tags.DB_INSTANCE; + // arbitrary (bucket-routed) tags + static final String CUSTOM_A = "custom.tag.a"; + static final String CUSTOM_B = "custom.tag.b"; + + @BeforeAll + static void registerResolver() { + // Generated ids are compile-time constants (literal), so a constant reference is inlined and + // never triggers KnownTags.. init() forces class-load -> KnownTagCodec.register. + KnownTags.init(); + assertTrue(KnownTagCodec.isActive(), "resolver must be live for the dense store to engage"); + assertTrue( + KnownTagCodec.isStored(KnownTagCodec.keyOf(BASE_SERVICE)), "base_service routes dense"); + assertFalse( + KnownTagCodec.isStored(KnownTagCodec.keyOf(CUSTOM_A)), "custom tag stays in buckets"); + assertFalse( + KnownTagCodec.isStored(KnownTagCodec.keyOf(Tags.ERROR)), "error is reserved, not stored"); + } + + private static TagMap map() { + return (TagMap) TagMap.create(); + } + + @Test + void knownTagRoundTripsThroughDenseStore() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(COMPONENT, "spring-web"); + + assertEquals("billing", map.getObject(BASE_SERVICE)); + assertEquals("spring-web", map.getString(COMPONENT)); + assertEquals("billing", map.getEntry(BASE_SERVICE).objectValue()); + assertTrue(map.containsKey(BASE_SERVICE)); + assertEquals(2, map.size()); + map.checkIntegrity(); + } + + @Test + void typedKnownValuesRoundTrip() { + TagMap map = map(); + map.set(DB_TYPE, "postgresql"); + map.set(HTTP_METHOD, "GET"); + map.set(Tags.PEER_PORT, 5432); + + assertEquals("postgresql", map.getString(DB_TYPE)); + assertEquals("GET", map.getString(HTTP_METHOD)); + assertEquals(5432, map.getInt(Tags.PEER_PORT)); + assertEquals(3, map.size()); + map.checkIntegrity(); + } + + @Test + void knownAndUnknownCoexist() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); // dense + map.set(CUSTOM_A, "alpha"); // bucket + map.set(DB_TYPE, "h2"); // dense + map.set(CUSTOM_B, "beta"); // bucket + + assertEquals("billing", map.getObject(BASE_SERVICE)); + assertEquals("alpha", map.getObject(CUSTOM_A)); + assertEquals("h2", map.getObject(DB_TYPE)); + assertEquals("beta", map.getObject(CUSTOM_B)); + assertEquals(4, map.size()); + assertFalse(map.isEmpty()); + map.checkIntegrity(); + + Map collected = new HashMap<>(); + map.fillMap(collected); + assertEquals(4, collected.size()); + assertEquals("billing", collected.get(BASE_SERVICE)); + assertEquals("alpha", collected.get(CUSTOM_A)); + assertEquals("h2", collected.get(DB_TYPE)); + assertEquals("beta", collected.get(CUSTOM_B)); + } + + @Test + void overwriteKnownReplacesInPlace() { + TagMap map = map(); + map.set(COMPONENT, "first"); + assertEquals("first", map.getObject(COMPONENT)); + map.set(COMPONENT, "second"); + assertEquals("second", map.getObject(COMPONENT)); + assertEquals(1, map.size()); // overwrite, not append + map.checkIntegrity(); + } + + @Test + void removeKnownClearsIt() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(DB_TYPE, "h2"); + map.set(CUSTOM_A, "alpha"); + assertEquals(3, map.size()); + + TagMap.Entry removed = map.getAndRemove(BASE_SERVICE); + assertEquals("billing", removed.objectValue()); + assertNull(map.getObject(BASE_SERVICE)); + assertEquals("h2", map.getObject(DB_TYPE)); // sibling dense entry intact + assertEquals("alpha", map.getObject(CUSTOM_A)); + assertEquals(2, map.size()); + map.checkIntegrity(); + } + + @Test + void forEachAndIteratorEmitDenseAndBucketEntries() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(COMPONENT, "web"); + map.set(CUSTOM_A, "alpha"); + + Map viaForEach = new HashMap<>(); + map.forEach(reader -> viaForEach.put(reader.tag(), reader.objectValue())); + assertEquals(3, viaForEach.size()); + assertEquals("billing", viaForEach.get(BASE_SERVICE)); + assertEquals("web", viaForEach.get(COMPONENT)); + assertEquals("alpha", viaForEach.get(CUSTOM_A)); + + Map viaIterator = new HashMap<>(); + for (TagMap.EntryReader reader : map) { + viaIterator.put(reader.tag(), reader.objectValue()); + } + assertEquals(viaForEach, viaIterator); + } + + @Test + void copyPreservesDenseStore() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(CUSTOM_A, "alpha"); + + TagMap copy = (TagMap) map.copy(); + assertEquals("billing", copy.getObject(BASE_SERVICE)); + assertEquals("alpha", copy.getObject(CUSTOM_A)); + assertEquals(2, copy.size()); + + // independence: mutating the copy doesn't touch the original's dense store + copy.set(BASE_SERVICE, "shipping"); + assertEquals("shipping", copy.getObject(BASE_SERVICE)); + assertEquals("billing", map.getObject(BASE_SERVICE)); + copy.checkIntegrity(); + map.checkIntegrity(); + } + + @Test + void clearEmptiesDenseStore() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); + map.set(CUSTOM_A, "alpha"); + map.clear(); + assertEquals(0, map.size()); + assertTrue(map.isEmpty()); + assertNull(map.getObject(BASE_SERVICE)); + map.checkIntegrity(); + } + + @Test + void putAllMergesDenseStore() { + TagMap src = map(); + src.set(BASE_SERVICE, "billing"); + src.set(DB_TYPE, "h2"); + src.set(CUSTOM_A, "alpha"); + + TagMap dst = map(); + dst.set(COMPONENT, "web"); // dense, distinct + dst.set(BASE_SERVICE, "old"); // dense, clobbered by src + dst.putAll((TagMap) src); + + assertEquals("billing", dst.getObject(BASE_SERVICE)); // src clobbers + assertEquals("h2", dst.getObject(DB_TYPE)); + assertEquals("web", dst.getObject(COMPONENT)); + assertEquals("alpha", dst.getObject(CUSTOM_A)); + assertEquals(4, dst.size()); + dst.checkIntegrity(); + } + + // ---- read-through union (dense parent + dense child) ---- + + private static TagMap frozenParent() { + TagMap parent = map(); + parent.set(BASE_SERVICE, "billing"); // dense + parent.set(COMPONENT, "web"); // dense + parent.set(CUSTOM_A, "alpha"); // bucket + parent.freeze(); + return parent; + } + + @Test + void childReadsThroughToParentDense() { + TagMap child = TagMap.createFromParent(frozenParent()); + child.set(DB_TYPE, "h2"); // child-only dense + child.set(CUSTOM_B, "beta"); // child-only bucket + + // inherited from parent + assertEquals("billing", child.getObject(BASE_SERVICE)); + assertEquals("web", child.getObject(COMPONENT)); + assertEquals("alpha", child.getObject(CUSTOM_A)); + // own + assertEquals("h2", child.getObject(DB_TYPE)); + assertEquals("beta", child.getObject(CUSTOM_B)); + // union size: 3 parent + 2 child + assertEquals(5, child.size()); + assertFalse(child.isEmpty()); + + Map union = new HashMap<>(); + child.forEach(reader -> union.put(reader.tag(), reader.objectValue())); + assertEquals(5, union.size()); + assertEquals("billing", union.get(BASE_SERVICE)); + assertEquals("h2", union.get(DB_TYPE)); + child.checkIntegrity(); + } + + @Test + void childDenseShadowsParentDense() { + TagMap child = TagMap.createFromParent(frozenParent()); + child.set(BASE_SERVICE, "shipping"); // shadows parent's dense base_service + + assertEquals("shipping", child.getObject(BASE_SERVICE)); // local wins + assertEquals("web", child.getObject(COMPONENT)); // still inherited + assertEquals(3, child.size()); // base_service counted once (shadowed, not doubled) + + Map union = new HashMap<>(); + child.forEach(reader -> union.put(reader.tag(), reader.objectValue())); + assertEquals(3, union.size()); + assertEquals("shipping", union.get(BASE_SERVICE)); // shadow value, parent suppressed + } + + @Test + void removingParentDenseKeyTombstonesIt() { + TagMap child = TagMap.createFromParent(frozenParent()); + + TagMap.Entry removed = child.getAndRemove(BASE_SERVICE); // parent-only dense key + assertEquals("billing", removed.objectValue()); // prior visible value was the parent's + assertNull(child.getObject(BASE_SERVICE)); // tombstoned: no read-through + assertEquals("web", child.getObject(COMPONENT)); // sibling still inherited + assertEquals(2, child.size()); // 3 parent - 1 tombstoned + + Map union = new HashMap<>(); + child.forEach(reader -> union.put(reader.tag(), reader.objectValue())); + assertEquals(2, union.size()); + assertFalse(union.containsKey(BASE_SERVICE)); + child.checkIntegrity(); + } + + @Test + void denseReaderExposesTagId() { + TagMap map = map(); + map.set(BASE_SERVICE, "billing"); // dense-routed + map.set(CUSTOM_A, "alpha"); // bucket-routed + + Map idsByTag = new HashMap<>(); + map.forEach(reader -> idsByTag.put(reader.tag(), reader.tagId())); + + // dense entry: the reader carries the real known-tag id directly + assertEquals(KnownTagCodec.keyOf(BASE_SERVICE), idsByTag.get(BASE_SERVICE).longValue()); + assertTrue(idsByTag.get(BASE_SERVICE) != 0L, "known tag has a non-zero id"); + // bucket entry for a custom tag: unknown -> 0L + assertEquals(0L, idsByTag.get(CUSTOM_A).longValue()); + } + + @Test + void bucketEntryResolvesTagIdLazily() { + TagMap map = map(); + map.set(CUSTOM_A, "alpha"); // custom tag stays a bucket Entry + + TagMap.Entry entry = map.getEntry(CUSTOM_A); + assertEquals(0L, entry.tagId()); // unknown tag -> 0L + assertEquals(0L, entry.tagId()); // second read hits the memoized field + } +} diff --git a/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java new file mode 100644 index 00000000000..6010499d7ea --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/TagMapDenseFuzzForkedTest.java @@ -0,0 +1,207 @@ +package datadog.trace.api; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.TagMapFuzzTest.MapAction; +import datadog.trace.api.TagMapFuzzTest.TestCase; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Supplier; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Fuzz test for the dense store under a LIVE resolver, across three key regimes. Reuses {@link + * TagMapFuzzTest}'s oracle machinery ({@code test(TestCase)} replays a random action sequence + * against a {@code HashMap}, verifying each step + {@code checkIntegrity}). + * + *

Uses a synthetic prefix resolver ({@code known-N} -> stored / dense, anything else -> bucket) + * rather than the real {@link KnownTags}: it gives an UNBOUNDED known key space, so the dense array + * actually grows past its initial capacity and the linear scan gets long, and it lets each test pin + * the known/custom ratio. The three regimes exercise paths the mixed run alone would miss: + * + *

    + *
  • known-only — the all-dense map (dense growth, dense-only putAll/copy/clear/iterate + * with no bucket phase, knownCount-only size). + *
  • custom-only — confirms the dense branches stay inert when nothing resolves, even + * with a resolver registered. + *
  • mixed — both regions and their interaction. + *
+ * + *

Forked (isolated JVM) because resolver registration is a global static with no un-register. + */ +class TagMapDenseFuzzForkedTest { + static final int SINGLE_MAP_CASES = 1500; + static final int MERGE_CASES = 400; + static final int MAX_ACTIONS = 40; + static final int MIN_ACTIONS = 8; + + // unbounded synthetic key spaces — large enough to grow the dense array past cap-8 several times + static final int KNOWN_SPACE = 48; + static final int CUSTOM_SPACE = 48; + + enum Regime { + KNOWN_ONLY, + CUSTOM_ONLY, + MIXED + } + + /** + * Synthetic resolver: {@code known-N} -> stored id (serial = FIRST_STORED_SERIAL + N); else 0. + */ + static final KnownTagCodec.Resolver FUZZ_RESOLVER = + new KnownTagCodec.Resolver() { + @Override + public long keyOf(String name) { + if (name.startsWith("known-")) { + int n = Integer.parseInt(name.substring("known-".length())); + return KnownTagCodec.makeTagId(KnownTagCodec.FIRST_STORED_SERIAL + n, n); + } + return 0L; + } + + @Override + public String nameOf(long tagId) { + int serial = KnownTagCodec.serialNum(tagId); + return serial >= KnownTagCodec.FIRST_STORED_SERIAL + ? "known-" + (serial - KnownTagCodec.FIRST_STORED_SERIAL) + : null; + } + + @Override + public String openTelemetryNameOf(long tagId) { + return null; // synthetic resolver declares no OpenTelemetry names + } + + @Override + public int slotCount() { + return 0; // positional unused + } + }; + + @BeforeAll + static void registerResolver() { + KnownTagCodec.register(FUZZ_RESOLVER); + assertTrue(KnownTagCodec.isActive(), "resolver must be live"); + assertTrue(KnownTagCodec.isStored(KnownTagCodec.keyOf("known-0")), "known- routes dense"); + assertFalse( + KnownTagCodec.isStored(KnownTagCodec.keyOf("custom-0")), "custom- stays in buckets"); + // round-trip the synthetic encoding + long id = KnownTagCodec.keyOf("known-7"); + assertTrue("known-7".equals(KnownTagCodec.nameOf(id)), "name<->id round-trips"); + } + + @Test + void knownOnlyFuzz() { + runRegime(Regime.KNOWN_ONLY); + } + + @Test + void customOnlyFuzz() { + runRegime(Regime.CUSTOM_ONLY); + } + + @Test + void mixedFuzz() { + runRegime(Regime.MIXED); + } + + private static void runRegime(Regime regime) { + for (int i = 0; i < SINGLE_MAP_CASES; ++i) { + TagMapFuzzTest.test(generateTest(regime)); + } + for (int i = 0; i < MERGE_CASES; ++i) { + TagMap mapA = TagMapFuzzTest.test(generateTest(regime)); + TagMap mapB = TagMapFuzzTest.test(generateTest(regime)); + + HashMap hashA = new HashMap<>(mapA); + HashMap hashB = new HashMap<>(mapB); + + mapA.putAll(mapB); + hashA.putAll(hashB); + + TagMapFuzzTest.assertMapEquals(hashA, mapA); + } + } + + // --- action generation (mirrors TagMapFuzzTest.randomAction, regime-driven key pool) --- + + private static TestCase generateTest(Regime regime) { + ThreadLocalRandom r = ThreadLocalRandom.current(); + int numActions = r.nextInt(MAX_ACTIONS - MIN_ACTIONS) + MIN_ACTIONS; + List actions = new ArrayList<>(numActions); + for (int i = 0; i < numActions; ++i) { + actions.add(randomAction(regime)); + } + return new TestCase(actions); + } + + private static MapAction randomAction(Regime regime) { + switch (randomChoice(0.02, 0.1, 0.2)) { + case 0: + return TagMapFuzzTest.clear(); + case 1: + return choose( + () -> TagMapFuzzTest.putAll(randomKeysAndValues(regime)), + () -> TagMapFuzzTest.putAllTagMap(randomKeysAndValues(regime)), + () -> TagMapFuzzTest.putAllLedger(randomKeysAndValues(regime))); + case 2: + return choose( + () -> TagMapFuzzTest.remove(randomKey(regime)), + () -> TagMapFuzzTest.removeLight(randomKey(regime)), + () -> TagMapFuzzTest.getAndRemove(randomKey(regime))); + default: + return choose( + () -> TagMapFuzzTest.put(randomKey(regime), randomValue()), + () -> TagMapFuzzTest.set(randomKey(regime), randomValue()), + () -> TagMapFuzzTest.getAndSet(randomKey(regime), randomValue())); + } + } + + private static String randomKey(Regime regime) { + ThreadLocalRandom r = ThreadLocalRandom.current(); + boolean known; + switch (regime) { + case KNOWN_ONLY: + known = true; + break; + case CUSTOM_ONLY: + known = false; + break; + default: + known = r.nextBoolean(); + } + return known ? "known-" + r.nextInt(KNOWN_SPACE) : "custom-" + r.nextInt(CUSTOM_SPACE); + } + + private static String randomValue() { + return "values-" + ThreadLocalRandom.current().nextInt(); + } + + private static String[] randomKeysAndValues(Regime regime) { + int numEntries = ThreadLocalRandom.current().nextInt(KNOWN_SPACE + CUSTOM_SPACE); + String[] keysAndValues = new String[numEntries << 1]; + for (int i = 0; i < keysAndValues.length; i += 2) { + keysAndValues[i] = randomKey(regime); + keysAndValues[i + 1] = randomValue(); + } + return keysAndValues; + } + + private static int randomChoice(double... proportions) { + double selector = ThreadLocalRandom.current().nextDouble(); + for (int i = 0; i < proportions.length; ++i) { + if (selector < proportions[i]) return i; + selector -= proportions[i]; + } + return proportions.length; + } + + @SafeVarargs + private static MapAction choose(Supplier... choices) { + return choices[ThreadLocalRandom.current().nextInt(choices.length)].get(); + } +} diff --git a/internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java b/internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java new file mode 100644 index 00000000000..c02ca085d53 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/util/FlatHashtableTest.java @@ -0,0 +1,158 @@ +package datadog.trace.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashSet; +import java.util.Set; +import org.junit.jupiter.api.Test; + +class FlatHashtableTest { + + /** + * Self-contained entry: carries its own key + cached spread hash (the FlatHashtable contract). + */ + static final class Entry { + final String key; + final int hash; + + Entry(String key, int hash) { + this.key = key; + this.hash = hash; + } + } + + /** Counts create() calls so tests can prove getOrCreate mints exactly once per key. */ + static final class CountingHelper extends FlatHashtable.StringHelper { + int creates; + + @Override + public boolean matches(String key, Entry value) { + return this.hash(key) == value.hash && key.equals(value.key); + } + + @Override + public Entry create(String key) { + this.creates++; + return new Entry(key, this.hash(key)); + } + } + + /** A helper whose keys all collide to slot 0, to exercise linear probing + fill-to-full. */ + static final class CollidingHelper extends FlatHashtable.Helper { + @Override + public int hash(String key) { + return 0; + } + + @Override + public boolean matches(String key, Entry value) { + return key.equals(value.key); + } + + @Override + public Entry create(String key) { + return new Entry(key, 0); + } + } + + @Test + void capacityForIsPowerOfTwoAtLeastTwiceLimit() { + for (int limit = 1; limit <= 1024; limit++) { + int cap = FlatHashtable.capacityFor(limit); + assertTrue(cap >= 2 * limit, "cap " + cap + " >= 2*" + limit); + assertEquals(0, cap & (cap - 1), "cap " + cap + " is a power of two"); + } + assertEquals(2, FlatHashtable.capacityFor(1)); + assertEquals(4, FlatHashtable.capacityFor(2)); + assertEquals(8, FlatHashtable.capacityFor(3)); + assertEquals(8, FlatHashtable.capacityFor(4)); + assertEquals(1024, FlatHashtable.capacityFor(512)); + } + + @Test + void capacityForRejectsNonPositive() { + assertThrows(IllegalArgumentException.class, () -> FlatHashtable.capacityFor(0)); + assertThrows(IllegalArgumentException.class, () -> FlatHashtable.capacityFor(-1)); + } + + @Test + void createAllocatesTypedArrayOfCapacity() { + Entry[] table = FlatHashtable.create(Entry.class, 512); + assertEquals(1024, table.length); + assertEquals(Entry.class, table.getClass().getComponentType()); + } + + @Test + void getReturnsNullOnEmptyTable() { + Entry[] table = FlatHashtable.create(Entry.class, 8); + assertNull(FlatHashtable.get(table, "absent", new CountingHelper())); + } + + @Test + void getOrCreateMintsOnceThenReturnsSameInstance() { + Entry[] table = FlatHashtable.create(Entry.class, 8); + CountingHelper helper = new CountingHelper(); + + Entry first = FlatHashtable.getOrCreate(table, "op", helper); + assertNotNull(first); + assertEquals(1, helper.creates); + + Entry again = FlatHashtable.getOrCreate(table, "op", helper); + assertSame(first, again, "second getOrCreate returns the existing entry"); + assertEquals(1, helper.creates, "no re-mint on hit"); + + assertSame(first, FlatHashtable.get(table, "op", helper), "get sees the inserted entry"); + } + + @Test + void storesManyDistinctKeysWithinBudget() { + int limit = 200; + Entry[] table = FlatHashtable.create(Entry.class, limit); + CountingHelper helper = new CountingHelper(); + + Set seen = new HashSet<>(); + for (int i = 0; i < limit; i++) { + Entry e = FlatHashtable.getOrCreate(table, "op-" + i, helper); + assertNotNull(e); + seen.add(e); + } + assertEquals(limit, seen.size()); + assertEquals(limit, helper.creates); + + // All still retrievable (probing across collisions works). + for (int i = 0; i < limit; i++) { + assertNotNull(FlatHashtable.get(table, "op-" + i, helper)); + } + } + + @Test + void getOrCreateReturnsNullWhenPhysicallyFull() { + // capacityFor(1) == 2 slots; all keys collide to slot 0 so 2 fills the table. + Entry[] table = FlatHashtable.create(Entry.class, 1); + assertEquals(2, table.length); + CollidingHelper helper = new CollidingHelper(); + + assertNotNull(FlatHashtable.getOrCreate(table, "a", helper)); + assertNotNull(FlatHashtable.getOrCreate(table, "b", helper)); + // Table is now full; a third distinct key has no empty slot. + assertNull(FlatHashtable.getOrCreate(table, "c", helper)); + // But existing keys are still found via the wrapped probe. + assertNotNull(FlatHashtable.get(table, "a", helper)); + assertNotNull(FlatHashtable.get(table, "b", helper)); + assertNull(FlatHashtable.get(table, "c", helper)); + } + + @Test + void stringHelperHashIsSpreadAndStable() { + CountingHelper helper = new CountingHelper(); + int h = helper.hash("component"); + assertEquals(h, helper.hash("component"), "hash is deterministic"); + int raw = "component".hashCode(); + assertEquals(raw ^ (raw >>> 16), h, "hash is the spread of String.hashCode()"); + } +} diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 850c6055d58..4c5bf4c8b0a 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -5641,6 +5641,14 @@ "aliases": [] } ], + "DD_TRACE_EXPERIMENTAL_DENSE_TAGS_ENABLED": [ + { + "version": "A", + "type": "boolean", + "default": "false", + "aliases": [] + } + ], "DD_TRACE_EXPERIMENTAL_FEATURES_ENABLED": [ { "version": "A", diff --git a/tag-conventions.java.yaml b/tag-conventions.java.yaml new file mode 100644 index 00000000000..fcb91ac04f0 --- /dev/null +++ b/tag-conventions.java.yaml @@ -0,0 +1,36 @@ +# dd-trace-java overlay — impl hints + special-tag registry. +# Consumed alongside the language-agnostic tag-conventions.yaml. Keyed by tag name +# (these are tag-intrinsic, not per-type). Only exceptions are listed. +# --------------------------------------------------------------------------- +# NOTE: the id coordinate (group-decl / field-decl) is NOT here — the generator assigns it from the +# declaration groups; the dense-vs-bucket split derives from the domain `required` level. + +# Tags whose set-path is handled by the Java TagInterceptor (side-effecting, but still stored). +# Generated ids carry the intercepted flag (bit 63). +intercepted: + - span.kind + - http.method + - http.url + - servlet.context + - db.statement + - peer.service + +# Reserved / special keys: accepted by the public setTag(...) API but ROUTED to a span field or a +# trace directive instead of tag storage. Reserved-tier ids (serial < FIRST_STORED_SERIAL, no slot). +# "Reserved" names the shared mechanism (the tracer reserves the key and handles it); the two kinds +# split on whether a value exists: structural has one (in a span/trace field), directive has none. +# The generated id->handler dispatch table is the data-driven replacement for the imperative +# TagInterceptor chain. +# kind: structural -> sets a span/trace field (`field:` names it) +# kind: directive -> triggers sampling/trace behavior +reserved: + - { tag: error, kind: structural, field: error } + - { tag: service, kind: structural, field: service, open-telemetry-name: service.name } + - { tag: resource.name, kind: structural, field: resource } + - { tag: span.type, kind: structural, field: type } + - { tag: origin, kind: structural, field: origin } # trace-level field + - { tag: sampling.priority, kind: directive } + - { tag: manual.keep, kind: directive } + - { tag: manual.drop, kind: directive } + - { tag: measured, kind: directive } + - { tag: analytics.sample_rate, kind: directive } # legacy diff --git a/tag-conventions.yaml b/tag-conventions.yaml new file mode 100644 index 00000000000..1639e0e8afc --- /dev/null +++ b/tag-conventions.yaml @@ -0,0 +1,134 @@ +# Tag conventions — LANGUAGE-AGNOSTIC domain spec (structure + semantics only) +# --------------------------------------------------------------------------- +# The code generator consumes THIS file (domain) plus a per-language overlay +# (tag-conventions..yaml: impl hints like `intercepted`, and the reserved/ +# special-tag registry) to emit each language's tag-id constants, id<->name +# resolver, and slot (bitmask-bit) assignment. +# +# TRACE-LEVEL is its own thing (its own TagMap "type" on the TraceSegment) — the process/trace +# constants + product flags that are set once per trace, NOT per span. Declared explicitly in the +# `trace_level` section below (a distinct tier), never inferred from `source`. +# +# SPAN TYPES compose three ways: +# extends — structural is-a inheritance (http.server is-a http is-a base). `base` is implicitly +# in every span; abstract layers exist only to be extended. +# include — a span type PULLS in a mixin it intrinsically has (has-a; core-owned). +# applies — a mixin PUSHES itself onto span types, gated by `enabled_by`. +# resolved_tags(type) = own + extends-chain (incl base) + included mixins + applied mixins (de-duped). +# +# tag fields (DOMAIN only): tag | type (string|int|long|boolean|double) +# | required (required|conditional|recommended|optional|opt_in) | open-telemetry-name. +# The id coordinate (group-decl / field-decl) is NOT authored here — the generator assigns it: each +# declaration source (the trace-level tier, each span type, each mixin) is a group, and within a +# group `field-decl` numbers the dense (required/conditional/recommended) tags; the rest are +# bucketed. See the design doc. +# --------------------------------------------------------------------------- + +# Trace-level tier: its own TagMap on the TraceSegment. Set once per trace, not per span. +trace_level: + tags: + - { tag: _dd.base_service, type: string, required: required } + - { tag: version, type: string, required: recommended } + - { tag: env, type: string, required: recommended } + - { tag: language, type: string, required: required } + - { tag: runtime-id, type: string, required: required } + - { tag: _dd.tracer_host, type: string, required: recommended } + - { tag: _dd.git.commit.sha, type: string, required: recommended } + - { tag: _dd.git.repository_url, type: string, required: recommended } + # product .enabled flags — process-constant; present on the trace segment regardless of whether + # the product is enabled (the flag carries the state), so always-present => recommended. + - { tag: _dd.profiling.enabled, type: boolean, required: recommended } + - { tag: _dd.dsm.enabled, type: boolean, required: recommended } + - { tag: _dd.appsec.enabled, type: boolean, required: recommended } + - { tag: _dd.djm.enabled, type: boolean, required: recommended } + - { tag: _dd.civisibility.enabled, type: boolean, required: recommended } + +span_types: + # root: per-span tags every span has (incl. the per-span core tags parent_id / integration / svc_src + # — core-set but per-span, so NOT trace-level). + base: + abstract: true + tags: + - { tag: _dd.parent_id, type: string, required: required } + - { tag: component, type: string, required: required } + - { tag: span.kind, type: string, required: required } + - { tag: _dd.integration, type: string, required: recommended } + - { tag: _dd.svc_src, type: string, required: optional } + - { tag: error.type, type: string, required: recommended } + - { tag: error.message, type: string, required: recommended } + - { tag: error.stack, type: string, required: recommended } + + http: + abstract: true + extends: base + tags: + - { tag: http.method, type: string, required: required, open-telemetry-name: http.request.method } + - { tag: http.status_code, type: int, required: conditional, open-telemetry-name: http.response.status_code } + - { tag: network.protocol.version, type: string, required: recommended } + + http.server: + extends: http + tags: + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } + - { tag: http.route, type: string, required: conditional } + - { tag: http.hostname, type: string, required: required, open-telemetry-name: server.address } + - { tag: http.useragent, type: string, required: recommended } + - { tag: http.query.string, type: string, required: recommended, open-telemetry-name: url.query } + - { tag: servlet.path, type: string, required: optional } + - { tag: servlet.context, type: string, required: optional } + + http.client: + extends: http + include: [ peer ] + tags: + - { tag: http.url, type: string, required: required, open-telemetry-name: url.full } + - { tag: http.resend_count, type: int, required: recommended } + + db.client: + extends: base + include: [ peer ] + tags: + - { tag: db.type, type: string, required: required, open-telemetry-name: db.system } + - { tag: db.instance, type: string, required: recommended } + - { tag: db.operation, type: string, required: recommended, open-telemetry-name: db.operation.name } + - { tag: db.user, type: string, required: recommended } + - { tag: db.pool.name, type: string, required: optional } + - { tag: db.statement, type: string, required: recommended, open-telemetry-name: db.query.text } + + view.render: + extends: base + tags: + - { tag: view.name, type: string, required: recommended } + +mixins: + # peer — outbound/remote-peer capability, PULLED via `include` by client span types. + peer: + tags: + - { tag: peer.service, type: string, required: recommended } + - { tag: _dd.peer.service.source, type: string, required: recommended } + - { tag: _dd.peer.service.remapped_from, type: string, required: recommended } + - { tag: peer.hostname, type: string, required: recommended } + - { tag: peer.ipv4, type: string } + - { tag: peer.ipv6, type: string } + - { tag: peer.port, type: int } + + # ci_visibility — per-span test tags. Its capability flag (_dd.civisibility.enabled) lives in + # trace_level, outside this mixin (general rule: capability flags are trace-level, mixins hold the + # per-span tags). Applies to the `test` span type (not modeled here yet). + ci_visibility: + enabled_by: dd.civisibility.enabled + applies: [ test ] + tags: + - { tag: test.name, type: string, required: recommended } + - { tag: test.suite, type: string, required: recommended } + - { tag: test.status, type: string, required: recommended } + - { tag: test.framework, type: string, required: recommended } + +# --------------------------------------------------------------------------- +# Notes +# - Product .enabled flags moved to `trace_level` (process-constant) — the old product mixins held +# only those flags, so they dissolved. `enabled_by`/attachment gating is a runtime concern. +# - span.kind enumerates: server | client | producer | consumer | internal | broker. +# - Reserved/special keys (service, resource.name, error, sampling.priority, ...) route to span +# fields/directives, not tag storage — they live in the per-language overlay, not here. +# ---------------------------------------------------------------------------