Add per-operation self-tuning dense-store sizing (phase 2)

buildSrc/build.gradle.kts

@@ -59,6 +59,11 @@ gradlePlugin {
       implementationClass = "datadog.gradle.plugin.config.SupportedConfigPlugin"
     }
 
+    create("tag-registry-generator") {
+      id = "dd-trace-java.tag-registry-generator"
+      implementationClass = "datadog.gradle.plugin.tags.TagRegistryGeneratorPlugin"
+    }
+
     create("supported-config-linter") {
       id = "dd-trace-java.config-inversion-linter"
       implementationClass = "datadog.gradle.plugin.config.ConfigInversionLinter"
@@ -107,6 +112,7 @@ dependencies {
   implementation("com.fasterxml.jackson.core:jackson-databind")
   implementation("com.fasterxml.jackson.core:jackson-annotations")
   implementation("com.fasterxml.jackson.core:jackson-core")
+  implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml")
 
   compileOnly(libs.develocity)
 }

buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/GenerateKnownTagsTask.kt

@@ -0,0 +1,38 @@
+package datadog.gradle.plugin.tags
+
+import javax.inject.Inject
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.file.RegularFileProperty
+import org.gradle.api.model.ObjectFactory
+import org.gradle.api.tasks.CacheableTask
+import org.gradle.api.tasks.InputFile
+import org.gradle.api.tasks.OutputDirectory
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.PathSensitivity
+import org.gradle.api.tasks.TaskAction
+
+/**
+ * Generates the committed tag registry (KnownTags.java + layout reports) from the language-agnostic
+ * {@code tag-conventions.yaml} + the Java overlay. The actual emit lives in [TagRegistryGenerator];
+ * this task just wires the inputs/outputs so Gradle can cache and up-to-date-check it.
+ */
+@CacheableTask
+abstract class GenerateKnownTagsTask @Inject constructor(objects: ObjectFactory) : DefaultTask() {
+  @get:InputFile
+  @get:PathSensitive(PathSensitivity.NONE)
+  val domainYaml: RegularFileProperty = objects.fileProperty()
+
+  @get:InputFile
+  @get:PathSensitive(PathSensitivity.NONE)
+  val overlayYaml: RegularFileProperty = objects.fileProperty()
+
+  @get:OutputDirectory val destinationDirectory: DirectoryProperty = objects.directoryProperty()
+
+  @TaskAction
+  fun generate() {
+    val outDir = destinationDirectory.get().asFile
+    TagRegistryGenerator.generate(domainYaml.get().asFile, overlayYaml.get().asFile, outDir)
+    logger.lifecycle("tag-registry: generated -> $outDir")
+  }
+}

buildSrc/src/main/kotlin/datadog/gradle/plugin/tags/KnownTagsEmitter.kt

@@ -0,0 +1,177 @@
+package datadog.gradle.plugin.tags
+
+import java.util.Locale
+
+/**
+ * Emits the generated `KnownTags.java` from a [TagRegistry]. Public API first — per-tag
+ * `<X>_NAME` (string) + `<X>_ID` (encoded long, literal) couplets with a trailing `// makeTagId(...)`
+ * derivation comment — then the package-private `<X>_SERIAL_NUM` constants, the
+ * `StringIndex.EmbeddingSupport` keyOf table, the `serialNum` switch `nameOf`, and resolver
+ * registration.
+ */
+object KnownTagsEmitter {
+
+  fun emit(reg: TagRegistry, pkg: String, className: String): String {
+    // Sanitize tag names into unique Java constant identifiers.
+    val used = HashSet<String>()
+    val cname = HashMap<String, String>()
+    fun mk(name: String): String {
+      var c = name.uppercase().replace(Regex("[^A-Za-z0-9]"), "_").replace(Regex("_+"), "_").trim('_')
+      if (c.isEmpty() || c[0].isDigit()) c = "T_$c"
+      var u = c
+      var n = 2
+      while (u in used) {
+        u = "${c}_$n"; n++
+      }
+      used.add(u)
+      cname[name] = u
+      return u
+    }
+    reg.reserved.forEach { mk(it.name) }
+    reg.stored.forEach { mk(it.name) }
+
+    // Constant names. Collapse a duplicated trailing token so e.g. "resource.name" yields NAME
+    // (not NAME_NAME) and "_dd.parent_id" yields ID (not ID_ID); the non-duplicating pairs
+    // (ID + _NAME -> ID_NAME, NAME + _ID -> NAME_ID) are kept as-is.
+    fun withSuffix(base: String, suffix: String) = if (base.endsWith(suffix)) base else "$base$suffix"
+    fun nameC(name: String) = withSuffix(cname[name]!!, "_NAME")
+    fun idC(name: String) = withSuffix(cname[name]!!, "_ID")
+    fun serialC(name: String) = withSuffix(cname[name]!!, "_SERIAL_NUM")
+
+    val order = reg.reserved.map { it.name } + reg.stored.map { it.name } // stable emit order
+    // canonical name -> OpenTelemetry name, for the reverse (openTelemetryNameOf) switch.
+    val otelName =
+      (reg.reserved.mapNotNull { v -> v.otelName?.let { v.name to it } } +
+          reg.stored.mapNotNull { t -> t.otelName?.let { t.name to it } })
+        .toMap()
+    val b = StringBuilder()
+    b.appendLine("package $pkg;")
+    b.appendLine()
+    b.appendLine("import datadog.trace.util.StringIndex;")
+    b.appendLine()
+    b.appendLine("// GENERATED by the tag-registry code generator (dd-trace-java.tag-registry-generator).")
+    b.appendLine("// DO NOT EDIT. Source: tag-conventions.yaml + tag-conventions.java.yaml.")
+    b.appendLine("public final class $className {")
+    b.appendLine("  static final int SLOT_COUNT = ${reg.slotCount};")
+    b.appendLine()
+
+    // Public API first (name + encoded id couplets), so readers see the useful parts up top; the
+    // serial ids and keyOf/resolver machinery follow below. Derivation is in the trailing comment.
+    b.appendLine("  // ---- reserved (routed to span fields or directives; not stored) ----")
+    for (v in reg.reserved) {
+      b.appendLine("  public static final String ${nameC(v.name)} = \"${v.name}\";")
+      b.appendLine("  public static final long ${idC(v.name)} = ${hex(v.id)};")
+      b.appendLine("  // makeTagId(serial=${v.serial}, slot=NO_SLOT) + intercepted  [${v.kind}${v.field?.let { " -> $it" } ?: ""}]")
+      b.appendLine()
+    }
+
+    b.appendLine("  // ---- stored (dense colored slot, or bucketed when slot=NO_SLOT) ----")
+    for (t in reg.stored) {
+      val slot = if (t.slotted) t.slot.toString() else "NO_SLOT"
+      b.appendLine("  public static final String ${nameC(t.name)} = \"${t.name}\";")
+      b.appendLine("  public static final long ${idC(t.name)} = ${hex(t.id)};")
+      b.appendLine("  // makeTagId(serial=${t.serial}, slot=$slot)${if (t.intercepted) " + intercepted" else ""}${if (t.traceLevel) " + trace-level" else ""}  <${t.required}>")
+      b.appendLine()
+    }
+
+    // Serial numbers (globalSerial per tag) — package-private, consumed by the resolver switch.
+    b.appendLine("  // ---- serial numbers ----")
+    for (v in reg.reserved) {
+      b.appendLine("  static final int ${serialC(v.name)} = ${v.serial};")
+    }
+    for (t in reg.stored) {
+      b.appendLine("  static final int ${serialC(t.name)} = ${t.serial};")
+    }
+    b.appendLine()
+
+    // OpenTelemetry name -> canonical tag name, for the tags that declare one. Deterministic order
+    // (by OTel name) so output stays byte-identical.
+    val otelByCanonical =
+      (reg.stored.mapNotNull { t -> t.otelName?.let { it to t.name } } +
+          reg.reserved.mapNotNull { v -> v.otelName?.let { it to v.name } })
+        .sortedBy { it.first }
+
+    // keyOf table (open-addressed, via StringIndex.EmbeddingSupport). Canonical names first, then
+    // OpenTelemetry names -- an OTel name resolves to its canonical tag's id (there is no distinct id
+    // for it), so keyOf(otelName) == keyOf(canonical); nameOf still returns the canonical name.
+    b.appendLine("  private static final String[] KEYOF_NAMES = {")
+    order.forEach { b.appendLine("    ${nameC(it)},") }
+    otelByCanonical.forEach { (otel, _) -> b.appendLine("    \"$otel\",") }
+    b.appendLine("  };")
+    b.appendLine("  private static final long[] KEYOF_VALUES = {")
+    order.forEach { b.appendLine("    ${idC(it)},") }
+    otelByCanonical.forEach { (_, canonical) -> b.appendLine("    ${idC(canonical)},") }
+    b.appendLine("  };")
+    b.appendLine("  private static final int[] KEYOF_HASHES;")
+    b.appendLine("  private static final String[] KEYOF_KEYS;")
+    b.appendLine("  private static final long[] KEYOF_IDS;")
+    b.appendLine()
+    b.appendLine("  static {")
+    b.appendLine("    StringIndex.Data data = StringIndex.EmbeddingSupport.create(KEYOF_NAMES);")
+    b.appendLine("    long[] ids = new long[data.names.length];")
+    b.appendLine("    for (int j = 0; j < KEYOF_NAMES.length; j++) {")
+    b.appendLine("      ids[StringIndex.EmbeddingSupport.indexOf(data.hashes, data.names, KEYOF_NAMES[j])] =")
+    b.appendLine("          KEYOF_VALUES[j];")
+    b.appendLine("    }")
+    b.appendLine("    KEYOF_HASHES = data.hashes;")
+    b.appendLine("    KEYOF_KEYS = data.names;")
+    b.appendLine("    KEYOF_IDS = ids;")
+    b.appendLine("  }")
+    b.appendLine()
+
+    // Resolver.
+    b.appendLine("  static final KnownTagCodec.Resolver RESOLVER =")
+    b.appendLine("      new KnownTagCodec.Resolver() {")
+    b.appendLine("        @Override")
+    b.appendLine("        public String nameOf(long tagId) {")
+    b.appendLine("          switch (KnownTagCodec.serialNum(tagId)) {")
+    for (name in order) {
+      b.appendLine("            case ${serialC(name)}:")
+      b.appendLine("              return ${nameC(name)};")
+    }
+    b.appendLine("            default:")
+    b.appendLine("              return null;")
+    b.appendLine("          }")
+    b.appendLine("        }")
+    b.appendLine()
+    // openTelemetryNameOf: canonical id -> OTel-namespace name, null when the tag has none. The
+    // caller (a serializer) owns any fall-back-to-Datadog-name policy; this stays a pure lookup.
+    b.appendLine("        @Override")
+    b.appendLine("        public String openTelemetryNameOf(long tagId) {")
+    b.appendLine("          switch (KnownTagCodec.serialNum(tagId)) {")
+    for (name in order) {
+      val otel = otelName[name] ?: continue
+      b.appendLine("            case ${serialC(name)}:")
+      b.appendLine("              return \"$otel\";")
+    }
+    b.appendLine("            default:")
+    b.appendLine("              return null;")
+    b.appendLine("          }")
+    b.appendLine("        }")
+    b.appendLine()
+    b.appendLine("        @Override")
+    b.appendLine("        public int slotCount() {")
+    b.appendLine("          return SLOT_COUNT;")
+    b.appendLine("        }")
+    b.appendLine()
+    b.appendLine("        @Override")
+    b.appendLine("        public long keyOf(String name) {")
+    b.appendLine("          int slot = StringIndex.EmbeddingSupport.indexOf(KEYOF_HASHES, KEYOF_KEYS, name);")
+    b.appendLine("          return slot < 0 ? 0L : KEYOF_IDS[slot];")
+    b.appendLine("        }")
+    b.appendLine("      };")
+    b.appendLine()
+    b.appendLine("  static {")
+    b.appendLine("    KnownTagCodec.register(RESOLVER);")
+    b.appendLine("  }")
+    b.appendLine()
+    b.appendLine("  /** Forces resolver registration by triggering <clinit>. Idempotent. */")
+    b.appendLine("  public static void init() {}")
+    b.appendLine()
+    b.appendLine("  private $className() {}")
+    b.appendLine("}")
+    return b.toString()
+  }
+
+  private fun hex(id: Long): String = "0x%016XL".format(Locale.ROOT, id)
+}