From f1d46cfe83047e788b9e22612c0a5a2e38c96b62 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Thu, 10 Sep 2026 11:00:34 -0400 Subject: [PATCH 1/5] Emit gen_ai.* attributes on APM spans Write the scalar gen_ai.* subset of an LLM Observability span onto the APM span, so model, provider, application, conversation and token usage become indexed, searchable APM tags. Co-Authored-By: Claude Opus 5 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 8 + .../domain/DDLLMObsSpanGenAiApmTagsTest.java | 128 ++++++++++++++ .../openai_java/OpenAiDecorator.java | 3 + .../groovy/ChatCompletionServiceTest.groovy | 10 ++ .../test/groovy/CompletionServiceTest.groovy | 7 + .../test/groovy/EmbeddingServiceTest.groovy | 6 + .../test/groovy/ResponseServiceTest.groovy | 9 + .../trace/api/llmobs/GenAiApmTags.java | 118 +++++++++++++ .../trace/api/llmobs/GenAiApmTagsTest.java | 164 ++++++++++++++++++ 9 files changed, 453 insertions(+) create mode 100644 dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java create mode 100644 internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java create mode 100644 internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index 473d118cdc3..a096a7d532a 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -6,6 +6,7 @@ import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.DDTraceId; import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.GenAiApmTags; import datadog.trace.api.llmobs.LLMObs; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.llmobs.LLMObsSampler; @@ -691,6 +692,13 @@ public void finish() { if (finished) { return; } + // Ahead of span.finish() while the span is still mutable, and in its own try/catch so a + // failure here cannot cost the LLM Observability event. + try { + GenAiApmTags.apply(span); + } catch (Throwable t) { + LOGGER.debug("failed to set gen_ai APM tags", t); + } span.finish(); if (standaloneApmScope != null) { standaloneApmScope.close(); diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java new file mode 100644 index 00000000000..00680cb083c --- /dev/null +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java @@ -0,0 +1,128 @@ +package datadog.trace.llmobs.domain; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.trace.agent.tooling.TracerInstaller; +import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.GenAiApmTags; +import datadog.trace.api.llmobs.LLMObsTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.CoreTracer; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Covers the scalar {@code gen_ai.*} attributes emitted onto the APM span at finish, which make + * model, provider, application, conversation and token usage searchable in APM. + */ +class DDLLMObsSpanGenAiApmTagsTest { + private static final Field SPAN_FIELD; + + private static CoreTracer tracer; + + static { + try { + SPAN_FIELD = DDLLMObsSpan.class.getDeclaredField("span"); + SPAN_FIELD.setAccessible(true); + } catch (ReflectiveOperationException error) { + throw new ExceptionInInitializerError(error); + } + } + + @BeforeAll + static void installTracer() { + tracer = CoreTracer.builder().build(); + TracerInstaller.forceInstallGlobalTracer(tracer); + } + + @AfterAll + static void closeTracer() { + TracerInstaller.forceInstallGlobalTracer(null); + tracer.close(); + } + + @Test + void llmSpanEmitsEveryScalarAndTokenUsage() { + DDLLMObsSpan llm = llmObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "chat", "sess-1"); + llm.setTag(LLMObsTags.MODEL_NAME, "gpt-4"); + llm.setTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + Map metrics = new HashMap<>(); + metrics.put("input_tokens", 10); + metrics.put("output_tokens", 20); + metrics.put("total_tokens", 30); + metrics.put("cache_read_input_tokens", 4); + metrics.put("cache_write_input_tokens", 5); + metrics.put("reasoning_output_tokens", 6); + llm.setMetrics(metrics); + llm.finish(); + + AgentSpan span = spanOf(llm); + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, span.getTag(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", span.getTag(GenAiApmTags.PROVIDER_NAME)); + assertEquals("test-ml-app", span.getTag(GenAiApmTags.APPLICATION_NAME)); + assertEquals("sess-1", span.getTag(GenAiApmTags.CONVERSATION_ID)); + assertEquals(10.0, span.getTag(GenAiApmTags.USAGE_INPUT_TOKENS)); + assertEquals(20.0, span.getTag(GenAiApmTags.USAGE_OUTPUT_TOKENS)); + assertEquals(30.0, span.getTag(GenAiApmTags.USAGE_TOTAL_TOKENS)); + assertEquals(4.0, span.getTag(GenAiApmTags.USAGE_CACHE_READ_INPUT_TOKENS)); + assertEquals(5.0, span.getTag(GenAiApmTags.USAGE_CACHE_WRITE_INPUT_TOKENS)); + assertEquals(6.0, span.getTag(GenAiApmTags.USAGE_REASONING_OUTPUT_TOKENS)); + } + + @Test + void modelBackedSpanWithoutModelFallsBackToCustom() { + DDLLMObsSpan embedding = llmObsSpan(Tags.LLMOBS_EMBEDDING_SPAN_KIND, "embed", null); + embedding.finish(); + + AgentSpan span = spanOf(embedding); + assertEquals("custom", span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", span.getTag(GenAiApmTags.PROVIDER_NAME)); + assertNull(span.getTag(GenAiApmTags.CONVERSATION_ID)); + } + + @Test + void nonModelBackedSpanEmitsNoModelFieldsOrTokenUsage() { + DDLLMObsSpan workflow = llmObsSpan(Tags.LLMOBS_WORKFLOW_SPAN_KIND, "flow", null); + workflow.setMetric("input_tokens", 10); + workflow.finish(); + + AgentSpan span = spanOf(workflow); + assertEquals(Tags.LLMOBS_WORKFLOW_SPAN_KIND, span.getTag(GenAiApmTags.OPERATION_NAME)); + assertNull(span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertNull(span.getTag(GenAiApmTags.PROVIDER_NAME)); + assertNull(span.getTag(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void nonModelBackedSpanKeepsExplicitModelFields() { + DDLLMObsSpan agent = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "agent", null); + agent.setTag(LLMObsTags.MODEL_NAME, "gpt-4"); + agent.setTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + agent.finish(); + + AgentSpan span = spanOf(agent); + assertEquals("gpt-4", span.getTag(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", span.getTag(GenAiApmTags.PROVIDER_NAME)); + } + + private static DDLLMObsSpan llmObsSpan(String kind, String name, String sessionId) { + WellKnownTags tags = + new WellKnownTags("runtime-id", "hostname", "test", "service", "version", "java"); + return new DDLLMObsSpan(kind, name, "test-ml-app", sessionId, "service", tags); + } + + private static AgentSpan spanOf(DDLLMObsSpan llmObsSpan) { + try { + return (AgentSpan) SPAN_FIELD.get(llmObsSpan); + } catch (IllegalAccessException error) { + throw new AssertionError(error); + } + } +} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 796490bdda5..b7b1bff9029 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -9,6 +9,7 @@ import datadog.trace.api.DDTags; import datadog.trace.api.DDTraceApiInfo; import datadog.trace.api.WellKnownTags; +import datadog.trace.api.llmobs.GenAiApmTags; import datadog.trace.api.llmobs.LLMObsContext; import datadog.trace.api.llmobs.LLMObsSampler; import datadog.trace.api.telemetry.LLMObsMetricCollector; @@ -179,6 +180,8 @@ protected void doBeforeFinish(@Nonnull Context context) { span.setTag(CommonTags.ERROR, span.isError() ? 1 : 0); span.setTag(CommonTags.ERROR_TYPE, span.getTag(DDTags.ERROR_TYPE)); + GenAiApmTags.apply(span); + Object spanKindTag = span.getTag(CommonTags.SPAN_KIND); if (spanKindTag != null) { String spanKind = spanKindTag.toString(); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 0a8553e7930..4a4d06e4784 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -401,6 +401,16 @@ class ChatCompletionServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "llm" + "gen_ai.request.model" String + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + if (!isStreaming) { + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.output_tokens" Double + "gen_ai.usage.total_tokens" Double + "gen_ai.usage.cache_read_input_tokens" Double + } "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String if (expectToolDefinitions) { diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy index 50529ee02a6..8723549ab00 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/CompletionServiceTest.groovy @@ -183,6 +183,13 @@ class CompletionServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "llm" + "gen_ai.request.model" String + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.output_tokens" Double + "gen_ai.usage.total_tokens" Double "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy index 41e341284f3..0d718ee5e3d 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/EmbeddingServiceTest.groovy @@ -76,6 +76,12 @@ class EmbeddingServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "embedding" + "gen_ai.request.model" "text-embedding-ada-002-v2" + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.total_tokens" Double "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String "$CommonTags.SOURCE" "integration" diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy index b22389a4d41..c455bffdd96 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ResponseServiceTest.groovy @@ -478,6 +478,15 @@ class ResponseServiceTest extends OpenAiTest { "_ml_obs_tag.sampling_decision" "1" "_ml_obs_tag.sample_rate" "1" "_ml_obs_tag.ml_app" String + "gen_ai.operation.name" "llm" + "gen_ai.request.model" String + "gen_ai.provider.name" "openai" + "gen_ai.application.name" String + "gen_ai.usage.input_tokens" Double + "gen_ai.usage.output_tokens" Double + "gen_ai.usage.total_tokens" Double + "gen_ai.usage.reasoning_output_tokens" Double + "gen_ai.usage.cache_read_input_tokens" Double "$CommonTags.INTEGRATION" "openai" "_ml_obs_tag.service" String "$CommonTags.DDTRACE_VERSION" String diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java new file mode 100644 index 00000000000..076fdd09f42 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -0,0 +1,118 @@ +package datadog.trace.api.llmobs; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.Locale; + +/** + * Emits the scalar {@code gen_ai.*} attributes of an LLM Observability span onto the APM span, so + * model, provider, application, conversation and token usage are indexed and searchable in APM. + * + *

Message bodies (input, output, tool definitions, retrieval documents) are deliberately left + * off the APM span and keep coming from the LLM Observability track. + * + *

Values are read back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} tags rather than + * passed in, because they are stamped by several decorators over a span's lifetime and are only all + * present at finish time. + */ +public final class GenAiApmTags { + public static final String OPERATION_NAME = "gen_ai.operation.name"; + public static final String REQUEST_MODEL = "gen_ai.request.model"; + public static final String PROVIDER_NAME = "gen_ai.provider.name"; + public static final String APPLICATION_NAME = "gen_ai.application.name"; + public static final String CONVERSATION_ID = "gen_ai.conversation.id"; + + public static final String USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + public static final String USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"; + public static final String USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"; + public static final String USAGE_CACHE_WRITE_INPUT_TOKENS = + "gen_ai.usage.cache_write_input_tokens"; + public static final String USAGE_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning_output_tokens"; + + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + private static final String SPAN_KIND_TAG = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND; + private static final String MODEL_NAME_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_NAME; + private static final String MODEL_PROVIDER_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.MODEL_PROVIDER; + private static final String ML_APP_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP; + private static final String SESSION_ID_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID; + + /** Matches the fallback the LLM Observability event uses for model-backed spans. */ + private static final String DEFAULT_MODEL = "custom"; + + /** LLM Observability metric name paired with the {@code gen_ai.usage.*} key it maps to. */ + private static final String[][] TOKEN_METRICS = { + {LLMOBS_METRIC_PREFIX + "input_tokens", USAGE_INPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "output_tokens", USAGE_OUTPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "total_tokens", USAGE_TOTAL_TOKENS}, + {LLMOBS_METRIC_PREFIX + "cache_read_input_tokens", USAGE_CACHE_READ_INPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "cache_write_input_tokens", USAGE_CACHE_WRITE_INPUT_TOKENS}, + {LLMOBS_METRIC_PREFIX + "reasoning_output_tokens", USAGE_REASONING_OUTPUT_TOKENS}, + }; + + /** + * Writes the {@code gen_ai.*} attributes onto the given span. Must be called before the span is + * finished, and is a no-op for a span that carries no LLM Observability span kind. + */ + public static void apply(AgentSpan span) { + if (span == null) { + return; + } + String spanKind = stringTag(span, SPAN_KIND_TAG); + if (spanKind == null) { + return; + } + span.setTag(OPERATION_NAME, spanKind); + + String modelName = stringTag(span, MODEL_NAME_TAG); + String modelProvider = stringTag(span, MODEL_PROVIDER_TAG); + boolean modelBacked = + Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) + || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); + if (modelBacked) { + span.setTag(REQUEST_MODEL, modelName == null ? DEFAULT_MODEL : modelName); + span.setTag( + PROVIDER_NAME, + (modelProvider == null ? DEFAULT_MODEL : modelProvider).toLowerCase(Locale.ROOT)); + } else { + if (modelName != null) { + span.setTag(REQUEST_MODEL, modelName); + } + if (modelProvider != null) { + span.setTag(PROVIDER_NAME, modelProvider.toLowerCase(Locale.ROOT)); + } + } + + String mlApp = stringTag(span, ML_APP_TAG); + if (mlApp != null) { + span.setTag(APPLICATION_NAME, mlApp); + } + String sessionId = stringTag(span, SESSION_ID_TAG); + if (sessionId != null) { + span.setTag(CONVERSATION_ID, sessionId); + } + + // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. + if (modelBacked) { + for (String[] metric : TOKEN_METRICS) { + Object value = span.getTag(metric[0]); + if (value instanceof Number) { + span.setMetric(metric[1], ((Number) value).doubleValue()); + } + } + } + } + + private static String stringTag(AgentSpan span, String key) { + Object value = span.getTag(key); + if (value == null) { + return null; + } + String string = String.valueOf(value); + return string.isEmpty() ? null : string; + } + + private GenAiApmTags() {} +} diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java new file mode 100644 index 00000000000..c257168ee1e --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -0,0 +1,164 @@ +package datadog.trace.api.llmobs; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class GenAiApmTagsTest { + private static final String LLMOBS_TAG_PREFIX = "_ml_obs_tag."; + private static final String LLMOBS_METRIC_PREFIX = "_ml_obs_metric."; + + private final Map tags = new HashMap<>(); + private AgentSpan span; + + @BeforeEach + void setUp() { + span = mock(AgentSpan.class); + when(span.getTag(anyString())).thenAnswer(call -> tags.get(call.getArgument(0))); + when(span.setTag(anyString(), anyString())) + .thenAnswer( + call -> { + tags.put(call.getArgument(0), call.getArgument(1)); + return span; + }); + when(span.setMetric(any(CharSequence.class), anyDouble())) + .thenAnswer( + call -> { + tags.put(call.getArgument(0).toString(), call.getArgument(1)); + return span; + }); + } + + @Test + void llmSpanEmitsEveryScalarAndTokenUsage() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-4"); + llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + llmObsTag(LLMObsTags.ML_APP, "my-app"); + llmObsTag(LLMObsTags.SESSION_ID, "sess-1"); + llmObsMetric("input_tokens", 10); + llmObsMetric("output_tokens", 20); + llmObsMetric("total_tokens", 30); + llmObsMetric("cache_read_input_tokens", 4); + llmObsMetric("cache_write_input_tokens", 5); + llmObsMetric("reasoning_output_tokens", 6); + + GenAiApmTags.apply(span); + + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); + assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); + assertEquals("sess-1", tags.get(GenAiApmTags.CONVERSATION_ID)); + assertEquals(10.0, tags.get(GenAiApmTags.USAGE_INPUT_TOKENS)); + assertEquals(20.0, tags.get(GenAiApmTags.USAGE_OUTPUT_TOKENS)); + assertEquals(30.0, tags.get(GenAiApmTags.USAGE_TOTAL_TOKENS)); + assertEquals(4.0, tags.get(GenAiApmTags.USAGE_CACHE_READ_INPUT_TOKENS)); + assertEquals(5.0, tags.get(GenAiApmTags.USAGE_CACHE_WRITE_INPUT_TOKENS)); + assertEquals(6.0, tags.get(GenAiApmTags.USAGE_REASONING_OUTPUT_TOKENS)); + } + + @Test + void modelBackedSpanWithoutModelFallsBackToCustom() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_EMBEDDING_SPAN_KIND); + + GenAiApmTags.apply(span); + + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void emptyModelValuesAreTreatedAsAbsent() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, ""); + llmObsTag(LLMObsTags.MODEL_PROVIDER, ""); + llmObsTag(LLMObsTags.ML_APP, ""); + llmObsTag(LLMObsTags.SESSION_ID, ""); + + GenAiApmTags.apply(span); + + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + assertNull(tags.get(GenAiApmTags.APPLICATION_NAME)); + assertNull(tags.get(GenAiApmTags.CONVERSATION_ID)); + } + + @Test + void nonModelBackedSpanKeepsExplicitModelFieldsWithoutFallbacks() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_AGENT_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-4"); + llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + + GenAiApmTags.apply(span); + + assertEquals(Tags.LLMOBS_AGENT_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void nonModelBackedSpanWithoutModelEmitsNoModelFields() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_WORKFLOW_SPAN_KIND); + + GenAiApmTags.apply(span); + + assertFalse(tags.containsKey(GenAiApmTags.REQUEST_MODEL)); + assertFalse(tags.containsKey(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void nonModelBackedSpanDropsTokenUsage() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_TASK_SPAN_KIND); + llmObsMetric("input_tokens", 10); + + GenAiApmTags.apply(span); + + assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void nonNumericTokenMetricIsSkipped() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_LLM_SPAN_KIND); + tags.put(LLMOBS_METRIC_PREFIX + "input_tokens", "not-a-number"); + + GenAiApmTags.apply(span); + + assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void spanWithoutLlmObsKindEmitsNothing() { + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-4"); + + GenAiApmTags.apply(span); + + assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); + } + + @Test + void nullSpanIsANoOp() { + GenAiApmTags.apply(null); + } + + private void llmObsTag(String key, String value) { + tags.put(LLMOBS_TAG_PREFIX + key, value); + } + + private void llmObsMetric(String key, Number value) { + tags.put(LLMOBS_METRIC_PREFIX + key, value); + } +} From abae3521358f7c01163d2824a16d6658f74d9d27 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 10:26:20 -0400 Subject: [PATCH 2/5] Emit available gen_ai.* attributes with LLM Observability off The openai-java instrumentation keeps tracing when LLM Observability is disabled, and the operation, model, provider and ml_app are all known on that path. Token usage and conversation id are not computed there, so they stay unreported. Co-Authored-By: Claude Opus 5 --- .../openai_java/OpenAiDecorator.java | 40 +++++++++ .../groovy/LlmObsDisabledForkedTest.groovy | 49 +++++++++++ .../trace/api/llmobs/GenAiApmTags.java | 84 ++++++++++++++----- .../trace/api/llmobs/GenAiApmTagsTest.java | 32 +++++++ 4 files changed, 183 insertions(+), 22 deletions(-) create mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index b7b1bff9029..8da3a318621 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -17,6 +17,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.ClientDecorator; import java.util.List; @@ -38,6 +39,8 @@ public class OpenAiDecorator extends ClientDecorator { private static final String TOKENS_LIMIT_METRIC = METRIC_PREFIX + "tokens.limit"; private static final String TOKENS_REMAINING_METRIC = METRIC_PREFIX + "tokens.remaining"; + private static final String EMBEDDINGS_ENDPOINT = "/v1/embeddings"; + private static final String HEADER_PREFIX = "x-ratelimit-"; private static final String LIMIT_REQUESTS_HEADER = HEADER_PREFIX + "limit-requests"; private static final String REMAINING_REQUESTS_HEADER = HEADER_PREFIX + "remaining-requests"; @@ -189,10 +192,47 @@ protected void doBeforeFinish(@Nonnull Context context) { LLMObsMetricCollector.get() .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } + } else if (span != null) { + // Tracing still runs with LLM Observability off, and model, provider and ml_app are known + // here. Token usage and session id are not computed on this path, so they stay unreported. + GenAiApmTags.applyWithoutLlmObs( + span, + operationName(span), + requestedModel(span), + stringTag(span, CommonTags.MODEL_PROVIDER), + Config.get().getLlmObsMlApp()); } super.doBeforeFinish(context); } + /** + * The LLM Observability span kind the endpoint maps to, or null if the span traced no request. + */ + private static String operationName(AgentSpan span) { + String endpoint = stringTag(span, CommonTags.OPENAI_REQUEST_ENDPOINT); + if (endpoint == null) { + return null; + } + return EMBEDDINGS_ENDPOINT.equals(endpoint) + ? Tags.LLMOBS_EMBEDDING_SPAN_KIND + : Tags.LLMOBS_LLM_SPAN_KIND; + } + + /** Prefers the model the response reports, which resolves aliases the request used. */ + private static String requestedModel(AgentSpan span) { + String model = stringTag(span, CommonTags.OPENAI_RESPONSE_MODEL); + return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); + } + + private static String stringTag(AgentSpan span, String key) { + Object value = span.getTag(key); + if (value == null) { + return null; + } + String string = value.toString(); + return string.isEmpty() ? null : string; + } + public void withHttpResponse(AgentSpan span, Headers headers) { if (!llmObsEnabled) { return; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy new file mode 100644 index 00000000000..1cc9d30fa29 --- /dev/null +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy @@ -0,0 +1,49 @@ +import datadog.trace.api.config.LlmObsConfig + +/** + * With LLM Observability off the instrumentation still traces, so the gen_ai.* attributes it can + * resolve without the LLMObs pipeline are emitted. Forked because OpenAiDecorator reads the + * llmobs.enabled flag once, into a final field, when the class is loaded. + */ +class LlmObsDisabledForkedTest extends OpenAiTest { + + @Override + void configurePreAgent() { + super.configurePreAgent() + injectSysConfig(LlmObsConfig.LLMOBS_ENABLED, "false") + } + + def "chat completion emits the gen_ai attributes available without LLMObs"() { + when: + openAiClient.chat().completions().create(chatCompletionCreateParams(false)) + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } + + then: + span.getTag("gen_ai.operation.name") == "llm" + span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") + span.getTag("gen_ai.provider.name") == "openai" + span.getTag("gen_ai.application.name") != null + + and: "neither is computed with LLMObs off" + span.getTag("gen_ai.conversation.id") == null + span.getTag("gen_ai.usage.input_tokens") == null + span.getTag("gen_ai.usage.output_tokens") == null + span.getTag("gen_ai.usage.total_tokens") == null + + and: "the LLMObs track stays off the span" + span.getTag("_ml_obs_tag.span.kind") == null + } + + def "embedding maps to the embedding operation"() { + when: + openAiClient.embeddings().create(embeddingCreateParams(false)) + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } + + then: + span.getTag("gen_ai.operation.name") == "embedding" + span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") + span.getTag("gen_ai.provider.name") == "openai" + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 076fdd09f42..46824594234 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -11,9 +11,10 @@ *

Message bodies (input, output, tool definitions, retrieval documents) are deliberately left * off the APM span and keep coming from the LLM Observability track. * - *

Values are read back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} tags rather than - * passed in, because they are stamped by several decorators over a span's lifetime and are only all - * present at finish time. + *

{@link #apply} reads the values back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} + * tags rather than taking them as arguments, because they are stamped by several decorators over a + * span's lifetime and are only all present at finish time. {@link #applyWithoutLlmObs} covers the + * instrumentation that still traces with LLM Observability off, where those tags do not exist. */ public final class GenAiApmTags { public static final String OPERATION_NAME = "gen_ai.operation.name"; @@ -64,14 +65,56 @@ public static void apply(AgentSpan span) { if (spanKind == null) { return; } - span.setTag(OPERATION_NAME, spanKind); - - String modelName = stringTag(span, MODEL_NAME_TAG); - String modelProvider = stringTag(span, MODEL_PROVIDER_TAG); - boolean modelBacked = - Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) - || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); - if (modelBacked) { + setScalars( + span, + spanKind, + stringTag(span, MODEL_NAME_TAG), + stringTag(span, MODEL_PROVIDER_TAG), + stringTag(span, ML_APP_TAG), + stringTag(span, SESSION_ID_TAG)); + + // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. + if (isModelBacked(spanKind)) { + for (String[] metric : TOKEN_METRICS) { + Object value = span.getTag(metric[0]); + if (value instanceof Number) { + span.setMetric(metric[1], ((Number) value).doubleValue()); + } + } + } + } + + /** + * Writes the subset of {@code gen_ai.*} attributes that instrumentation can supply with LLM + * Observability disabled. Token usage and conversation id are left out: neither is computed on + * that path, so there is nothing to report. + * + * @param operationName the LLM Observability span kind this operation maps to + */ + public static void applyWithoutLlmObs( + AgentSpan span, String operationName, String modelName, String modelProvider, String mlApp) { + if (span == null || operationName == null) { + return; + } + setScalars( + span, + operationName, + emptyToNull(modelName), + emptyToNull(modelProvider), + emptyToNull(mlApp), + null); + } + + private static void setScalars( + AgentSpan span, + String operationName, + String modelName, + String modelProvider, + String mlApp, + String sessionId) { + span.setTag(OPERATION_NAME, operationName); + + if (isModelBacked(operationName)) { span.setTag(REQUEST_MODEL, modelName == null ? DEFAULT_MODEL : modelName); span.setTag( PROVIDER_NAME, @@ -85,24 +128,21 @@ public static void apply(AgentSpan span) { } } - String mlApp = stringTag(span, ML_APP_TAG); if (mlApp != null) { span.setTag(APPLICATION_NAME, mlApp); } - String sessionId = stringTag(span, SESSION_ID_TAG); if (sessionId != null) { span.setTag(CONVERSATION_ID, sessionId); } + } - // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. - if (modelBacked) { - for (String[] metric : TOKEN_METRICS) { - Object value = span.getTag(metric[0]); - if (value instanceof Number) { - span.setMetric(metric[1], ((Number) value).doubleValue()); - } - } - } + private static boolean isModelBacked(String spanKind) { + return Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) + || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); + } + + private static String emptyToNull(String value) { + return value == null || value.isEmpty() ? null : value; } private static String stringTag(AgentSpan span, String key) { diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java index c257168ee1e..81fa87c0c6c 100644 --- a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -154,6 +154,38 @@ void nullSpanIsANoOp() { GenAiApmTags.apply(null); } + @Test + void withoutLlmObsEmitsScalarsButNoUsageOrConversation() { + GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "OpenAI", "my-app"); + + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("openai", tags.get(GenAiApmTags.PROVIDER_NAME)); + assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); + assertFalse(tags.containsKey(GenAiApmTags.CONVERSATION_ID)); + assertFalse(tags.containsKey(GenAiApmTags.USAGE_INPUT_TOKENS)); + } + + @Test + void withoutLlmObsFallsBackToCustomForModelBackedKinds() { + GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "", "app"); + + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + } + + @Test + void withoutLlmObsIgnoresSpanWithNoOperationName() { + GenAiApmTags.applyWithoutLlmObs(span, null, "gpt-4", "openai", "my-app"); + + assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); + } + + @Test + void withoutLlmObsNullSpanIsANoOp() { + GenAiApmTags.applyWithoutLlmObs(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "openai", "app"); + } + private void llmObsTag(String key, String value) { tags.put(LLMOBS_TAG_PREFIX + key, value); } From 1527dca26575f4748d1f76ef7620d0a50d27df18 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 11:01:46 -0400 Subject: [PATCH 3/5] Rewrite the LLMObs-disabled openai test in Java The repository does not accept new .groovy files. The module already has a JUnit forked-test base for LLMObs configurations, so the coverage moves there. Co-Authored-By: Claude Opus 5 --- .../groovy/LlmObsDisabledForkedTest.groovy | 49 --------------- .../LlmObsContextPropagationForkedTest.java | 63 +++++++++++++++++++ 2 files changed, 63 insertions(+), 49 deletions(-) delete mode 100644 dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy deleted file mode 100644 index 1cc9d30fa29..00000000000 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/LlmObsDisabledForkedTest.groovy +++ /dev/null @@ -1,49 +0,0 @@ -import datadog.trace.api.config.LlmObsConfig - -/** - * With LLM Observability off the instrumentation still traces, so the gen_ai.* attributes it can - * resolve without the LLMObs pipeline are emitted. Forked because OpenAiDecorator reads the - * llmobs.enabled flag once, into a final field, when the class is loaded. - */ -class LlmObsDisabledForkedTest extends OpenAiTest { - - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(LlmObsConfig.LLMOBS_ENABLED, "false") - } - - def "chat completion emits the gen_ai attributes available without LLMObs"() { - when: - openAiClient.chat().completions().create(chatCompletionCreateParams(false)) - TEST_WRITER.waitForTraces(1) - def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } - - then: - span.getTag("gen_ai.operation.name") == "llm" - span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") - span.getTag("gen_ai.provider.name") == "openai" - span.getTag("gen_ai.application.name") != null - - and: "neither is computed with LLMObs off" - span.getTag("gen_ai.conversation.id") == null - span.getTag("gen_ai.usage.input_tokens") == null - span.getTag("gen_ai.usage.output_tokens") == null - span.getTag("gen_ai.usage.total_tokens") == null - - and: "the LLMObs track stays off the span" - span.getTag("_ml_obs_tag.span.kind") == null - } - - def "embedding maps to the embedding operation"() { - when: - openAiClient.embeddings().create(embeddingCreateParams(false)) - TEST_WRITER.waitForTraces(1) - def span = TEST_WRITER.flatten().find { it.operationName.toString() == "openai.request" } - - then: - span.getTag("gen_ai.operation.name") == "embedding" - span.getTag("gen_ai.request.model") == span.getTag("openai.response.model") - span.getTag("gen_ai.provider.name") == "openai" - } -} diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java index 66d4f6aa285..840426bd7c8 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java @@ -9,6 +9,8 @@ import com.openai.credential.BearerTokenCredential; import com.openai.models.ChatModel; import com.openai.models.chat.completions.ChatCompletionCreateParams; +import com.openai.models.embeddings.EmbeddingCreateParams; +import com.openai.models.embeddings.EmbeddingModel; import com.sun.net.httpserver.HttpServer; import datadog.context.ContextScope; import datadog.trace.agent.test.AbstractInstrumentationTest; @@ -76,6 +78,13 @@ protected static ChatCompletionCreateParams buildMinimalChatParams() { .build(); } + protected static EmbeddingCreateParams buildMinimalEmbeddingParams() { + return EmbeddingCreateParams.builder() + .model(EmbeddingModel.TEXT_EMBEDDING_ADA_002) + .input("") + .build(); + } + protected static DDSpan findSpanByOperationName(List> traces, String operationName) { return traces.stream() .flatMap(List::stream) @@ -318,3 +327,57 @@ void parentlessOpenAiRequestSpanIsDroppedAtZeroSampleRate() throws Exception { assertEquals("0", openAiSpan.getTag("_ml_obs_tag.sample_rate")); } } + +/** + * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled. + * The instrumentation still traces on that path, so operation, model, provider and application are + * resolvable, while token usage and conversation id are never computed and must stay absent. + */ +@WithConfig(key = "llmobs.enabled", value = "false") +class LlmObsDisabledForkedTest extends AbstractLlmObsOpenAiForkedTest { + + @Test + void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Exception { + try { + openAiClient.chat().completions().create(buildMinimalChatParams()); + } catch (Exception ignored) { + // Mock server returns no body — the SDK may throw on parse. The span is already created. + } + + writer.waitForTraces(1); + DDSpan openAiSpan = findSpanByOperationName(writer, "openai.request"); + assertNotNull(openAiSpan, "openai.request span should have been created"); + + assertEquals("llm", openAiSpan.getTag("gen_ai.operation.name")); + // The mock returns no body, so there is no response model and the request model stands in. + assertEquals( + openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); + assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); + assertNotNull(openAiSpan.getTag("gen_ai.application.name")); + + assertNull(openAiSpan.getTag("gen_ai.conversation.id")); + assertNull(openAiSpan.getTag("gen_ai.usage.input_tokens")); + assertNull(openAiSpan.getTag("gen_ai.usage.output_tokens")); + assertNull(openAiSpan.getTag("gen_ai.usage.total_tokens")); + + assertNull(openAiSpan.getTag("_ml_obs_tag.span.kind")); + } + + @Test + void embeddingMapsToTheEmbeddingOperation() throws Exception { + try { + openAiClient.embeddings().create(buildMinimalEmbeddingParams()); + } catch (Exception ignored) { + // Mock server returns no body — the SDK may throw on parse. The span is already created. + } + + writer.waitForTraces(1); + DDSpan openAiSpan = findSpanByOperationName(writer, "openai.request"); + assertNotNull(openAiSpan, "openai.request span should have been created"); + + assertEquals("embedding", openAiSpan.getTag("gen_ai.operation.name")); + assertEquals( + openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); + assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); + } +} From faac2ec6f81098d36e1d5f50964c7bc59b1adb23 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 13:35:41 -0400 Subject: [PATCH 4/5] Trim comments and docstrings Co-Authored-By: Claude Opus 5 --- .../trace/llmobs/domain/DDLLMObsSpan.java | 3 +-- .../domain/DDLLMObsSpanGenAiApmTagsTest.java | 5 +--- .../openai_java/OpenAiDecorator.java | 8 ++---- .../LlmObsContextPropagationForkedTest.java | 11 ++++---- .../trace/api/llmobs/GenAiApmTags.java | 26 ++++++------------- 5 files changed, 17 insertions(+), 36 deletions(-) diff --git a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java index a096a7d532a..3aaac795186 100644 --- a/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java +++ b/dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java @@ -692,8 +692,7 @@ public void finish() { if (finished) { return; } - // Ahead of span.finish() while the span is still mutable, and in its own try/catch so a - // failure here cannot cost the LLM Observability event. + // While the span is still mutable, and isolated so a failure cannot cost the event. try { GenAiApmTags.apply(span); } catch (Throwable t) { diff --git a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java index 00680cb083c..7b4720699ae 100644 --- a/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java +++ b/dd-java-agent/agent-llmobs/src/test/java/datadog/trace/llmobs/domain/DDLLMObsSpanGenAiApmTagsTest.java @@ -17,10 +17,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -/** - * Covers the scalar {@code gen_ai.*} attributes emitted onto the APM span at finish, which make - * model, provider, application, conversation and token usage searchable in APM. - */ +/** Covers the {@code gen_ai.*} attributes a manual LLM Observability span emits at finish. */ class DDLLMObsSpanGenAiApmTagsTest { private static final Field SPAN_FIELD; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 8da3a318621..9ae3f33466f 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -193,8 +193,7 @@ protected void doBeforeFinish(@Nonnull Context context) { .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } } else if (span != null) { - // Tracing still runs with LLM Observability off, and model, provider and ml_app are known - // here. Token usage and session id are not computed on this path, so they stay unreported. + // Tracing still runs with LLM Observability off, where these four remain resolvable. GenAiApmTags.applyWithoutLlmObs( span, operationName(span), @@ -205,9 +204,6 @@ protected void doBeforeFinish(@Nonnull Context context) { super.doBeforeFinish(context); } - /** - * The LLM Observability span kind the endpoint maps to, or null if the span traced no request. - */ private static String operationName(AgentSpan span) { String endpoint = stringTag(span, CommonTags.OPENAI_REQUEST_ENDPOINT); if (endpoint == null) { @@ -218,7 +214,7 @@ private static String operationName(AgentSpan span) { : Tags.LLMOBS_LLM_SPAN_KIND; } - /** Prefers the model the response reports, which resolves aliases the request used. */ + /** The response model resolves aliases the request used, so it wins. */ private static String requestedModel(AgentSpan span) { String model = stringTag(span, CommonTags.OPENAI_RESPONSE_MODEL); return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java index 840426bd7c8..16830ca0d2a 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/java/datadog/trace/instrumentation/openai_java/LlmObsContextPropagationForkedTest.java @@ -329,9 +329,8 @@ void parentlessOpenAiRequestSpanIsDroppedAtZeroSampleRate() throws Exception { } /** - * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled. - * The instrumentation still traces on that path, so operation, model, provider and application are - * resolvable, while token usage and conversation id are never computed and must stay absent. + * Verifies the gen_ai.* attributes an openai.request span carries with LLM Observability disabled: + * operation, model, provider and application, but never token usage or conversation id. */ @WithConfig(key = "llmobs.enabled", value = "false") class LlmObsDisabledForkedTest extends AbstractLlmObsOpenAiForkedTest { @@ -341,7 +340,7 @@ void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Except try { openAiClient.chat().completions().create(buildMinimalChatParams()); } catch (Exception ignored) { - // Mock server returns no body — the SDK may throw on parse. The span is already created. + // The mock server returns no body, so the SDK may throw while parsing the response. } writer.waitForTraces(1); @@ -349,7 +348,7 @@ void chatCompletionEmitsTheGenAiAttributesAvailableWithoutLlmObs() throws Except assertNotNull(openAiSpan, "openai.request span should have been created"); assertEquals("llm", openAiSpan.getTag("gen_ai.operation.name")); - // The mock returns no body, so there is no response model and the request model stands in. + // The mock returns no body, so the request model stands in for the absent response model. assertEquals( openAiSpan.getTag("openai.request.model"), openAiSpan.getTag("gen_ai.request.model")); assertEquals("openai", openAiSpan.getTag("gen_ai.provider.name")); @@ -368,7 +367,7 @@ void embeddingMapsToTheEmbeddingOperation() throws Exception { try { openAiClient.embeddings().create(buildMinimalEmbeddingParams()); } catch (Exception ignored) { - // Mock server returns no body — the SDK may throw on parse. The span is already created. + // The mock server returns no body, so the SDK may throw while parsing the response. } writer.waitForTraces(1); diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 46824594234..9a4aabe331e 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -6,15 +6,8 @@ /** * Emits the scalar {@code gen_ai.*} attributes of an LLM Observability span onto the APM span, so - * model, provider, application, conversation and token usage are indexed and searchable in APM. - * - *

Message bodies (input, output, tool definitions, retrieval documents) are deliberately left - * off the APM span and keep coming from the LLM Observability track. - * - *

{@link #apply} reads the values back from the {@code _ml_obs_tag.} / {@code _ml_obs_metric.} - * tags rather than taking them as arguments, because they are stamped by several decorators over a - * span's lifetime and are only all present at finish time. {@link #applyWithoutLlmObs} covers the - * instrumentation that still traces with LLM Observability off, where those tags do not exist. + * model, provider, application, conversation and token usage are searchable in APM. Message bodies + * stay off the APM span and keep coming from the LLM Observability track. */ public final class GenAiApmTags { public static final String OPERATION_NAME = "gen_ai.operation.name"; @@ -40,7 +33,7 @@ public final class GenAiApmTags { private static final String ML_APP_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.ML_APP; private static final String SESSION_ID_TAG = LLMOBS_TAG_PREFIX + LLMObsTags.SESSION_ID; - /** Matches the fallback the LLM Observability event uses for model-backed spans. */ + /** Matches the fallback the LLM Observability event uses. */ private static final String DEFAULT_MODEL = "custom"; /** LLM Observability metric name paired with the {@code gen_ai.usage.*} key it maps to. */ @@ -54,8 +47,8 @@ public final class GenAiApmTags { }; /** - * Writes the {@code gen_ai.*} attributes onto the given span. Must be called before the span is - * finished, and is a no-op for a span that carries no LLM Observability span kind. + * Writes the attributes onto a span that is not yet finished, reading them back from its {@code + * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. No-op for a span with no LLM Observability kind. */ public static void apply(AgentSpan span) { if (span == null) { @@ -73,7 +66,7 @@ public static void apply(AgentSpan span) { stringTag(span, ML_APP_TAG), stringTag(span, SESSION_ID_TAG)); - // Other span kinds carry unrelated metrics that would be misleading under a gen_ai.usage.* key. + // Other kinds carry unrelated metrics that a gen_ai.usage.* key would misrepresent. if (isModelBacked(spanKind)) { for (String[] metric : TOKEN_METRICS) { Object value = span.getTag(metric[0]); @@ -85,11 +78,8 @@ public static void apply(AgentSpan span) { } /** - * Writes the subset of {@code gen_ai.*} attributes that instrumentation can supply with LLM - * Observability disabled. Token usage and conversation id are left out: neither is computed on - * that path, so there is nothing to report. - * - * @param operationName the LLM Observability span kind this operation maps to + * Writes the subset available with LLM Observability disabled. Token usage and conversation id + * are never computed on that path, so they are left out. */ public static void applyWithoutLlmObs( AgentSpan span, String operationName, String modelName, String modelProvider, String mlApp) { From 324b5bbd7fd30def57807b1df8a16dfe14834ba0 Mon Sep 17 00:00:00 2001 From: Max Zhang Date: Fri, 11 Sep 2026 16:32:02 -0400 Subject: [PATCH 5/5] Address review: single apply method and shared tag reader Co-Authored-By: Claude Opus 5 --- .../openai_java/OpenAiDecorator.java | 20 +--- .../groovy/ChatCompletionServiceTest.groovy | 1 + .../trace/api/llmobs/GenAiApmTags.java | 111 +++++++----------- .../trace/api/llmobs/GenAiApmTagsTest.java | 33 ++++-- 4 files changed, 67 insertions(+), 98 deletions(-) diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java index 9ae3f33466f..a31cf5c95b3 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/main/java/datadog/trace/instrumentation/openai_java/OpenAiDecorator.java @@ -1,5 +1,6 @@ package datadog.trace.instrumentation.openai_java; +import static datadog.trace.api.llmobs.GenAiApmTags.stringTag; import static datadog.trace.bootstrap.instrumentation.api.AgentSpan.fromContext; import com.openai.core.ClientOptions; @@ -193,13 +194,9 @@ protected void doBeforeFinish(@Nonnull Context context) { .recordSpanFinished(INTEGRATION, spanKind, isRootSpan, true, span.isError(), false); } } else if (span != null) { - // Tracing still runs with LLM Observability off, where these four remain resolvable. - GenAiApmTags.applyWithoutLlmObs( - span, - operationName(span), - requestedModel(span), - stringTag(span, CommonTags.MODEL_PROVIDER), - Config.get().getLlmObsMlApp()); + // Tracing still runs with LLM Observability off, where these remain resolvable. + GenAiApmTags.apply( + span, operationName(span), requestedModel(span), Config.get().getLlmObsMlApp()); } super.doBeforeFinish(context); } @@ -220,15 +217,6 @@ private static String requestedModel(AgentSpan span) { return model != null ? model : stringTag(span, CommonTags.OPENAI_REQUEST_MODEL); } - private static String stringTag(AgentSpan span, String key) { - Object value = span.getTag(key); - if (value == null) { - return null; - } - String string = value.toString(); - return string.isEmpty() ? null : string; - } - public void withHttpResponse(AgentSpan span, Headers headers) { if (!llmObsEnabled) { return; diff --git a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy index 4a4d06e4784..37ba7178988 100644 --- a/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy +++ b/dd-java-agent/instrumentation/openai-java/openai-java-3.0/src/test/groovy/ChatCompletionServiceTest.groovy @@ -406,6 +406,7 @@ class ChatCompletionServiceTest extends OpenAiTest { "gen_ai.provider.name" "openai" "gen_ai.application.name" String if (!isStreaming) { + // streamed completions missing usage data "gen_ai.usage.input_tokens" Double "gen_ai.usage.output_tokens" Double "gen_ai.usage.total_tokens" Double diff --git a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java index 9a4aabe331e..1324ec98a7a 100644 --- a/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java +++ b/internal-api/src/main/java/datadog/trace/api/llmobs/GenAiApmTags.java @@ -46,96 +46,61 @@ public final class GenAiApmTags { {LLMOBS_METRIC_PREFIX + "reasoning_output_tokens", USAGE_REASONING_OUTPUT_TOKENS}, }; + public static void apply(AgentSpan span) { + apply(span, null, null, null); + } + /** - * Writes the attributes onto a span that is not yet finished, reading them back from its {@code - * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. No-op for a span with no LLM Observability kind. + * Writes the attributes onto a span that is not yet finished, reading them from its {@code + * _ml_obs_tag.} / {@code _ml_obs_metric.} tags. The arguments take precedence over those tags and + * cover instrumentation that traces with LLM Observability disabled, where they are not all set. + * No-op for a span with no resolvable operation. */ - public static void apply(AgentSpan span) { + public static void apply(AgentSpan span, String operationName, String modelName, String mlApp) { if (span == null) { return; } - String spanKind = stringTag(span, SPAN_KIND_TAG); - if (spanKind == null) { + String operation = firstNonEmpty(operationName, stringTag(span, SPAN_KIND_TAG)); + if (operation == null) { return; } - setScalars( - span, - spanKind, - stringTag(span, MODEL_NAME_TAG), - stringTag(span, MODEL_PROVIDER_TAG), - stringTag(span, ML_APP_TAG), - stringTag(span, SESSION_ID_TAG)); + span.setTag(OPERATION_NAME, operation); - // Other kinds carry unrelated metrics that a gen_ai.usage.* key would misrepresent. - if (isModelBacked(spanKind)) { - for (String[] metric : TOKEN_METRICS) { - Object value = span.getTag(metric[0]); - if (value instanceof Number) { - span.setMetric(metric[1], ((Number) value).doubleValue()); - } - } - } - } + boolean modelBacked = + Tags.LLMOBS_LLM_SPAN_KIND.equals(operation) + || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(operation); - /** - * Writes the subset available with LLM Observability disabled. Token usage and conversation id - * are never computed on that path, so they are left out. - */ - public static void applyWithoutLlmObs( - AgentSpan span, String operationName, String modelName, String modelProvider, String mlApp) { - if (span == null || operationName == null) { - return; + String model = firstNonEmpty(modelName, stringTag(span, MODEL_NAME_TAG)); + if (model != null || modelBacked) { + span.setTag(REQUEST_MODEL, model == null ? DEFAULT_MODEL : model); } - setScalars( - span, - operationName, - emptyToNull(modelName), - emptyToNull(modelProvider), - emptyToNull(mlApp), - null); - } - - private static void setScalars( - AgentSpan span, - String operationName, - String modelName, - String modelProvider, - String mlApp, - String sessionId) { - span.setTag(OPERATION_NAME, operationName); - - if (isModelBacked(operationName)) { - span.setTag(REQUEST_MODEL, modelName == null ? DEFAULT_MODEL : modelName); + String provider = stringTag(span, MODEL_PROVIDER_TAG); + if (provider != null || modelBacked) { span.setTag( - PROVIDER_NAME, - (modelProvider == null ? DEFAULT_MODEL : modelProvider).toLowerCase(Locale.ROOT)); - } else { - if (modelName != null) { - span.setTag(REQUEST_MODEL, modelName); - } - if (modelProvider != null) { - span.setTag(PROVIDER_NAME, modelProvider.toLowerCase(Locale.ROOT)); - } + PROVIDER_NAME, (provider == null ? DEFAULT_MODEL : provider).toLowerCase(Locale.ROOT)); } - - if (mlApp != null) { - span.setTag(APPLICATION_NAME, mlApp); + String application = firstNonEmpty(mlApp, stringTag(span, ML_APP_TAG)); + if (application != null) { + span.setTag(APPLICATION_NAME, application); } + String sessionId = stringTag(span, SESSION_ID_TAG); if (sessionId != null) { span.setTag(CONVERSATION_ID, sessionId); } - } - private static boolean isModelBacked(String spanKind) { - return Tags.LLMOBS_LLM_SPAN_KIND.equals(spanKind) - || Tags.LLMOBS_EMBEDDING_SPAN_KIND.equals(spanKind); - } - - private static String emptyToNull(String value) { - return value == null || value.isEmpty() ? null : value; + // Other kinds carry unrelated metrics that a gen_ai.usage.* key would misrepresent. + if (modelBacked) { + for (String[] metric : TOKEN_METRICS) { + Object value = span.getTag(metric[0]); + if (value instanceof Number) { + span.setMetric(metric[1], ((Number) value).doubleValue()); + } + } + } } - private static String stringTag(AgentSpan span, String key) { + /** The value of {@code key} as a non-empty string, or null. */ + public static String stringTag(AgentSpan span, String key) { Object value = span.getTag(key); if (value == null) { return null; @@ -144,5 +109,9 @@ private static String stringTag(AgentSpan span, String key) { return string.isEmpty() ? null : string; } + private static String firstNonEmpty(String preferred, String fallback) { + return preferred == null || preferred.isEmpty() ? fallback : preferred; + } + private GenAiApmTags() {} } diff --git a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java index 81fa87c0c6c..98783a6badd 100644 --- a/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java +++ b/internal-api/src/test/java/datadog/trace/api/llmobs/GenAiApmTagsTest.java @@ -152,11 +152,14 @@ void spanWithoutLlmObsKindEmitsNothing() { @Test void nullSpanIsANoOp() { GenAiApmTags.apply(null); + GenAiApmTags.apply(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "app"); } @Test - void withoutLlmObsEmitsScalarsButNoUsageOrConversation() { - GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "OpenAI", "my-app"); + void argumentsStandInForMissingLlmObsTags() { + llmObsTag(LLMObsTags.MODEL_PROVIDER, "OpenAI"); + + GenAiApmTags.apply(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "my-app"); assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); @@ -167,23 +170,31 @@ void withoutLlmObsEmitsScalarsButNoUsageOrConversation() { } @Test - void withoutLlmObsFallsBackToCustomForModelBackedKinds() { - GenAiApmTags.applyWithoutLlmObs(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "", "app"); + void argumentsTakePrecedenceOverLlmObsTags() { + llmObsTag(Tags.SPAN_KIND, Tags.LLMOBS_WORKFLOW_SPAN_KIND); + llmObsTag(LLMObsTags.MODEL_NAME, "gpt-3.5"); + llmObsTag(LLMObsTags.ML_APP, "tagged-app"); - assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); - assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); + GenAiApmTags.apply(span, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "my-app"); + + assertEquals(Tags.LLMOBS_LLM_SPAN_KIND, tags.get(GenAiApmTags.OPERATION_NAME)); + assertEquals("gpt-4", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("my-app", tags.get(GenAiApmTags.APPLICATION_NAME)); } @Test - void withoutLlmObsIgnoresSpanWithNoOperationName() { - GenAiApmTags.applyWithoutLlmObs(span, null, "gpt-4", "openai", "my-app"); + void modelBackedArgumentsFallBackToCustom() { + GenAiApmTags.apply(span, Tags.LLMOBS_EMBEDDING_SPAN_KIND, null, "app"); - assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); + assertEquals("custom", tags.get(GenAiApmTags.REQUEST_MODEL)); + assertEquals("custom", tags.get(GenAiApmTags.PROVIDER_NAME)); } @Test - void withoutLlmObsNullSpanIsANoOp() { - GenAiApmTags.applyWithoutLlmObs(null, Tags.LLMOBS_LLM_SPAN_KIND, "gpt-4", "openai", "app"); + void spanWithNoResolvableOperationEmitsNothing() { + GenAiApmTags.apply(span, null, "gpt-4", "my-app"); + + assertTrue(tags.keySet().stream().noneMatch(key -> key.startsWith("gen_ai."))); } private void llmObsTag(String key, String value) {