From 02f2a4d7fd6cbd7b2a67c39889e8b3a8d56b101c Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Wed, 12 Aug 2026 13:00:06 -0400 Subject: [PATCH 01/30] Populate source.name and source.version on flag_evaluations and exposures EVP FeatureFlagEvpContext.from builds the top-level context map shared by both the flagevaluation and exposures EVP writers. Add source.name ("dd-trace-java") and source.version (TracerVersion.TRACER_VERSION) so the SDK identity facets are populated on both EVP streams. This closes the gap noted in the Feature Flag Observability Telemetry Roadmap where the Java server SDK emitted no SDK/tracer name or version on the flag_evaluations EVP stream. Co-Authored-By: Claude --- .../com/datadog/featureflag/FeatureFlagEvpContext.java | 10 +++++++++- .../com/datadog/featureflag/ExposureWriterTests.java | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java index c964efa6c7f..84de91330bb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.Config; import java.util.HashMap; import java.util.Map; @@ -8,8 +9,11 @@ final class FeatureFlagEvpContext { private FeatureFlagEvpContext() {} + /** The name of the SDK emitting the feature flag evaluation/exposure EVP data. */ + private static final String SOURCE_NAME = "dd-trace-java"; + static Map from(final Config config) { - final Map context = new HashMap<>(4); + final Map context = new HashMap<>(6); context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); if (config.getEnv() != null) { context.put("env", config.getEnv()); @@ -17,6 +21,10 @@ static Map from(final Config config) { if (config.getVersion() != null) { context.put("version", config.getVersion()); } + // SDK identity — populates the `source.name` / `source.version` facets on the + // `flag_evaluations` and `exposures` EVP streams. See FFL-2995. + context.put("source.name", SOURCE_NAME); + context.put("source.version", TracerVersion.TRACER_VERSION); return context; } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 76b9e2602d8..73fee60fdc8 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -15,6 +15,7 @@ import com.squareup.moshi.Moshi; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.ddagent.TracerVersion; import datadog.trace.agent.test.server.http.JavaTestHttpServer; import datadog.trace.agent.test.server.http.JavaTestHttpServer.HandlerApi; import datadog.trace.api.Config; @@ -287,6 +288,9 @@ private static void assertContext( assertEquals(service == null ? "unknown" : service, context.get("service")); assertOptionalContextValue(context, "env", env); assertOptionalContextValue(context, "version", version); + // SDK identity populated by FeatureFlagEvpContext (FFL-2995). + assertEquals("dd-trace-java", context.get("source.name")); + assertEquals(TracerVersion.TRACER_VERSION, context.get("source.version")); } private static void assertOptionalContextValue( From 8ed8522f9ff7c1465532e9ca92fde949dc37ab5f Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Wed, 12 Aug 2026 13:12:09 -0400 Subject: [PATCH 02/30] Tighten comments: drop ticket refs and redundant javadoc --- .../java/com/datadog/featureflag/FeatureFlagEvpContext.java | 5 ++--- .../java/com/datadog/featureflag/ExposureWriterTests.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java index 84de91330bb..181dab8495b 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java @@ -9,7 +9,6 @@ final class FeatureFlagEvpContext { private FeatureFlagEvpContext() {} - /** The name of the SDK emitting the feature flag evaluation/exposure EVP data. */ private static final String SOURCE_NAME = "dd-trace-java"; static Map from(final Config config) { @@ -21,8 +20,8 @@ static Map from(final Config config) { if (config.getVersion() != null) { context.put("version", config.getVersion()); } - // SDK identity — populates the `source.name` / `source.version` facets on the - // `flag_evaluations` and `exposures` EVP streams. See FFL-2995. + // SDK identity — populates the `source.name` / `source.version` facets on both the + // `flag_evaluations` and `exposures` EVP streams (this map is shared by both writers). context.put("source.name", SOURCE_NAME); context.put("source.version", TracerVersion.TRACER_VERSION); return context; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 73fee60fdc8..4c47133d8d7 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -288,7 +288,7 @@ private static void assertContext( assertEquals(service == null ? "unknown" : service, context.get("service")); assertOptionalContextValue(context, "env", env); assertOptionalContextValue(context, "version", version); - // SDK identity populated by FeatureFlagEvpContext (FFL-2995). + // SDK identity populated by FeatureFlagEvpContext. assertEquals("dd-trace-java", context.get("source.name")); assertEquals(TracerVersion.TRACER_VERSION, context.get("source.version")); } From 4aa5a6f90a25fa38a4455a3fc7930e835c9947fd Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Fri, 14 Aug 2026 11:34:03 -0400 Subject: [PATCH 03/30] fix(openfeature): emit source.name/version per-event, not in batch context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flagevaluation track schema (logs-backend flagevaluation.conf) declares source.name/source.version as top-level per-event fields, siblings of flag/variant/targeting_key. The previous implementation put them in the batch context envelope alongside service/env/version, which the EVP indexer maps to context.source.* — an undeclared facet that causes the indexer to drop the entire event. Move source to the FlagEvaluationEvent top level (as a nested source object {name,version}) so it lands on the declared source.name/source.version facets. Verified end-to-end via ffe-dogfooding against staging: Java flagevaluation events now index in the staging flag_evaluations data source. Generated with Claude Code Co-Authored-By: Claude --- .../featureflag/FeatureFlagEvpContext.java | 13 +++++-------- .../featureflag/FlagEvaluationPayloads.java | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java index 181dab8495b..255c2377905 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpContext.java @@ -1,6 +1,5 @@ package com.datadog.featureflag; -import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.Config; import java.util.HashMap; import java.util.Map; @@ -9,10 +8,8 @@ final class FeatureFlagEvpContext { private FeatureFlagEvpContext() {} - private static final String SOURCE_NAME = "dd-trace-java"; - static Map from(final Config config) { - final Map context = new HashMap<>(6); + final Map context = new HashMap<>(4); context.put("service", config.getServiceName() == null ? "unknown" : config.getServiceName()); if (config.getEnv() != null) { context.put("env", config.getEnv()); @@ -20,10 +17,10 @@ static Map from(final Config config) { if (config.getVersion() != null) { context.put("version", config.getVersion()); } - // SDK identity — populates the `source.name` / `source.version` facets on both the - // `flag_evaluations` and `exposures` EVP streams (this map is shared by both writers). - context.put("source.name", SOURCE_NAME); - context.put("source.version", TracerVersion.TRACER_VERSION); + // SDK identity (source.name / source.version) is emitted per-event at the top level + // (sibling of flag/variant/targeting_key), matching the flagevaluation track schema in + // logs-backend. Putting it in the batch context would map it to context.source.*, which is + // not a declared facet and causes the indexer to drop the event. return context; } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java index 3a8734fd516..95ce3e0b082 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import datadog.communication.ddagent.TracerVersion; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; @@ -157,6 +158,9 @@ private byte[] toByteArray() { } } + private static final String SOURCE_NAME = "dd-trace-java"; + private static final String SOURCE_VERSION = TracerVersion.TRACER_VERSION; + static class FlagEvaluationEvent { public final long timestamp; public final FlagKeyObject flag; @@ -168,6 +172,7 @@ static class FlagEvaluationEvent { public final String targeting_key; public final Boolean runtime_default_used; public final EventContext context; + public final SourceObject source; public final ErrorObject error; FlagEvaluationEvent( @@ -196,6 +201,7 @@ static class FlagEvaluationEvent { (evaluationAttrs != null && !evaluationAttrs.isEmpty()) ? new EventContext(evaluationAttrs) : null; + this.source = new SourceObject(SOURCE_NAME, SOURCE_VERSION); this.error = (errorMessage != null && !errorMessage.isEmpty()) ? new ErrorObject(errorMessage) : null; } @@ -284,6 +290,16 @@ static class ErrorObject { } } + static class SourceObject { + public final String name; + public final String version; + + SourceObject(final String name, final String version) { + this.name = name; + this.version = version; + } + } + static class EventContext { public final Map evaluation; From a745b610c367a9ba59b9b9a67729f003ea7b2678 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 27 Aug 2026 20:35:53 -0400 Subject: [PATCH 04/30] test(openfeature): verify flag evaluation source metadata Generated with Claude Code --- .../featureflag/FlagEvaluationPayloadsTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java index d4ca517fffd..abf59462e9a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -10,6 +10,7 @@ import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; +import datadog.communication.ddagent.TracerVersion; import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; @@ -46,6 +47,7 @@ void fullTierPayloadUsesWorkerWireShape() throws Exception { assertObjectWithKey(ev.get("variant"), "on"); assertObjectWithKey(ev.get("allocation"), "alloc-x"); assertObjectWithKey(ev.get("flag"), "my-flag"); + assertSource(ev); final Map ctx = (Map) ev.get("context"); assertNotNull(ctx); final Map evalAttrs = (Map) ctx.get("evaluation"); @@ -222,6 +224,7 @@ void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); assertNull(ev.get("targeting_key")); assertNull(ev.get("context")); + assertSource(ev); } @Test @@ -439,6 +442,13 @@ private static void assertObjectWithKey(final Object object, final String expect assertEquals(expectedKey, ((Map) object).get("key")); } + private static void assertSource(final Map event) { + assertTrue(event.get("source") instanceof Map); + final Map source = (Map) event.get("source"); + assertEquals("dd-trace-java", source.get("name")); + assertEquals(TracerVersion.TRACER_VERSION, source.get("version")); + } + private static String repeat(final char c, final int count) { final char[] chars = new char[count]; java.util.Arrays.fill(chars, c); From b4724e93aa90dd1e91190d80d2ee0706d1a989c0 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 27 Aug 2026 20:50:07 -0400 Subject: [PATCH 05/30] fix(openfeature): send flag evaluation source headers Generated with Claude Code --- .../datadog/communication/BackendApi.java | 18 +++++++++ .../datadog/communication/EvpProxyApi.java | 19 ++++++++++ .../java/datadog/communication/IntakeApi.java | 19 ++++++++++ .../communication/EvpProxyApiTest.java | 37 ++++++++++++++++++ .../datadog/communication/IntakeApiTest.java | 34 +++++++++++++++++ .../AgentlessFeatureFlagBackendApi.java | 20 +++++++++- .../featureflag/FeatureFlagEvpPublisher.java | 21 +++++++++- .../featureflag/FlagEvaluationPayloads.java | 16 -------- .../AgentlessFeatureFlagBackendApiTest.java | 38 +++++++++++++++++++ .../featureflag/ExposureWriterTests.java | 16 +++++--- .../FeatureFlagEvpPublisherTest.java | 31 ++++++++++++++- .../FlagEvaluationPayloadsTest.java | 12 +----- .../FlagEvaluationTestSupport.java | 4 +- .../FlagEvaluationWriterImplTest.java | 26 ++++++++----- 14 files changed, 265 insertions(+), 46 deletions(-) diff --git a/communication/src/main/java/datadog/communication/BackendApi.java b/communication/src/main/java/datadog/communication/BackendApi.java index aa4385d6d75..b7b5d49eca7 100644 --- a/communication/src/main/java/datadog/communication/BackendApi.java +++ b/communication/src/main/java/datadog/communication/BackendApi.java @@ -4,6 +4,7 @@ import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; +import java.util.Map; import javax.annotation.Nullable; import okhttp3.RequestBody; @@ -17,4 +18,21 @@ T post( @Nullable OkHttpUtils.CustomListener requestListener, boolean requestCompression) throws IOException; + + /** + * Posts an HTTP request with caller-supplied headers. + * + *

The default implementation preserves compatibility with backends that do not support custom + * headers. + */ + default T post( + String uri, + RequestBody requestBody, + IOThrowingFunction responseParser, + @Nullable OkHttpUtils.CustomListener requestListener, + boolean requestCompression, + Map requestHeaders) + throws IOException { + return post(uri, requestBody, responseParser, requestListener, requestCompression); + } } diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 8bb768b7e4e..2dae897dc50 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -1,10 +1,13 @@ package datadog.communication; +import static java.util.Collections.emptyMap; + import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; +import java.util.Map; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; import okhttp3.HttpUrl; @@ -57,6 +60,18 @@ public T post( @Nullable OkHttpUtils.CustomListener requestListener, boolean requestCompression) throws IOException { + return post(uri, requestBody, responseParser, requestListener, requestCompression, emptyMap()); + } + + @Override + public T post( + String uri, + RequestBody requestBody, + IOThrowingFunction responseParser, + @Nullable OkHttpUtils.CustomListener requestListener, + boolean requestCompression, + Map requestHeaders) + throws IOException { final HttpUrl url = evpProxyUrl.resolve(uri); Request.Builder requestBuilder = @@ -66,6 +81,10 @@ public T post( .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); + for (Map.Entry header : requestHeaders.entrySet()) { + requestBuilder.addHeader(header.getKey(), header.getValue()); + } + if (requestListener != null) { requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener); } diff --git a/communication/src/main/java/datadog/communication/IntakeApi.java b/communication/src/main/java/datadog/communication/IntakeApi.java index 1a6f3f91bc7..285627381dd 100644 --- a/communication/src/main/java/datadog/communication/IntakeApi.java +++ b/communication/src/main/java/datadog/communication/IntakeApi.java @@ -1,10 +1,13 @@ package datadog.communication; +import static java.util.Collections.emptyMap; + import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; +import java.util.Map; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; import okhttp3.HttpUrl; @@ -57,6 +60,18 @@ public T post( @Nullable OkHttpUtils.CustomListener requestListener, boolean requestCompression) throws IOException { + return post(uri, requestBody, responseParser, requestListener, requestCompression, emptyMap()); + } + + @Override + public T post( + String uri, + RequestBody requestBody, + IOThrowingFunction responseParser, + @Nullable OkHttpUtils.CustomListener requestListener, + boolean requestCompression, + Map requestHeaders) + throws IOException { HttpUrl url = hostUrl.resolve(uri); Request.Builder requestBuilder = new Request.Builder() @@ -66,6 +81,10 @@ public T post( .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); + for (Map.Entry header : requestHeaders.entrySet()) { + requestBuilder.addHeader(header.getKey(), header.getValue()); + } + if (requestListener != null) { requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener); } diff --git a/communication/src/test/java/datadog/communication/EvpProxyApiTest.java b/communication/src/test/java/datadog/communication/EvpProxyApiTest.java index 14c6962bf8e..79adb0fa3cd 100644 --- a/communication/src/test/java/datadog/communication/EvpProxyApiTest.java +++ b/communication/src/test/java/datadog/communication/EvpProxyApiTest.java @@ -1,10 +1,13 @@ package datadog.communication; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import datadog.communication.http.HttpRetryPolicy; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.RequestBody; @@ -61,5 +64,39 @@ void reportsHttpStatusForRejectedRequest() throws Exception { final RecordedRequest request = server.takeRequest(); assertEquals("/evp_proxy/v4/api/v2/exposures", request.getPath()); assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); + assertNull(request.getHeader("DD-EVP-ORIGIN")); + assertNull(request.getHeader("DD-EVP-ORIGIN-VERSION")); + } + + @Test + void addsCustomHeadersWithoutReplacingEvpHeaders() throws Exception { + server.enqueue(new MockResponse().setResponseCode(200)); + final EvpProxyApi api = + new EvpProxyApi( + "123", + server.url("/evp_proxy/v4/"), + "event-platform-intake", + HttpRetryPolicy.Factory.NEVER_RETRY, + client, + false); + final Map requestHeaders = new HashMap<>(); + requestHeaders.put("DD-EVP-ORIGIN", "dd-trace-java"); + requestHeaders.put("DD-EVP-ORIGIN-VERSION", "1.2.3"); + + api.post( + "flagevaluation", + RequestBody.create(MediaType.parse("application/json"), "{}"), + stream -> null, + null, + false, + requestHeaders); + + final RecordedRequest request = server.takeRequest(); + assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); + assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); + assertEquals("123", request.getHeader("x-datadog-trace-id")); + assertEquals("123", request.getHeader("x-datadog-parent-id")); + assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); + assertEquals("1.2.3", request.getHeader("DD-EVP-ORIGIN-VERSION")); } } diff --git a/communication/src/test/java/datadog/communication/IntakeApiTest.java b/communication/src/test/java/datadog/communication/IntakeApiTest.java index 326cf21ca84..0f2f1d552c7 100644 --- a/communication/src/test/java/datadog/communication/IntakeApiTest.java +++ b/communication/src/test/java/datadog/communication/IntakeApiTest.java @@ -4,6 +4,8 @@ import datadog.communication.http.HttpRetryPolicy; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.RequestBody; @@ -45,6 +47,38 @@ void requestsIdentityResponseEncodingWhenCompressionIsDisabled() throws Exceptio assertEquals("identity", postAndReadAcceptEncoding(false)); } + @Test + void addsCustomHeadersWithoutReplacingIntakeHeaders() throws Exception { + server.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + final IntakeApi api = + new IntakeApi( + server.url("/api/v2/"), + "api-key", + "123", + HttpRetryPolicy.Factory.NEVER_RETRY, + client, + false); + final Map requestHeaders = new HashMap<>(); + requestHeaders.put("DD-EVP-ORIGIN", "dd-trace-java"); + requestHeaders.put("DD-EVP-ORIGIN-VERSION", "1.2.3"); + + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}"), + responseBody -> null, + null, + false, + requestHeaders); + + final RecordedRequest request = server.takeRequest(); + assertEquals("/api/v2/flagevaluation", request.getPath()); + assertEquals("api-key", request.getHeader("dd-api-key")); + assertEquals("123", request.getHeader("x-datadog-trace-id")); + assertEquals("123", request.getHeader("x-datadog-parent-id")); + assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); + assertEquals("1.2.3", request.getHeader("DD-EVP-ORIGIN-VERSION")); + } + private String postAndReadAcceptEncoding(final boolean responseCompression) throws Exception { server.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); final IntakeApi api = diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java index 769ebfd1dd1..45e08a37594 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java @@ -1,5 +1,7 @@ package com.datadog.featureflag; +import static java.util.Collections.emptyMap; + import datadog.communication.BackendApi; import datadog.communication.HttpResponseException; import datadog.communication.http.OkHttpUtils; @@ -7,6 +9,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; +import java.util.Map; import java.util.function.Supplier; import javax.annotation.Nullable; import okhttp3.RequestBody; @@ -43,10 +46,22 @@ public T post( @Nullable final OkHttpUtils.CustomListener requestListener, final boolean requestCompression) throws IOException { + return post(uri, requestBody, responseParser, requestListener, requestCompression, emptyMap()); + } + + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression, + final Map requestHeaders) + throws IOException { final BackendApi selectedApi = activeApi; try { return selectedApi.post( - uri, requestBody, responseParser, requestListener, requestCompression); + uri, requestBody, responseParser, requestListener, requestCompression, requestHeaders); } catch (final IOException exception) { if (selectedApi != proxyApi || !isDefinitiveRejection(exception)) { throw exception; @@ -56,7 +71,8 @@ public T post( if (directApi == null) { throw exception; } - return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); + return directApi.post( + uri, requestBody, responseParser, requestListener, requestCompression, requestHeaders); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java index 8a024b52f43..9543317568f 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java @@ -1,12 +1,18 @@ package com.datadog.featureflag; +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; + import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.intake.Intake; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; import java.util.function.Supplier; import okhttp3.MediaType; import okhttp3.RequestBody; @@ -14,6 +20,8 @@ final class FeatureFlagEvpPublisher { private static final MediaType JSON = MediaType.parse("application/json"); + private static final String FLAG_EVALUATION_ROUTE = "flagevaluation"; + private static final Map FLAG_EVALUATION_HEADERS = flagEvaluationHeaders(); private final Supplier backendApiSupplier; private final JsonAdapter jsonAdapter; @@ -58,7 +66,18 @@ void post(final String route, final byte[] json) throws IOException { throw new IllegalStateException("EVP Proxy not available"); } final RequestBody requestBody = RequestBody.create(JSON, json); - evp.post(route, requestBody, stream -> null, null, false); + evp.post(route, requestBody, stream -> null, null, false, requestHeaders(route)); + } + + private static Map requestHeaders(final String route) { + return FLAG_EVALUATION_ROUTE.equals(route) ? FLAG_EVALUATION_HEADERS : emptyMap(); + } + + private static Map flagEvaluationHeaders() { + final Map headers = new HashMap<>(2); + headers.put("DD-EVP-ORIGIN", "dd-trace-java"); + headers.put("DD-EVP-ORIGIN-VERSION", TracerVersion.TRACER_VERSION); + return unmodifiableMap(headers); } static byte[] utf8Bytes(final String json) { diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java index 95ce3e0b082..3a8734fd516 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationPayloads.java @@ -1,6 +1,5 @@ package com.datadog.featureflag; -import datadog.communication.ddagent.TracerVersion; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; @@ -158,9 +157,6 @@ private byte[] toByteArray() { } } - private static final String SOURCE_NAME = "dd-trace-java"; - private static final String SOURCE_VERSION = TracerVersion.TRACER_VERSION; - static class FlagEvaluationEvent { public final long timestamp; public final FlagKeyObject flag; @@ -172,7 +168,6 @@ static class FlagEvaluationEvent { public final String targeting_key; public final Boolean runtime_default_used; public final EventContext context; - public final SourceObject source; public final ErrorObject error; FlagEvaluationEvent( @@ -201,7 +196,6 @@ static class FlagEvaluationEvent { (evaluationAttrs != null && !evaluationAttrs.isEmpty()) ? new EventContext(evaluationAttrs) : null; - this.source = new SourceObject(SOURCE_NAME, SOURCE_VERSION); this.error = (errorMessage != null && !errorMessage.isEmpty()) ? new ErrorObject(errorMessage) : null; } @@ -290,16 +284,6 @@ static class ErrorObject { } } - static class SourceObject { - public final String name; - public final String version; - - SourceObject(final String name, final String version) { - this.name = name; - this.version = version; - } - } - static class EventContext { public final Map evaluation; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java index 117c72ba2a1..6d57b3e5cba 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java @@ -1,5 +1,7 @@ package com.datadog.featureflag; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -15,6 +17,7 @@ import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -58,6 +61,22 @@ void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throw assertSame(secondBody, direct.requestBodies.get(1)); } + @Test + void preservesRequestHeadersWhenReplayingRejectedBatch() throws Exception { + final RecordingBackendApi local = + new RecordingBackendApi(new HttpResponseException(404, "rejected")); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi(local, () -> direct, "flag evaluation"); + final Map requestHeaders = singletonMap("DD-EVP-ORIGIN", "dd-trace-java"); + + api.post( + "flagevaluation", requestBody("evaluation"), stream -> null, null, false, requestHeaders); + + assertSame(requestHeaders, local.requestHeaders.get(0)); + assertSame(requestHeaders, direct.requestHeaders.get(0)); + } + @ParameterizedTest @MethodSource("featureFlagRoutes") void fallsBackAfterConnectionRefusal(final String route, final String eventType) @@ -167,6 +186,7 @@ private static Stream featureFlagRoutes() { private static final class RecordingBackendApi implements BackendApi { private IOException failure; private final List requestBodies = new ArrayList<>(); + private final List> requestHeaders = new ArrayList<>(); private int calls; private RecordingBackendApi() { @@ -185,8 +205,26 @@ public T post( @Nullable final OkHttpUtils.CustomListener requestListener, final boolean requestCompression) throws IOException { + return record(requestBody, emptyMap()); + } + + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression, + final Map requestHeaders) + throws IOException { + return record(requestBody, requestHeaders); + } + + private T record(final RequestBody requestBody, final Map requestHeaders) + throws IOException { calls++; requestBodies.add(requestBody); + this.requestHeaders.add(requestHeaders); if (failure != null) { throw failure; } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index daaf213ebfb..0c22b61bcf8 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -313,7 +314,7 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, true)) .thenReturn(directApi); - when(proxyApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))) + when(proxyApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap())) .thenThrow(new SocketTimeoutException("ambiguous timeout")) .thenThrow(new ConnectException("definitive refusal")); final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = @@ -328,21 +329,24 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception poll.eventually( () -> verify(proxyApi) - .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))); + .post( + eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap())); MILLISECONDS.sleep(300); verify(proxyApi, times(1)) - .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap()); verify(directApi, never()) - .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap()); writer.accept(exposures.get(1)); poll.eventually( () -> verify(directApi) - .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))); + .post( + eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap())); final ArgumentCaptor directBody = ArgumentCaptor.forClass(RequestBody.class); - verify(directApi).post(eq("exposures"), directBody.capture(), any(), any(), eq(false)); + verify(directApi) + .post(eq("exposures"), directBody.capture(), any(), any(), eq(false), anyMap()); final Buffer buffer = new Buffer(); directBody.getValue().writeTo(buffer); final ExposuresRequest directRequest = diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java index 379dd49e444..4d94a982ad8 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import static java.util.Collections.emptyMap; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -12,7 +13,10 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.intake.Intake; +import java.util.HashMap; +import java.util.Map; import okhttp3.RequestBody; import org.junit.jupiter.api.Test; @@ -46,7 +50,25 @@ void responseCompressionCanBeDisabled() throws Exception { verify(factory).createBackendApi(Intake.EVENT_PLATFORM, false); verify(backendApi) - .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); + .post( + eq("flagevaluation"), + any(RequestBody.class), + any(), + isNull(), + eq(false), + eq(flagEvaluationHeaders())); + } + + @Test + void exposureRequestsDoNotIncludeFlagEvaluationHeaders() throws Exception { + final BackendApi backendApi = mock(BackendApi.class); + final FeatureFlagEvpPublisher publisher = + new FeatureFlagEvpPublisher<>(() -> backendApi, TestRequest.class); + + publisher.post("exposures", new TestRequest("value")); + + verify(backendApi) + .post(eq("exposures"), any(RequestBody.class), any(), isNull(), eq(false), eq(emptyMap())); } @Test @@ -61,6 +83,13 @@ void postThrowsWhenEvpBackendApiCannotBeCreated() { () -> publisher.post("flagevaluation", FeatureFlagEvpPublisher.utf8Bytes("{}"))); } + private static Map flagEvaluationHeaders() { + final Map headers = new HashMap<>(); + headers.put("DD-EVP-ORIGIN", "dd-trace-java"); + headers.put("DD-EVP-ORIGIN-VERSION", TracerVersion.TRACER_VERSION); + return headers; + } + static class TestRequest { public final String value; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java index abf59462e9a..9c5ed201e0c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -10,7 +10,6 @@ import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import com.squareup.moshi.Types; -import datadog.communication.ddagent.TracerVersion; import java.lang.reflect.Type; import java.util.Arrays; import java.util.HashMap; @@ -47,13 +46,13 @@ void fullTierPayloadUsesWorkerWireShape() throws Exception { assertObjectWithKey(ev.get("variant"), "on"); assertObjectWithKey(ev.get("allocation"), "alloc-x"); assertObjectWithKey(ev.get("flag"), "my-flag"); - assertSource(ev); final Map ctx = (Map) ev.get("context"); assertNotNull(ctx); final Map evalAttrs = (Map) ctx.get("evaluation"); assertNotNull(evalAttrs); assertEquals("us-east-1", evalAttrs.get("region")); assertFalse(ev.containsKey("reason")); + assertFalse(ev.containsKey("source")); } @Test @@ -98,6 +97,7 @@ void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { final Map ev = firstEvent(json); assertNull(ev.get("targeting_key")); assertNull(ev.get("context")); + assertFalse(ev.containsKey("source")); } @Test @@ -224,7 +224,6 @@ void oversizedFullPayloadRowIsDegradedBeforeDrop() throws Exception { assertEquals(2.0, ((Number) ev.get("evaluation_count")).doubleValue()); assertNull(ev.get("targeting_key")); assertNull(ev.get("context")); - assertSource(ev); } @Test @@ -442,13 +441,6 @@ private static void assertObjectWithKey(final Object object, final String expect assertEquals(expectedKey, ((Map) object).get("key")); } - private static void assertSource(final Map event) { - assertTrue(event.get("source") instanceof Map); - final Map source = (Map) event.get("source"); - assertEquals("dd-trace-java", source.get("name")); - assertEquals(TracerVersion.TRACER_VERSION, source.get("version")); - } - private static String repeat(final char c, final int count) { final char[] chars = new char[count]; java.util.Arrays.fill(chars, c); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java index 502f0e76c69..b1f94fc0e3e 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -132,7 +133,8 @@ static CapturedJson flushAndCapture(final TestWriterSetup setup) throws Exceptio static List flushAndCaptureAll(final TestWriterSetup setup) throws Exception { final List captured = new ArrayList<>(); - when(setup.mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + when(setup.mockEvp.post( + eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) .thenAnswer( inv -> { captured.add(inv.getArgument(1)); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index ba3d03ec6d5..da19e448dd2 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -22,6 +22,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; @@ -377,7 +378,8 @@ void finalFlushRunsWithoutTheInterruptFlagSet() throws Exception { final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); final boolean[] interruptedDuringPost = {true}; final BackendApi mockEvp = mock(BackendApi.class); - when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + when(mockEvp.post( + eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) .thenAnswer( inv -> { interruptedDuringPost[0] = Thread.currentThread().isInterrupted(); @@ -406,7 +408,8 @@ void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); final RequestBody[] captured = {null}; final BackendApi mockEvp = mock(BackendApi.class); - when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + when(mockEvp.post( + eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) .thenAnswer( inv -> { captured[0] = inv.getArgument(1); @@ -448,7 +451,8 @@ void continuousTrafficFlushesWithoutWaitingForIdle() throws Exception { writer.enqueue(simpleEvent("busy-flag", "on")); try { verify(mockEvp, atLeastOnce()) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + .post( + eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); posted = true; break; } catch (AssertionError ignored) { @@ -472,7 +476,8 @@ void flushPostsToFlagevaluationEndpoint() throws Exception { setup.handler.flush(); verify(setup.factory).createBackendApi(Intake.EVENT_PLATFORM, false); - verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + verify(mockEvp) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); } @Test @@ -489,7 +494,7 @@ void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { return null; }) .when(mockEvp) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); for (int i = 0; i < 4; i++) { final Map attrs = new HashMap<>(); @@ -555,7 +560,8 @@ void eventConsentFalseStaysHashedEvenWhenGatewayLaterReportsTrue() throws Except setup.handler.drainAndAggregate(); final java.util.List captured = new java.util.ArrayList<>(); - when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + when(mockEvp.post( + eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) .thenAnswer( inv -> { captured.add(inv.getArgument(1)); @@ -588,7 +594,8 @@ void eventConsentTrueStaysRawEvenWhenGatewayLaterReportsFalse() throws Exception setup.handler.drainAndAggregate(); final java.util.List captured = new java.util.ArrayList<>(); - when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) + when(mockEvp.post( + eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) .thenAnswer( inv -> { captured.add(inv.getArgument(1)); @@ -654,13 +661,14 @@ void encodeFailureClearsAggregatorSoLaterFlushesRecover() throws Exception { setup.handler.drainAndAggregate(); setup.handler.flush(); verify(mockEvp, org.mockito.Mockito.never()) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); // The bucket must not survive the failed flush. A follow-up healthy event flushes cleanly. setup.handler.add(simpleEvent("healthy-flag", "on")); setup.handler.drainAndAggregate(); setup.handler.flush(); - verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); + verify(mockEvp) + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); } @Test From 38e9e1b4159384989d1f2c01f8339e4cb07cd3b5 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 27 Aug 2026 21:34:07 -0400 Subject: [PATCH 06/30] test(communication): cover backend header fallback Generated with Claude Code --- .../datadog/communication/BackendApiTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 communication/src/test/java/datadog/communication/BackendApiTest.java diff --git a/communication/src/test/java/datadog/communication/BackendApiTest.java b/communication/src/test/java/datadog/communication/BackendApiTest.java new file mode 100644 index 00000000000..225892e847b --- /dev/null +++ b/communication/src/test/java/datadog/communication/BackendApiTest.java @@ -0,0 +1,45 @@ +package datadog.communication; + +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import datadog.communication.http.OkHttpUtils; +import datadog.communication.util.IOThrowingFunction; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import javax.annotation.Nullable; +import okhttp3.RequestBody; +import org.junit.jupiter.api.Test; + +class BackendApiTest { + + @Test + void defaultPostFallsBackToPostWithoutHeaders() throws IOException { + final BackendApi api = new TestBackendApi(); + + final String response = + api.post( + "flagevaluation", + null, + input -> "response", + null, + false, + singletonMap("DD-EVP-ORIGIN", "dd-trace-java")); + + assertEquals("response", response); + } + + private static final class TestBackendApi implements BackendApi { + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression) + throws IOException { + return responseParser.apply(new ByteArrayInputStream(new byte[0])); + } + } +} From 3b940be549e83e9a8b13c8c6585693a8c3be90a9 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 27 Aug 2026 21:48:37 -0400 Subject: [PATCH 07/30] refactor(openfeature): configure SDK headers per backend Send SDK identity headers on both flag evaluation and exposure requests while preserving them across proxy-to-direct fallback. Keep the BackendApi request signature unchanged by configuring headers on feature-flag backend instances.\n\nGenerated with Claude Code --- .../datadog/communication/BackendApi.java | 18 -------- .../communication/BackendApiFactory.java | 20 ++++++++- .../datadog/communication/EvpProxyApi.java | 34 +++++++++----- .../java/datadog/communication/IntakeApi.java | 27 ++++++----- .../communication/BackendApiFactoryTest.java | 6 ++- .../datadog/communication/BackendApiTest.java | 45 ------------------- .../communication/EvpProxyApiTest.java | 37 --------------- .../datadog/communication/IntakeApiTest.java | 25 +++-------- .../AgentlessFeatureFlagBackendApi.java | 20 +-------- .../FeatureFlagBackendApiFactory.java | 17 ++++++- .../featureflag/FeatureFlagEvpPublisher.java | 21 +-------- .../AgentlessFeatureFlagBackendApiTest.java | 38 ---------------- .../featureflag/ExposureWriterTests.java | 16 +++---- .../FeatureFlagBackendApiFactoryTest.java | 11 +++++ .../FeatureFlagEvpPublisherTest.java | 31 +------------ .../FlagEvaluationPayloadsTest.java | 2 - .../FlagEvaluationTestSupport.java | 4 +- .../FlagEvaluationWriterImplTest.java | 26 ++++------- 18 files changed, 113 insertions(+), 285 deletions(-) delete mode 100644 communication/src/test/java/datadog/communication/BackendApiTest.java diff --git a/communication/src/main/java/datadog/communication/BackendApi.java b/communication/src/main/java/datadog/communication/BackendApi.java index b7b5d49eca7..aa4385d6d75 100644 --- a/communication/src/main/java/datadog/communication/BackendApi.java +++ b/communication/src/main/java/datadog/communication/BackendApi.java @@ -4,7 +4,6 @@ import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; -import java.util.Map; import javax.annotation.Nullable; import okhttp3.RequestBody; @@ -18,21 +17,4 @@ T post( @Nullable OkHttpUtils.CustomListener requestListener, boolean requestCompression) throws IOException; - - /** - * Posts an HTTP request with caller-supplied headers. - * - *

The default implementation preserves compatibility with backends that do not support custom - * headers. - */ - default T post( - String uri, - RequestBody requestBody, - IOThrowingFunction responseParser, - @Nullable OkHttpUtils.CustomListener requestListener, - boolean requestCompression, - Map requestHeaders) - throws IOException { - return post(uri, requestBody, responseParser, requestListener, requestCompression); - } } diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 2ac0447fc7d..347b5d4c42d 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -1,11 +1,16 @@ package datadog.communication; +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; + import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; import datadog.trace.util.throwable.FatalAgentMisconfigurationError; +import java.util.HashMap; +import java.util.Map; import javax.annotation.Nullable; import okhttp3.HttpUrl; import org.slf4j.Logger; @@ -17,10 +22,19 @@ public class BackendApiFactory { private final Config config; private final SharedCommunicationObjects sharedCommunicationObjects; + private final Map requestHeaders; public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommunicationObjects) { + this(config, sharedCommunicationObjects, emptyMap()); + } + + public BackendApiFactory( + Config config, + SharedCommunicationObjects sharedCommunicationObjects, + Map requestHeaders) { this.config = config; this.sharedCommunicationObjects = sharedCommunicationObjects; + this.requestHeaders = unmodifiableMap(new HashMap<>(requestHeaders)); } public @Nullable BackendApi createBackendApi(Intake intake) { @@ -61,7 +75,8 @@ public BackendApi createDirectIntakeApi(Intake intake, boolean responseCompressi traceId, retryPolicyFactory(), sharedCommunicationObjects.getIntakeHttpClient(), - responseCompression); + responseCompression, + requestHeaders); } /** Creates an API client that uses the specified retry policy with a compatible local proxy. */ @@ -99,7 +114,8 @@ public BackendApi createDirectIntakeApi(Intake intake, boolean responseCompressi subdomain, retryPolicyFactory, sharedCommunicationObjects.agentHttpClient, - responseCompression); + responseCompression, + requestHeaders); } private static HttpRetryPolicy.Factory retryPolicyFactory() { diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 2dae897dc50..48adf199739 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -1,12 +1,14 @@ package datadog.communication; import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; @@ -36,6 +38,7 @@ public class EvpProxyApi implements BackendApi { private final String subdomain; private final OkHttpClient httpClient; private final boolean responseCompression; + private final Map requestHeaders; public EvpProxyApi( String traceId, @@ -44,12 +47,31 @@ public EvpProxyApi( HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient, boolean responseCompression) { + this( + traceId, + evpProxyUrl, + subdomain, + retryPolicyFactory, + httpClient, + responseCompression, + emptyMap()); + } + + public EvpProxyApi( + String traceId, + HttpUrl evpProxyUrl, + String subdomain, + HttpRetryPolicy.Factory retryPolicyFactory, + OkHttpClient httpClient, + boolean responseCompression, + Map requestHeaders) { this.traceId = traceId; this.evpProxyUrl = evpProxyUrl.resolve("api/" + API_VERSION + "/"); this.subdomain = subdomain; this.retryPolicyFactory = retryPolicyFactory; this.httpClient = httpClient; this.responseCompression = responseCompression; + this.requestHeaders = unmodifiableMap(new HashMap<>(requestHeaders)); } @Override @@ -60,18 +82,6 @@ public T post( @Nullable OkHttpUtils.CustomListener requestListener, boolean requestCompression) throws IOException { - return post(uri, requestBody, responseParser, requestListener, requestCompression, emptyMap()); - } - - @Override - public T post( - String uri, - RequestBody requestBody, - IOThrowingFunction responseParser, - @Nullable OkHttpUtils.CustomListener requestListener, - boolean requestCompression, - Map requestHeaders) - throws IOException { final HttpUrl url = evpProxyUrl.resolve(uri); Request.Builder requestBuilder = diff --git a/communication/src/main/java/datadog/communication/IntakeApi.java b/communication/src/main/java/datadog/communication/IntakeApi.java index 285627381dd..fa431a29786 100644 --- a/communication/src/main/java/datadog/communication/IntakeApi.java +++ b/communication/src/main/java/datadog/communication/IntakeApi.java @@ -1,12 +1,14 @@ package datadog.communication; import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.Map; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; @@ -36,6 +38,7 @@ public class IntakeApi implements BackendApi { private final boolean responseCompression; private final HttpUrl hostUrl; private final OkHttpClient httpClient; + private final Map requestHeaders; public IntakeApi( HttpUrl hostUrl, @@ -44,12 +47,24 @@ public IntakeApi( HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient, boolean responseCompression) { + this(hostUrl, apiKey, traceId, retryPolicyFactory, httpClient, responseCompression, emptyMap()); + } + + public IntakeApi( + HttpUrl hostUrl, + String apiKey, + String traceId, + HttpRetryPolicy.Factory retryPolicyFactory, + OkHttpClient httpClient, + boolean responseCompression, + Map requestHeaders) { this.hostUrl = hostUrl; this.apiKey = apiKey; this.traceId = traceId; this.retryPolicyFactory = retryPolicyFactory; this.responseCompression = responseCompression; this.httpClient = httpClient; + this.requestHeaders = unmodifiableMap(new HashMap<>(requestHeaders)); } @Override @@ -60,18 +75,6 @@ public T post( @Nullable OkHttpUtils.CustomListener requestListener, boolean requestCompression) throws IOException { - return post(uri, requestBody, responseParser, requestListener, requestCompression, emptyMap()); - } - - @Override - public T post( - String uri, - RequestBody requestBody, - IOThrowingFunction responseParser, - @Nullable OkHttpUtils.CustomListener requestListener, - boolean requestCompression, - Map requestHeaders) - throws IOException { HttpUrl url = hostUrl.resolve(uri); Request.Builder requestBuilder = new Request.Builder() diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index 726c34a7f73..5368e4ad4ab 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -1,6 +1,7 @@ package datadog.communication; import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -45,7 +46,9 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT); final BackendApiFactory factory = new BackendApiFactory( - Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + Config.get(), + sharedCommunicationObjects(discovery, agent.url("/")), + singletonMap("DD-EVP-ORIGIN", "dd-trace-java")); final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, false); assertNotNull(api); @@ -58,6 +61,7 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce final RecordedRequest request = agent.takeRequest(); assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); + assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); } finally { agent.shutdown(); } diff --git a/communication/src/test/java/datadog/communication/BackendApiTest.java b/communication/src/test/java/datadog/communication/BackendApiTest.java deleted file mode 100644 index 225892e847b..00000000000 --- a/communication/src/test/java/datadog/communication/BackendApiTest.java +++ /dev/null @@ -1,45 +0,0 @@ -package datadog.communication; - -import static java.util.Collections.singletonMap; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import datadog.communication.http.OkHttpUtils; -import datadog.communication.util.IOThrowingFunction; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import javax.annotation.Nullable; -import okhttp3.RequestBody; -import org.junit.jupiter.api.Test; - -class BackendApiTest { - - @Test - void defaultPostFallsBackToPostWithoutHeaders() throws IOException { - final BackendApi api = new TestBackendApi(); - - final String response = - api.post( - "flagevaluation", - null, - input -> "response", - null, - false, - singletonMap("DD-EVP-ORIGIN", "dd-trace-java")); - - assertEquals("response", response); - } - - private static final class TestBackendApi implements BackendApi { - @Override - public T post( - final String uri, - final RequestBody requestBody, - final IOThrowingFunction responseParser, - @Nullable final OkHttpUtils.CustomListener requestListener, - final boolean requestCompression) - throws IOException { - return responseParser.apply(new ByteArrayInputStream(new byte[0])); - } - } -} diff --git a/communication/src/test/java/datadog/communication/EvpProxyApiTest.java b/communication/src/test/java/datadog/communication/EvpProxyApiTest.java index 79adb0fa3cd..14c6962bf8e 100644 --- a/communication/src/test/java/datadog/communication/EvpProxyApiTest.java +++ b/communication/src/test/java/datadog/communication/EvpProxyApiTest.java @@ -1,13 +1,10 @@ package datadog.communication; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import datadog.communication.http.HttpRetryPolicy; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.RequestBody; @@ -64,39 +61,5 @@ void reportsHttpStatusForRejectedRequest() throws Exception { final RecordedRequest request = server.takeRequest(); assertEquals("/evp_proxy/v4/api/v2/exposures", request.getPath()); assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); - assertNull(request.getHeader("DD-EVP-ORIGIN")); - assertNull(request.getHeader("DD-EVP-ORIGIN-VERSION")); - } - - @Test - void addsCustomHeadersWithoutReplacingEvpHeaders() throws Exception { - server.enqueue(new MockResponse().setResponseCode(200)); - final EvpProxyApi api = - new EvpProxyApi( - "123", - server.url("/evp_proxy/v4/"), - "event-platform-intake", - HttpRetryPolicy.Factory.NEVER_RETRY, - client, - false); - final Map requestHeaders = new HashMap<>(); - requestHeaders.put("DD-EVP-ORIGIN", "dd-trace-java"); - requestHeaders.put("DD-EVP-ORIGIN-VERSION", "1.2.3"); - - api.post( - "flagevaluation", - RequestBody.create(MediaType.parse("application/json"), "{}"), - stream -> null, - null, - false, - requestHeaders); - - final RecordedRequest request = server.takeRequest(); - assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); - assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); - assertEquals("123", request.getHeader("x-datadog-trace-id")); - assertEquals("123", request.getHeader("x-datadog-parent-id")); - assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); - assertEquals("1.2.3", request.getHeader("DD-EVP-ORIGIN-VERSION")); } } diff --git a/communication/src/test/java/datadog/communication/IntakeApiTest.java b/communication/src/test/java/datadog/communication/IntakeApiTest.java index 0f2f1d552c7..f34a22a952c 100644 --- a/communication/src/test/java/datadog/communication/IntakeApiTest.java +++ b/communication/src/test/java/datadog/communication/IntakeApiTest.java @@ -1,11 +1,10 @@ package datadog.communication; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import datadog.communication.http.HttpRetryPolicy; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.RequestBody; @@ -48,7 +47,7 @@ void requestsIdentityResponseEncodingWhenCompressionIsDisabled() throws Exceptio } @Test - void addsCustomHeadersWithoutReplacingIntakeHeaders() throws Exception { + void addsConfiguredRequestHeaders() throws Exception { server.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); final IntakeApi api = new IntakeApi( @@ -57,26 +56,14 @@ void addsCustomHeadersWithoutReplacingIntakeHeaders() throws Exception { "123", HttpRetryPolicy.Factory.NEVER_RETRY, client, - false); - final Map requestHeaders = new HashMap<>(); - requestHeaders.put("DD-EVP-ORIGIN", "dd-trace-java"); - requestHeaders.put("DD-EVP-ORIGIN-VERSION", "1.2.3"); + false, + singletonMap("DD-EVP-ORIGIN", "dd-trace-java")); - api.post( - "flagevaluation", - RequestBody.create(JSON, "{}"), - responseBody -> null, - null, - false, - requestHeaders); + api.post("exposures", RequestBody.create(JSON, "{}"), responseBody -> null, null, false); final RecordedRequest request = server.takeRequest(); - assertEquals("/api/v2/flagevaluation", request.getPath()); - assertEquals("api-key", request.getHeader("dd-api-key")); - assertEquals("123", request.getHeader("x-datadog-trace-id")); - assertEquals("123", request.getHeader("x-datadog-parent-id")); assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); - assertEquals("1.2.3", request.getHeader("DD-EVP-ORIGIN-VERSION")); + assertEquals("api-key", request.getHeader("dd-api-key")); } private String postAndReadAcceptEncoding(final boolean responseCompression) throws Exception { diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java index 45e08a37594..769ebfd1dd1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java @@ -1,7 +1,5 @@ package com.datadog.featureflag; -import static java.util.Collections.emptyMap; - import datadog.communication.BackendApi; import datadog.communication.HttpResponseException; import datadog.communication.http.OkHttpUtils; @@ -9,7 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; -import java.util.Map; import java.util.function.Supplier; import javax.annotation.Nullable; import okhttp3.RequestBody; @@ -46,22 +43,10 @@ public T post( @Nullable final OkHttpUtils.CustomListener requestListener, final boolean requestCompression) throws IOException { - return post(uri, requestBody, responseParser, requestListener, requestCompression, emptyMap()); - } - - @Override - public T post( - final String uri, - final RequestBody requestBody, - final IOThrowingFunction responseParser, - @Nullable final OkHttpUtils.CustomListener requestListener, - final boolean requestCompression, - final Map requestHeaders) - throws IOException { final BackendApi selectedApi = activeApi; try { return selectedApi.post( - uri, requestBody, responseParser, requestListener, requestCompression, requestHeaders); + uri, requestBody, responseParser, requestListener, requestCompression); } catch (final IOException exception) { if (selectedApi != proxyApi || !isDefinitiveRejection(exception)) { throw exception; @@ -71,8 +56,7 @@ public T post( if (directApi == null) { throw exception; } - return directApi.post( - uri, requestBody, responseParser, requestListener, requestCompression, requestHeaders); + return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java index 0dbf9c74254..8d5314d911c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -1,13 +1,17 @@ package com.datadog.featureflag; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; +import static java.util.Collections.unmodifiableMap; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.ddagent.TracerVersion; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; +import java.util.HashMap; +import java.util.Map; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,6 +20,7 @@ final class FeatureFlagBackendApiFactory { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlagBackendApiFactory.class); + static final Map REQUEST_HEADERS = requestHeaders(); private final Config config; private final BackendApiFactory backendApiFactory; @@ -25,7 +30,10 @@ final class FeatureFlagBackendApiFactory { final Config config, final SharedCommunicationObjects sharedCommunicationObjects, final FeatureFlagEventType eventType) { - this(config, new BackendApiFactory(config, sharedCommunicationObjects), eventType); + this( + config, + new BackendApiFactory(config, sharedCommunicationObjects, REQUEST_HEADERS), + eventType); } FeatureFlagBackendApiFactory( @@ -78,6 +86,13 @@ BackendApi create() { return null; } + private static Map requestHeaders() { + final Map headers = new HashMap<>(2); + headers.put("DD-EVP-ORIGIN", "dd-trace-java"); + headers.put("DD-EVP-ORIGIN-VERSION", TracerVersion.TRACER_VERSION); + return unmodifiableMap(headers); + } + private boolean hasDirectCredentials() { final String apiKey = config.getApiKey(); return apiKey != null && !apiKey.isEmpty(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java index 9543317568f..8a024b52f43 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagEvpPublisher.java @@ -1,18 +1,12 @@ package com.datadog.featureflag; -import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; - import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.intake.Intake; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.util.HashMap; -import java.util.Map; import java.util.function.Supplier; import okhttp3.MediaType; import okhttp3.RequestBody; @@ -20,8 +14,6 @@ final class FeatureFlagEvpPublisher { private static final MediaType JSON = MediaType.parse("application/json"); - private static final String FLAG_EVALUATION_ROUTE = "flagevaluation"; - private static final Map FLAG_EVALUATION_HEADERS = flagEvaluationHeaders(); private final Supplier backendApiSupplier; private final JsonAdapter jsonAdapter; @@ -66,18 +58,7 @@ void post(final String route, final byte[] json) throws IOException { throw new IllegalStateException("EVP Proxy not available"); } final RequestBody requestBody = RequestBody.create(JSON, json); - evp.post(route, requestBody, stream -> null, null, false, requestHeaders(route)); - } - - private static Map requestHeaders(final String route) { - return FLAG_EVALUATION_ROUTE.equals(route) ? FLAG_EVALUATION_HEADERS : emptyMap(); - } - - private static Map flagEvaluationHeaders() { - final Map headers = new HashMap<>(2); - headers.put("DD-EVP-ORIGIN", "dd-trace-java"); - headers.put("DD-EVP-ORIGIN-VERSION", TracerVersion.TRACER_VERSION); - return unmodifiableMap(headers); + evp.post(route, requestBody, stream -> null, null, false); } static byte[] utf8Bytes(final String json) { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java index 6d57b3e5cba..117c72ba2a1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java @@ -1,7 +1,5 @@ package com.datadog.featureflag; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -17,7 +15,6 @@ import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -61,22 +58,6 @@ void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throw assertSame(secondBody, direct.requestBodies.get(1)); } - @Test - void preservesRequestHeadersWhenReplayingRejectedBatch() throws Exception { - final RecordingBackendApi local = - new RecordingBackendApi(new HttpResponseException(404, "rejected")); - final RecordingBackendApi direct = new RecordingBackendApi(); - final AgentlessFeatureFlagBackendApi api = - new AgentlessFeatureFlagBackendApi(local, () -> direct, "flag evaluation"); - final Map requestHeaders = singletonMap("DD-EVP-ORIGIN", "dd-trace-java"); - - api.post( - "flagevaluation", requestBody("evaluation"), stream -> null, null, false, requestHeaders); - - assertSame(requestHeaders, local.requestHeaders.get(0)); - assertSame(requestHeaders, direct.requestHeaders.get(0)); - } - @ParameterizedTest @MethodSource("featureFlagRoutes") void fallsBackAfterConnectionRefusal(final String route, final String eventType) @@ -186,7 +167,6 @@ private static Stream featureFlagRoutes() { private static final class RecordingBackendApi implements BackendApi { private IOException failure; private final List requestBodies = new ArrayList<>(); - private final List> requestHeaders = new ArrayList<>(); private int calls; private RecordingBackendApi() { @@ -205,26 +185,8 @@ public T post( @Nullable final OkHttpUtils.CustomListener requestListener, final boolean requestCompression) throws IOException { - return record(requestBody, emptyMap()); - } - - @Override - public T post( - final String uri, - final RequestBody requestBody, - final IOThrowingFunction responseParser, - @Nullable final OkHttpUtils.CustomListener requestListener, - final boolean requestCompression, - final Map requestHeaders) - throws IOException { - return record(requestBody, requestHeaders); - } - - private T record(final RequestBody requestBody, final Map requestHeaders) - throws IOException { calls++; requestBodies.add(requestBody); - this.requestHeaders.add(requestHeaders); if (failure != null) { throw failure; } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 0c22b61bcf8..daaf213ebfb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -10,7 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -314,7 +313,7 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, true)) .thenReturn(directApi); - when(proxyApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap())) + when(proxyApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))) .thenThrow(new SocketTimeoutException("ambiguous timeout")) .thenThrow(new ConnectException("definitive refusal")); final FeatureFlagBackendApiFactory featureFlagBackendApiFactory = @@ -329,24 +328,21 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception poll.eventually( () -> verify(proxyApi) - .post( - eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap())); + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))); MILLISECONDS.sleep(300); verify(proxyApi, times(1)) - .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); verify(directApi, never()) - .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); writer.accept(exposures.get(1)); poll.eventually( () -> verify(directApi) - .post( - eq("exposures"), any(RequestBody.class), any(), any(), eq(false), anyMap())); + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))); final ArgumentCaptor directBody = ArgumentCaptor.forClass(RequestBody.class); - verify(directApi) - .post(eq("exposures"), directBody.capture(), any(), any(), eq(false), anyMap()); + verify(directApi).post(eq("exposures"), directBody.capture(), any(), any(), eq(false)); final Buffer buffer = new Buffer(); directBody.getValue().writeTo(buffer); final ExposuresRequest directRequest = diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index 8b117742715..3544e66b02c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -4,6 +4,7 @@ import static com.datadog.featureflag.FeatureFlagEventType.FLAG_EVALUATION; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -14,6 +15,7 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.TracerVersion; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; @@ -21,6 +23,15 @@ class FeatureFlagBackendApiFactoryTest { + @Test + void configuresSdkIdentityHeadersForAllFeatureFlagEventTypes() { + assertEquals( + "dd-trace-java", FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN")); + assertEquals( + TracerVersion.TRACER_VERSION, + FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN-VERSION")); + } + @Test void remoteConfigUsesOnlyLocalEvpProxy() { final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java index 4d94a982ad8..379dd49e444 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagEvpPublisherTest.java @@ -1,6 +1,5 @@ package com.datadog.featureflag; -import static java.util.Collections.emptyMap; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -13,10 +12,7 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.intake.Intake; -import java.util.HashMap; -import java.util.Map; import okhttp3.RequestBody; import org.junit.jupiter.api.Test; @@ -50,25 +46,7 @@ void responseCompressionCanBeDisabled() throws Exception { verify(factory).createBackendApi(Intake.EVENT_PLATFORM, false); verify(backendApi) - .post( - eq("flagevaluation"), - any(RequestBody.class), - any(), - isNull(), - eq(false), - eq(flagEvaluationHeaders())); - } - - @Test - void exposureRequestsDoNotIncludeFlagEvaluationHeaders() throws Exception { - final BackendApi backendApi = mock(BackendApi.class); - final FeatureFlagEvpPublisher publisher = - new FeatureFlagEvpPublisher<>(() -> backendApi, TestRequest.class); - - publisher.post("exposures", new TestRequest("value")); - - verify(backendApi) - .post(eq("exposures"), any(RequestBody.class), any(), isNull(), eq(false), eq(emptyMap())); + .post(eq("flagevaluation"), any(RequestBody.class), any(), isNull(), eq(false)); } @Test @@ -83,13 +61,6 @@ void postThrowsWhenEvpBackendApiCannotBeCreated() { () -> publisher.post("flagevaluation", FeatureFlagEvpPublisher.utf8Bytes("{}"))); } - private static Map flagEvaluationHeaders() { - final Map headers = new HashMap<>(); - headers.put("DD-EVP-ORIGIN", "dd-trace-java"); - headers.put("DD-EVP-ORIGIN-VERSION", TracerVersion.TRACER_VERSION); - return headers; - } - static class TestRequest { public final String value; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java index 9c5ed201e0c..d4ca517fffd 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationPayloadsTest.java @@ -52,7 +52,6 @@ void fullTierPayloadUsesWorkerWireShape() throws Exception { assertNotNull(evalAttrs); assertEquals("us-east-1", evalAttrs.get("region")); assertFalse(ev.containsKey("reason")); - assertFalse(ev.containsKey("source")); } @Test @@ -97,7 +96,6 @@ void degradedTierEventOmitsTargetingKeyAndContext() throws Exception { final Map ev = firstEvent(json); assertNull(ev.get("targeting_key")); assertNull(ev.get("context")); - assertFalse(ev.containsKey("source")); } @Test diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java index b1f94fc0e3e..502f0e76c69 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationTestSupport.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -133,8 +132,7 @@ static CapturedJson flushAndCapture(final TestWriterSetup setup) throws Exceptio static List flushAndCaptureAll(final TestWriterSetup setup) throws Exception { final List captured = new ArrayList<>(); - when(setup.mockEvp.post( - eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) + when(setup.mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) .thenAnswer( inv -> { captured.add(inv.getArgument(1)); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java index da19e448dd2..ba3d03ec6d5 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FlagEvaluationWriterImplTest.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyMap; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; @@ -378,8 +377,7 @@ void finalFlushRunsWithoutTheInterruptFlagSet() throws Exception { final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); final boolean[] interruptedDuringPost = {true}; final BackendApi mockEvp = mock(BackendApi.class); - when(mockEvp.post( - eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) .thenAnswer( inv -> { interruptedDuringPost[0] = Thread.currentThread().isInterrupted(); @@ -408,8 +406,7 @@ void closeDrainsAndFinalFlushesQueuedEvents() throws Exception { final java.util.concurrent.CountDownLatch posted = new java.util.concurrent.CountDownLatch(1); final RequestBody[] captured = {null}; final BackendApi mockEvp = mock(BackendApi.class); - when(mockEvp.post( - eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) .thenAnswer( inv -> { captured[0] = inv.getArgument(1); @@ -451,8 +448,7 @@ void continuousTrafficFlushesWithoutWaitingForIdle() throws Exception { writer.enqueue(simpleEvent("busy-flag", "on")); try { verify(mockEvp, atLeastOnce()) - .post( - eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); posted = true; break; } catch (AssertionError ignored) { @@ -476,8 +472,7 @@ void flushPostsToFlagevaluationEndpoint() throws Exception { setup.handler.flush(); verify(setup.factory).createBackendApi(Intake.EVENT_PLATFORM, false); - verify(mockEvp) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); } @Test @@ -494,7 +489,7 @@ void splitPostFailureDoesNotRetryAlreadySentPayloads() throws Exception { return null; }) .when(mockEvp) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); for (int i = 0; i < 4; i++) { final Map attrs = new HashMap<>(); @@ -560,8 +555,7 @@ void eventConsentFalseStaysHashedEvenWhenGatewayLaterReportsTrue() throws Except setup.handler.drainAndAggregate(); final java.util.List captured = new java.util.ArrayList<>(); - when(mockEvp.post( - eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) .thenAnswer( inv -> { captured.add(inv.getArgument(1)); @@ -594,8 +588,7 @@ void eventConsentTrueStaysRawEvenWhenGatewayLaterReportsFalse() throws Exception setup.handler.drainAndAggregate(); final java.util.List captured = new java.util.ArrayList<>(); - when(mockEvp.post( - eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap())) + when(mockEvp.post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false))) .thenAnswer( inv -> { captured.add(inv.getArgument(1)); @@ -661,14 +654,13 @@ void encodeFailureClearsAggregatorSoLaterFlushesRecover() throws Exception { setup.handler.drainAndAggregate(); setup.handler.flush(); verify(mockEvp, org.mockito.Mockito.never()) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); // The bucket must not survive the failed flush. A follow-up healthy event flushes cleanly. setup.handler.add(simpleEvent("healthy-flag", "on")); setup.handler.drainAndAggregate(); setup.handler.flush(); - verify(mockEvp) - .post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false), anyMap()); + verify(mockEvp).post(eq("flagevaluation"), any(RequestBody.class), any(), any(), eq(false)); } @Test From 1cb2c70e1936c02d0a36f4bd09980914df8b484f Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Thu, 27 Aug 2026 21:58:21 -0400 Subject: [PATCH 08/30] refactor(openfeature): inject SDK headers with HTTP client Scope the SDK identity headers to feature-flag HTTP clients so both evaluation and exposure requests retain them across proxy and direct intake without modifying transport implementations.\n\nGenerated with Claude Code --- .../communication/BackendApiFactory.java | 29 +++++++++++++++---- .../datadog/communication/EvpProxyApi.java | 29 ------------------- .../java/datadog/communication/IntakeApi.java | 22 -------------- .../communication/BackendApiFactoryTest.java | 2 ++ .../datadog/communication/IntakeApiTest.java | 21 -------------- 5 files changed, 25 insertions(+), 78 deletions(-) diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 347b5d4c42d..4ea35cb8596 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -13,6 +13,8 @@ import java.util.Map; import javax.annotation.Nullable; import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,9 +76,8 @@ public BackendApi createDirectIntakeApi(Intake intake, boolean responseCompressi apiKey, traceId, retryPolicyFactory(), - sharedCommunicationObjects.getIntakeHttpClient(), - responseCompression, - requestHeaders); + withRequestHeaders(sharedCommunicationObjects.getIntakeHttpClient()), + responseCompression); } /** Creates an API client that uses the specified retry policy with a compatible local proxy. */ @@ -113,9 +114,25 @@ public BackendApi createDirectIntakeApi(Intake intake, boolean responseCompressi evpProxyUrl, subdomain, retryPolicyFactory, - sharedCommunicationObjects.agentHttpClient, - responseCompression, - requestHeaders); + withRequestHeaders(sharedCommunicationObjects.agentHttpClient), + responseCompression); + } + + private OkHttpClient withRequestHeaders(final OkHttpClient httpClient) { + if (requestHeaders.isEmpty()) { + return httpClient; + } + return httpClient + .newBuilder() + .addInterceptor( + chain -> { + final Request.Builder requestBuilder = chain.request().newBuilder(); + for (Map.Entry header : requestHeaders.entrySet()) { + requestBuilder.header(header.getKey(), header.getValue()); + } + return chain.proceed(requestBuilder.build()); + }) + .build(); } private static HttpRetryPolicy.Factory retryPolicyFactory() { diff --git a/communication/src/main/java/datadog/communication/EvpProxyApi.java b/communication/src/main/java/datadog/communication/EvpProxyApi.java index 48adf199739..8bb768b7e4e 100644 --- a/communication/src/main/java/datadog/communication/EvpProxyApi.java +++ b/communication/src/main/java/datadog/communication/EvpProxyApi.java @@ -1,15 +1,10 @@ package datadog.communication; -import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; - import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; import okhttp3.HttpUrl; @@ -38,7 +33,6 @@ public class EvpProxyApi implements BackendApi { private final String subdomain; private final OkHttpClient httpClient; private final boolean responseCompression; - private final Map requestHeaders; public EvpProxyApi( String traceId, @@ -47,31 +41,12 @@ public EvpProxyApi( HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient, boolean responseCompression) { - this( - traceId, - evpProxyUrl, - subdomain, - retryPolicyFactory, - httpClient, - responseCompression, - emptyMap()); - } - - public EvpProxyApi( - String traceId, - HttpUrl evpProxyUrl, - String subdomain, - HttpRetryPolicy.Factory retryPolicyFactory, - OkHttpClient httpClient, - boolean responseCompression, - Map requestHeaders) { this.traceId = traceId; this.evpProxyUrl = evpProxyUrl.resolve("api/" + API_VERSION + "/"); this.subdomain = subdomain; this.retryPolicyFactory = retryPolicyFactory; this.httpClient = httpClient; this.responseCompression = responseCompression; - this.requestHeaders = unmodifiableMap(new HashMap<>(requestHeaders)); } @Override @@ -91,10 +66,6 @@ public T post( .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); - for (Map.Entry header : requestHeaders.entrySet()) { - requestBuilder.addHeader(header.getKey(), header.getValue()); - } - if (requestListener != null) { requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener); } diff --git a/communication/src/main/java/datadog/communication/IntakeApi.java b/communication/src/main/java/datadog/communication/IntakeApi.java index fa431a29786..1a6f3f91bc7 100644 --- a/communication/src/main/java/datadog/communication/IntakeApi.java +++ b/communication/src/main/java/datadog/communication/IntakeApi.java @@ -1,15 +1,10 @@ package datadog.communication; -import static java.util.Collections.emptyMap; -import static java.util.Collections.unmodifiableMap; - import datadog.communication.http.HttpRetryPolicy; import datadog.communication.http.OkHttpUtils; import datadog.communication.util.IOThrowingFunction; import java.io.IOException; import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; import java.util.zip.GZIPInputStream; import javax.annotation.Nullable; import okhttp3.HttpUrl; @@ -38,7 +33,6 @@ public class IntakeApi implements BackendApi { private final boolean responseCompression; private final HttpUrl hostUrl; private final OkHttpClient httpClient; - private final Map requestHeaders; public IntakeApi( HttpUrl hostUrl, @@ -47,24 +41,12 @@ public IntakeApi( HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient, boolean responseCompression) { - this(hostUrl, apiKey, traceId, retryPolicyFactory, httpClient, responseCompression, emptyMap()); - } - - public IntakeApi( - HttpUrl hostUrl, - String apiKey, - String traceId, - HttpRetryPolicy.Factory retryPolicyFactory, - OkHttpClient httpClient, - boolean responseCompression, - Map requestHeaders) { this.hostUrl = hostUrl; this.apiKey = apiKey; this.traceId = traceId; this.retryPolicyFactory = retryPolicyFactory; this.responseCompression = responseCompression; this.httpClient = httpClient; - this.requestHeaders = unmodifiableMap(new HashMap<>(requestHeaders)); } @Override @@ -84,10 +66,6 @@ public T post( .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); - for (Map.Entry header : requestHeaders.entrySet()) { - requestBuilder.addHeader(header.getKey(), header.getValue()); - } - if (requestListener != null) { requestBuilder.tag(OkHttpUtils.CustomListener.class, requestListener); } diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index 5368e4ad4ab..d81000d43d2 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -61,7 +61,9 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce final RecordedRequest request = agent.takeRequest(); assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); + assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); + assertEquals("identity", request.getHeader("Accept-Encoding")); } finally { agent.shutdown(); } diff --git a/communication/src/test/java/datadog/communication/IntakeApiTest.java b/communication/src/test/java/datadog/communication/IntakeApiTest.java index f34a22a952c..326cf21ca84 100644 --- a/communication/src/test/java/datadog/communication/IntakeApiTest.java +++ b/communication/src/test/java/datadog/communication/IntakeApiTest.java @@ -1,6 +1,5 @@ package datadog.communication; -import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import datadog.communication.http.HttpRetryPolicy; @@ -46,26 +45,6 @@ void requestsIdentityResponseEncodingWhenCompressionIsDisabled() throws Exceptio assertEquals("identity", postAndReadAcceptEncoding(false)); } - @Test - void addsConfiguredRequestHeaders() throws Exception { - server.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); - final IntakeApi api = - new IntakeApi( - server.url("/api/v2/"), - "api-key", - "123", - HttpRetryPolicy.Factory.NEVER_RETRY, - client, - false, - singletonMap("DD-EVP-ORIGIN", "dd-trace-java")); - - api.post("exposures", RequestBody.create(JSON, "{}"), responseBody -> null, null, false); - - final RecordedRequest request = server.takeRequest(); - assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); - assertEquals("api-key", request.getHeader("dd-api-key")); - } - private String postAndReadAcceptEncoding(final boolean responseCompression) throws Exception { server.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); final IntakeApi api = From b4a467d4fd6808ae3736d2a73a7dd792f7d9f271 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Wed, 9 Sep 2026 16:48:46 -0400 Subject: [PATCH 09/30] refactor(openfeature): share EVP origin header constants Move the DD-EVP-ORIGIN and DD-EVP-ORIGIN-VERSION header names and the dd-trace-java origin value into EvpProxy, next to the existing subdomain header constant. ProfileUploader and CrashUploader already declare private copies of these names, so the feature-flagging code used string literals as a third copy. Replace the FeatureFlagBackendApiFactory test that asserted the contents of the REQUEST_HEADERS map, which restated the implementation and would pass even if the headers never reached the wire. The field is private again. Cover both transports with MockWebServer assertions instead. The direct intake test uses followRedirects=false to match the feature-flagging caller, so it also covers the redirect-scoped client wrapped by the header interceptor. Removing the interceptor from the direct intake path makes the new test fail. Co-Authored-By: Claude Opus 5 --- .../java/datadog/communication/EvpProxy.java | 9 +++ .../communication/BackendApiFactoryTest.java | 78 +++++++++++++++++-- .../FeatureFlagBackendApiFactory.java | 9 ++- .../FeatureFlagBackendApiFactoryTest.java | 11 --- 4 files changed, 88 insertions(+), 19 deletions(-) diff --git a/communication/src/main/java/datadog/communication/EvpProxy.java b/communication/src/main/java/datadog/communication/EvpProxy.java index c2453bccb25..05ce9757339 100644 --- a/communication/src/main/java/datadog/communication/EvpProxy.java +++ b/communication/src/main/java/datadog/communication/EvpProxy.java @@ -5,6 +5,15 @@ public final class EvpProxy { public static final String SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; + /** Identifies the SDK that produced an EVP request. */ + public static final String ORIGIN_HEADER = "DD-EVP-ORIGIN"; + + /** Identifies the version of the SDK that produced an EVP request. */ + public static final String ORIGIN_VERSION_HEADER = "DD-EVP-ORIGIN-VERSION"; + + /** Origin header value identifying this tracing library. */ + public static final String JAVA_TRACING_LIBRARY = "dd-trace-java"; + /** * Default SDK-side target for uncompressed EVP request bodies. Writers may split batches at or * below this size to keep Agent proxy requests comfortably bounded. diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index f3b7576184a..a519e519a93 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -1,6 +1,10 @@ package datadog.communication; +import static datadog.communication.EvpProxy.JAVA_TRACING_LIBRARY; +import static datadog.communication.EvpProxy.ORIGIN_HEADER; import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL; +import static datadog.trace.api.config.GeneralConfig.API_KEY; import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -17,6 +21,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Locale; +import java.util.Properties; import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -136,9 +141,7 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT); final BackendApiFactory factory = new BackendApiFactory( - Config.get(), - sharedCommunicationObjects(discovery, agent.url("/")), - singletonMap("DD-EVP-ORIGIN", "dd-trace-java")); + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, false); assertNotNull(api); @@ -151,14 +154,79 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce final RecordedRequest request = agent.takeRequest(); assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); - assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); - assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); + assertEquals("event-platform-intake", request.getHeader(EvpProxy.SUBDOMAIN_HEADER)); assertEquals("identity", request.getHeader("Accept-Encoding")); } finally { agent.shutdown(); } } + @Test + void evpProxySendsConfiguredRequestHeaders() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), + sharedCommunicationObjects(discovery, agent.url("/")), + singletonMap(ORIGIN_HEADER, JAVA_TRACING_LIBRARY)); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = agent.takeRequest(); + assertEquals(JAVA_TRACING_LIBRARY, request.getHeader(ORIGIN_HEADER)); + } finally { + agent.shutdown(); + } + } + + @Test + void directIntakeSendsConfiguredRequestHeaders() throws Exception { + final MockWebServer intake = new MockWebServer(); + intake.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + intake.start(); + try { + final Properties properties = new Properties(); + properties.setProperty(API_KEY, "api-key"); + properties.setProperty( + CIVISIBILITY_AGENTLESS_URL, intake.url("/").toString().replaceAll("/$", "")); + final Config config = Config.get(properties); + final BackendApiFactory factory = + new BackendApiFactory( + config, + sharedCommunicationObjects(new FakeFeaturesDiscovery(null), null), + singletonMap(ORIGIN_HEADER, JAVA_TRACING_LIBRARY)); + + // followRedirects=false mirrors the feature-flagging caller, so this also covers the + // interaction between the redirect-scoped client and the header interceptor. + final BackendApi api = factory.createDirectIntakeApi(Intake.API, false, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + final RecordedRequest request = intake.takeRequest(); + assertEquals(JAVA_TRACING_LIBRARY, request.getHeader(ORIGIN_HEADER)); + assertEquals("api-key", request.getHeader("DD-API-KEY")); + } finally { + intake.shutdown(); + } + } + @Test void explicitNoRetryProxyPolicyDoesNotReplayAmbiguousFailure() throws Exception { final MockWebServer agent = new MockWebServer(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java index 06934d23e34..87ddcaaae16 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -1,5 +1,8 @@ package com.datadog.featureflag; +import static datadog.communication.EvpProxy.JAVA_TRACING_LIBRARY; +import static datadog.communication.EvpProxy.ORIGIN_HEADER; +import static datadog.communication.EvpProxy.ORIGIN_VERSION_HEADER; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static java.util.Collections.unmodifiableMap; @@ -20,7 +23,7 @@ final class FeatureFlagBackendApiFactory { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlagBackendApiFactory.class); - static final Map REQUEST_HEADERS = requestHeaders(); + private static final Map REQUEST_HEADERS = requestHeaders(); private final Config config; private final BackendApiFactory backendApiFactory; @@ -88,8 +91,8 @@ BackendApi create() { private static Map requestHeaders() { final Map headers = new HashMap<>(2); - headers.put("DD-EVP-ORIGIN", "dd-trace-java"); - headers.put("DD-EVP-ORIGIN-VERSION", TracerVersion.TRACER_VERSION); + headers.put(ORIGIN_HEADER, JAVA_TRACING_LIBRARY); + headers.put(ORIGIN_VERSION_HEADER, TracerVersion.TRACER_VERSION); return unmodifiableMap(headers); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index f4d950bd7ce..b89cf3ce2b1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -4,7 +4,6 @@ import static com.datadog.featureflag.FeatureFlagEventType.FLAG_EVALUATION; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -15,7 +14,6 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.TracerVersion; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; @@ -23,15 +21,6 @@ class FeatureFlagBackendApiFactoryTest { - @Test - void configuresSdkIdentityHeadersForAllFeatureFlagEventTypes() { - assertEquals( - "dd-trace-java", FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN")); - assertEquals( - TracerVersion.TRACER_VERSION, - FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN-VERSION")); - } - @Test void remoteConfigUsesOnlyLocalEvpProxy() { final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); From 49eada3526b00d45930aee76b7a9cc0b2e5a2258 Mon Sep 17 00:00:00 2001 From: Vickie Boettcher Date: Wed, 9 Sep 2026 16:55:02 -0400 Subject: [PATCH 10/30] test(communication): drop unrelated EVP proxy assertions The subdomain and Accept-Encoding assertions were added by this branch to a pre-existing test that only checked the request path. Neither relates to SDK identity headers, which evpProxySendsConfiguredRequestHeaders now covers. Co-Authored-By: Claude Opus 5 --- .../test/java/datadog/communication/BackendApiFactoryTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index a519e519a93..a7fb7642a35 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -154,8 +154,6 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce final RecordedRequest request = agent.takeRequest(); assertEquals("/evp_proxy/v4/api/v2/flagevaluation", request.getPath()); - assertEquals("event-platform-intake", request.getHeader(EvpProxy.SUBDOMAIN_HEADER)); - assertEquals("identity", request.getHeader("Accept-Encoding")); } finally { agent.shutdown(); } From a4b447eeea720f94d38c4c205691dbd0d629f930 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 9 Sep 2026 21:34:32 -0400 Subject: [PATCH 11/30] Publish dd-openfeature PR snapshots (#12400) Publish dd-openfeature PR snapshots Expose the OpenFeature provider beside the agent in the existing public S3 snapshot job. Environment: Datadog workspace Merge branch 'master' into leo.romanovsky/publish-openfeature-snapshot-artifacts Co-authored-by: devflow.devflow-routing-intake --- .gitlab-ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa7c927a625..0e26911aa0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -355,7 +355,7 @@ build: script: - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi - ./gradlew --version - - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests -x spotlessCheck $GRADLE_ARGS + - ./gradlew clean :dd-java-agent:shadowJar :dd-java-agent:check :dd-trace-api:jar :dd-trace-ot:shadowJar :products:feature-flagging:feature-flagging-api:jar -PskipTests -x spotlessCheck $GRADLE_ARGS - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env @@ -365,6 +365,7 @@ build: - 'workspace/dd-java-agent/build/libs/*.jar' - 'workspace/dd-trace-api/build/libs/*.jar' - 'workspace/dd-trace-ot/build/libs/*.jar' + - 'workspace/products/feature-flagging/feature-flagging-api/build/libs/*.jar' - 'upstream.env' - '.gradle/daemon/*/*.out.log' reports: @@ -481,9 +482,11 @@ publish-artifacts-to-s3: - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar + - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-openfeature.jar - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar + - aws s3 cp workspace/products/feature-flagging/feature-flagging-api/build/libs/dd-openfeature-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar - | cat << EOF > links.json { @@ -493,6 +496,12 @@ publish-artifacts-to-s3: "label": "Public Link to dd-java-agent.jar", "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar" } + }, + { + "external_link": { + "label": "Public Link to dd-openfeature.jar", + "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-openfeature.jar" + } } ] } From 59b0330c0cdbbc069a528611a062f4800a76e2fb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bempel Date: Thu, 10 Sep 2026 09:25:10 +0200 Subject: [PATCH 12/30] Fix NullPointerException in Exception Replay (#12429) Fix NullPointerException in Exception Replay For FastThrow, getStackTrace can return null instead of empty StackTraceElement array call only once getStackTrace fix possible NPE from chained exceptions Co-authored-by: devflow.devflow-routing-intake --- .../debugger/exception/AbstractExceptionDebugger.java | 7 +++++-- .../debugger/exception/DefaultExceptionDebugger.java | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/AbstractExceptionDebugger.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/AbstractExceptionDebugger.java index eb8abc565de..e92f65ef223 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/AbstractExceptionDebugger.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/AbstractExceptionDebugger.java @@ -93,9 +93,12 @@ public void handleException(Throwable t, AgentSpan span) { Throwable throwable; int chainedExceptionIdx = 0; while ((throwable = chainedExceptions.pollFirst()) != null) { + StackTraceElement[] stackTrace = throwable.getStackTrace(); + if (stackTrace == null || stackTrace.length == 0) { + continue; + } ExceptionProbeManager.CreationResult creationResult = - exceptionProbeManager.createProbesForException( - throwable.getStackTrace(), chainedExceptionIdx); + exceptionProbeManager.createProbesForException(stackTrace, chainedExceptionIdx); if (creationResult.probesCreated > 0) { if (!applyConfigAsync) { applyExceptionConfiguration(fingerprint); diff --git a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java index e2921f490bd..601fca62d52 100644 --- a/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java +++ b/dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/DefaultExceptionDebugger.java @@ -54,7 +54,8 @@ protected boolean shouldHandleException(Throwable t, AgentSpan span) { } // do not handle exception with no stacktrace. cannot capture anything for it. // includes also FastThrow ones - if (t.getStackTrace().length == 0) { + StackTraceElement[] stackTrace = t.getStackTrace(); + if (stackTrace == null || stackTrace.length == 0) { return false; } return circuitBreaker.trip(); From 2df9ed50743490d20ea5df518584fea63ccfebd2 Mon Sep 17 00:00:00 2001 From: Charles de Beauchesne Date: Thu, 10 Sep 2026 17:32:33 +0200 Subject: [PATCH 13/30] Add team freeze guard workflow (#12437) Add team freeze guard workflow Co-Authored-By: Claude Sonnet 5 Rename file to .yaml Update .github/workflows/README.md, Update TFG to 0.0.4 Co-authored-by: devflow.devflow-routing-intake --- .github/workflows/README.md | 10 ++++++ .github/workflows/team-freeze-guard.yaml | 41 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 .github/workflows/team-freeze-guard.yaml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index e378e35e4de..d1f8e2af197 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -129,6 +129,16 @@ _Recovery:_ Re-write the new Groovy files in Java / JUnit. To override this chec _Notes:_ The override label skips the workflow entirely. +### team-freeze-guard [🔗](team-freeze-guard.yaml) + +_Trigger:_ When a pull request is opened, reopened, synchronized, labeled, unlabeled, or marked ready for review. + +_Action:_ Fail the check if the pull request author or the last committer belongs to a team currently listed in the `frozen-teams` input, blocking the PR from merging. + +_Recovery:_ If your PR is not dedicated to fix CI issue, it'll be block until the incident is resolved. If your Pr aims to fix CI issues, add one of the configured labels: `comp: testing`, `comp: tooling`, `tag: flaky test`, `tag: flaky test/disabled` to the pull request. + +_Notes:_ Configure frozen teams by editing the `frozen-teams` input in the workflow file (empty string means no team is frozen; otherwise list `@DataDog/` entries one per line). + ## Code Quality and Security ### analyze-changes [🔗](analyze-changes.yaml) diff --git a/.github/workflows/team-freeze-guard.yaml b/.github/workflows/team-freeze-guard.yaml new file mode 100644 index 00000000000..32e1a50db59 --- /dev/null +++ b/.github/workflows/team-freeze-guard.yaml @@ -0,0 +1,41 @@ +# How to configure: +# +# When no team is frozen, set `frozen-teams` to an empty string: +# frozen-teams: "" +# +# When one or more teams are frozen, add them one per line with the @DataDog prefix: +# +# frozen-teams: | +# @DataDog/team-a +# @DataDog/team-b + +name: Team freeze guard + +on: + pull_request_target: + types: + - opened + - reopened + - synchronize + - labeled + - unlabeled + - ready_for_review + +permissions: + id-token: write + contents: read + +jobs: + team-freeze-guard: + name: Team freeze guard + runs-on: ubuntu-latest + + steps: + - uses: DataDog/team-freeze-guard@e4ff51a72cac229b238e5a302af0d3d955ae5b54 # v0.0.4 + with: + bypass-labels: | + comp: testing + comp: tooling + tag: flaky test + tag: flaky test/disabled + frozen-teams: "" From 7c6ce517d636d80a79e8a6a64d3a0195745bab14 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 10 Sep 2026 14:37:19 -0400 Subject: [PATCH 14/30] Fix NPE in DatabaseClientDecorator when JDBC connection has no db type (#12371) Fix NPE in DatabaseClientDecorator when JDBC connection has no db type processDatabaseType/dbService dereferenced the NamingEntry returned by CACHE.computeIfAbsent(dbType, NamingEntry::new) without checking for null. FixedSizeCache.computeIfAbsent returns null for a null key without invoking the producer, so a null dbType() (e.g. an undetermined DBInfo.getType()) led to a NullPointerException in tracing instrumentation instead of a graceful no-op. Guard on dbType == null directly instead of inferring it from the cache's return value, and add a test exercising both methods with a null dbType. Co-Authored-By: Claude Sonnet 5 Merge branch 'master' into dougqh/fix-database-client-decorator-npe Merge branch 'master' into dougqh/fix-database-client-decorator-npe Merge branch 'master' into dougqh/fix-database-client-decorator-npe Co-authored-by: devflow.devflow-routing-intake --- .../decorator/DatabaseClientDecorator.java | 12 +++- ...DatabaseClientDecoratorNullDbTypeTest.java | 70 +++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecoratorNullDbTypeTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecorator.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecorator.java index fa105776113..79d08f4139a 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecorator.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecorator.java @@ -96,8 +96,10 @@ public String dbService(final String dbType, final String instanceName) { if (instanceName != null && Config.get().isDbClientSplitByInstance()) { return dbClientService(instanceName); } - final NamingEntry entry = CACHE.computeIfAbsent(dbType, NamingEntry::new); - return entry.getService(); + if (dbType == null) { + return null; + } + return CACHE.computeIfAbsent(dbType, NamingEntry::new).getService(); } public String dbClientService(final String instanceName) { @@ -144,11 +146,15 @@ public void onRawStatement(AgentSpan span, String sql) { } protected void processDatabaseType(AgentSpan span, String dbType) { + if (dbType == null) { + return; + } + final NamingEntry namingEntry = CACHE.computeIfAbsent(dbType, NamingEntry::new); span.setTag(DB_TYPE, namingEntry.dbType); postProcessServiceAndOperationName(span, namingEntry); - if (Config.get().isAppSecRaspEnabled() && dbType != null) { + if (Config.get().isAppSecRaspEnabled()) { BiConsumer connectDbCallback = AgentTracer.get() .getCallbackProvider(RequestContextSlot.APPSEC) diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecoratorNullDbTypeTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecoratorNullDbTypeTest.java new file mode 100644 index 00000000000..681ceb460a2 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/decorator/DatabaseClientDecoratorNullDbTypeTest.java @@ -0,0 +1,70 @@ +package datadog.trace.bootstrap.instrumentation.decorator; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import org.junit.jupiter.api.Test; + +class DatabaseClientDecoratorNullDbTypeTest { + + private final AgentSpan span = mock(AgentSpan.class); + private final DatabaseClientDecorator decorator = + new DatabaseClientDecorator() { + @Override + protected String[] instrumentationNames() { + return new String[] {"test"}; + } + + @Override + protected CharSequence spanType() { + return "test-type"; + } + + @Override + protected CharSequence component() { + return "test-component"; + } + + @Override + protected String service() { + return "test-service"; + } + + @Override + protected String dbType() { + return null; + } + + @Override + protected String dbUser(Object connection) { + return null; + } + + @Override + protected String dbInstance(Object connection) { + return null; + } + + @Override + protected CharSequence dbHostname(Object connection) { + return null; + } + }; + + @Test + void processDatabaseTypeWithNullDbTypeDoesNotThrowOrTag() { + assertDoesNotThrow(() -> decorator.processDatabaseType(span, null)); + + verify(span, never()).setTag(anyString(), anyString()); + } + + @Test + void dbServiceWithNullDbTypeReturnsNull() { + assertNull(decorator.dbService(null, null)); + } +} From 20463f5ce913e4964e56cd96bfd771f46993e50d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Bempel Date: Thu, 10 Sep 2026 22:26:47 +0200 Subject: [PATCH 15/30] Migrate telemetry groovy files to java part 1 (#12430) Migrate telemetry groovy files to java part 1 we migrate 11 tests: - BufferedEventsTest - DisabledDependencyServiceTest - EventSourceTest - ExtendedHeartbeatDataTest - HostInfoTest - TelemetryClientTest - TelemetryRequestBodyTest - TelemetryRouterTest - TelemetryRunnableTest - TelemetryServiceTest - TelemetrySystemTest fix review comment address review coments missing when/then Co-authored-by: devflow.devflow-routing-intake --- .../skills/migrate-groovy-to-java/SKILL.md | 1 + .../datadog/telemetry/TelemetrySystem.java | 6 + .../BufferedEventsSpecification.groovy | 122 ---- .../DisabledDependencyServiceTest.groovy | 24 - .../datadog/telemetry/EventSourceTest.groovy | 68 -- .../ExtendedHeartbeatDataSpecification.groovy | 84 --- .../datadog/telemetry/HostInfoTest.groovy | 55 -- .../telemetry/TelemetryClientTest.groovy | 54 -- .../TelemetryRequestBodySpecification.groovy | 191 ----- .../TelemetryRouterSpecification.groovy | 449 ------------ .../TelemetryRunnableSpecification.groovy | 392 ---------- .../TelemetryServiceSpecification.groovy | 511 ------------- .../TelemetrySystemSpecification.groovy | 73 -- .../telemetry/TestTelemetryRouter.groovy | 446 ------------ .../datadog/telemetry/BufferedEventsTest.java | 127 ++++ .../DisabledDependencyServiceTest.java | 28 + .../datadog/telemetry/EventSourceTest.java | 107 +++ .../telemetry/ExtendedHeartbeatDataTest.java | 89 +++ .../java/datadog/telemetry/HostInfoTest.java | 80 ++ .../telemetry/TelemetryClientTest.java | 76 ++ .../telemetry/TelemetryRequestBodyTest.java | 222 ++++++ .../telemetry/TelemetryRouterTest.java | 524 +++++++++++++ .../telemetry/TelemetryRunnableTest.java | 437 +++++++++++ .../telemetry/TelemetryServiceTest.java | 686 ++++++++++++++++++ .../telemetry/TelemetrySystemTest.java | 85 +++ .../telemetry/TestTelemetryRouter.java | 513 +++++++++++++ 26 files changed, 2981 insertions(+), 2469 deletions(-) delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/BufferedEventsSpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/DisabledDependencyServiceTest.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/EventSourceTest.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/ExtendedHeartbeatDataSpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/HostInfoTest.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TelemetryClientTest.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TelemetryRequestBodySpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TelemetryRouterSpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TelemetryRunnableSpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TelemetryServiceSpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TelemetrySystemSpecification.groovy delete mode 100644 telemetry/src/test/groovy/datadog/telemetry/TestTelemetryRouter.groovy create mode 100644 telemetry/src/test/java/datadog/telemetry/BufferedEventsTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/DisabledDependencyServiceTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/EventSourceTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/ExtendedHeartbeatDataTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/HostInfoTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TelemetryClientTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TelemetryRequestBodyTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TelemetryRouterTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TelemetryRunnableTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TelemetryServiceTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TelemetrySystemTest.java create mode 100644 telemetry/src/test/java/datadog/telemetry/TestTelemetryRouter.java diff --git a/.agents/skills/migrate-groovy-to-java/SKILL.md b/.agents/skills/migrate-groovy-to-java/SKILL.md index 7c8b4842ef5..a772200e2d5 100644 --- a/.agents/skills/migrate-groovy-to-java/SKILL.md +++ b/.agents/skills/migrate-groovy-to-java/SKILL.md @@ -35,6 +35,7 @@ When converting Groovy code to Java code, make sure that: - Migrate the named Spock clauses if they exist as inline comments in the Java unit test - When Groovy tests navigate a JSON request body through helpers like `asMap()` / `asLong()` / `asList()`, check whether `json-unit-assertj` (`libs.json.unit.assertj`) is already in the module's build file. If it is, add a method that returns the raw JSON string and use `assertThatJson(json).node("some.nested.field").isEqualTo(value)` directly instead of the map traversal. - Groovy's `[key: val]` map literals use a `LinkedHashMap`. When the test doesn't care about insertion order, use `singletonMap` for a single entry or `HashMap` for two or more. If a helper method builds these maps, add a two-arg overload rather than scattering `new LinkedHashMap<>()` constructions through test bodies. +- The Spock construct `clean:` section needs to wrap the unit test with a try...finally block to respect the behavior. TableTest usage Import: `import org.tabletest.junit.TableTest;` diff --git a/telemetry/src/main/java/datadog/telemetry/TelemetrySystem.java b/telemetry/src/main/java/datadog/telemetry/TelemetrySystem.java index 667fe414cb6..4b233aa3037 100644 --- a/telemetry/src/main/java/datadog/telemetry/TelemetrySystem.java +++ b/telemetry/src/main/java/datadog/telemetry/TelemetrySystem.java @@ -26,6 +26,7 @@ import datadog.trace.api.InstrumenterConfig; import datadog.trace.api.civisibility.config.BazelMode; import datadog.trace.api.iast.telemetry.Verbosity; +import datadog.trace.api.internal.VisibleForTesting; import datadog.trace.api.rum.RumInjector; import datadog.trace.util.AgentThreadFactory; import java.lang.instrument.Instrumentation; @@ -171,4 +172,9 @@ public static void stop() { } } } + + @VisibleForTesting + static Thread getTelemetryThread() { + return TELEMETRY_THREAD; + } } diff --git a/telemetry/src/test/groovy/datadog/telemetry/BufferedEventsSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/BufferedEventsSpecification.groovy deleted file mode 100644 index 4dcd8cd50b1..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/BufferedEventsSpecification.groovy +++ /dev/null @@ -1,122 +0,0 @@ -package datadog.telemetry - -import datadog.telemetry.api.DistributionSeries -import datadog.telemetry.api.Integration -import datadog.telemetry.api.LogMessage -import datadog.telemetry.api.Metric -import datadog.telemetry.dependency.Dependency -import datadog.trace.api.ConfigOrigin -import datadog.trace.api.ConfigSetting -import datadog.trace.api.telemetry.Endpoint -import datadog.trace.test.util.DDSpecification - -class BufferedEventsSpecification extends DDSpecification { - - def 'empty events'() { - def events = new BufferedEvents() - - expect: - events.isEmpty() - !events.hasConfigChangeEvent() - !events.hasDependencyEvent() - !events.hasDistributionSeriesEvent() - !events.hasIntegrationEvent() - !events.hasLogMessageEvent() - !events.hasMetricEvent() - !events.hasEndpoint() - } - - def 'return added events'() { - def events = new BufferedEvents() - def configSetting = ConfigSetting.of("key", "value", ConfigOrigin.DEFAULT) - def dependency = new Dependency("name", "version", "source", "hash") - def series = new DistributionSeries() - def integration = new Integration("integration-name", true) - def logMessage = new LogMessage() - def metric = new Metric() - def endpoint = new Endpoint() - - when: - events.addConfigChangeEvent(configSetting) - - then: - !events.isEmpty() - events.hasConfigChangeEvent() - events.nextConfigChangeEvent() == configSetting - !events.hasConfigChangeEvent() - events.isEmpty() - - when: - events.addDependencyEvent(dependency) - - then: - !events.isEmpty() - events.hasDependencyEvent() - events.nextDependencyEvent() == dependency - !events.hasDependencyEvent() - events.isEmpty() - - when: - events.addDistributionSeriesEvent(series) - - then: - !events.isEmpty() - events.hasDistributionSeriesEvent() - events.nextDistributionSeriesEvent() == series - !events.hasDistributionSeriesEvent() - events.isEmpty() - - when: - events.addIntegrationEvent(integration) - - then: - !events.isEmpty() - events.hasIntegrationEvent() - events.nextIntegrationEvent() == integration - !events.hasIntegrationEvent() - events.isEmpty() - - when: - events.addLogMessageEvent(logMessage) - - then: - !events.isEmpty() - events.hasLogMessageEvent() - events.nextLogMessageEvent() == logMessage - !events.hasLogMessageEvent() - events.isEmpty() - - when: - events.addMetricEvent(metric) - - then: - !events.isEmpty() - events.hasMetricEvent() - events.nextMetricEvent() == metric - !events.hasMetricEvent() - events.isEmpty() - - when: - events.addEndpointEvent(endpoint) - - then: - !events.isEmpty() - events.hasEndpoint() - events.nextEndpoint() == endpoint - !events.hasEndpoint() - events.isEmpty() - } - - def 'noop sink'() { - def sink = EventSink.NOOP - - expect: - sink.addMetricEvent(null) - sink.addLogMessageEvent(null) - sink.addIntegrationEvent(null) - sink.addDistributionSeriesEvent(null) - sink.addDependencyEvent(null) - sink.addConfigChangeEvent(null) - sink.addEndpointEvent(null) - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/DisabledDependencyServiceTest.groovy b/telemetry/src/test/groovy/datadog/telemetry/DisabledDependencyServiceTest.groovy deleted file mode 100644 index ae2c3723f44..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/DisabledDependencyServiceTest.groovy +++ /dev/null @@ -1,24 +0,0 @@ -package datadog.telemetry - -import datadog.telemetry.dependency.LocationsCollectingTransformer -import datadog.trace.test.util.DDSpecification - -import java.lang.instrument.Instrumentation - -class DisabledDependencyServiceTest extends DDSpecification{ - - Instrumentation inst = Mock() - - void setup(){ - injectSysConfig("dd.telemetry.dependency-collection.enabled", "false") - } - - void 'installs disabled dependency service and verify transformer'() { - when: - def depService = TelemetrySystem.createDependencyService(inst) - - then: - 0 * inst.addTransformer(_ as LocationsCollectingTransformer) - null == depService - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/EventSourceTest.groovy b/telemetry/src/test/groovy/datadog/telemetry/EventSourceTest.groovy deleted file mode 100644 index 5da3f0a3ea1..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/EventSourceTest.groovy +++ /dev/null @@ -1,68 +0,0 @@ -package datadog.telemetry - -import datadog.telemetry.api.DistributionSeries -import datadog.telemetry.api.Integration -import datadog.telemetry.api.LogMessage -import datadog.telemetry.api.Metric -import datadog.telemetry.dependency.Dependency -import datadog.trace.api.ConfigOrigin -import datadog.trace.api.ConfigSetting -import datadog.trace.api.telemetry.Endpoint -import datadog.trace.api.telemetry.ProductChange -import datadog.trace.test.util.DDSpecification - -import java.util.concurrent.LinkedBlockingQueue - -class EventSourceTest extends DDSpecification{ - - void "test isEmpty when adding and clearing #eventType"() { - setup: - final eventQueues = [ - configChangeQueue : new LinkedBlockingQueue(), - integrationQueue : new LinkedBlockingQueue(), - dependencyQueue : new LinkedBlockingQueue(), - metricQueue : new LinkedBlockingQueue(), - distributionSeriesQueue: new LinkedBlockingQueue(), - logMessageQueue : new LinkedBlockingQueue(), - productChanges : new LinkedBlockingQueue(), - endpointQueue : new LinkedBlockingQueue() - ] - - def eventSource = new EventSource.Queued( - eventQueues.configChangeQueue, - eventQueues.integrationQueue, - eventQueues.dependencyQueue, - eventQueues.metricQueue, - eventQueues.distributionSeriesQueue, - eventQueues.logMessageQueue, - eventQueues.productChanges, - eventQueues.endpointQueue - ) - - expect: - eventSource.isEmpty() - - when: "add an event to the queue" - eventQueues[eventQueueName].add(eventInstance) - - then: "eventSource should not be empty" - !eventSource.isEmpty() - - when: "clear the queue" - eventQueues[eventQueueName].clear() - - then: "eventSource should be empty again" - eventSource.isEmpty() - - where: - eventType | eventQueueName | eventInstance - "Config Change" | "configChangeQueue" | ConfigSetting.of("key", "value", ConfigOrigin.ENV) - "Integration" | "integrationQueue" | new Integration("name", true) - "Dependency" | "dependencyQueue" | new Dependency("name", "version", "type", null) - "Metric" | "metricQueue" | new Metric() - "Distribution Series" | "distributionSeriesQueue" | new DistributionSeries() - "Log Message" | "logMessageQueue" | new LogMessage() - "Product Change" | "productChanges" | new ProductChange() - "Endpoint" | "endpointQueue" | new Endpoint() - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/ExtendedHeartbeatDataSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/ExtendedHeartbeatDataSpecification.groovy deleted file mode 100644 index 98d83c7572f..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/ExtendedHeartbeatDataSpecification.groovy +++ /dev/null @@ -1,84 +0,0 @@ -package datadog.telemetry - -import datadog.telemetry.api.Integration -import datadog.telemetry.dependency.Dependency -import datadog.trace.api.ConfigOrigin -import datadog.trace.api.ConfigSetting -import spock.lang.Specification - -class ExtendedHeartbeatDataSpecification extends Specification { - - def dependency = new Dependency("name", "version", "source", "hash") - def configSetting = ConfigSetting.of("key", "value", ConfigOrigin.DEFAULT) - def integration = new Integration("integration", true) - - def 'discard dependencies after exceeding limit'() { - setup: - def extHeartbeatData = new ExtendedHeartbeatData(limit) - - when: - (limit + 1).times { - extHeartbeatData.pushDependency(dependency) - } - - then: - def snapshot = extHeartbeatData.snapshot() - int i = 0 - while (snapshot.hasDependencyEvent()) { - snapshot.nextDependencyEvent() - i++ - } - i == limit - - where: - limit << [0, 2, 10] - } - - def 'return all collected data'() { - setup: - def extHeartbeatData = new ExtendedHeartbeatData() - - when: - def s0 = extHeartbeatData.snapshot() - - then: - s0.isEmpty() - - when: - extHeartbeatData.pushDependency(dependency) - extHeartbeatData.pushConfigSetting(configSetting) - extHeartbeatData.pushIntegration(integration) - - then: - def s1 = extHeartbeatData.snapshot() - - !s1.isEmpty() - - s1.hasDependencyEvent() - s1.nextDependencyEvent() == dependency - !s1.hasDependencyEvent() - - !s1.isEmpty() - - s1.hasConfigChangeEvent() - s1.nextConfigChangeEvent() == configSetting - !s1.hasConfigChangeEvent() - - !s1.isEmpty() - - s1.hasIntegrationEvent() - s1.nextIntegrationEvent() == integration - !s1.hasIntegrationEvent() - - s1.isEmpty() - - when: 'another snapshot includes all data' - def s2 = extHeartbeatData.snapshot() - - then: - !s2.isEmpty() - s2.hasDependencyEvent() - s2.hasConfigChangeEvent() - s2.hasIntegrationEvent() - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/HostInfoTest.groovy b/telemetry/src/test/groovy/datadog/telemetry/HostInfoTest.groovy deleted file mode 100644 index eddb474baad..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/HostInfoTest.groovy +++ /dev/null @@ -1,55 +0,0 @@ -package datadog.telemetry - -import datadog.environment.OperatingSystem -import spock.lang.Specification - -import static org.junit.jupiter.api.Assumptions.assumeTrue - -class HostInfoTest extends Specification { - void 'getHostname'() { - when: - final hostname = HostInfo.getHostname() - - then: - hostname != null - !hostname.trim().isEmpty() - } - - void 'getOsName'() { - when: - final osName = HostInfo.getOsName() - - then: - ["Linux", "Windows", "Darwin"].contains(osName) - } - - void 'getOsVersion'() { - when: - final osVersion = HostInfo.getOsVersion() - - then: - osVersion != null - !osVersion.trim().isEmpty() - } - - void 'compare to uname'() { - assumeTrue('uname -a'.execute().waitFor() == 0) - - expect: - HostInfo.getHostname() == 'uname -n'.execute().text.trim() - HostInfo.getOsName() == 'uname -s'.execute().text.trim() - HostInfo.getKernelName() == 'uname -s'.execute().text.trim() - if (OperatingSystem.isMacOs()) { - // uname -r will return X.Y.Z version, while JVM will report just X.Y - 'uname -r'.execute().text.trim().startsWith(HostInfo.getKernelRelease()) - - // No /proc in macOS - HostInfo.getKernelVersion() == '' - } - else { - HostInfo.getKernelRelease() == 'uname -r'.execute().text.trim() - // Ideally, this would be equal, but for now, we'll compromise to startWith. - 'uname -v'.execute().text.trim().startsWith(HostInfo.getKernelVersion()) - } - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TelemetryClientTest.groovy b/telemetry/src/test/groovy/datadog/telemetry/TelemetryClientTest.groovy deleted file mode 100644 index e4327d5010e..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TelemetryClientTest.groovy +++ /dev/null @@ -1,54 +0,0 @@ -package datadog.telemetry - -import datadog.communication.http.HttpRetryPolicy -import datadog.telemetry.api.RequestType -import datadog.trace.api.Config -import okhttp3.HttpUrl -import okhttp3.OkHttpClient -import spock.lang.Specification - -class TelemetryClientTest extends Specification { - - def "Intake client uses CI Visibility agentless URL if configured to do so"() { - setup: - def config = Spy(Config.get()) - config.getApiKey() >> "dummy-key" - config.getAgentTimeout() >> 123 - config.getSite() >> "datad0g.com" - config.isCiVisibilityEnabled() >> ciVisEnabled - config.isCiVisibilityAgentlessEnabled() >> ciVisAgentlessEnabled - config.getCiVisibilityAgentlessUrl() >> ciVisAgentlessUrl - - when: - def intakeClient = TelemetryClient.buildIntakeClient(config, HttpRetryPolicy.Factory.NEVER_RETRY) - - then: - intakeClient.getUrl().toString() == expectedUrl - - where: - ciVisEnabled | ciVisAgentlessEnabled | ciVisAgentlessUrl | expectedUrl - true | true | "http://ci.visibility.agentless.url" | "http://ci.visibility.agentless.url/api/v2/apmtelemetry" - false | true | "http://ci.visibility.agentless.url" | "https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry" - true | false | "http://ci.visibility.agentless.url" | "https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry" - true | true | null | "https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry" - } - - def "Intake client retries telemetry request if configured to do so"() { - setup: - def httpClient = Mock(OkHttpClient) - def httpRetryPolicy = new HttpRetryPolicy.Factory(2, 50, 1.5, true) - def httpUrl = HttpUrl.get("https://intake.example.com") - def intakeClient = new TelemetryClient(httpClient, httpRetryPolicy, httpUrl, "dummy-api-key") - - when: - intakeClient.sendHttpRequest(dummyRequest()) - - then: - // original request + 2 retries - 3 * httpClient.newCall(_) >> { throw new ConnectException("exception") } - } - - def dummyRequest() { - return new TelemetryRequest(Mock(EventSource), Mock(EventSink), 1000, RequestType.APP_STARTED, false).httpRequest() - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TelemetryRequestBodySpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/TelemetryRequestBodySpecification.groovy deleted file mode 100644 index 462b6cec0f0..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TelemetryRequestBodySpecification.groovy +++ /dev/null @@ -1,191 +0,0 @@ -package datadog.telemetry - - -import com.squareup.moshi.Moshi -import com.squareup.moshi.Types -import datadog.telemetry.api.RequestType -import datadog.trace.api.ConfigOrigin -import datadog.trace.api.ConfigSetting -import datadog.trace.api.ProcessTags -import datadog.trace.api.telemetry.ProductChange -import datadog.trace.test.util.DDSpecification -import okhttp3.RequestBody -import okio.Buffer - -import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED -import static datadog.trace.api.telemetry.ProductChange.ProductType.APPSEC -import static datadog.trace.api.telemetry.ProductChange.ProductType.DYNAMIC_INSTRUMENTATION -import static datadog.trace.api.telemetry.ProductChange.ProductType.PROFILER - -/** - * This test only verifies non-functional specifics that are not covered in TelemetryServiceSpecification - */ -class TelemetryRequestBodySpecification extends DDSpecification { - - def 'throw SerializationException in case of JSON nesting problem'() { - setup: - def req = new TelemetryRequestBody(RequestType.APP_STARTED) - - when: - req.beginRequest(false) - req.beginRequest(false) - - then: - TelemetryRequestBody.SerializationException ex = thrown() - ex.message == "Failed serializing Telemetry begin-request part!" - ex.cause != null - } - - def 'throw SerializationException in case of more than one top-level JSON value'() { - setup: - def req = new TelemetryRequestBody(RequestType.APP_STARTED) - - when: - req.beginRequest(false) - req.endRequest() - req.beginRequest(false) - - then: - TelemetryRequestBody.SerializationException ex = thrown() - ex.message == "Failed serializing Telemetry begin-request part!" - ex.cause != null - } - - def 'writeConfig must support values of Boolean, String, Number, and null'() { - setup: - TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_CLIENT_CONFIGURATION_CHANGE) - Map map = new HashMap<>() - map.put("key1", "value1") - map.put("key2", Double.parseDouble("432.32")) - map.put("key3", 324) - - when: - req.beginRequest(false) - // exclude request header to simplify assertion - drainToString(req) - - then: - req.beginConfiguration() - [ - ConfigSetting.of("string", "bar", ConfigOrigin.REMOTE), - ConfigSetting.of("int", 2342, ConfigOrigin.DEFAULT), - ConfigSetting.of("double", Double.valueOf("123.456"), ConfigOrigin.ENV), - ConfigSetting.of("map", map, ConfigOrigin.JVM_PROP), - ConfigSetting.of("list", Arrays.asList("1", "2", 3), ConfigOrigin.DEFAULT), - // make sure null values are serialized - ConfigSetting.of("null", null, ConfigOrigin.DEFAULT) - ].forEach { cc -> req.writeConfiguration(cc) } - req.endConfiguration() - - then: - drainToString(req) == ',"configuration":[' + - '{"name":"DD_STRING","value":"bar","origin":"remote_config","seq_id":0},' + - '{"name":"DD_INT","value":"2342","origin":"default","seq_id":0},' + - '{"name":"DD_DOUBLE","value":"123.456","origin":"env_var","seq_id":0},' + - '{"name":"DD_MAP","value":"key1:value1,key2:432.32,key3:324","origin":"jvm_prop","seq_id":0},' + - '{"name":"DD_LIST","value":"1,2,3","origin":"default","seq_id":0},' + - '{"name":"DD_NULL","value":null,"origin":"default","seq_id":0}]' - } - - def 'use environment variable for setting keys'() { - setup: - TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_CLIENT_CONFIGURATION_CHANGE) - - when: - req.beginRequest(false) - // exclude request header to simplify assertion - drainToString(req) - - then: - req.beginConfiguration() - req.writeConfiguration(ConfigSetting.of("this.is.a.key", "value", ConfigOrigin.REMOTE)) - req.endConfiguration() - - then: - drainToString(req) == ',"configuration":[{"name":"DD_THIS_IS_A_KEY","value":"value","origin":"remote_config","seq_id":0}]' - } - - def 'add debug flag'() { - setup: - TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_STARTED) - - when: - req.beginRequest(true) - req.endRequest() - - then: - drainToString(req).contains("\"debug\":true") - } - - void 'test writeProducts'() { - setup: - TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_PRODUCT_CHANGE) - final products = new HashMap() - if (appsecChange) { - products.put(APPSEC, appsecEnabled) - } - if (profilerChange) { - products.put(PROFILER, profilerEnabled) - } - if (dynamicInstrumentationChange) { - products.put(DYNAMIC_INSTRUMENTATION, dynamicInstrumentationEnabled) - } - - when: - req.beginRequest(false) - req.writeProducts(products) - req.endRequest() - - then: - final result = drainToString(req) - result.contains("\"appsec\":{\"enabled\":${appsecEnabled}}") == appsecChange - result.contains("\"profiler\":{\"enabled\":${profilerEnabled}}") == profilerChange - result.contains("\"dynamic_instrumentation\":{\"enabled\":${dynamicInstrumentationEnabled}}") == dynamicInstrumentationChange - - where: - appsecChange | profilerChange | dynamicInstrumentationChange | appsecEnabled | profilerEnabled | dynamicInstrumentationEnabled - true | true | true | true | true | true - true | true | true | false | false | false - false | false | false | true | true | true - false | true | true | true | true | true - true | false | true | true | true | true - true | true | false | true | true | true - } - - String drainToString(RequestBody body) { - Buffer buf = new Buffer() - body.writeTo(buf) - byte[] bytes = new byte[buf.size()] - buf.read(bytes) - return new String(bytes) - } - - def 'Should propagate process tags when enabled #processTagsEnabled'() { - setup: - injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "$processTagsEnabled") - ProcessTags.reset() - TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_STARTED) - - when: - req.beginRequest(true) - req.endRequest() - - then: - def type = Types.newParameterizedType(Map, String, Object) - def adapter = new Moshi.Builder().build().adapter(type) - def parsed = (Map)adapter.fromJson(drainToString(req)) - def parsedTags = ((Map)parsed.get("application")).get("process_tags") - if (processTagsEnabled) { - assert parsedTags == ProcessTags.tagsForSerialization.toString() - } else { - assert parsedTags == null - } - - cleanup: - injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "false") - ProcessTags.reset() - - where: - processTagsEnabled << [true, false] - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TelemetryRouterSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/TelemetryRouterSpecification.groovy deleted file mode 100644 index 5fa71ea4b23..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TelemetryRouterSpecification.groovy +++ /dev/null @@ -1,449 +0,0 @@ -package datadog.telemetry - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.communication.http.HttpRetryPolicy -import datadog.telemetry.api.RequestType -import okhttp3.Call -import okhttp3.HttpUrl -import okhttp3.MediaType -import okhttp3.OkHttpClient -import okhttp3.Protocol -import okhttp3.Request -import okhttp3.Response -import okhttp3.ResponseBody -import spock.lang.Specification - -class TelemetryRouterSpecification extends Specification { - - def dummyRequest() { - return new TelemetryRequest(Mock(EventSource), Mock(EventSink), 1000, RequestType.APP_STARTED, false) - } - - Call mockResponse(int code) { - Stub(Call) { - execute() >> { - new Response.Builder() - .request(new Request.Builder().url(HttpUrl.get("https://example.com")).build()) - .protocol(Protocol.HTTP_1_1) - .message("OK") - .body(ResponseBody.create(MediaType.get("text/plain"), "OK")) - .code(code) - .build() - } - } - } - - static HttpUrl agentUrl = HttpUrl.get("https://agent.example.com") - static HttpUrl agentTelemetryUrl = agentUrl.resolve("telemetry/proxy/api/v2/apmtelemetry") - static HttpUrl intakeUrl = HttpUrl.get("https://intake.example.com") - static String apiKey = "api-key" - static String apiKeyHeader = "DD-API-KEY" - - OkHttpClient okHttpClient = Mock() - DDAgentFeaturesDiscovery ddAgentFeaturesDiscovery = Mock() - - def agentTelemetryClient = TelemetryClient.buildAgentClient(okHttpClient, agentUrl, HttpRetryPolicy.Factory.NEVER_RETRY) - def intakeTelemetryClient = new TelemetryClient(okHttpClient, HttpRetryPolicy.Factory.NEVER_RETRY, intakeUrl, apiKey) - def httpClient = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, false) - - def 'map an http status code to the correct send result'() { - when: - def result = httpClient.sendRequest(dummyRequest()) - - then: - result == sendResult - 1 * okHttpClient.newCall(_) >> mockResponse(httpCode) - - where: - httpCode | sendResult - 100 | TelemetryClient.Result.FAILURE - 202 | TelemetryClient.Result.SUCCESS - 404 | TelemetryClient.Result.NOT_FOUND - 500 | TelemetryClient.Result.FAILURE - } - - def 'catch IOException from OkHttpClient and return FAILURE'() { - when: - def result = httpClient.sendRequest(dummyRequest()) - - then: - result == TelemetryClient.Result.FAILURE - 1 * okHttpClient.newCall(_) >> { throw new IOException("exception") } - } - - def 'catch InterruptedIOException from OkHttpClient and return INTERRUPTED'() { - when: - def result = httpClient.sendRequest(dummyRequest()) - - then: - result == TelemetryClient.Result.INTERRUPTED - 1 * okHttpClient.newCall(_) >> { throw new InterruptedIOException("interrupted") } - } - - def 'keep trying to send telemetry to Agent despite of return code when Intake client is null'() { - setup: - def httpClient = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, null, false) - - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >>> [true, false] - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> [false, true] - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - where: - returnCode | _ - 200 | _ - 404 | _ - 500 | _ - } - - def 'switch to Intake when Agent stops supporting telemetry proxy and telemetry requests start failing'() { - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> true - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - where: - returnCode | _ - 404 | _ - 500 | _ - } - - def 'when configured to prefer Intake: use Intake client from the start'() { - Request request - - setup: - def telemetryRouter = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true) - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(200) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - } - - def 'when configured to prefer Intake: do not switch to Agent if Intake request succeeds, even if Agent supports telemetry proxy'() { - Request request - - setup: - def telemetryRouter = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true) - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> true - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(200) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(200) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - } - - def 'when configured to prefer Intake: do not switch to Agent if request is interrupted'() { - Request request - - setup: - def telemetryRouter = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true) - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> true - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; throw new InterruptedIOException("interrupted") } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(200) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - } - - def 'when configured to prefer Intake: switch to Agent if Intake request fails'() { - Request request - - setup: - def telemetryRouter = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true) - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> true - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(403) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - when: - telemetryRouter.sendRequest(dummyRequest()) - - then: - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(200) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - } - - def 'do not switch to Intake when Agent stops supporting telemetry proxy but accepts telemetry requests'() { - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> true - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(200) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(201) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - } - - def 'switch to Intake when Agent fails to receive telemetry requests'() { - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >>> [true, false] - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >>> [false, true] - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - where: - returnCode | _ - 404 | _ - 500 | _ - } - - def 'use Agent when Intake is not available'() { - setup: - def httpClient = new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, null, false) - - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> - request = args[0]; mockResponse(returnCode) - } - request.url() == expectedUrl - request.header(apiKeyHeader) == expectedApiKey - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == expectedUrl - request.header(apiKeyHeader) == expectedApiKey - - where: - returnCode | expectedApiKey | expectedUrl - 404 | null | agentTelemetryUrl - 500 | null | agentTelemetryUrl - } - - def 'switch to Intake then back to Agent when both fail to receive telemetry requests'() { - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: 'always send first telemetry request to Agent' - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> - request = args[0]; mockResponse(returnCode) - } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: 'switch to Intake if sending a telemetry request to Agent failed or Agent supports telemetry proxy' - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - when: - httpClient.sendRequest(dummyRequest()) - - then: 'switch back to Agent if Intake request fails' - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - where: - returnCode | _ - 404 | _ - 500 | _ - } - - def 'single-client constructor skips feature discovery and delegates to the given client'() { - setup: - def singleClient = Mock(TelemetryClient) - def router = new TelemetryRouter(singleClient) - - when: - def result = router.sendRequest(dummyRequest()) - - then: - result == TelemetryClient.Result.SUCCESS - 1 * singleClient.sendHttpRequest(_) >> TelemetryClient.Result.SUCCESS - 0 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 0 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() - } - - def 'single-client constructor does not switch clients on failure'() { - setup: - def singleClient = Mock(TelemetryClient) - def router = new TelemetryRouter(singleClient) - - when: 'first request fails' - def firstResult = router.sendRequest(dummyRequest()) - - then: - firstResult == TelemetryClient.Result.FAILURE - 1 * singleClient.sendHttpRequest(_) >> TelemetryClient.Result.FAILURE - - when: 'second request goes to the same client' - def secondResult = router.sendRequest(dummyRequest()) - - then: - secondResult == TelemetryClient.Result.SUCCESS - 1 * singleClient.sendHttpRequest(_) >> TelemetryClient.Result.SUCCESS - 0 * ddAgentFeaturesDiscovery._ - } - - def 'switch back to Agent if it starts supporting telemetry'() { - Request request - - when: - httpClient.sendRequest(dummyRequest()) - - then: 'always send first telemetry request to Agent' - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> - request = args[0]; mockResponse(returnCode) - } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - when: - httpClient.sendRequest(dummyRequest()) - - then: 'switch to Intake if sending a telemetry request to Agent failed or Agent supports telemetry proxy' - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> true - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(201) } - request.url() == intakeUrl - request.header(apiKeyHeader) == apiKey - - when: - httpClient.sendRequest(dummyRequest()) - - then: 'switch back to Agent if it starts supporting telemetry proxy' - 1 * ddAgentFeaturesDiscovery.discoverIfOutdated() - 1 * ddAgentFeaturesDiscovery.supportsTelemetryProxy() >> false - 1 * okHttpClient.newCall(_) >> { args -> request = args[0]; mockResponse(returnCode) } - request.url() == agentTelemetryUrl - request.header(apiKeyHeader) == null - - where: - returnCode | _ - 404 | _ - 500 | _ - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TelemetryRunnableSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/TelemetryRunnableSpecification.groovy deleted file mode 100644 index 16e392b872d..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TelemetryRunnableSpecification.groovy +++ /dev/null @@ -1,392 +0,0 @@ -package datadog.telemetry - -import datadog.telemetry.metric.MetricPeriodicAction -import datadog.trace.api.config.GeneralConfig -import datadog.trace.api.telemetry.MetricCollector -import datadog.trace.api.time.TimeSource -import datadog.trace.test.util.DDSpecification -import datadog.trace.util.ConfigStrings - -import java.util.concurrent.CyclicBarrier -import java.util.concurrent.TimeUnit - -class TelemetryRunnableSpecification extends DDSpecification { - - static class TickSleeper implements TelemetryRunnable.ThreadSleeper { - CyclicBarrier sleeped = new CyclicBarrier(2) - CyclicBarrier go = new CyclicBarrier(2) - TelemetryRunnable.ThreadSleeper delegate - - @Override - void sleep(long timeoutMs) { - delegate?.sleep(timeoutMs) - sleeped.await(10, TimeUnit.SECONDS) - go.await(10, TimeUnit.SECONDS) - } - } - - Thread t = null - - void cleanup() { - if (t?.isAlive()) { - t.interrupt() - t.join() - } - } - - void 'happy path'() { - setup: - injectEnvConfig(ConfigStrings.toEnvVar(GeneralConfig.TELEMETRY_EXTENDED_HEARTBEAT_INTERVAL), "65") - TelemetryRunnable.ThreadSleeper sleeperMock = Mock() - TickSleeper sleeper = new TickSleeper(delegate: sleeperMock) - TimeSource timeSource = Mock() - TelemetryService telemetryService = Mock(TelemetryService) - MetricCollector metricCollector = Mock(MetricCollector) - MetricPeriodicAction metricAction = Stub(MetricPeriodicAction) { - collector() >> metricCollector - } - TelemetryRunnable.TelemetryPeriodicAction periodicAction = Mock(TelemetryRunnable.TelemetryPeriodicAction) - TelemetryRunnable runnable = new TelemetryRunnable(telemetryService, [metricAction, periodicAction], sleeper, timeSource) - t = new Thread(runnable) - - when: 'initial iteration before the first sleep (metrics and heartbeat)' - t.start() - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: 'two unsuccessful attempts to send app-started with the following successful attempt' - 3 * telemetryService.sendAppStartedEvent() >>> [false, false, true] - 1 * timeSource.getCurrentTimeMillis() >> 60 * 1000 - _ * telemetryService.addConfiguration(_) - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * metricCollector.drain() >> [] - 1 * metricCollector.drainDistributionSeries() >> [] - 1 * periodicAction.doIteration(telemetryService) - - then: 'two partial and one final telemetry data requests' - 3 * telemetryService.sendTelemetryEvents() >>> [true, true, false] - 1 * timeSource.getCurrentTimeMillis() >> 60 * 1000 + 1 - 1 * sleeperMock.sleep(9999) - 0 * _ - - when: 'second iteration (10 seconds, metrics)' - sleeper.go.await(10, TimeUnit.SECONDS) - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 70 * 1000 - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 70 * 1000 + 2 - 1 * sleeperMock.sleep(9998) - 0 * _ - - when: 'third iteration (20 seconds, metrics)' - sleeper.go.await(10, TimeUnit.SECONDS) - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 80 * 1000 - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 80 * 1000 + 3 - 1 * sleeperMock.sleep(9997) - 0 * _ - - when: 'fourth iteration (30 seconds, metrics)' - sleeper.go.await(10, TimeUnit.SECONDS) - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 90 * 1000 - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 90 * 1000 + 4 - 1 * sleeperMock.sleep(9996) - 0 * _ - - when: 'fifth iteration (40 seconds, metrics)' - sleeper.go.await(10, TimeUnit.SECONDS) - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 100 * 1000 - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 100 * 1000 + 5 - 1 * sleeperMock.sleep(9995) - 0 * _ - - when: 'sixth iteration (50 seconds, metrics)' - sleeper.go.await(10, TimeUnit.SECONDS) - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 110 * 1000 - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 110 * 1000 + 6 - 1 * sleeperMock.sleep(9994) - 0 * _ - - when: 'seventh iteration (60 seconds, metrics, heartbeat)' - sleeper.go.await(10, TimeUnit.SECONDS) - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 120 * 1000 - - then: - 1 * metricCollector.prepareMetrics() - - then: - 1 * metricCollector.drain() >> [] - 1 * metricCollector.drainDistributionSeries() >> [] - 1 * periodicAction.doIteration(telemetryService) - - then: - 1 * telemetryService.sendTelemetryEvents() - 1 * timeSource.getCurrentTimeMillis() >> 120 * 1000 + 7 - 1 * sleeperMock.sleep(9993) - - when: 'eights iteration (65 seconds, extended-heartbeat)' - sleeper.go.await(5, TimeUnit.SECONDS) - sleeper.sleeped.await(5, TimeUnit.SECONDS) - - then: - 1 * timeSource.getCurrentTimeMillis() >> 125 * 1000 - - then: - 1 * telemetryService.sendExtendedHeartbeat() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 125 * 1000 + 8 - 1 * sleeperMock.sleep(4992) - 0 * _ - - when: - t.interrupt() - t.join() - - // flush pending data before shutdown - then: - 1 * metricCollector.prepareMetrics() - 1 * metricCollector.drain() >> [] - 1 * metricCollector.drainDistributionSeries() >> [] - 1 * periodicAction.doIteration(telemetryService) - 1 * telemetryService.sendTelemetryEvents() - - then: - 1 * telemetryService.sendAppClosingEvent() - 0 * _ - } - - void 'do not reattempt app-started event until next cycle'() { - setup: - TelemetryRunnable.ThreadSleeper sleeperMock = Mock() - TickSleeper sleeper = new TickSleeper(delegate: sleeperMock) - TimeSource timeSource = Mock() - TelemetryService telemetryService = Mock(TelemetryService) - MetricCollector metricCollector = Mock(MetricCollector) - MetricPeriodicAction metricAction = Stub(MetricPeriodicAction) { - collector() >> metricCollector - } - TelemetryRunnable.TelemetryPeriodicAction periodicAction = Mock(TelemetryRunnable.TelemetryPeriodicAction) - TelemetryRunnable runnable = new TelemetryRunnable(telemetryService, [metricAction, periodicAction], sleeper, timeSource) - t = new Thread(runnable) - - when: 'initial iteration before the first sleep (metrics and heartbeat)' - t.start() - sleeper.sleeped.await(10, TimeUnit.SECONDS) - - then: 'three unsuccessful attempts to send app-started (TelemetryRunnable.MAX_APP_STARTED_RETRIES) with following successful attempt' - 3 * telemetryService.sendAppStartedEvent() >>> [false, false, false] - 2 * timeSource.getCurrentTimeMillis() >> 60 * 1000 - 1 * sleeperMock.sleep(10000) - } - - void 'scheduler skips metrics intervals'() { - setup: - TimeSource timeSource = Mock() - TickSleeper sleeper = Mock() - TelemetryRunnable.Scheduler scheduler = new TelemetryRunnable.Scheduler(timeSource, sleeper, 60 * 1000, 10 * 1000, 0) - - when: 'first iteration' - scheduler.init() - - then: 'run everything' - timeSource.getCurrentTimeMillis() >> 0 - scheduler.shouldRunMetrics() - scheduler.shouldRunHeartbeat() - 0 * _ - - when: - scheduler.sleepUntilNextIteration() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 1 - 1 * sleeper.sleep(10 * 1000 - 1) - 1 * timeSource.getCurrentTimeMillis() >> 10 * 1000 - 0 * _ - - when: 'one metrics interval is exceeded' - assert scheduler.shouldRunMetrics() - assert !scheduler.shouldRunHeartbeat() - scheduler.sleepUntilNextIteration() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 20 * 1000 + 1 - 1 * sleeper.sleep(9999) - 1 * timeSource.getCurrentTimeMillis() >> 30 * 1000 - 0 * _ - - when: 'two metrics interval are exceeded' - assert scheduler.shouldRunMetrics() - assert !scheduler.shouldRunHeartbeat() - scheduler.sleepUntilNextIteration() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 50 * 1000 + 2 - 1 * sleeper.sleep(9998) - 1 * timeSource.getCurrentTimeMillis() >> 60 * 1000 - 0 * _ - scheduler.shouldRunMetrics() - scheduler.shouldRunHeartbeat() - } - - void 'scheduler skips heartbeat intervals'() { - setup: - TimeSource timeSource = Mock() - TickSleeper sleeper = Mock() - TelemetryRunnable.Scheduler scheduler = new TelemetryRunnable.Scheduler(timeSource, sleeper, 60 * 1000, 10 * 1000, 0) - - when: 'first iteration' - scheduler.init() - - then: 'run everything' - timeSource.getCurrentTimeMillis() >> 0 - scheduler.shouldRunMetrics() - scheduler.shouldRunHeartbeat() - 0 * _ - - when: - scheduler.sleepUntilNextIteration() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 1 - 1 * sleeper.sleep(10 * 1000 - 1) - 1 * timeSource.getCurrentTimeMillis() >> 10 * 1000 - 0 * _ - - when: 'heartbeat interval is exceeded' - assert scheduler.shouldRunMetrics() - assert !scheduler.shouldRunHeartbeat() - scheduler.sleepUntilNextIteration() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 70 * 1000 - 0 * _ - scheduler.shouldRunMetrics() - scheduler.shouldRunHeartbeat() - - when: 'metrics interval has been adjusted' - scheduler.sleepUntilNextIteration() - - then: - 1 * timeSource.getCurrentTimeMillis() >> 70 * 1000 + 1 - 1 * sleeper.sleep(10 * 1000 - 1) - 1 * timeSource.getCurrentTimeMillis() >> 80 * 1000 - 0 * _ - scheduler.shouldRunMetrics() - !scheduler.shouldRunHeartbeat() - } - - void 'scheduler with heartbeat #heartbeatSecs and metrics #metricsSecs and extended-heartbeat #extHeartbeatSecs'() { - setup: - TimeSourceAndSleeper timing = new TimeSourceAndSleeper() - TelemetryRunnable.Scheduler scheduler = new TelemetryRunnable.Scheduler(timing, timing, heartbeatSecs * 1000, metricsSecs * 1000, extHeartbeatSecs * 1000) - def metricsRun = [] - def heartbeatsRun = [] - def extHeartbeatsRun = [] - - when: - scheduler.init() - - and: - iters.times { - metricsRun.add(scheduler.shouldRunMetrics()) - heartbeatsRun.add(scheduler.shouldRunHeartbeat()) - def runExtHeartbeat = scheduler.shouldRunExtendedHeartbeat() - extHeartbeatsRun.add(runExtHeartbeat) - if (runExtHeartbeat) { - // need to manually advance to retry next iteration if extended-heartbeat request failed - scheduler.scheduleNextExtendedHeartbeat() - } - scheduler.sleepUntilNextIteration() - } - - then: - metricsRun.size() == iters - heartbeatsRun.size() == iters - metricsRun.count { it } == expectedMetrics - heartbeatsRun.count { it } == expectedHeartbeats - extHeartbeatsRun.count { it } == expectedExtHeartbeats - - where: - iters | metricsSecs | heartbeatSecs | extHeartbeatSecs | expectedMetrics | expectedHeartbeats | expectedExtHeartbeats - 10 | 0 | 0 | 0 | 10 | 10 | 10 - 10 | 1 | 1 | 1 | 10 | 10 | 9 - 12 | 10 | 60 | 60 | 12 | 2 | 1 - 12 | 60 | 10 | 10 | 2 | 12 | 11 - 6 | 3 | 5 | 5 | 4 | 3 | 2 - 6 | 5 | 3 | 3 | 3 | 4 | 3 - } - - class TimeSourceAndSleeper implements TimeSource, TelemetryRunnable.ThreadSleeper { - - private long currentTime = 0 - - @Override - void sleep(long timeoutMs) { - currentTime += timeoutMs - } - - @Override - long getCurrentTimeMillis() { - return currentTime - } - - @Override - long getNanoTicks() { - throw new RuntimeException("NOT IMPLEMENTED") - } - - @Override - long getCurrentTimeMicros() { - throw new RuntimeException("NOT IMPLEMENTED") - } - - @Override - long getCurrentTimeNanos() { - throw new RuntimeException("NOT IMPLEMENTED") - } - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TelemetryServiceSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/TelemetryServiceSpecification.groovy deleted file mode 100644 index 63702b4148d..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TelemetryServiceSpecification.groovy +++ /dev/null @@ -1,511 +0,0 @@ -package datadog.telemetry - -import datadog.telemetry.api.DistributionSeries -import datadog.telemetry.api.Integration -import datadog.telemetry.api.LogMessage -import datadog.telemetry.api.LogMessageLevel -import datadog.telemetry.api.Metric -import datadog.telemetry.api.RequestType -import datadog.telemetry.dependency.Dependency -import datadog.trace.api.ConfigOrigin -import datadog.trace.api.ConfigSetting -import datadog.trace.api.config.AppSecConfig -import datadog.trace.api.config.DebuggerConfig -import datadog.trace.api.config.ProfilingConfig -import datadog.trace.api.telemetry.Endpoint -import datadog.trace.api.telemetry.ProductChange -import datadog.trace.test.util.DDSpecification -import datadog.trace.util.ConfigStrings - -class TelemetryServiceSpecification extends DDSpecification { - def confKeyOrigin = ConfigOrigin.DEFAULT - def confKeyValue = ConfigSetting.of("confkey", "confvalue", confKeyOrigin) - def configuration = [confKeyOrigin: [confkey: confKeyValue]] - def integration = new Integration("integration", true) - def dependency = new Dependency("dependency", "1.0.0", "src", "hash") - def metric = new Metric().namespace("tracers").metric("metric").points([[1, 2]]).tags(["tag1", "tag2"]) - def distribution = new DistributionSeries().namespace("tracers").metric("distro").points([1, 2, 3]).tags(["tag1", "tag2"]).common(false) - def logMessage = new LogMessage().message("log-message").tags("tag1:tag2").level(LogMessageLevel.DEBUG).stackTrace("stack-trace").tracerTime(32423).count(1) - def productChange = new ProductChange().productType(ProductChange.ProductType.APPSEC).enabled(true) - def endpoint = new Endpoint().first(true).type('REST').method("GET").operation('http.request').resource("GET /test").path("/test").requestBodyType(['application/json']).responseBodyType(['application/json']).responseCode([200]).authentication(['JWT']) - - def 'happy path without data'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - when: 'first iteration' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: 'app-started' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products() - testHttpClient.assertNoMoreRequests() - - when: 'second iteration' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: 'app-heartbeat only' - testHttpClient.assertRequestBody(RequestType.APP_HEARTBEAT) - testHttpClient.assertNoMoreRequests() - - when: 'third iteration' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: 'app-heartbeat only' - testHttpClient.assertRequestBody(RequestType.APP_HEARTBEAT) - testHttpClient.assertNoMoreRequests() - } - - def 'happy path with data'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - when: 'add data before first iteration' - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - telemetryService.addMetric(metric) - telemetryService.addDistributionSeries(distribution) - telemetryService.addLogMessage(logMessage) - telemetryService.addProductChange(productChange) - telemetryService.addEndpoint(endpoint) - - and: 'send messages' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload() - .products() - .configuration([confKeyValue]) - - when: - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - .assertBatch(8) - .assertFirstMessage(RequestType.APP_HEARTBEAT).hasNoPayload() - // no configuration here as it has already been sent with the app-started event - .assertNextMessage(RequestType.APP_INTEGRATIONS_CHANGE).hasPayload().integrations([integration]) - .assertNextMessage(RequestType.APP_DEPENDENCIES_LOADED).hasPayload().dependencies([dependency]) - .assertNextMessage(RequestType.GENERATE_METRICS).hasPayload().namespace("tracers").metrics([metric]) - .assertNextMessage(RequestType.DISTRIBUTIONS).hasPayload().namespace("tracers").distributionSeries([distribution]) - .assertNextMessage(RequestType.LOGS).hasPayload().logs([logMessage]) - .assertNextMessage(RequestType.APP_PRODUCT_CHANGE).hasPayload().productChange(productChange) - .assertNextMessage(RequestType.APP_ENDPOINTS).hasPayload().endpoint(endpoint) - .assertNoMoreMessages() - testHttpClient.assertNoMoreRequests() - - when: 'second iteration heartbeat only' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: - testHttpClient.assertRequestBody(RequestType.APP_HEARTBEAT).assertNoPayload() - testHttpClient.assertNoMoreRequests() - - when: 'third iteration metrics data' - telemetryService.addMetric(metric) - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - .assertBatch(2) - .assertFirstMessage(RequestType.APP_HEARTBEAT).hasNoPayload() - .assertNextMessage(RequestType.GENERATE_METRICS).hasPayload().namespace("tracers").metrics([metric]) - .assertNoMoreMessages() - testHttpClient.assertNoMoreRequests() - } - - def 'happy path with data after app-started'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - when: 'send messages' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products() - testHttpClient.assertNoMoreRequests() - - when: 'add data after first iteration' - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - telemetryService.addMetric(metric) - telemetryService.addDistributionSeries(distribution) - telemetryService.addLogMessage(logMessage) - telemetryService.addProductChange(productChange) - telemetryService.addEndpoint(endpoint) - - and: 'send messages' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - .assertBatch(9) - .assertFirstMessage(RequestType.APP_HEARTBEAT).hasNoPayload() - .assertNextMessage(RequestType.APP_CLIENT_CONFIGURATION_CHANGE).hasPayload().configuration([confKeyValue]) - .assertNextMessage(RequestType.APP_INTEGRATIONS_CHANGE).hasPayload().integrations([integration]) - .assertNextMessage(RequestType.APP_DEPENDENCIES_LOADED).hasPayload().dependencies([dependency]) - .assertNextMessage(RequestType.GENERATE_METRICS).hasPayload().namespace("tracers").metrics([metric]) - .assertNextMessage(RequestType.DISTRIBUTIONS).hasPayload().namespace("tracers").distributionSeries([distribution]) - .assertNextMessage(RequestType.LOGS).hasPayload().logs([logMessage]) - .assertNextMessage(RequestType.APP_PRODUCT_CHANGE).hasPayload().productChange(productChange) - .assertNextMessage(RequestType.APP_ENDPOINTS).hasPayload().endpoint(endpoint) - .assertNoMoreMessages() - testHttpClient.assertNoMoreRequests() - } - - def 'do not discard data for app-started event until it has been successfully sent'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - telemetryService.addConfiguration(configuration) - - when: 'attempt with 404 error' - testHttpClient.expectRequest(TelemetryClient.Result.NOT_FOUND) - !telemetryService.sendAppStartedEvent() - - then: 'app-started is attempted' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products().configuration([confKeyValue]) - testHttpClient.assertNoMoreRequests() - - when: 'attempt with 500 error' - testHttpClient.expectRequest(TelemetryClient.Result.FAILURE) - !telemetryService.sendAppStartedEvent() - - then: 'app-started is attempted' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products().configuration([confKeyValue]) - testHttpClient.assertNoMoreRequests() - - when: 'attempt with unexpected FAILURE (not valid)' - testHttpClient.expectRequest(TelemetryClient.Result.FAILURE) - !telemetryService.sendAppStartedEvent() - - then: 'app-started is attempted' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products().configuration([confKeyValue]) - testHttpClient.assertNoMoreRequests() - - when: 'attempt with success' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: 'app-started is attempted' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products().configuration([confKeyValue]) - testHttpClient.assertNoMoreRequests() - } - - def 'resend data on successful attempt after a failure'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - telemetryService.addMetric(metric) - telemetryService.addDistributionSeries(distribution) - telemetryService.addLogMessage(logMessage) - telemetryService.addProductChange(productChange) - telemetryService.addEndpoint(endpoint) - - when: 'attempt with NOT_FOUND error' - testHttpClient.expectRequest(TelemetryClient.Result.NOT_FOUND) - !telemetryService.sendAppStartedEvent() - - then: 'app-started attempted with config' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products().configuration([confKeyValue]) - testHttpClient.assertNoMoreRequests() - - when: 'successful app-started attempt' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: 'attempt app-started with SUCCESS' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products().configuration([confKeyValue]) - - when: 'successful batch attempt' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: 'attempt batch with SUCCESS' - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - .assertBatch(8) - .assertFirstMessage(RequestType.APP_HEARTBEAT).hasNoPayload() - // no configuration here as it has already been sent with the app-started event - .assertNextMessage(RequestType.APP_INTEGRATIONS_CHANGE).hasPayload().integrations([integration]) - .assertNextMessage(RequestType.APP_DEPENDENCIES_LOADED).hasPayload().dependencies([dependency]) - .assertNextMessage(RequestType.GENERATE_METRICS).hasPayload().namespace("tracers").metrics([metric]) - .assertNextMessage(RequestType.DISTRIBUTIONS).hasPayload().namespace("tracers").distributionSeries([distribution]) - .assertNextMessage(RequestType.LOGS).hasPayload().logs([logMessage]) - .assertNextMessage(RequestType.APP_PRODUCT_CHANGE).hasPayload().productChange(productChange) - .assertNextMessage(RequestType.APP_ENDPOINTS).hasPayload().endpoint(endpoint) - .assertNoMoreMessages() - testHttpClient.assertNoMoreRequests() - - when: 'attempt with NOT_FOUND error' - testHttpClient.expectRequest(TelemetryClient.Result.NOT_FOUND) - telemetryService.sendTelemetryEvents() - - then: 'message-batch attempted with heartbeat' - testHttpClient.assertRequestBody(RequestType.APP_HEARTBEAT).assertNoPayload() - testHttpClient.assertNoMoreRequests() - } - - def 'send closing event request'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - when: - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppClosingEvent() - - then: - testHttpClient.assertRequestBody(RequestType.APP_CLOSING) - testHttpClient.assertNoMoreRequests() - } - - def 'report when both OTel and OT are enabled'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = Spy(new TelemetryService(testHttpClient, 1000, false)) - def otel = new Integration("opentelemetry-1", otelEnabled) - def ot = new Integration("opentracing", otEnabled) - - when: - telemetryService.addIntegration(otel) - - then: - 0 * telemetryService.warnAboutExclusiveIntegrations() - - when: - telemetryService.addIntegration(ot) - - then: - warnining * telemetryService.warnAboutExclusiveIntegrations() - - where: - otelEnabled | otEnabled | warnining - true | true | 1 - true | false | 0 - false | true | 0 - false | false | 0 - } - - def 'split telemetry requests if the size above the limit'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 5000, false) - - when: 'send a heartbeat request without telemetry data to measure body size to set stable request size limit' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: 'get body size' - def bodySize = testHttpClient.assertRequestBody(RequestType.APP_HEARTBEAT).bodySize() - bodySize > 0 - - when: 'sending first part of data' - telemetryService = new TelemetryService(testHttpClient, bodySize + 512, false) - - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - telemetryService.addMetric(metric) - telemetryService.addDistributionSeries(distribution) - telemetryService.addLogMessage(logMessage) - telemetryService.addProductChange(productChange) - telemetryService.addEndpoint(endpoint) - - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendTelemetryEvents() - - then: 'attempt with SUCCESS' - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - .assertBatch(5) - .assertFirstMessage(RequestType.APP_HEARTBEAT).hasNoPayload() - .assertNextMessage(RequestType.APP_CLIENT_CONFIGURATION_CHANGE).hasPayload().configuration([confKeyValue]) - .assertNextMessage(RequestType.APP_INTEGRATIONS_CHANGE).hasPayload().integrations([integration]) - .assertNextMessage(RequestType.APP_DEPENDENCIES_LOADED).hasPayload().dependencies([dependency]) - .assertNextMessage(RequestType.GENERATE_METRICS).hasPayload().namespace("tracers").metrics([metric]) - // no more data fit this message is sent in the next message - .assertNoMoreMessages() - - when: 'sending second part of data' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - !telemetryService.sendTelemetryEvents() - - then: - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - .assertBatch(5) - .assertFirstMessage(RequestType.APP_HEARTBEAT).hasNoPayload() - .assertNextMessage(RequestType.DISTRIBUTIONS).hasPayload().namespace("tracers").distributionSeries([distribution]) - .assertNextMessage(RequestType.LOGS).hasPayload().logs([logMessage]) - .assertNextMessage(RequestType.APP_PRODUCT_CHANGE).hasPayload().productChange(productChange) - .assertNextMessage(RequestType.APP_ENDPOINTS).hasPayload().endpoint(endpoint) - .assertNoMoreMessages() - testHttpClient.assertNoMoreRequests() - } - - def 'send all collected data with extended-heartbeat request every time'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - - when: - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendExtendedHeartbeat() - - then: - testHttpClient.assertRequestBody(RequestType.APP_EXTENDED_HEARTBEAT) - .assertPayload() - .configuration([confKeyValue]) - .integrations([integration]) - .dependencies([dependency]) - testHttpClient.assertNoMoreRequests() - - when: - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendExtendedHeartbeat() - - then: - testHttpClient.assertRequestBody(RequestType.APP_EXTENDED_HEARTBEAT) - .assertPayload() - .configuration([confKeyValue, confKeyValue]) - .integrations([integration, integration]) - .dependencies([dependency, dependency]) - testHttpClient.assertNoMoreRequests() - } - - def 'send extended-heartbeat request, even if data already has been sent or attempted as part of another telemetry events'() { - setup: - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - telemetryService.addConfiguration(configuration) - telemetryService.addIntegration(integration) - telemetryService.addDependency(dependency) - - when: - testHttpClient.expectRequest(resultCode) - telemetryService.sendTelemetryEvents() - - then: - testHttpClient.assertRequestBody(RequestType.MESSAGE_BATCH) - - when: - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendExtendedHeartbeat() - - then: - testHttpClient.assertRequestBody(RequestType.APP_EXTENDED_HEARTBEAT) - .assertPayload() - .configuration([confKeyValue]) - .integrations([integration]) - .dependencies([dependency]) - testHttpClient.assertNoMoreRequests() - - where: - resultCode << [ - TelemetryClient.Result.SUCCESS, - TelemetryClient.Result.FAILURE, - TelemetryClient.Result.NOT_FOUND - ] - } - - def 'app can propagate configuration id'() { - setup: - String instrKey = 'instrumentation_config_id' - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - def configMap = [(instrKey): ConfigSetting.of(instrKey, id, ConfigOrigin.ENV)] - telemetryService.addConfiguration([(ConfigOrigin.ENV): configMap]) - - when: 'first iteration' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: 'app-started' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().instrumentationConfigId(id) - testHttpClient.assertNoMoreRequests() - - where: - id << ["foo", null, ""] - } - - def 'app started must have install signature'() { - setup: - injectEnvConfig("INSTRUMENTATION_INSTALL_ID", installId) - injectEnvConfig("INSTRUMENTATION_INSTALL_TYPE", installType) - injectEnvConfig("INSTRUMENTATION_INSTALL_TIME", installTime) - - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - when: 'first iteration' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: 'app-started' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().installSignature(installId, installType, installTime) - testHttpClient.assertNoMoreRequests() - - where: - installId | installType | installTime - null | null | null - null | null | "1703188334" - null | "k8s_single_step" | null - null | "k8s_single_step" | "1703188212" - "68e75c99-57ca-4a12-adfc-575c4b05fcbe" | null | null - "68e75c48-57ca-4a12-adfc-575c4b05bfff" | null | "1704183412" - "68e75c55-57ca-4a12-adfc-575c4b05aaaa" | "k8s_single_step" | null - "68e75c77-57ca-4a12-adfc-575c4b05fc44" | "k8s_single_step" | "1993188215" - } - - def 'app-started must include activated products info'() { - setup: - injectEnvConfig(ConfigStrings.toEnvVar(AppSecConfig.APPSEC_ENABLED), appsecConfig) - injectEnvConfig(ConfigStrings.toEnvVar(ProfilingConfig.PROFILING_ENABLED), profilingConfig) - injectEnvConfig(ConfigStrings.toEnvVar(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED), dynInstrConfig) - - TestTelemetryRouter testHttpClient = new TestTelemetryRouter() - TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false) - - when: 'first iteration' - testHttpClient.expectRequest(TelemetryClient.Result.SUCCESS) - telemetryService.sendAppStartedEvent() - - then: 'app-started' - testHttpClient.assertRequestBody(RequestType.APP_STARTED).assertPayload().products(appsecEnabled, profilingEnabled, dynInstrEnabled) - testHttpClient.assertNoMoreRequests() - - where: - appsecConfig | appsecEnabled | profilingConfig | profilingEnabled | dynInstrConfig | dynInstrEnabled - "1" | true | "1" | true | "1" | true - "1" | true | "1" | true | "0" | false - "1" | true | "0" | false | "1" | true - "1" | true | "0" | false | "0" | false - "0" | false | "0" | false | "0" | false - "inactive" | true | "0" | false | "0" | false - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TelemetrySystemSpecification.groovy b/telemetry/src/test/groovy/datadog/telemetry/TelemetrySystemSpecification.groovy deleted file mode 100644 index e9afa34c058..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TelemetrySystemSpecification.groovy +++ /dev/null @@ -1,73 +0,0 @@ -package datadog.telemetry - -import datadog.communication.ddagent.DDAgentFeaturesDiscovery -import datadog.communication.ddagent.SharedCommunicationObjects -import datadog.metrics.api.Monitoring -import datadog.telemetry.dependency.DependencyService -import datadog.telemetry.dependency.LocationsCollectingTransformer -import datadog.trace.api.config.GeneralConfig -import datadog.trace.test.util.DDSpecification -import datadog.trace.util.ConfigStrings -import okhttp3.HttpUrl -import okhttp3.OkHttpClient -import java.lang.instrument.Instrumentation - -class TelemetrySystemSpecification extends DDSpecification { - Instrumentation inst = Mock() - - void 'installs dependencies transformer'() { - when: - def depService = TelemetrySystem.createDependencyService(inst) - - then: - 1 * inst.addTransformer(_ as LocationsCollectingTransformer) - - cleanup: - depService.stop() - } - - void 'create telemetry thread'() { - setup: - def telemetryService = Mock(TelemetryService) - def depService = Mock(DependencyService) - - when: - def thread = TelemetrySystem.createTelemetryRunnable(telemetryService, depService, true) - - then: - thread != null - - cleanup: - TelemetrySystem.stop() - } - - void 'start-stop telemetry system'() { - setup: - injectEnvConfig(ConfigStrings.toEnvVar(GeneralConfig.SITE), "datad0g.com") - injectEnvConfig(ConfigStrings.toEnvVar(GeneralConfig.API_KEY), "api-key") - def instrumentation = Mock(Instrumentation) - - when: - TelemetrySystem.startTelemetry(instrumentation, sharedCommunicationObjects()) - - then: - TelemetrySystem.TELEMETRY_THREAD != null - - when: - TelemetrySystem.stop() - - then: - TelemetrySystem.TELEMETRY_THREAD == null || - TelemetrySystem.TELEMETRY_THREAD.isInterrupted() || - !TelemetrySystem.TELEMETRY_THREAD.isAlive() - } - - private SharedCommunicationObjects sharedCommunicationObjects() { - new SharedCommunicationObjects( - agentHttpClient: Mock(OkHttpClient), - monitoring: Mock(Monitoring), - agentUrl: HttpUrl.get('https://example.com'), - featuresDiscovery: Mock(DDAgentFeaturesDiscovery) - ) - } -} diff --git a/telemetry/src/test/groovy/datadog/telemetry/TestTelemetryRouter.groovy b/telemetry/src/test/groovy/datadog/telemetry/TestTelemetryRouter.groovy deleted file mode 100644 index 78017872b4f..00000000000 --- a/telemetry/src/test/groovy/datadog/telemetry/TestTelemetryRouter.groovy +++ /dev/null @@ -1,446 +0,0 @@ -package datadog.telemetry - -import datadog.communication.ddagent.TracerVersion -import datadog.telemetry.dependency.Dependency -import datadog.telemetry.api.Integration -import datadog.telemetry.api.DistributionSeries -import datadog.telemetry.api.LogMessage -import datadog.telemetry.api.Metric -import datadog.telemetry.api.RequestType -import datadog.trace.api.Config -import datadog.trace.api.ConfigSetting -import datadog.trace.api.telemetry.Endpoint -import datadog.trace.api.telemetry.ProductChange -import groovy.json.JsonSlurper -import okhttp3.Request -import okio.Buffer - -class TestTelemetryRouter extends TelemetryRouter { - private Queue mockResults = new LinkedList<>() - private Queue requests = new LinkedList<>() - - TestTelemetryRouter() { - super(null, null, null, false) - } - - @Override - TelemetryClient.Result sendRequest(TelemetryRequest request) { - if (mockResults.isEmpty()) { - throw new IllegalStateException("Unexpected request has been sent. State expectations with `expectRequests` prior sending requests.") - } - - def requestBuilder = request.httpRequest() - requestBuilder.url("https://example.com") - requests.add(new RequestAssertions(requestBuilder.build())) - return mockResults.poll() - } - - void expectRequest(TelemetryClient.Result mockResult) { - expectRequests(1, mockResult) - } - - void expectRequests(int requestNumber, TelemetryClient.Result mockResult) { - for (int i=0; i < requestNumber; i++) { - mockResults.add(mockResult) - } - } - - RequestAssertions assertRequest() { - if (this.mockResults.size() > 0) { - throw new IllegalStateException("Expected ${this.mockResults.size()} more sendRequest calls") - } - if (this.requests.size() == 0) { - throw new IllegalStateException("No more requests have been sent.") - } - return this.requests.poll() - } - - BodyAssertions assertRequestBody(RequestType rt) { - return assertRequest().headers(rt).assertBody().commonParts(rt) - } - - void assertNoMoreRequests() { - if (this.mockResults.size() > 0) { - throw new IllegalStateException("Still expect ${this.mockResults.size()} request(s)") - } - if (this.requests.size() > 0) { - throw new IllegalStateException("Still have ${this.requests.size()} requests when none expected.") - } - } - - static class RequestAssertions { - private final static JsonSlurper SLURPER = new JsonSlurper() - - private Request request - - RequestAssertions(Request request) { - this.request = request - } - - RequestAssertions headers(RequestType requestType) { - assert this.request.method() == 'POST' - assert this.request.headers().names().containsAll([ - 'Content-Type', - 'Content-Length', - 'DD-Client-Library-Language', - 'DD-Client-Library-Version', - 'DD-Telemetry-API-Version', - 'DD-Telemetry-Request-Type', - 'DD-Session-ID' - ]) - assert this.request.header('Content-Type') == 'application/json; charset=utf-8' - assert this.request.header('Content-Length').toInteger() > 0 - assert this.request.header('DD-Client-Library-Language') == 'jvm' - assert this.request.header('DD-Client-Library-Version') == TracerVersion.TRACER_VERSION - assert this.request.header('DD-Telemetry-API-Version') == 'v2' - assert this.request.header('DD-Telemetry-Request-Type') == requestType.toString() - def entityId = this.request.header('Datadog-Entity-ID') - assert entityId == null || entityId.startsWith("in-") || entityId.startsWith("cin-") - def sessionId = this.request.header('DD-Session-ID') - assert sessionId =~ /[\da-f]{8}-([\da-f]{4}-){3}[\da-f]{12}/ - assert sessionId == Config.get().getRuntimeId() - // DD-Root-Session-ID should only be present when inherited from a parent process - // (i.e., when rootSessionId != runtimeId). In normal test context, they're equal. - def rootSessionId = this.request.header('DD-Root-Session-ID') - if (Config.get().getRootSessionId() == Config.get().getRuntimeId()) { - assert rootSessionId == null - } else { - assert rootSessionId == Config.get().getRootSessionId() - } - return this - } - - BodyAssertions assertBody() { - Buffer buf = new Buffer() - this.request.body().writeTo(buf) - byte[] bytes = new byte[buf.size()] - buf.read(bytes) - def parsed = SLURPER.parse(bytes) as Map - return new BodyAssertions(parsed, bytes) - } - } - - static class BodyAssertions { - private final Map body - private final byte[] bodyBytes - - BodyAssertions(Map body, byte[] bodyBytes) { - this.body = body - this.bodyBytes = bodyBytes - } - - int bodySize() { - return this.bodyBytes.length - } - - BodyAssertions commonParts(RequestType requestType) { - assert body['api_version'] == 'v2' - - def app = body['application'] - assert app['env'] != null - assert app['language_name'] == 'jvm' - assert app['language_version'] =~ /\d+/ - assert app['runtime_name'] != null - assert app['runtime_version'] != null - assert app['service_name'] != null - assert app['tracer_version'] == '0.42.0' - - def host = body['host'] - assert host['hostname'] != null - assert host['os'] != null - assert host['os_version'] != null - assert host['kernel_name'] != null - assert host['kernel_release'] != null - assert host['kernel_version'] != null - - assert body['runtime_id'] =~ /[\da-f]{8}-([\da-f]{4}-){3}[\da-f]{12}/ - assert body['seq_id'] > 0 - assert body['tracer_time'] > 0 - assert body['request_type'] == requestType.toString() - return this - } - - PayloadAssertions assertPayload() { - def payload = body['payload'] as Map - assert payload != null - return new PayloadAssertions(payload) - } - - BatchAssertions assertBatch(int expectedNumberOfPayloads) { - List> payloads = body['payload'] - assert payloads != null && payloads.size() == expectedNumberOfPayloads - return new BatchAssertions(payloads) - } - - void assertNoPayload() { - assert body['payload'] == null - } - } - - static class BatchAssertions { - private List> messages - - BatchAssertions(List> messages) { - this.messages = messages - } - - BatchMessageAssertions assertFirstMessage(RequestType expected) { - return assertMessage(0, expected) - } - - private BatchMessageAssertions assertMessage(int index, RequestType expected) { - if (index > messages.size()) { - throw new IllegalStateException("Asserted more messages than available (${messages.size()}) in the batch") - } - def message = messages[index] - assert message['request_type'] == String.valueOf(expected) - return new BatchMessageAssertions(this, index, message) - } - } - - static class BatchMessageAssertions { - private BatchAssertions batchAssertions - private int messageIndex - private Map message - - BatchMessageAssertions(BatchAssertions batchAssertions, int messageIndex, Map message) { - this.batchAssertions = batchAssertions - this.messageIndex = messageIndex - this.message = message - } - - BatchMessageAssertions hasNoPayload() { - assert message['payload'] == null - return this - } - - BatchMessageAssertions assertNextMessage(RequestType expected) { - messageIndex += 1 - if (messageIndex >= batchAssertions.messages.size()) { - throw new IllegalStateException("No more messages available") - } - return batchAssertions.assertMessage(messageIndex, expected) - } - - PayloadAssertions hasPayload() { - def payload = message['payload'] as Map - assert payload != null - return new PayloadAssertions(payload, this) - } - - void assertNoMoreMessages() { - assert messageIndex == batchAssertions.messages.size() - 1 - } - } - - static class PayloadAssertions { - private Map payload - private BatchMessageAssertions batch - - PayloadAssertions(Map payload) { - this(payload, null) - } - - PayloadAssertions(Map payload, BatchMessageAssertions batch) { - this.payload = payload - this.batch = batch - } - - PayloadAssertions configuration(List configuration) { - def expected = configuration == null ? null : [] - if (configuration != null) { - for (ConfigSetting cs : configuration) { - def item = [name: cs.key, value: cs.stringValue(), origin: cs.origin.value, 'seq_id': cs.seqId] - expected.add(item) - } - } - assert this.payload['configuration'] == expected - return this - } - - PayloadAssertions instrumentationConfigId(String id) { - boolean checked = false - this.payload['configuration'].each { v -> - if (v['name'] == 'DD_INSTRUMENTATION_CONFIG_ID') { - assert v['value'] == id - checked = true - } - } - - if (!checked) { - assert id == null - } - - return this - } - - PayloadAssertions productChange(ProductChange product) { - def name = product.getProductType().getName() - def expected = [ - (name) : [enabled: product.isEnabled()] - ] - assert this.payload['products'] == expected - return this - } - - PayloadAssertions endpoint(final Endpoint... endpoints) { - def expected = [] - endpoints.each { - final item = [ - 'operation_name': it.operation, - 'resource_name' : it.method + ' ' + it.path, - ] as Map - if (it.type) { - item['type'] = it.type - } - if (it.method) { - item['method'] = it.method - } - if (it.path) { - item['path'] = it.path - } - if (it.requestBodyType) { - item['request_body_type'] = it.requestBodyType - } - if (it.responseBodyType) { - item['response_body_type'] = it.responseBodyType - } - if (it.authentication) { - item['authentication'] = it.authentication - } - if (it.responseCode) { - item['response_code'] = it.responseCode - } - if (it.metadata) { - item['metadata'] = it.metadata - } - expected.add(item) - } - assert this.payload['endpoints'] == expected - return this - } - - PayloadAssertions products(boolean appsecEnabled = true, boolean profilerEnabled = false, boolean dynamicInstrumentationEnabled = false) { - def expected = [ - appsec: [enabled: appsecEnabled], - profiler: [enabled: profilerEnabled], - dynamic_instrumentation: [enabled: dynamicInstrumentationEnabled] - ] - assert this.payload['products'] == expected - return this - } - - PayloadAssertions dependencies(List dependencies) { - def expected = [] - for (Dependency d : dependencies) { - expected.add([hash: d.hash, name: d.name, version: d.version]) - } - assert this.payload['dependencies'] == expected - return this - } - - PayloadAssertions integrations(List integrations) { - def expected = [] - for (Integration i : integrations) { - Map map = new HashMap() - map.put("enabled", i.enabled) - map.put("name", i.name) - expected.add(map) - } - assert this.payload['integrations'] == expected - return this - } - - PayloadAssertions namespace(String namespace) { - assert this.payload['namespace'] == namespace - return this - } - - PayloadAssertions metrics(List metrics) { - def expected = [] - for (Metric m : metrics) { - List> points = [] - for (List ps: m.getPoints()) { - points.add(ps) - } - Map obj = new HashMap() - obj.put("namespace", m.getNamespace()) - if (m.getCommon() != null) { - obj.put("common", m.getCommon()) - } - obj.put("metric", m.getMetric()) - obj.put("points", points) - if (m.getType() != null) { - obj.put("type", m.getType()) - } - obj.put("tags", m.getTags()) - expected.add(obj) - } - assert this.payload['series'] == expected - return this - } - - PayloadAssertions distributionSeries(List ds) { - def expected = [] - for (DistributionSeries d : ds) { - Map obj = new HashMap() - obj.put("namespace", d.getNamespace()) - if (d.getCommon() != null) { - obj.put("common", d.getCommon()) - } - obj.put("metric", d.getMetric()) - obj.put("points", d.getPoints()) - obj.put("tags", d.getTags()) - expected.add(obj) - } - assert this.payload['series'] == expected - return this - } - - PayloadAssertions logs(List ls) { - def expected = [] - for (LogMessage l : ls) { - Map map = new HashMap() - map.put("message", l.getMessage()) - map.put("level", l.getLevel().toString()) - map.put("tags", l.getTags()) - if (l.getStackTrace() != null) { - map.put("stack_trace", l.getStackTrace()) - } - if (l.getTracerTime() != null) { - map.put("tracer_time", l.getTracerTime()) - } - map.put("count", l.getCount()) - expected.add(map) - } - assert this.payload['logs'] == expected - return this - } - - BatchMessageAssertions assertNextMessage(RequestType requestType) { - return batch.assertNextMessage(requestType) - } - - void assertNoMoreMessages() { - batch.assertNoMoreMessages() - } - - void installSignature(String installId, String installType, String installTime) { - if (installId == null && installType == null && installTime == null) { - assert this.payload['install_signature'] == null - return - } - LinkedHashMap expected = [:] - if (installId != null) { - expected.put("install_id", installId) - } - if (installType != null) { - expected.put("install_type", installType) - } - if (installTime != null) { - expected.put("install_time", installTime) - } - assert this.payload['install_signature'] == expected - } - } -} diff --git a/telemetry/src/test/java/datadog/telemetry/BufferedEventsTest.java b/telemetry/src/test/java/datadog/telemetry/BufferedEventsTest.java new file mode 100644 index 00000000000..a85206287f5 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/BufferedEventsTest.java @@ -0,0 +1,127 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.telemetry.api.DistributionSeries; +import datadog.telemetry.api.Integration; +import datadog.telemetry.api.LogMessage; +import datadog.telemetry.api.Metric; +import datadog.telemetry.dependency.Dependency; +import datadog.trace.api.ConfigOrigin; +import datadog.trace.api.ConfigSetting; +import datadog.trace.api.telemetry.Endpoint; +import org.junit.jupiter.api.Test; + +class BufferedEventsTest { + + @Test + void emptyEvents() { + BufferedEvents events = new BufferedEvents(); + + assertTrue(events.isEmpty()); + assertFalse(events.hasConfigChangeEvent()); + assertFalse(events.hasDependencyEvent()); + assertFalse(events.hasDistributionSeriesEvent()); + assertFalse(events.hasIntegrationEvent()); + assertFalse(events.hasLogMessageEvent()); + assertFalse(events.hasMetricEvent()); + assertFalse(events.hasEndpoint()); + } + + @Test + void returnAddedEvents() { + BufferedEvents events = new BufferedEvents(); + ConfigSetting configSetting = ConfigSetting.of("key", "value", ConfigOrigin.DEFAULT); + Dependency dependency = new Dependency("name", "version", "source", "hash"); + DistributionSeries series = new DistributionSeries(); + Integration integration = new Integration("integration-name", true); + LogMessage logMessage = new LogMessage(); + Metric metric = new Metric(); + Endpoint endpoint = new Endpoint(); + + // when + events.addConfigChangeEvent(configSetting); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasConfigChangeEvent()); + assertEquals(configSetting, events.nextConfigChangeEvent()); + assertFalse(events.hasConfigChangeEvent()); + assertTrue(events.isEmpty()); + + // when + events.addDependencyEvent(dependency); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasDependencyEvent()); + assertEquals(dependency, events.nextDependencyEvent()); + assertFalse(events.hasDependencyEvent()); + assertTrue(events.isEmpty()); + + // when + events.addDistributionSeriesEvent(series); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasDistributionSeriesEvent()); + assertEquals(series, events.nextDistributionSeriesEvent()); + assertFalse(events.hasDistributionSeriesEvent()); + assertTrue(events.isEmpty()); + + // when + events.addIntegrationEvent(integration); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasIntegrationEvent()); + assertEquals(integration, events.nextIntegrationEvent()); + assertFalse(events.hasIntegrationEvent()); + assertTrue(events.isEmpty()); + + // when + events.addLogMessageEvent(logMessage); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasLogMessageEvent()); + assertEquals(logMessage, events.nextLogMessageEvent()); + assertFalse(events.hasLogMessageEvent()); + assertTrue(events.isEmpty()); + + // when + events.addMetricEvent(metric); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasMetricEvent()); + assertEquals(metric, events.nextMetricEvent()); + assertFalse(events.hasMetricEvent()); + assertTrue(events.isEmpty()); + + // when + events.addEndpointEvent(endpoint); + + // then + assertFalse(events.isEmpty()); + assertTrue(events.hasEndpoint()); + assertEquals(endpoint, events.nextEndpoint()); + assertFalse(events.hasEndpoint()); + assertTrue(events.isEmpty()); + } + + @Test + void noopSink() { + EventSink sink = EventSink.NOOP; + + sink.addMetricEvent(null); + sink.addLogMessageEvent(null); + sink.addIntegrationEvent(null); + sink.addDistributionSeriesEvent(null); + sink.addDependencyEvent(null); + sink.addConfigChangeEvent(null); + sink.addEndpointEvent(null); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/DisabledDependencyServiceTest.java b/telemetry/src/test/java/datadog/telemetry/DisabledDependencyServiceTest.java new file mode 100644 index 00000000000..b019bba1738 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/DisabledDependencyServiceTest.java @@ -0,0 +1,28 @@ +package datadog.telemetry; + +import static datadog.trace.api.config.GeneralConfig.TELEMETRY_DEPENDENCY_COLLECTION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.telemetry.dependency.DependencyService; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import java.lang.instrument.Instrumentation; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(WithConfigExtension.class) +@WithConfig(key = TELEMETRY_DEPENDENCY_COLLECTION_ENABLED, value = "false") +class DisabledDependencyServiceTest { + + @Test + void installsDisabledDependencyServiceAndVerifyTransformer() { + Instrumentation instrumentation = mock(Instrumentation.class); + + DependencyService dependencyService = TelemetrySystem.createDependencyService(instrumentation); + + verifyNoInteractions(instrumentation); + assertNull(dependencyService); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/EventSourceTest.java b/telemetry/src/test/java/datadog/telemetry/EventSourceTest.java new file mode 100644 index 00000000000..3625f0a04df --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/EventSourceTest.java @@ -0,0 +1,107 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import datadog.telemetry.api.DistributionSeries; +import datadog.telemetry.api.Integration; +import datadog.telemetry.api.LogMessage; +import datadog.telemetry.api.Metric; +import datadog.telemetry.dependency.Dependency; +import datadog.trace.api.ConfigOrigin; +import datadog.trace.api.ConfigSetting; +import datadog.trace.api.telemetry.Endpoint; +import datadog.trace.api.telemetry.ProductChange; +import java.util.Queue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class EventSourceTest { + + private static class EventQueues { + final Queue configChangeQueue = new LinkedBlockingQueue<>(); + final Queue integrationQueue = new LinkedBlockingQueue<>(); + final Queue dependencyQueue = new LinkedBlockingQueue<>(); + final Queue metricQueue = new LinkedBlockingQueue<>(); + final Queue distributionSeriesQueue = new LinkedBlockingQueue<>(); + final Queue logMessageQueue = new LinkedBlockingQueue<>(); + final Queue productChanges = new LinkedBlockingQueue<>(); + final Queue endpointQueue = new LinkedBlockingQueue<>(); + } + + @SuppressWarnings("unchecked") + private static void addInstance(Queue queue, Object instance) { + queue.add((T) instance); + } + + @ParameterizedTest(name = "test isEmpty when adding and clearing {0}") + @MethodSource("testIsEmptyWhenAddingAndClearingArguments") + void testIsEmptyWhenAddingAndClearing( + String eventType, Function> queueSelector, Object eventInstance) { + EventQueues eventQueues = new EventQueues(); + EventSource eventSource = + new EventSource.Queued( + eventQueues.configChangeQueue, + eventQueues.integrationQueue, + eventQueues.dependencyQueue, + eventQueues.metricQueue, + eventQueues.distributionSeriesQueue, + eventQueues.logMessageQueue, + eventQueues.productChanges, + eventQueues.endpointQueue); + + assertTrue(eventSource.isEmpty()); + + // add an event to the queue + Queue queue = queueSelector.apply(eventQueues); + addInstance(queue, eventInstance); + + // eventSource should not be empty + assertFalse(eventSource.isEmpty()); + + // clear the queue + queue.clear(); + + // eventSource should be empty again + assertTrue(eventSource.isEmpty()); + } + + private static Stream testIsEmptyWhenAddingAndClearingArguments() { + return Stream.of( + arguments( + "Config Change", + (Function>) queues -> queues.configChangeQueue, + ConfigSetting.of("key", "value", ConfigOrigin.ENV)), + arguments( + "Integration", + (Function>) queues -> queues.integrationQueue, + new Integration("name", true)), + arguments( + "Dependency", + (Function>) queues -> queues.dependencyQueue, + new Dependency("name", "version", "type", null)), + arguments( + "Metric", (Function>) queues -> queues.metricQueue, new Metric()), + arguments( + "Distribution Series", + (Function>) queues -> queues.distributionSeriesQueue, + new DistributionSeries()), + arguments( + "Log Message", + (Function>) queues -> queues.logMessageQueue, + new LogMessage()), + arguments( + "Product Change", + (Function>) queues -> queues.productChanges, + new ProductChange()), + arguments( + "Endpoint", + (Function>) queues -> queues.endpointQueue, + new Endpoint())); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/ExtendedHeartbeatDataTest.java b/telemetry/src/test/java/datadog/telemetry/ExtendedHeartbeatDataTest.java new file mode 100644 index 00000000000..057972fabc2 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/ExtendedHeartbeatDataTest.java @@ -0,0 +1,89 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.telemetry.api.Integration; +import datadog.telemetry.dependency.Dependency; +import datadog.trace.api.ConfigOrigin; +import datadog.trace.api.ConfigSetting; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class ExtendedHeartbeatDataTest { + + private final Dependency dependency = new Dependency("name", "version", "source", "hash"); + private final ConfigSetting configSetting = + ConfigSetting.of("key", "value", ConfigOrigin.DEFAULT); + private final Integration integration = new Integration("integration", true); + + @TableTest({ + "scenario | limit", + "no limit | 0 ", + "small limit | 2 ", + "large limit | 10 " + }) + void discardDependenciesAfterExceedingLimit(int limit) { + ExtendedHeartbeatData extHeartbeatData = new ExtendedHeartbeatData(limit); + + for (int pushed = 0; pushed < limit + 1; pushed++) { + extHeartbeatData.pushDependency(dependency); + } + + EventSource snapshot = extHeartbeatData.snapshot(); + int dependencyCount = 0; + while (snapshot.hasDependencyEvent()) { + snapshot.nextDependencyEvent(); + dependencyCount++; + } + assertEquals(limit, dependencyCount); + } + + @Test + void returnAllCollectedData() { + ExtendedHeartbeatData extHeartbeatData = new ExtendedHeartbeatData(); + + // when + EventSource emptySnapshot = extHeartbeatData.snapshot(); + // then + assertTrue(emptySnapshot.isEmpty()); + + // when + extHeartbeatData.pushDependency(dependency); + extHeartbeatData.pushConfigSetting(configSetting); + extHeartbeatData.pushIntegration(integration); + + // then + EventSource snapshot = extHeartbeatData.snapshot(); + + assertFalse(snapshot.isEmpty()); + + assertTrue(snapshot.hasDependencyEvent()); + assertEquals(dependency, snapshot.nextDependencyEvent()); + assertFalse(snapshot.hasDependencyEvent()); + + assertFalse(snapshot.isEmpty()); + + assertTrue(snapshot.hasConfigChangeEvent()); + assertEquals(configSetting, snapshot.nextConfigChangeEvent()); + assertFalse(snapshot.hasConfigChangeEvent()); + + assertFalse(snapshot.isEmpty()); + + assertTrue(snapshot.hasIntegrationEvent()); + assertEquals(integration, snapshot.nextIntegrationEvent()); + assertFalse(snapshot.hasIntegrationEvent()); + + assertTrue(snapshot.isEmpty()); + + // when another snapshot includes all data + EventSource anotherSnapshot = extHeartbeatData.snapshot(); + + // then + assertFalse(anotherSnapshot.isEmpty()); + assertTrue(anotherSnapshot.hasDependencyEvent()); + assertTrue(anotherSnapshot.hasConfigChangeEvent()); + assertTrue(anotherSnapshot.hasIntegrationEvent()); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/HostInfoTest.java b/telemetry/src/test/java/datadog/telemetry/HostInfoTest.java new file mode 100644 index 00000000000..d4266fa7bfb --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/HostInfoTest.java @@ -0,0 +1,80 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import datadog.environment.OperatingSystem; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +class HostInfoTest { + + @Test + void getHostname() { + String hostname = HostInfo.getHostname(); + + assertNotNull(hostname); + assertFalse(hostname.trim().isEmpty()); + } + + @Test + void getOsName() { + String osName = HostInfo.getOsName(); + + assertTrue(Arrays.asList("Linux", "Windows", "Darwin").contains(osName)); + } + + @Test + void getOsVersion() { + String osVersion = HostInfo.getOsVersion(); + + assertNotNull(osVersion); + assertFalse(osVersion.trim().isEmpty()); + } + + @Test + void compareToUname() throws IOException, InterruptedException { + assumeTrue(exitCode("uname", "-a") == 0); + + assertEquals(runCommand("uname", "-n"), HostInfo.getHostname()); + assertEquals(runCommand("uname", "-s"), HostInfo.getOsName()); + assertEquals(runCommand("uname", "-s"), HostInfo.getKernelName()); + if (OperatingSystem.isMacOs()) { + // uname -r will return X.Y.Z version, while JVM will report just X.Y + // disabled, the uname -r gives the Kernel version which is different from Mac OS version + // assertTrue(runCommand("uname", "-r").startsWith(HostInfo.getKernelRelease())); + + // No /proc in macOS, so using property os.version like for KernelRelease + assertEquals(HostInfo.getKernelRelease(), HostInfo.getKernelVersion()); + } else { + assertEquals(runCommand("uname", "-r"), HostInfo.getKernelRelease()); + // Ideally, this would be equal, but for now, we'll compromise to startWith. + assertTrue(runCommand("uname", "-v").startsWith(HostInfo.getKernelVersion())); + } + } + + private static String runCommand(String... command) throws IOException, InterruptedException { + Process process = new ProcessBuilder(command).start(); + String output; + try (BufferedReader reader = + new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + process.waitFor(); + return output.trim(); + } + + private static int exitCode(String... command) throws IOException, InterruptedException { + Process process = new ProcessBuilder(command).start(); + return process.waitFor(); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TelemetryClientTest.java b/telemetry/src/test/java/datadog/telemetry/TelemetryClientTest.java new file mode 100644 index 00000000000..37f5a54dbc2 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TelemetryClientTest.java @@ -0,0 +1,76 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import datadog.communication.http.HttpRetryPolicy; +import datadog.telemetry.api.RequestType; +import datadog.trace.api.Config; +import java.net.ConnectException; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class TelemetryClientTest { + + @TableTest({ + "scenario | ciVisEnabled | ciVisAgentlessEnabled | ciVisAgentlessUrl | expectedUrl ", + "ci visibility agentless enabled | true | true | http://ci.visibility.agentless.url | http://ci.visibility.agentless.url/api/v2/apmtelemetry ", + "ci visibility disabled | false | true | http://ci.visibility.agentless.url | https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry", + "ci visibility agentless disabled | true | false | http://ci.visibility.agentless.url | https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry", + "ci visibility agentless url is null | true | true | | https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry" + }) + void intakeClientUsesCiVisibilityAgentlessUrlIfConfiguredToDoSo( + boolean ciVisEnabled, + boolean ciVisAgentlessEnabled, + String ciVisAgentlessUrl, + String expectedUrl) { + Config config = spy(Config.get()); + doReturn("dummy-key").when(config).getApiKey(); + doReturn(123).when(config).getAgentTimeout(); + doReturn("datad0g.com").when(config).getSite(); + doReturn(ciVisEnabled).when(config).isCiVisibilityEnabled(); + doReturn(ciVisAgentlessEnabled).when(config).isCiVisibilityAgentlessEnabled(); + doReturn(ciVisAgentlessUrl).when(config).getCiVisibilityAgentlessUrl(); + + TelemetryClient intakeClient = + TelemetryClient.buildIntakeClient(config, HttpRetryPolicy.Factory.NEVER_RETRY); + + assertEquals(expectedUrl, intakeClient.getUrl().toString()); + } + + @Test + void intakeClientRetriesTelemetryRequestIfConfiguredToDoSo() { + OkHttpClient httpClient = mock(OkHttpClient.class); + HttpRetryPolicy.Factory httpRetryPolicy = new HttpRetryPolicy.Factory(2, 50, 1.5, true); + HttpUrl httpUrl = HttpUrl.get("https://intake.example.com"); + TelemetryClient intakeClient = + new TelemetryClient(httpClient, httpRetryPolicy, httpUrl, "dummy-api-key"); + + doAnswer( + invocation -> { + throw new ConnectException("exception"); + }) + .when(httpClient) + .newCall(any()); + + intakeClient.sendHttpRequest(dummyRequest()); + + // original request + 2 retries + verify(httpClient, times(3)).newCall(any()); + } + + private Request.Builder dummyRequest() { + return new TelemetryRequest( + mock(EventSource.class), mock(EventSink.class), 1000, RequestType.APP_STARTED, false) + .httpRequest(); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TelemetryRequestBodyTest.java b/telemetry/src/test/java/datadog/telemetry/TelemetryRequestBodyTest.java new file mode 100644 index 00000000000..b1c4600eaa5 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TelemetryRequestBodyTest.java @@ -0,0 +1,222 @@ +package datadog.telemetry; + +import static datadog.trace.api.config.GeneralConfig.EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED; +import static datadog.trace.api.telemetry.ProductChange.ProductType.APPSEC; +import static datadog.trace.api.telemetry.ProductChange.ProductType.DYNAMIC_INSTRUMENTATION; +import static datadog.trace.api.telemetry.ProductChange.ProductType.PROFILER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.telemetry.api.RequestType; +import datadog.trace.api.Config; +import datadog.trace.api.ConfigOrigin; +import datadog.trace.api.ConfigSetting; +import datadog.trace.api.ProcessTags; +import datadog.trace.api.telemetry.ProductChange.ProductType; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import java.io.IOException; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import okhttp3.RequestBody; +import okio.Buffer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.tabletest.junit.TableTest; + +/** This test only verifies non-functional specifics that are not covered in TelemetryServiceTest */ +@ExtendWith(WithConfigExtension.class) +class TelemetryRequestBodyTest { + + @AfterEach + void resetProcessTags() { + ProcessTags.reset(Config.get()); + } + + @Test + void throwSerializationExceptionInCaseOfJsonNestingProblem() { + TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_STARTED); + + req.beginRequest(false); + TelemetryRequestBody.SerializationException exception = + assertThrows( + TelemetryRequestBody.SerializationException.class, () -> req.beginRequest(false)); + + assertEquals("Failed serializing Telemetry begin-request part!", exception.getMessage()); + assertNotNull(exception.getCause()); + } + + @Test + void throwSerializationExceptionInCaseOfMoreThanOneTopLevelJsonValue() { + TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_STARTED); + + req.beginRequest(false); + req.endRequest(); + TelemetryRequestBody.SerializationException exception = + assertThrows( + TelemetryRequestBody.SerializationException.class, () -> req.beginRequest(false)); + + assertEquals("Failed serializing Telemetry begin-request part!", exception.getMessage()); + assertNotNull(exception.getCause()); + } + + @Test + void writeConfigMustSupportValuesOfBooleanStringNumberAndNull() throws IOException { + TelemetryRequestBody req = + new TelemetryRequestBody(RequestType.APP_CLIENT_CONFIGURATION_CHANGE); + Map map = new HashMap<>(); + map.put("key1", "value1"); + map.put("key2", Double.parseDouble("432.32")); + map.put("key3", 324); + + req.beginRequest(false); + // exclude request header to simplify assertion + drainToString(req); + + req.beginConfiguration(); + List configSettings = + Arrays.asList( + ConfigSetting.of("string", "bar", ConfigOrigin.REMOTE), + ConfigSetting.of("int", 2342, ConfigOrigin.DEFAULT), + ConfigSetting.of("double", Double.valueOf("123.456"), ConfigOrigin.ENV), + ConfigSetting.of("map", map, ConfigOrigin.JVM_PROP), + ConfigSetting.of("list", Arrays.asList("1", "2", 3), ConfigOrigin.DEFAULT), + // make sure null values are serialized + ConfigSetting.of("null", null, ConfigOrigin.DEFAULT)); + for (ConfigSetting configSetting : configSettings) { + req.writeConfiguration(configSetting); + } + req.endConfiguration(); + + String expectedJson = + ",\"configuration\":[" + + "{\"name\":\"DD_STRING\",\"value\":\"bar\",\"origin\":\"remote_config\",\"seq_id\":0}," + + "{\"name\":\"DD_INT\",\"value\":\"2342\",\"origin\":\"default\",\"seq_id\":0}," + + "{\"name\":\"DD_DOUBLE\",\"value\":\"123.456\",\"origin\":\"env_var\",\"seq_id\":0}," + + "{\"name\":\"DD_MAP\",\"value\":\"key1:value1,key2:432.32,key3:324\",\"origin\":\"jvm_prop\",\"seq_id\":0}," + + "{\"name\":\"DD_LIST\",\"value\":\"1,2,3\",\"origin\":\"default\",\"seq_id\":0}," + + "{\"name\":\"DD_NULL\",\"value\":null,\"origin\":\"default\",\"seq_id\":0}]"; + assertEquals(expectedJson, drainToString(req)); + } + + @Test + void useEnvironmentVariableForSettingKeys() throws IOException { + TelemetryRequestBody req = + new TelemetryRequestBody(RequestType.APP_CLIENT_CONFIGURATION_CHANGE); + + req.beginRequest(false); + // exclude request header to simplify assertion + drainToString(req); + + req.beginConfiguration(); + req.writeConfiguration(ConfigSetting.of("this.is.a.key", "value", ConfigOrigin.REMOTE)); + req.endConfiguration(); + + assertEquals( + ",\"configuration\":[{\"name\":\"DD_THIS_IS_A_KEY\",\"value\":\"value\",\"origin\":\"remote_config\",\"seq_id\":0}]", + drainToString(req)); + } + + @Test + void addDebugFlag() throws IOException { + TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_STARTED); + + req.beginRequest(true); + req.endRequest(); + + assertTrue(drainToString(req).contains("\"debug\":true")); + } + + @TableTest({ + "scenario | appsecChange | profilerChange | dynamicInstrumentationChange | appsecEnabled | profilerEnabled | dynamicInstrumentationEnabled", + "all products changed and enabled | true | true | true | true | true | true ", + "all products changed and disabled | true | true | true | false | false | false ", + "no product changed | false | false | false | true | true | true ", + "only profiler and dynamic instrumentation changed | false | true | true | true | true | true ", + "only appsec and dynamic instrumentation changed | true | false | true | true | true | true ", + "only appsec and profiler changed | true | true | false | true | true | true " + }) + void writeProducts( + boolean appsecChange, + boolean profilerChange, + boolean dynamicInstrumentationChange, + boolean appsecEnabled, + boolean profilerEnabled, + boolean dynamicInstrumentationEnabled) + throws IOException { + TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_PRODUCT_CHANGE); + Map products = new EnumMap<>(ProductType.class); + if (appsecChange) { + products.put(APPSEC, appsecEnabled); + } + if (profilerChange) { + products.put(PROFILER, profilerEnabled); + } + if (dynamicInstrumentationChange) { + products.put(DYNAMIC_INSTRUMENTATION, dynamicInstrumentationEnabled); + } + + req.beginRequest(false); + req.writeProducts(products); + req.endRequest(); + + String result = drainToString(req); + assertEquals(appsecChange, result.contains("\"appsec\":{\"enabled\":" + appsecEnabled + "}")); + assertEquals( + profilerChange, result.contains("\"profiler\":{\"enabled\":" + profilerEnabled + "}")); + assertEquals( + dynamicInstrumentationChange, + result.contains( + "\"dynamic_instrumentation\":{\"enabled\":" + dynamicInstrumentationEnabled + "}")); + } + + @TableTest({ + "scenario | processTagsEnabled", + "enabled | true ", + "disabled | false " + }) + @SuppressWarnings("unchecked") + void shouldPropagateProcessTagsWhenEnabled(boolean processTagsEnabled) throws IOException { + WithConfigExtension.injectSysConfig( + EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, String.valueOf(processTagsEnabled)); + try { + ProcessTags.reset(Config.get()); + TelemetryRequestBody req = new TelemetryRequestBody(RequestType.APP_STARTED); + + req.beginRequest(true); + req.endRequest(); + + JsonAdapter adapter = + new Moshi.Builder() + .build() + .adapter(Types.newParameterizedType(Map.class, String.class, Object.class)); + Map parsed = (Map) adapter.fromJson(drainToString(req)); + Map application = (Map) parsed.get("application"); + Object parsedTags = application.get("process_tags"); + if (processTagsEnabled) { + assertEquals(ProcessTags.getTagsForSerialization().toString(), parsedTags); + } else { + assertNull(parsedTags); + } + } finally { + WithConfigExtension.injectSysConfig(EXPERIMENTAL_PROPAGATE_PROCESS_TAGS_ENABLED, "false"); + } + } + + private static String drainToString(RequestBody body) throws IOException { + Buffer buf = new Buffer(); + body.writeTo(buf); + byte[] bytes = new byte[(int) buf.size()]; + buf.read(bytes); + return new String(bytes); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TelemetryRouterTest.java b/telemetry/src/test/java/datadog/telemetry/TelemetryRouterTest.java new file mode 100644 index 00000000000..5cc474ce23a --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TelemetryRouterTest.java @@ -0,0 +1,524 @@ +package datadog.telemetry; + +import static datadog.communication.http.HttpRetryPolicy.Factory.NEVER_RETRY; +import static datadog.telemetry.TelemetryClient.Result.FAILURE; +import static datadog.telemetry.TelemetryClient.Result.SUCCESS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.telemetry.api.RequestType; +import java.io.IOException; +import java.io.InterruptedIOException; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class TelemetryRouterTest { + + private static final HttpUrl AGENT_URL = HttpUrl.get("https://agent.example.com"); + private static final HttpUrl AGENT_TELEMETRY_URL = + AGENT_URL.resolve("telemetry/proxy/api/v2/apmtelemetry"); + private static final HttpUrl INTAKE_URL = HttpUrl.get("https://intake.example.com"); + private static final String API_KEY = "api-key"; + private static final String API_KEY_HEADER = "DD-API-KEY"; + + private final OkHttpClient okHttpClient = mock(OkHttpClient.class); + private final DDAgentFeaturesDiscovery ddAgentFeaturesDiscovery = + mock(DDAgentFeaturesDiscovery.class); + + private TelemetryClient agentTelemetryClient; + private TelemetryClient intakeTelemetryClient; + private TelemetryRouter router; + + @BeforeEach + void setup() { + agentTelemetryClient = TelemetryClient.buildAgentClient(okHttpClient, AGENT_URL, NEVER_RETRY); + intakeTelemetryClient = new TelemetryClient(okHttpClient, NEVER_RETRY, INTAKE_URL, API_KEY); + router = + new TelemetryRouter( + ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, false); + } + + @TableTest({ + "scenario | httpCode | sendResult", + "informational status is a failure | 100 | FAILURE ", + "accepted status is a success | 202 | SUCCESS ", + "not found status is not found | 404 | NOT_FOUND ", + "server error status is a failure | 500 | FAILURE " + }) + void mapAnHttpStatusCodeToTheCorrectSendResult(int httpCode, String sendResult) { + stubNewCallReturning(httpCode); + + TelemetryClient.Result result = router.sendRequest(dummyRequest()); + + assertEquals(TelemetryClient.Result.valueOf(sendResult), result); + verify(okHttpClient, times(1)).newCall(any()); + } + + @Test + void catchIOExceptionFromOkHttpClientAndReturnFailure() { + stubNewCallThrowing(new IOException("exception")); + + TelemetryClient.Result result = router.sendRequest(dummyRequest()); + + assertEquals(FAILURE, result); + verify(okHttpClient, times(1)).newCall(any()); + } + + @Test + void catchInterruptedIOExceptionFromOkHttpClientAndReturnInterrupted() { + stubNewCallThrowing(new InterruptedIOException("interrupted")); + + TelemetryClient.Result result = router.sendRequest(dummyRequest()); + + assertEquals(TelemetryClient.Result.INTERRUPTED, result); + verify(okHttpClient, times(1)).newCall(any()); + } + + @TableTest({ + "returnCode", + "200 ", + "404 ", + "500 " + }) + void keepTryingToSendTelemetryToAgentDespiteOfReturnCodeWhenIntakeClientIsNull(int returnCode) { + TelemetryRouter agentOnlyRouter = + new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, null, false); + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + agentOnlyRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + agentOnlyRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + } + + @TableTest({ + "returnCode", + "404 ", + "500 " + }) + void switchToIntakeWhenAgentStopsSupportingTelemetryProxyAndTelemetryRequestsStartFailing( + int returnCode) { + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + } + + @Test + void whenConfiguredToPreferIntakeUseIntakeClientFromTheStart() { + TelemetryRouter telemetryRouter = + new TelemetryRouter( + ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true); + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, 200); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + } + + @Test + void + whenConfiguredToPreferIntakeDoNotSwitchToAgentIfIntakeRequestSucceedsEvenIfAgentSupportsTelemetryProxy() { + TelemetryRouter telemetryRouter = + new TelemetryRouter( + ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true); + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, 200); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + } + + @Test + void whenConfiguredToPreferIntakeDoNotSwitchToAgentIfRequestIsInterrupted() { + TelemetryRouter telemetryRouter = + new TelemetryRouter( + ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true); + Request[] capturedRequest = new Request[1]; + doAnswer( + invocation -> { + capturedRequest[0] = invocation.getArgument(0); + throw new InterruptedIOException("interrupted"); + }) + .when(okHttpClient) + .newCall(any()); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + stubNewCallCapturingRequest(capturedRequest, 200); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + } + + @Test + void whenConfiguredToPreferIntakeSwitchToAgentIfIntakeRequestFails() { + TelemetryRouter telemetryRouter = + new TelemetryRouter( + ddAgentFeaturesDiscovery, agentTelemetryClient, intakeTelemetryClient, true); + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, 403); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + stubNewCallCapturingRequest(capturedRequest, 200); + + telemetryRouter.sendRequest(dummyRequest()); + + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + } + + @Test + void doNotSwitchToIntakeWhenAgentStopsSupportingTelemetryProxyButAcceptsTelemetryRequests() { + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, 200); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + stubNewCallCapturingRequest(capturedRequest, 201); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + } + + @TableTest({ + "returnCode", + "404 ", + "500 " + }) + void switchToIntakeWhenAgentFailsToReceiveTelemetryRequests(int returnCode) { + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true, false); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + } + + @TableTest({ + "returnCode | expectedApiKey | expectedUrlIsAgent", + "404 | | true ", + "500 | | true " + }) + void useAgentWhenIntakeIsNotAvailable( + int returnCode, String expectedApiKey, boolean expectedUrlIsAgent) { + HttpUrl expectedUrl = expectedUrlIsAgent ? AGENT_TELEMETRY_URL : INTAKE_URL; + TelemetryRouter agentOnlyRouter = + new TelemetryRouter(ddAgentFeaturesDiscovery, agentTelemetryClient, null, false); + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + agentOnlyRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(expectedUrl, capturedRequest[0].url()); + assertEquals(expectedApiKey, capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + agentOnlyRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + agentOnlyRouter.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(expectedUrl, capturedRequest[0].url()); + assertEquals(expectedApiKey, capturedRequest[0].header(API_KEY_HEADER)); + } + + @TableTest({ + "returnCode", + "404 ", + "500 " + }) + void switchToIntakeThenBackToAgentWhenBothFailToReceiveTelemetryRequests(int returnCode) { + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + // always send first telemetry request to Agent + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + // switch to Intake if sending a telemetry request to Agent failed or Agent supports + // telemetry proxy + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + // switch back to Agent if Intake request fails + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + } + + @Test + void singleClientConstructorSkipsFeatureDiscoveryAndDelegatesToTheGivenClient() { + TelemetryClient singleClient = mock(TelemetryClient.class); + TelemetryRouter singleClientRouter = new TelemetryRouter(singleClient); + when(singleClient.sendHttpRequest(any())).thenReturn(SUCCESS); + + TelemetryClient.Result result = singleClientRouter.sendRequest(dummyRequest()); + + assertEquals(SUCCESS, result); + verify(singleClient, times(1)).sendHttpRequest(any()); + verify(ddAgentFeaturesDiscovery, never()).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, never()).supportsTelemetryProxy(); + } + + @Test + void singleClientConstructorDoesNotSwitchClientsOnFailure() { + TelemetryClient singleClient = mock(TelemetryClient.class); + TelemetryRouter singleClientRouter = new TelemetryRouter(singleClient); + + // first request fails + when(singleClient.sendHttpRequest(any())).thenReturn(FAILURE); + TelemetryClient.Result firstResult = singleClientRouter.sendRequest(dummyRequest()); + + assertEquals(FAILURE, firstResult); + verify(singleClient, times(1)).sendHttpRequest(any()); + + // second request goes to the same client + when(singleClient.sendHttpRequest(any())).thenReturn(SUCCESS); + TelemetryClient.Result secondResult = singleClientRouter.sendRequest(dummyRequest()); + + assertEquals(SUCCESS, secondResult); + verify(singleClient, times(2)).sendHttpRequest(any()); + verifyNoMoreInteractions(ddAgentFeaturesDiscovery); + } + + @TableTest({ + "returnCode", + "404 ", + "500 " + }) + void switchBackToAgentIfItStartsSupportingTelemetry(int returnCode) { + Request[] capturedRequest = new Request[1]; + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + // always send first telemetry request to Agent + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + // switch to Intake if sending a telemetry request to Agent failed or Agent supports + // telemetry proxy + stubNewCallCapturingRequest(capturedRequest, 201); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(true); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(INTAKE_URL, capturedRequest[0].url()); + assertEquals(API_KEY, capturedRequest[0].header(API_KEY_HEADER)); + clearInvocations(okHttpClient, ddAgentFeaturesDiscovery); + + // switch back to Agent if it starts supporting telemetry proxy + stubNewCallCapturingRequest(capturedRequest, returnCode); + when(ddAgentFeaturesDiscovery.supportsTelemetryProxy()).thenReturn(false); + + router.sendRequest(dummyRequest()); + + verify(ddAgentFeaturesDiscovery, times(1)).discoverIfOutdated(); + verify(ddAgentFeaturesDiscovery, times(1)).supportsTelemetryProxy(); + verify(okHttpClient, times(1)).newCall(any()); + assertEquals(AGENT_TELEMETRY_URL, capturedRequest[0].url()); + assertNull(capturedRequest[0].header(API_KEY_HEADER)); + } + + private TelemetryRequest dummyRequest() { + return new TelemetryRequest( + mock(EventSource.class), mock(EventSink.class), 1000, RequestType.APP_STARTED, false); + } + + private static Call mockCall(int code) throws IOException { + Call call = mock(Call.class); + when(call.execute()) + .thenReturn( + new Response.Builder() + .request(new Request.Builder().url(HttpUrl.get("https://example.com")).build()) + .protocol(Protocol.HTTP_1_1) + .message("OK") + .body(ResponseBody.create(MediaType.get("text/plain"), "OK")) + .code(code) + .build()); + return call; + } + + // stubs newCall() via doAnswer rather than when().thenAnswer() so that re-stubbing across + // multiple stages of the same test does not trigger a previously configured throwing answer + private void stubNewCallReturning(int code) { + doAnswer(invocation -> mockCall(code)).when(okHttpClient).newCall(any()); + } + + private void stubNewCallCapturingRequest(Request[] capturedRequest, int code) { + doAnswer( + invocation -> { + capturedRequest[0] = invocation.getArgument(0); + return mockCall(code); + }) + .when(okHttpClient) + .newCall(any()); + } + + private void stubNewCallThrowing(IOException exception) { + doAnswer( + invocation -> { + throw exception; + }) + .when(okHttpClient) + .newCall(any()); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TelemetryRunnableTest.java b/telemetry/src/test/java/datadog/telemetry/TelemetryRunnableTest.java new file mode 100644 index 00000000000..25b4d30f064 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TelemetryRunnableTest.java @@ -0,0 +1,437 @@ +package datadog.telemetry; + +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import datadog.telemetry.metric.MetricPeriodicAction; +import datadog.trace.api.telemetry.MetricCollector; +import datadog.trace.api.time.TimeSource; +import datadog.trace.test.junit.utils.config.WithConfig; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +class TelemetryRunnableTest { + + private Thread thread; + + @AfterEach + void cleanup() throws InterruptedException { + if (thread != null && thread.isAlive()) { + thread.interrupt(); + thread.join(); + } + } + + @Test + @WithConfig(key = "TELEMETRY_EXTENDED_HEARTBEAT_INTERVAL", value = "65", env = true) + void happyPath() throws InterruptedException, BrokenBarrierException, TimeoutException { + TelemetryRunnable.ThreadSleeper sleeperMock = mock(TelemetryRunnable.ThreadSleeper.class); + TickSleeper sleeper = new TickSleeper(sleeperMock); + TimeSource timeSource = mock(TimeSource.class); + TelemetryService telemetryService = mock(TelemetryService.class); + MetricCollector metricCollector = mock(MetricCollector.class); + MetricPeriodicAction metricAction = mock(MetricPeriodicAction.class); + when(metricAction.collector()).thenReturn(metricCollector); + TelemetryRunnable.TelemetryPeriodicAction periodicAction = + mock(TelemetryRunnable.TelemetryPeriodicAction.class); + TelemetryRunnable runnable = + new TelemetryRunnable( + telemetryService, asList(metricAction, periodicAction), sleeper, timeSource); + thread = new Thread(runnable); + + // when initial iteration before the first sleep (metrics and heartbeat) + when(telemetryService.sendAppStartedEvent()).thenReturn(false, false, true); + when(timeSource.getCurrentTimeMillis()).thenReturn(60L * 1000, 60L * 1000 + 1); + when(metricCollector.drain()).thenReturn(asList()); + when(metricCollector.drainDistributionSeries()).thenReturn(asList()); + when(telemetryService.sendTelemetryEvents()).thenReturn(true, true, false); + thread.start(); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then two unsuccessful attempts to send app-started with the following successful attempt + verify(telemetryService, times(3)).sendAppStartedEvent(); + verify(timeSource, times(2)).getCurrentTimeMillis(); + // then two partial and one final telemetry data requests + verify(metricCollector, times(1)).prepareMetrics(); + verify(metricCollector, times(1)).drain(); + verify(metricCollector, times(1)).drainDistributionSeries(); + verify(periodicAction, times(1)).doIteration(telemetryService); + verify(telemetryService, times(3)).sendTelemetryEvents(); + verify(sleeperMock, times(1)).sleep(9999); + verify(telemetryService, atLeast(0)).addConfiguration(any()); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when second iteration (10 seconds, metrics) + when(timeSource.getCurrentTimeMillis()).thenReturn(70L * 1000, 70L * 1000 + 2); + sleeper.go.await(10, TimeUnit.SECONDS); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(metricCollector, times(1)).prepareMetrics(); + verify(sleeperMock, times(1)).sleep(9998); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when third iteration (20 seconds, metrics) + when(timeSource.getCurrentTimeMillis()).thenReturn(80L * 1000, 80L * 1000 + 3); + sleeper.go.await(10, TimeUnit.SECONDS); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(metricCollector, times(1)).prepareMetrics(); + verify(sleeperMock, times(1)).sleep(9997); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when fourth iteration (30 seconds, metrics) + when(timeSource.getCurrentTimeMillis()).thenReturn(90L * 1000, 90L * 1000 + 4); + sleeper.go.await(10, TimeUnit.SECONDS); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(metricCollector, times(1)).prepareMetrics(); + verify(sleeperMock, times(1)).sleep(9996); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when fifth iteration (40 seconds, metrics) + when(timeSource.getCurrentTimeMillis()).thenReturn(100L * 1000, 100L * 1000 + 5); + sleeper.go.await(10, TimeUnit.SECONDS); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(metricCollector, times(1)).prepareMetrics(); + verify(sleeperMock, times(1)).sleep(9995); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when sixth iteration (50 seconds, metrics) + when(timeSource.getCurrentTimeMillis()).thenReturn(110L * 1000, 110L * 1000 + 6); + sleeper.go.await(10, TimeUnit.SECONDS); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(metricCollector, times(1)).prepareMetrics(); + verify(sleeperMock, times(1)).sleep(9994); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when seventh iteration (60 seconds, metrics, heartbeat) + when(timeSource.getCurrentTimeMillis()).thenReturn(120L * 1000, 120L * 1000 + 7); + sleeper.go.await(10, TimeUnit.SECONDS); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(metricCollector, times(1)).prepareMetrics(); + verify(metricCollector, times(1)).drain(); + verify(metricCollector, times(1)).drainDistributionSeries(); + verify(periodicAction, times(1)).doIteration(telemetryService); + verify(telemetryService, times(1)).sendTelemetryEvents(); + verify(sleeperMock, times(1)).sleep(9993); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when eighth iteration (65 seconds, extended-heartbeat) + when(timeSource.getCurrentTimeMillis()).thenReturn(125L * 1000, 125L * 1000 + 8); + sleeper.go.await(5, TimeUnit.SECONDS); + sleeper.sleeped.await(5, TimeUnit.SECONDS); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(telemetryService, times(1)).sendExtendedHeartbeat(); + verify(sleeperMock, times(1)).sleep(4992); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + clearInvocations(telemetryService, timeSource, metricCollector, periodicAction, sleeperMock); + + // when + thread.interrupt(); + thread.join(); + + // then + // flush pending data before shutdown + verify(metricCollector, times(1)).prepareMetrics(); + verify(metricCollector, times(1)).drain(); + verify(metricCollector, times(1)).drainDistributionSeries(); + verify(periodicAction, times(1)).doIteration(telemetryService); + verify(telemetryService, times(1)).sendTelemetryEvents(); + verify(telemetryService, times(1)).sendAppClosingEvent(); + verifyNoMoreInteractions( + telemetryService, timeSource, sleeperMock, metricCollector, periodicAction); + } + + @Test + void doNotReattemptAppStartedEventUntilNextCycle() + throws InterruptedException, BrokenBarrierException, TimeoutException { + TelemetryRunnable.ThreadSleeper sleeperMock = mock(TelemetryRunnable.ThreadSleeper.class); + TickSleeper sleeper = new TickSleeper(sleeperMock); + TimeSource timeSource = mock(TimeSource.class); + TelemetryService telemetryService = mock(TelemetryService.class); + MetricCollector metricCollector = mock(MetricCollector.class); + MetricPeriodicAction metricAction = mock(MetricPeriodicAction.class); + when(metricAction.collector()).thenReturn(metricCollector); + TelemetryRunnable.TelemetryPeriodicAction periodicAction = + mock(TelemetryRunnable.TelemetryPeriodicAction.class); + TelemetryRunnable runnable = + new TelemetryRunnable( + telemetryService, asList(metricAction, periodicAction), sleeper, timeSource); + thread = new Thread(runnable); + + // three unsuccessful attempts to send app-started (TelemetryRunnable.MAX_APP_STARTED_RETRIES) + when(telemetryService.sendAppStartedEvent()).thenReturn(false, false, false); + when(timeSource.getCurrentTimeMillis()).thenReturn(60L * 1000); + + thread.start(); + sleeper.sleeped.await(10, TimeUnit.SECONDS); + + verify(telemetryService, times(3)).sendAppStartedEvent(); + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(sleeperMock, times(1)).sleep(10000); + } + + @Test + void schedulerSkipsMetricsIntervals() { + TimeSource timeSource = mock(TimeSource.class); + TelemetryRunnable.ThreadSleeper sleeper = mock(TelemetryRunnable.ThreadSleeper.class); + TelemetryRunnable.Scheduler scheduler = + new TelemetryRunnable.Scheduler(timeSource, sleeper, 60 * 1000, 10 * 1000, 0); + + // first iteration: run everything + when(timeSource.getCurrentTimeMillis()).thenReturn(0L); + + scheduler.init(); + + assertTrue(scheduler.shouldRunMetrics()); + assertTrue(scheduler.shouldRunHeartbeat()); + verify(timeSource, times(1)).getCurrentTimeMillis(); + verifyNoMoreInteractions(timeSource, sleeper); + clearInvocations(timeSource, sleeper); + + when(timeSource.getCurrentTimeMillis()).thenReturn(1L, 10L * 1000); + + scheduler.sleepUntilNextIteration(); + + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(sleeper, times(1)).sleep(10 * 1000 - 1); + verifyNoMoreInteractions(timeSource, sleeper); + clearInvocations(timeSource, sleeper); + + // one metrics interval is exceeded + assertTrue(scheduler.shouldRunMetrics()); + assertFalse(scheduler.shouldRunHeartbeat()); + + when(timeSource.getCurrentTimeMillis()).thenReturn(20L * 1000 + 1, 30L * 1000); + + scheduler.sleepUntilNextIteration(); + + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(sleeper, times(1)).sleep(9999); + verifyNoMoreInteractions(timeSource, sleeper); + clearInvocations(timeSource, sleeper); + + // two metrics intervals are exceeded + assertTrue(scheduler.shouldRunMetrics()); + assertFalse(scheduler.shouldRunHeartbeat()); + + when(timeSource.getCurrentTimeMillis()).thenReturn(50L * 1000 + 2, 60L * 1000); + + scheduler.sleepUntilNextIteration(); + + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(sleeper, times(1)).sleep(9998); + verifyNoMoreInteractions(timeSource, sleeper); + assertTrue(scheduler.shouldRunMetrics()); + assertTrue(scheduler.shouldRunHeartbeat()); + } + + @Test + void schedulerSkipsHeartbeatIntervals() { + TimeSource timeSource = mock(TimeSource.class); + TelemetryRunnable.ThreadSleeper sleeper = mock(TelemetryRunnable.ThreadSleeper.class); + TelemetryRunnable.Scheduler scheduler = + new TelemetryRunnable.Scheduler(timeSource, sleeper, 60 * 1000, 10 * 1000, 0); + + // first iteration + when(timeSource.getCurrentTimeMillis()).thenReturn(0L); + scheduler.init(); + + // run everything + assertTrue(scheduler.shouldRunMetrics()); + assertTrue(scheduler.shouldRunHeartbeat()); + verify(timeSource, times(1)).getCurrentTimeMillis(); + verifyNoMoreInteractions(timeSource, sleeper); + clearInvocations(timeSource, sleeper); + + // when + when(timeSource.getCurrentTimeMillis()).thenReturn(1L, 10L * 1000); + scheduler.sleepUntilNextIteration(); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(sleeper, times(1)).sleep(10 * 1000 - 1); + verifyNoMoreInteractions(timeSource, sleeper); + clearInvocations(timeSource, sleeper); + + // when heartbeat interval is exceeded + assertTrue(scheduler.shouldRunMetrics()); + assertFalse(scheduler.shouldRunHeartbeat()); + when(timeSource.getCurrentTimeMillis()).thenReturn(70L * 1000); + scheduler.sleepUntilNextIteration(); + + // then + verify(timeSource, times(1)).getCurrentTimeMillis(); + verifyNoMoreInteractions(timeSource, sleeper); + assertTrue(scheduler.shouldRunMetrics()); + assertTrue(scheduler.shouldRunHeartbeat()); + clearInvocations(timeSource, sleeper); + + // when metrics interval has been adjusted + when(timeSource.getCurrentTimeMillis()).thenReturn(70L * 1000 + 1, 80L * 1000); + scheduler.sleepUntilNextIteration(); + + // then + verify(timeSource, times(2)).getCurrentTimeMillis(); + verify(sleeper, times(1)).sleep(10 * 1000 - 1); + verifyNoMoreInteractions(timeSource, sleeper); + assertTrue(scheduler.shouldRunMetrics()); + assertFalse(scheduler.shouldRunHeartbeat()); + } + + @TableTest({ + "scenario | iters | metricsSecs | heartbeatSecs | extHeartbeatSecs | expectedMetrics | expectedHeartbeats | expectedExtHeartbeats", + "no intervals configured | 10 | 0 | 0 | 0 | 10 | 10 | 10 ", + "one second intervals | 10 | 1 | 1 | 1 | 10 | 10 | 9 ", + "metrics runs more frequently than heartbeat | 12 | 10 | 60 | 60 | 12 | 2 | 1 ", + "heartbeat runs more frequently than metrics | 12 | 60 | 10 | 10 | 2 | 12 | 11 ", + "metrics and heartbeat intervals close together (3, 5) | 6 | 3 | 5 | 5 | 4 | 3 | 2 ", + "metrics and heartbeat intervals close together (5, 3) | 6 | 5 | 3 | 3 | 3 | 4 | 3 " + }) + void schedulerWithHeartbeatMetricsAndExtendedHeartbeatIntervals( + int iters, + int metricsSecs, + int heartbeatSecs, + int extHeartbeatSecs, + int expectedMetrics, + int expectedHeartbeats, + int expectedExtHeartbeats) { + TimeSourceAndSleeper timing = new TimeSourceAndSleeper(); + TelemetryRunnable.Scheduler scheduler = + new TelemetryRunnable.Scheduler( + timing, timing, heartbeatSecs * 1000L, metricsSecs * 1000L, extHeartbeatSecs * 1000L); + int metricsRunCount = 0; + int heartbeatsRunCount = 0; + int extHeartbeatsRunCount = 0; + + scheduler.init(); + for (int i = 0; i < iters; i++) { + if (scheduler.shouldRunMetrics()) { + metricsRunCount++; + } + if (scheduler.shouldRunHeartbeat()) { + heartbeatsRunCount++; + } + boolean runExtHeartbeat = scheduler.shouldRunExtendedHeartbeat(); + if (runExtHeartbeat) { + extHeartbeatsRunCount++; + // need to manually advance to retry next iteration if extended-heartbeat request failed + scheduler.scheduleNextExtendedHeartbeat(); + } + scheduler.sleepUntilNextIteration(); + } + + assertEquals(expectedMetrics, metricsRunCount); + assertEquals(expectedHeartbeats, heartbeatsRunCount); + assertEquals(expectedExtHeartbeats, extHeartbeatsRunCount); + } + + // wraps a ThreadSleeper delegate with two barriers so the test thread can step the background + // runnable one iteration at a time + private static final class TickSleeper implements TelemetryRunnable.ThreadSleeper { + final CyclicBarrier sleeped = new CyclicBarrier(2); + final CyclicBarrier go = new CyclicBarrier(2); + final TelemetryRunnable.ThreadSleeper delegate; + + TickSleeper(TelemetryRunnable.ThreadSleeper delegate) { + this.delegate = delegate; + } + + @Override + public void sleep(long timeoutMs) { + if (delegate != null) { + delegate.sleep(timeoutMs); + } + try { + sleeped.await(10, TimeUnit.SECONDS); + go.await(10, TimeUnit.SECONDS); + } catch (InterruptedException | BrokenBarrierException | TimeoutException e) { + // Thread.interrupt() on the background thread trips this barrier; rethrow the real + // exception unchecked so it reaches TelemetryRunnable.run()'s catch (InterruptedException) + // block exactly like the interrupted CyclicBarrier.await() would in the JVM. + sneakyThrow(e); + } + } + + @SuppressWarnings("unchecked") + private static void sneakyThrow(Throwable t) throws T { + throw (T) t; + } + } + + private static final class TimeSourceAndSleeper + implements TimeSource, TelemetryRunnable.ThreadSleeper { + + private long currentTime = 0; + + @Override + public void sleep(long timeoutMs) { + currentTime += timeoutMs; + } + + @Override + public long getCurrentTimeMillis() { + return currentTime; + } + + @Override + public long getNanoTicks() { + throw new UnsupportedOperationException("NOT IMPLEMENTED"); + } + + @Override + public long getCurrentTimeMicros() { + throw new UnsupportedOperationException("NOT IMPLEMENTED"); + } + + @Override + public long getCurrentTimeNanos() { + throw new UnsupportedOperationException("NOT IMPLEMENTED"); + } + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TelemetryServiceTest.java b/telemetry/src/test/java/datadog/telemetry/TelemetryServiceTest.java new file mode 100644 index 00000000000..661ef36f474 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TelemetryServiceTest.java @@ -0,0 +1,686 @@ +package datadog.telemetry; + +import static datadog.telemetry.TelemetryClient.Result.FAILURE; +import static datadog.telemetry.TelemetryClient.Result.NOT_FOUND; +import static datadog.telemetry.TelemetryClient.Result.SUCCESS; +import static datadog.telemetry.api.RequestType.APP_DEPENDENCIES_LOADED; +import static datadog.telemetry.api.RequestType.APP_ENDPOINTS; +import static datadog.telemetry.api.RequestType.APP_EXTENDED_HEARTBEAT; +import static datadog.telemetry.api.RequestType.APP_HEARTBEAT; +import static datadog.telemetry.api.RequestType.APP_INTEGRATIONS_CHANGE; +import static datadog.telemetry.api.RequestType.APP_PRODUCT_CHANGE; +import static datadog.telemetry.api.RequestType.APP_STARTED; +import static datadog.telemetry.api.RequestType.DISTRIBUTIONS; +import static datadog.telemetry.api.RequestType.GENERATE_METRICS; +import static datadog.telemetry.api.RequestType.LOGS; +import static datadog.telemetry.api.RequestType.MESSAGE_BATCH; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import datadog.telemetry.api.DistributionSeries; +import datadog.telemetry.api.Integration; +import datadog.telemetry.api.LogMessage; +import datadog.telemetry.api.LogMessageLevel; +import datadog.telemetry.api.Metric; +import datadog.telemetry.api.RequestType; +import datadog.telemetry.dependency.Dependency; +import datadog.trace.api.ConfigOrigin; +import datadog.trace.api.ConfigSetting; +import datadog.trace.api.config.AppSecConfig; +import datadog.trace.api.config.DebuggerConfig; +import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.api.telemetry.Endpoint; +import datadog.trace.api.telemetry.ProductChange; +import datadog.trace.test.junit.utils.config.WithConfigExtension; +import datadog.trace.util.ConfigStrings; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.tabletest.junit.TableTest; + +@ExtendWith(WithConfigExtension.class) +class TelemetryServiceTest { + + private final ConfigOrigin confKeyOrigin = ConfigOrigin.DEFAULT; + private final ConfigSetting confKeyValue = + ConfigSetting.of("confkey", "confvalue", confKeyOrigin); + private final Map> configuration = + singletonMap(confKeyOrigin, singletonMap("confkey", confKeyValue)); + private final Integration integration = new Integration("integration", true); + private final Dependency dependency = new Dependency("dependency", "1.0.0", "src", "hash"); + private final Metric metric = + new Metric() + .namespace("tracers") + .metric("metric") + .points(Collections.>singletonList(Arrays.asList(1, 2))) + .tags(Arrays.asList("tag1", "tag2")); + private final DistributionSeries distribution = + new DistributionSeries() + .namespace("tracers") + .metric("distro") + .points(Arrays.asList(1, 2, 3)) + .tags(Arrays.asList("tag1", "tag2")) + .common(false); + private final LogMessage logMessage = + new LogMessage() + .message("log-message") + .tags("tag1:tag2") + .level(LogMessageLevel.DEBUG) + .stackTrace("stack-trace") + .tracerTime(32423L) + .count(1); + private final ProductChange productChange = + new ProductChange().productType(ProductChange.ProductType.APPSEC).enabled(true); + private final Endpoint endpoint = + new Endpoint() + .first(true) + .type("REST") + .method("GET") + .operation("http.request") + .resource("GET /test") + .path("/test") + .requestBodyType(singletonList("application/json")) + .responseBodyType(singletonList("application/json")) + .responseCode(singletonList(200)) + .authentication(singletonList("JWT")); + + @Test + void happyPathWithoutData() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + // first iteration + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + // app-started + testHttpClient.assertRequestBody(APP_STARTED).assertPayload().products(); + testHttpClient.assertNoMoreRequests(); + + // second iteration + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + // app-heartbeat only + testHttpClient.assertRequestBody(APP_HEARTBEAT); + testHttpClient.assertNoMoreRequests(); + + // third iteration + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + // app-heartbeat only + testHttpClient.assertRequestBody(APP_HEARTBEAT); + testHttpClient.assertNoMoreRequests(); + } + + @Test + void happyPathWithData() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + // add data before first iteration + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + telemetryService.addMetric(metric); + telemetryService.addDistributionSeries(distribution); + telemetryService.addLogMessage(logMessage); + telemetryService.addProductChange(productChange); + telemetryService.addEndpoint(endpoint); + + // send messages + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + testHttpClient + .assertRequestBody(MESSAGE_BATCH) + .assertBatch(8) + .assertFirstMessage(APP_HEARTBEAT) + .hasNoPayload() + // no configuration here as it has already been sent with the app-started event + .assertNextMessage(APP_INTEGRATIONS_CHANGE) + .hasPayload() + .integrations(singletonList(integration)) + .assertNextMessage(APP_DEPENDENCIES_LOADED) + .hasPayload() + .dependencies(singletonList(dependency)) + .assertNextMessage(GENERATE_METRICS) + .hasPayload() + .namespace("tracers") + .metrics(singletonList(metric)) + .assertNextMessage(DISTRIBUTIONS) + .hasPayload() + .namespace("tracers") + .distributionSeries(singletonList(distribution)) + .assertNextMessage(LOGS) + .hasPayload() + .logs(singletonList(logMessage)) + .assertNextMessage(APP_PRODUCT_CHANGE) + .hasPayload() + .productChange(productChange) + .assertNextMessage(APP_ENDPOINTS) + .hasPayload() + .endpoint(endpoint) + .assertNoMoreMessages(); + testHttpClient.assertNoMoreRequests(); + + // second iteration heartbeat only + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + testHttpClient.assertRequestBody(APP_HEARTBEAT).assertNoPayload(); + testHttpClient.assertNoMoreRequests(); + + // third iteration metrics data + telemetryService.addMetric(metric); + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + testHttpClient + .assertRequestBody(MESSAGE_BATCH) + .assertBatch(2) + .assertFirstMessage(APP_HEARTBEAT) + .hasNoPayload() + .assertNextMessage(GENERATE_METRICS) + .hasPayload() + .namespace("tracers") + .metrics(singletonList(metric)) + .assertNoMoreMessages(); + testHttpClient.assertNoMoreRequests(); + } + + @Test + void happyPathWithDataAfterAppStarted() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + // send messages + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + testHttpClient.assertRequestBody(APP_STARTED).assertPayload().products(); + testHttpClient.assertNoMoreRequests(); + + // add data after first iteration + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + telemetryService.addMetric(metric); + telemetryService.addDistributionSeries(distribution); + telemetryService.addLogMessage(logMessage); + telemetryService.addProductChange(productChange); + telemetryService.addEndpoint(endpoint); + + // send messages + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + testHttpClient + .assertRequestBody(MESSAGE_BATCH) + .assertBatch(9) + .assertFirstMessage(APP_HEARTBEAT) + .hasNoPayload() + .assertNextMessage(RequestType.APP_CLIENT_CONFIGURATION_CHANGE) + .hasPayload() + .configuration(singletonList(confKeyValue)) + .assertNextMessage(APP_INTEGRATIONS_CHANGE) + .hasPayload() + .integrations(singletonList(integration)) + .assertNextMessage(APP_DEPENDENCIES_LOADED) + .hasPayload() + .dependencies(singletonList(dependency)) + .assertNextMessage(GENERATE_METRICS) + .hasPayload() + .namespace("tracers") + .metrics(singletonList(metric)) + .assertNextMessage(DISTRIBUTIONS) + .hasPayload() + .namespace("tracers") + .distributionSeries(singletonList(distribution)) + .assertNextMessage(LOGS) + .hasPayload() + .logs(singletonList(logMessage)) + .assertNextMessage(APP_PRODUCT_CHANGE) + .hasPayload() + .productChange(productChange) + .assertNextMessage(APP_ENDPOINTS) + .hasPayload() + .endpoint(endpoint) + .assertNoMoreMessages(); + testHttpClient.assertNoMoreRequests(); + } + + @Test + void doNotDiscardDataForAppStartedEventUntilItHasBeenSuccessfullySent() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + telemetryService.addConfiguration(configuration); + + // attempt with 404 error + testHttpClient.expectRequest(NOT_FOUND); + assertFalse(telemetryService.sendAppStartedEvent()); + + // app-started is attempted + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + testHttpClient.assertNoMoreRequests(); + + // attempt with 500 error + testHttpClient.expectRequest(FAILURE); + assertFalse(telemetryService.sendAppStartedEvent()); + + // app-started is attempted + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + testHttpClient.assertNoMoreRequests(); + + // attempt with unexpected FAILURE (not valid) + testHttpClient.expectRequest(FAILURE); + assertFalse(telemetryService.sendAppStartedEvent()); + + // app-started is attempted + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + testHttpClient.assertNoMoreRequests(); + + // attempt with success + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + // app-started is attempted + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + testHttpClient.assertNoMoreRequests(); + } + + @Test + void resendDataOnSuccessfulAttemptAfterAFailure() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + telemetryService.addMetric(metric); + telemetryService.addDistributionSeries(distribution); + telemetryService.addLogMessage(logMessage); + telemetryService.addProductChange(productChange); + telemetryService.addEndpoint(endpoint); + + // attempt with NOT_FOUND error + testHttpClient.expectRequest(NOT_FOUND); + assertFalse(telemetryService.sendAppStartedEvent()); + + // app-started attempted with config + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + testHttpClient.assertNoMoreRequests(); + + // successful app-started attempt + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + // attempt app-started with SUCCESS + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products() + .configuration(singletonList(confKeyValue)); + + // successful batch attempt + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + // attempt batch with SUCCESS + testHttpClient + .assertRequestBody(MESSAGE_BATCH) + .assertBatch(8) + .assertFirstMessage(APP_HEARTBEAT) + .hasNoPayload() + // no configuration here as it has already been sent with the app-started event + .assertNextMessage(APP_INTEGRATIONS_CHANGE) + .hasPayload() + .integrations(singletonList(integration)) + .assertNextMessage(APP_DEPENDENCIES_LOADED) + .hasPayload() + .dependencies(singletonList(dependency)) + .assertNextMessage(GENERATE_METRICS) + .hasPayload() + .namespace("tracers") + .metrics(singletonList(metric)) + .assertNextMessage(DISTRIBUTIONS) + .hasPayload() + .namespace("tracers") + .distributionSeries(singletonList(distribution)) + .assertNextMessage(LOGS) + .hasPayload() + .logs(singletonList(logMessage)) + .assertNextMessage(APP_PRODUCT_CHANGE) + .hasPayload() + .productChange(productChange) + .assertNextMessage(APP_ENDPOINTS) + .hasPayload() + .endpoint(endpoint) + .assertNoMoreMessages(); + testHttpClient.assertNoMoreRequests(); + + // attempt with NOT_FOUND error + testHttpClient.expectRequest(NOT_FOUND); + telemetryService.sendTelemetryEvents(); + + // message-batch attempted with heartbeat + testHttpClient.assertRequestBody(APP_HEARTBEAT).assertNoPayload(); + testHttpClient.assertNoMoreRequests(); + } + + @Test + void sendClosingEventRequest() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppClosingEvent(); + + testHttpClient.assertRequestBody(RequestType.APP_CLOSING); + testHttpClient.assertNoMoreRequests(); + } + + @TableTest({ + "scenario | otelEnabled | otEnabled | expectedWarnings", + "both otel and ot enabled | true | true | 1 ", + "only otel enabled | true | false | 0 ", + "only ot enabled | false | true | 0 ", + "neither otel nor ot enabled | false | false | 0 " + }) + void reportWhenBothOTelAndOTAreEnabled( + boolean otelEnabled, boolean otEnabled, int expectedWarnings) { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = spy(new TelemetryService(testHttpClient, 1000, false)); + Integration otel = new Integration("opentelemetry-1", otelEnabled); + Integration ot = new Integration("opentracing", otEnabled); + + telemetryService.addIntegration(otel); + + verify(telemetryService, times(0)).warnAboutExclusiveIntegrations(); + + telemetryService.addIntegration(ot); + + verify(telemetryService, times(expectedWarnings)).warnAboutExclusiveIntegrations(); + } + + @Test + void splitTelemetryRequestsIfTheSizeAboveTheLimit() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 5000, false); + + // send a heartbeat request without telemetry data to measure body size to set stable request + // size limit + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + // get body size + int bodySize = testHttpClient.assertRequestBody(APP_HEARTBEAT).bodySize(); + assertTrue(bodySize > 0); + + // sending first part of data + telemetryService = new TelemetryService(testHttpClient, bodySize + 512, false); + + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + telemetryService.addMetric(metric); + telemetryService.addDistributionSeries(distribution); + telemetryService.addLogMessage(logMessage); + telemetryService.addProductChange(productChange); + telemetryService.addEndpoint(endpoint); + + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendTelemetryEvents(); + + // attempt with SUCCESS + testHttpClient + .assertRequestBody(MESSAGE_BATCH) + .assertBatch(5) + .assertFirstMessage(APP_HEARTBEAT) + .hasNoPayload() + .assertNextMessage(RequestType.APP_CLIENT_CONFIGURATION_CHANGE) + .hasPayload() + .configuration(singletonList(confKeyValue)) + .assertNextMessage(APP_INTEGRATIONS_CHANGE) + .hasPayload() + .integrations(singletonList(integration)) + .assertNextMessage(APP_DEPENDENCIES_LOADED) + .hasPayload() + .dependencies(singletonList(dependency)) + .assertNextMessage(GENERATE_METRICS) + .hasPayload() + .namespace("tracers") + .metrics(singletonList(metric)) + // no more data fit this message is sent in the next message + .assertNoMoreMessages(); + + // sending second part of data + testHttpClient.expectRequest(SUCCESS); + assertFalse(telemetryService.sendTelemetryEvents()); + + testHttpClient + .assertRequestBody(MESSAGE_BATCH) + .assertBatch(5) + .assertFirstMessage(APP_HEARTBEAT) + .hasNoPayload() + .assertNextMessage(DISTRIBUTIONS) + .hasPayload() + .namespace("tracers") + .distributionSeries(singletonList(distribution)) + .assertNextMessage(LOGS) + .hasPayload() + .logs(singletonList(logMessage)) + .assertNextMessage(APP_PRODUCT_CHANGE) + .hasPayload() + .productChange(productChange) + .assertNextMessage(APP_ENDPOINTS) + .hasPayload() + .endpoint(endpoint) + .assertNoMoreMessages(); + testHttpClient.assertNoMoreRequests(); + } + + @Test + void sendAllCollectedDataWithExtendedHeartbeatRequestEveryTime() throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendExtendedHeartbeat(); + + testHttpClient + .assertRequestBody(APP_EXTENDED_HEARTBEAT) + .assertPayload() + .configuration(singletonList(confKeyValue)) + .integrations(singletonList(integration)) + .dependencies(singletonList(dependency)); + testHttpClient.assertNoMoreRequests(); + + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendExtendedHeartbeat(); + + testHttpClient + .assertRequestBody(APP_EXTENDED_HEARTBEAT) + .assertPayload() + .configuration(Arrays.asList(confKeyValue, confKeyValue)) + .integrations(Arrays.asList(integration, integration)) + .dependencies(Arrays.asList(dependency, dependency)); + testHttpClient.assertNoMoreRequests(); + } + + @TableTest({ + "scenario | resultCode", + "success | SUCCESS ", + "failure | FAILURE ", + "notFound | NOT_FOUND " + }) + void + sendExtendedHeartbeatRequestEvenIfDataAlreadyHasBeenSentOrAttemptedAsPartOfAnotherTelemetryEvents( + String resultCode) throws IOException { + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + telemetryService.addConfiguration(configuration); + telemetryService.addIntegration(integration); + telemetryService.addDependency(dependency); + + testHttpClient.expectRequest(TelemetryClient.Result.valueOf(resultCode)); + telemetryService.sendTelemetryEvents(); + + testHttpClient.assertRequestBody(MESSAGE_BATCH); + + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendExtendedHeartbeat(); + + testHttpClient + .assertRequestBody(APP_EXTENDED_HEARTBEAT) + .assertPayload() + .configuration(singletonList(confKeyValue)) + .integrations(singletonList(integration)) + .dependencies(singletonList(dependency)); + testHttpClient.assertNoMoreRequests(); + } + + @TableTest({ + "scenario | id ", + "with value | foo", + "null value | ", + "empty value | '' " + }) + void appCanPropagateConfigurationId(String id) throws IOException { + String instrumentationConfigIdKey = "instrumentation_config_id"; + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + Map configMap = + singletonMap( + instrumentationConfigIdKey, + ConfigSetting.of(instrumentationConfigIdKey, id, ConfigOrigin.ENV)); + telemetryService.addConfiguration(singletonMap(ConfigOrigin.ENV, configMap)); + + // first iteration + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + // app-started + testHttpClient.assertRequestBody(APP_STARTED).assertPayload().instrumentationConfigId(id); + testHttpClient.assertNoMoreRequests(); + } + + @TableTest({ + "scenario | installId | installType | installTime", + "no install data | | | ", + "install time only | | | 1703188334 ", + "install type only | | k8s_single_step | ", + "install type and time | | k8s_single_step | 1703188212 ", + "install id only | 68e75c99-57ca-4a12-adfc-575c4b05fcbe | | ", + "install id and time | 68e75c48-57ca-4a12-adfc-575c4b05bfff | | 1704183412 ", + "install id and type | 68e75c55-57ca-4a12-adfc-575c4b05aaaa | k8s_single_step | ", + "install id, type and time | 68e75c77-57ca-4a12-adfc-575c4b05fc44 | k8s_single_step | 1993188215 " + }) + void appStartedMustHaveInstallSignature(String installId, String installType, String installTime) + throws IOException { + WithConfigExtension.injectEnvConfig("INSTRUMENTATION_INSTALL_ID", installId); + WithConfigExtension.injectEnvConfig("INSTRUMENTATION_INSTALL_TYPE", installType); + WithConfigExtension.injectEnvConfig("INSTRUMENTATION_INSTALL_TIME", installTime); + + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + // first iteration + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + // app-started + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .installSignature(installId, installType, installTime); + testHttpClient.assertNoMoreRequests(); + } + + @TableTest({ + "scenario | appsecConfig | appsecEnabled | profilingConfig | profilingEnabled | dynInstrConfig | dynInstrEnabled", + "all products enabled | 1 | true | 1 | true | 1 | true ", + "dynamic instrumentation disabled | 1 | true | 1 | true | 0 | false ", + "profiling disabled | 1 | true | 0 | false | 1 | true ", + "profiling and dyn instr disabled | 1 | true | 0 | false | 0 | false ", + "all products disabled | 0 | false | 0 | false | 0 | false ", + "appsec inactive value treated enabled | inactive | true | 0 | false | 0 | false " + }) + void appStartedMustIncludeActivatedProductsInfo( + String appsecConfig, + boolean appsecEnabled, + String profilingConfig, + boolean profilingEnabled, + String dynInstrConfig, + boolean dynInstrEnabled) + throws IOException { + WithConfigExtension.injectEnvConfig( + ConfigStrings.toEnvVar(AppSecConfig.APPSEC_ENABLED), appsecConfig); + WithConfigExtension.injectEnvConfig( + ConfigStrings.toEnvVar(ProfilingConfig.PROFILING_ENABLED), profilingConfig); + WithConfigExtension.injectEnvConfig( + ConfigStrings.toEnvVar(DebuggerConfig.DYNAMIC_INSTRUMENTATION_ENABLED), dynInstrConfig); + + TestTelemetryRouter testHttpClient = new TestTelemetryRouter(); + TelemetryService telemetryService = new TelemetryService(testHttpClient, 10000, false); + + // first iteration + testHttpClient.expectRequest(SUCCESS); + telemetryService.sendAppStartedEvent(); + + // app-started + testHttpClient + .assertRequestBody(APP_STARTED) + .assertPayload() + .products(appsecEnabled, profilingEnabled, dynInstrEnabled); + testHttpClient.assertNoMoreRequests(); + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TelemetrySystemTest.java b/telemetry/src/test/java/datadog/telemetry/TelemetrySystemTest.java new file mode 100644 index 00000000000..76c57f9d842 --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TelemetrySystemTest.java @@ -0,0 +1,85 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import datadog.communication.ddagent.DDAgentFeaturesDiscovery; +import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.metrics.api.Monitoring; +import datadog.telemetry.dependency.DependencyService; +import datadog.trace.api.config.GeneralConfig; +import datadog.trace.test.junit.utils.config.WithConfig; +import java.lang.instrument.ClassFileTransformer; +import java.lang.instrument.Instrumentation; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class TelemetrySystemTest { + + @AfterEach + void cleanup() { + TelemetrySystem.stop(); + } + + @Test + void installsDependenciesTransformer() { + Instrumentation instrumentation = mock(Instrumentation.class); + + DependencyService dependencyService = TelemetrySystem.createDependencyService(instrumentation); + try { + ArgumentCaptor transformerCaptor = + ArgumentCaptor.forClass(ClassFileTransformer.class); + verify(instrumentation, times(1)).addTransformer(transformerCaptor.capture()); + assertEquals( + "datadog.telemetry.dependency.LocationsCollectingTransformer", + transformerCaptor.getValue().getClass().getName()); + } finally { + dependencyService.stop(); + } + } + + @Test + void createTelemetryThread() { + TelemetryService telemetryService = mock(TelemetryService.class); + DependencyService dependencyService = mock(DependencyService.class); + + Thread thread = + TelemetrySystem.createTelemetryRunnable(telemetryService, dependencyService, true); + + assertNotNull(thread); + } + + @Test + @WithConfig(key = GeneralConfig.SITE, value = "datad0g.com") + @WithConfig(key = GeneralConfig.API_KEY, value = "api-key") + void startStopTelemetrySystem() { + Instrumentation instrumentation = mock(Instrumentation.class); + + TelemetrySystem.startTelemetry(instrumentation, sharedCommunicationObjects()); + + assertNotNull(TelemetrySystem.getTelemetryThread()); + + TelemetrySystem.stop(); + + assertTrue( + TelemetrySystem.getTelemetryThread() == null + || TelemetrySystem.getTelemetryThread().isInterrupted() + || !TelemetrySystem.getTelemetryThread().isAlive()); + } + + private SharedCommunicationObjects sharedCommunicationObjects() { + SharedCommunicationObjects sco = new SharedCommunicationObjects(); + sco.agentHttpClient = mock(OkHttpClient.class); + sco.monitoring = mock(Monitoring.class); + sco.agentUrl = HttpUrl.get("https://example.com"); + sco.setFeaturesDiscovery(mock(DDAgentFeaturesDiscovery.class)); + return sco; + } +} diff --git a/telemetry/src/test/java/datadog/telemetry/TestTelemetryRouter.java b/telemetry/src/test/java/datadog/telemetry/TestTelemetryRouter.java new file mode 100644 index 00000000000..5038614045d --- /dev/null +++ b/telemetry/src/test/java/datadog/telemetry/TestTelemetryRouter.java @@ -0,0 +1,513 @@ +package datadog.telemetry; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.squareup.moshi.Moshi; +import com.squareup.moshi.Types; +import datadog.communication.ddagent.TracerVersion; +import datadog.telemetry.api.DistributionSeries; +import datadog.telemetry.api.Integration; +import datadog.telemetry.api.LogMessage; +import datadog.telemetry.api.Metric; +import datadog.telemetry.api.RequestType; +import datadog.telemetry.dependency.Dependency; +import datadog.trace.api.Config; +import datadog.trace.api.ConfigSetting; +import datadog.trace.api.telemetry.Endpoint; +import datadog.trace.api.telemetry.ProductChange; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import okhttp3.Request; +import okio.Buffer; + +class TestTelemetryRouter extends TelemetryRouter { + private final Queue mockResults = new LinkedList<>(); + private final Queue requests = new LinkedList<>(); + + TestTelemetryRouter() { + super(null, null, null, false); + } + + @Override + public TelemetryClient.Result sendRequest(TelemetryRequest request) { + if (mockResults.isEmpty()) { + throw new IllegalStateException( + "Unexpected request has been sent. State expectations with `expectRequests` prior sending requests."); + } + + Request.Builder requestBuilder = request.httpRequest(); + requestBuilder.url("https://example.com"); + requests.add(new RequestAssertions(requestBuilder.build())); + return mockResults.poll(); + } + + void expectRequest(TelemetryClient.Result mockResult) { + expectRequests(1, mockResult); + } + + void expectRequests(int requestNumber, TelemetryClient.Result mockResult) { + for (int i = 0; i < requestNumber; i++) { + mockResults.add(mockResult); + } + } + + RequestAssertions assertRequest() { + if (!mockResults.isEmpty()) { + throw new IllegalStateException("Expected " + mockResults.size() + " more sendRequest calls"); + } + if (requests.isEmpty()) { + throw new IllegalStateException("No more requests have been sent."); + } + return requests.poll(); + } + + BodyAssertions assertRequestBody(RequestType requestType) throws IOException { + return assertRequest().headers(requestType).assertBody().commonParts(requestType); + } + + void assertNoMoreRequests() { + if (!mockResults.isEmpty()) { + throw new IllegalStateException("Still expect " + mockResults.size() + " request(s)"); + } + if (!requests.isEmpty()) { + throw new IllegalStateException( + "Still have " + requests.size() + " requests when none expected."); + } + } + + private static List toDoubles(List numbers) { + if (numbers == null) { + return null; + } + List doubles = new ArrayList<>(numbers.size()); + for (Number number : numbers) { + doubles.add(number.doubleValue()); + } + return doubles; + } + + static class RequestAssertions { + private final Request request; + + RequestAssertions(Request request) { + this.request = request; + } + + RequestAssertions headers(RequestType requestType) { + assertEquals("POST", request.method()); + assertTrue( + request + .headers() + .names() + .containsAll( + Arrays.asList( + "Content-Type", + "Content-Length", + "DD-Client-Library-Language", + "DD-Client-Library-Version", + "DD-Telemetry-API-Version", + "DD-Telemetry-Request-Type", + "DD-Session-ID"))); + assertEquals("application/json; charset=utf-8", request.header("Content-Type")); + assertTrue(Integer.parseInt(request.header("Content-Length")) > 0); + assertEquals("jvm", request.header("DD-Client-Library-Language")); + assertEquals(TracerVersion.TRACER_VERSION, request.header("DD-Client-Library-Version")); + assertEquals("v2", request.header("DD-Telemetry-API-Version")); + assertEquals(requestType.toString(), request.header("DD-Telemetry-Request-Type")); + String entityId = request.header("Datadog-Entity-ID"); + assertTrue(entityId == null || entityId.startsWith("in-") || entityId.startsWith("cin-")); + String sessionId = request.header("DD-Session-ID"); + assertTrue( + sessionId != null && sessionId.matches("[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}")); + assertEquals(Config.get().getRuntimeId(), sessionId); + // DD-Root-Session-ID should only be present when inherited from a parent process + // (i.e., when rootSessionId != runtimeId). In normal test context, they're equal. + String rootSessionId = request.header("DD-Root-Session-ID"); + if (Config.get().getRootSessionId().equals(Config.get().getRuntimeId())) { + assertNull(rootSessionId); + } else { + assertEquals(Config.get().getRootSessionId(), rootSessionId); + } + return this; + } + + @SuppressWarnings("unchecked") + BodyAssertions assertBody() throws IOException { + Buffer buf = new Buffer(); + request.body().writeTo(buf); + byte[] bytes = new byte[(int) buf.size()]; + buf.read(bytes); + Map parsed = + (Map) + new Moshi.Builder() + .build() + .adapter(Types.newParameterizedType(Map.class, String.class, Object.class)) + .fromJson(new String(bytes, StandardCharsets.UTF_8)); + return new BodyAssertions(parsed, bytes); + } + } + + static class BodyAssertions { + private final Map body; + private final byte[] bodyBytes; + + BodyAssertions(Map body, byte[] bodyBytes) { + this.body = body; + this.bodyBytes = bodyBytes; + } + + int bodySize() { + return bodyBytes.length; + } + + @SuppressWarnings("unchecked") + BodyAssertions commonParts(RequestType requestType) { + assertEquals("v2", body.get("api_version")); + + Map app = (Map) body.get("application"); + assertNotNull(app.get("env")); + assertEquals("jvm", app.get("language_name")); + assertTrue(((String) app.get("language_version")).matches("\\d+.*")); + assertNotNull(app.get("runtime_name")); + assertNotNull(app.get("runtime_version")); + assertNotNull(app.get("service_name")); + assertEquals("0.42.0", app.get("tracer_version")); + + Map host = (Map) body.get("host"); + assertNotNull(host.get("hostname")); + assertNotNull(host.get("os")); + assertNotNull(host.get("os_version")); + assertNotNull(host.get("kernel_name")); + assertNotNull(host.get("kernel_release")); + assertNotNull(host.get("kernel_version")); + + assertTrue( + ((String) body.get("runtime_id")).matches("[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}")); + assertTrue(((Number) body.get("seq_id")).doubleValue() > 0); + assertTrue(((Number) body.get("tracer_time")).doubleValue() > 0); + assertEquals(requestType.toString(), body.get("request_type")); + return this; + } + + @SuppressWarnings("unchecked") + PayloadAssertions assertPayload() { + Map payload = (Map) body.get("payload"); + assertNotNull(payload); + return new PayloadAssertions(payload); + } + + @SuppressWarnings("unchecked") + BatchAssertions assertBatch(int expectedNumberOfPayloads) { + List> payloads = (List>) body.get("payload"); + assertNotNull(payloads); + assertEquals(expectedNumberOfPayloads, payloads.size()); + return new BatchAssertions(payloads); + } + + void assertNoPayload() { + assertNull(body.get("payload")); + } + } + + static class BatchAssertions { + private final List> messages; + + BatchAssertions(List> messages) { + this.messages = messages; + } + + BatchMessageAssertions assertFirstMessage(RequestType expected) { + return assertMessage(0, expected); + } + + private BatchMessageAssertions assertMessage(int index, RequestType expected) { + if (index > messages.size()) { + throw new IllegalStateException( + "Asserted more messages than available (" + messages.size() + ") in the batch"); + } + Map message = messages.get(index); + assertEquals(String.valueOf(expected), message.get("request_type")); + return new BatchMessageAssertions(this, index, message); + } + } + + static class BatchMessageAssertions { + private final BatchAssertions batchAssertions; + private int messageIndex; + private final Map message; + + BatchMessageAssertions( + BatchAssertions batchAssertions, int messageIndex, Map message) { + this.batchAssertions = batchAssertions; + this.messageIndex = messageIndex; + this.message = message; + } + + BatchMessageAssertions hasNoPayload() { + assertNull(message.get("payload")); + return this; + } + + BatchMessageAssertions assertNextMessage(RequestType expected) { + messageIndex += 1; + if (messageIndex >= batchAssertions.messages.size()) { + throw new IllegalStateException("No more messages available"); + } + return batchAssertions.assertMessage(messageIndex, expected); + } + + @SuppressWarnings("unchecked") + PayloadAssertions hasPayload() { + Map payload = (Map) message.get("payload"); + assertNotNull(payload); + return new PayloadAssertions(payload, this); + } + + void assertNoMoreMessages() { + assertEquals(batchAssertions.messages.size() - 1, messageIndex); + } + } + + static class PayloadAssertions { + private final Map payload; + private final BatchMessageAssertions batch; + + PayloadAssertions(Map payload) { + this(payload, null); + } + + PayloadAssertions(Map payload, BatchMessageAssertions batch) { + this.payload = payload; + this.batch = batch; + } + + PayloadAssertions configuration(List configuration) { + List> expected = configuration == null ? null : new ArrayList<>(); + if (configuration != null) { + for (ConfigSetting cs : configuration) { + Map item = new HashMap<>(); + item.put("name", cs.key); + item.put("value", cs.stringValue()); + item.put("origin", cs.origin.value); + item.put("seq_id", (double) cs.seqId); + expected.add(item); + } + } + assertEquals(expected, payload.get("configuration")); + return this; + } + + @SuppressWarnings("unchecked") + PayloadAssertions instrumentationConfigId(String id) { + boolean checked = false; + List> configuration = + (List>) payload.get("configuration"); + for (Map entry : configuration) { + if ("DD_INSTRUMENTATION_CONFIG_ID".equals(entry.get("name"))) { + assertEquals(id, entry.get("value")); + checked = true; + } + } + if (!checked) { + assertNull(id); + } + return this; + } + + PayloadAssertions productChange(ProductChange product) { + String name = product.getProductType().getName(); + Map expected = new HashMap<>(); + expected.put(name, Collections.singletonMap("enabled", product.isEnabled())); + assertEquals(expected, payload.get("products")); + return this; + } + + PayloadAssertions endpoint(Endpoint... endpoints) { + List> expected = new ArrayList<>(); + for (Endpoint endpoint : endpoints) { + Map item = new HashMap<>(); + item.put("operation_name", endpoint.getOperation()); + item.put("resource_name", endpoint.getMethod() + " " + endpoint.getPath()); + if (endpoint.getType() != null) { + item.put("type", endpoint.getType()); + } + if (endpoint.getMethod() != null) { + item.put("method", endpoint.getMethod()); + } + if (endpoint.getPath() != null) { + item.put("path", endpoint.getPath()); + } + if (endpoint.getRequestBodyType() != null) { + item.put("request_body_type", endpoint.getRequestBodyType()); + } + if (endpoint.getResponseBodyType() != null) { + item.put("response_body_type", endpoint.getResponseBodyType()); + } + if (endpoint.getAuthentication() != null) { + item.put("authentication", endpoint.getAuthentication()); + } + if (endpoint.getResponseCode() != null) { + item.put("response_code", toDoubles(endpoint.getResponseCode())); + } + if (endpoint.getMetadata() != null) { + item.put("metadata", endpoint.getMetadata()); + } + expected.add(item); + } + assertEquals(expected, payload.get("endpoints")); + return this; + } + + PayloadAssertions products() { + return products(true, false, false); + } + + PayloadAssertions products(boolean appsecEnabled) { + return products(appsecEnabled, false, false); + } + + PayloadAssertions products(boolean appsecEnabled, boolean profilerEnabled) { + return products(appsecEnabled, profilerEnabled, false); + } + + PayloadAssertions products( + boolean appsecEnabled, boolean profilerEnabled, boolean dynamicInstrumentationEnabled) { + Map expected = new HashMap<>(); + expected.put("appsec", Collections.singletonMap("enabled", appsecEnabled)); + expected.put("profiler", Collections.singletonMap("enabled", profilerEnabled)); + expected.put( + "dynamic_instrumentation", + Collections.singletonMap("enabled", dynamicInstrumentationEnabled)); + assertEquals(expected, payload.get("products")); + return this; + } + + PayloadAssertions dependencies(List dependencies) { + List> expected = new ArrayList<>(); + for (Dependency dependency : dependencies) { + Map item = new HashMap<>(); + item.put("hash", dependency.hash); + item.put("name", dependency.name); + item.put("version", dependency.version); + expected.add(item); + } + assertEquals(expected, payload.get("dependencies")); + return this; + } + + PayloadAssertions integrations(List integrations) { + List> expected = new ArrayList<>(); + for (Integration integration : integrations) { + Map item = new HashMap<>(); + item.put("enabled", integration.enabled); + item.put("name", integration.name); + expected.add(item); + } + assertEquals(expected, payload.get("integrations")); + return this; + } + + PayloadAssertions namespace(String namespace) { + assertEquals(namespace, payload.get("namespace")); + return this; + } + + PayloadAssertions metrics(List metrics) { + List> expected = new ArrayList<>(); + for (Metric metric : metrics) { + Map item = new HashMap<>(); + item.put("namespace", metric.getNamespace()); + if (metric.getCommon() != null) { + item.put("common", metric.getCommon()); + } + item.put("metric", metric.getMetric()); + List> points = new ArrayList<>(); + for (List point : metric.getPoints()) { + points.add(toDoubles(point)); + } + item.put("points", points); + if (metric.getType() != null) { + item.put("type", metric.getType()); + } + item.put("tags", metric.getTags()); + expected.add(item); + } + assertEquals(expected, payload.get("series")); + return this; + } + + PayloadAssertions distributionSeries(List distributionSeriesList) { + List> expected = new ArrayList<>(); + for (DistributionSeries distribution : distributionSeriesList) { + Map item = new HashMap<>(); + item.put("namespace", distribution.getNamespace()); + if (distribution.getCommon() != null) { + item.put("common", distribution.getCommon()); + } + item.put("metric", distribution.getMetric()); + item.put("points", toDoubles(distribution.getPoints())); + item.put("tags", distribution.getTags()); + expected.add(item); + } + assertEquals(expected, payload.get("series")); + return this; + } + + PayloadAssertions logs(List logMessages) { + List> expected = new ArrayList<>(); + for (LogMessage logMessage : logMessages) { + Map item = new HashMap<>(); + item.put("message", logMessage.getMessage()); + item.put("level", logMessage.getLevel().toString()); + item.put("tags", logMessage.getTags()); + if (logMessage.getStackTrace() != null) { + item.put("stack_trace", logMessage.getStackTrace()); + } + if (logMessage.getTracerTime() != null) { + item.put("tracer_time", logMessage.getTracerTime().doubleValue()); + } + item.put("count", (double) logMessage.getCount()); + expected.add(item); + } + assertEquals(expected, payload.get("logs")); + return this; + } + + BatchMessageAssertions assertNextMessage(RequestType requestType) { + return batch.assertNextMessage(requestType); + } + + void assertNoMoreMessages() { + batch.assertNoMoreMessages(); + } + + void installSignature(String installId, String installType, String installTime) { + if (installId == null && installType == null && installTime == null) { + assertNull(payload.get("install_signature")); + return; + } + Map expected = new HashMap<>(); + if (installId != null) { + expected.put("install_id", installId); + } + if (installType != null) { + expected.put("install_type", installType); + } + if (installTime != null) { + expected.put("install_time", installTime); + } + assertEquals(expected, payload.get("install_signature")); + } + } +} From 4b3e21628e5da59f9aa573d1b3cc9e14598bb075 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Thu, 10 Sep 2026 17:36:40 -0400 Subject: [PATCH 16/30] Return empty method lines instead of throwing when class bytecode resource is missing (#12380) Return empty method lines instead of throwing when class bytecode resource is missing Utils.getClassStream() is documented @Nullable and can return null for classes whose .class resource can't be located on the classpath (e.g. Mockito-generated proxy classes). ByteCodeLinesResolver.ClassMethodLines.parse passed that possibly-null stream straight into ClassReader, which threw an IOException that got wrapped and logged at ERROR, then swallowed. This has been recurring in error tracking for CI Visibility test runs across many tracer versions. Short-circuit on a null stream and log at debug instead, matching the existing handling in Utils.getFileName for the same condition. Co-Authored-By: Claude Sonnet 5 Flag transient-failure caching risk; sharpen regression test Note (no behavior change) on the null-classStream branch: it's cached as a permanent empty result via computeIfAbsent, which is correct for a generated/proxy class with no bytecode resource, but ClassLoader#getResourceAsStream also swallows IOException and returns null on transient failures (e.g. OOM mid-read) -- seen regularly in production exception tracking. Not fixing that here; flagging it for whoever picks this up next. Also fix testReturnsEmptyMethodLinesWhenClassResourceIsMissing to call ClassMethodLines.parse() directly instead of going through getMethodLines(), whose outer catch(Exception) already masks whether the null-stream guard under test is doing anything. Co-Authored-By: Claude Sonnet 5 Co-authored-by: devflow.devflow-routing-intake --- .../source/ByteCodeLinesResolver.java | 11 ++ .../source/ByteCodeLinesResolverTest.groovy | 85 ------------- .../source/ByteCodeLinesResolverTest.java | 115 ++++++++++++++++++ .../source/NullResourceClassLoader.java | 36 ++++++ 4 files changed, 162 insertions(+), 85 deletions(-) delete mode 100644 dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.groovy create mode 100644 dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.java create mode 100644 dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/NullResourceClassLoader.java diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/ByteCodeLinesResolver.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/ByteCodeLinesResolver.java index c77a1f92e5d..cb014802fe5 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/ByteCodeLinesResolver.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/ByteCodeLinesResolver.java @@ -66,6 +66,17 @@ public static ClassMethodLines parse(Class clazz) { try { ClassMethodLines classMethodLines = new ClassMethodLines(); try (InputStream classStream = Utils.getClassStream(clazz)) { + if (classStream == null) { + // Cached below via computeIfAbsent, as a permanent empty result for this class. + // That's correct for the case this guards against -- a generated/proxy class that + // will never have a bytecode resource -- but ClassLoader#getResourceAsStream also + // swallows IOException and returns null, so in principle a transient failure (I/O + // error, OOM while reading the class bytes) could hit this same branch and get + // pinned as a permanent negative. Not handling that here; flagging it for whoever + // next touches this if transient-failure caching turns out to matter in practice. + log.debug("Could not get input stream for class {}", clazz.getName()); + return classMethodLines; + } ClassReader classReader = new ClassReader(classStream); MethodLocator methodLocator = new MethodLocator(classMethodLines); classReader.accept(methodLocator, ClassReader.SKIP_FRAMES); diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.groovy deleted file mode 100644 index 0960a2a40ab..00000000000 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.groovy +++ /dev/null @@ -1,85 +0,0 @@ -package datadog.trace.civisibility.source - -import org.spockframework.util.IoUtil -import spock.lang.Specification - -class ByteCodeLinesResolverTest extends Specification { - - def "test method lines resolution"() { - setup: - def aTestMethod = NestedClass.getDeclaredMethod("aTestMethod") - - when: - def linesResolver = new ByteCodeLinesResolver() - def methodLines = linesResolver.getMethodLines(aTestMethod) - - then: - methodLines.isValid() - methodLines.startLineNumber > 0 - methodLines.endLineNumber > methodLines.startLineNumber - } - - def "test always invalid class lines resolution" () { - when: - def linesResolver = new ByteCodeLinesResolver() - def classLines = linesResolver.getClassLines(NestedClass) - - then: - !classLines.isValid() - } - - def "test invalid method lines resolution"() { - setup: - def aTestMethod = NestedClass.getDeclaredMethod("abstractMethod") - - when: - def linesResolver = new ByteCodeLinesResolver() - def methodLines = linesResolver.getMethodLines(aTestMethod) - - then: - !methodLines.isValid() - } - - def "test returns empty method lines when class cannot be loaded"() { - setup: - def misbehavingClassLoader = new MisbehavingClassLoader() - - Utils.getClassStream(NestedClass).withCloseable { stream -> - def baos = new ByteArrayOutputStream() - IoUtil.copyStream(stream, baos) - misbehavingClassLoader.putClass(NestedClass.name, baos.toByteArray()) - } - - def misbehavingClass = misbehavingClassLoader.loadClass(NestedClass.name) - def misbehavingMethod = misbehavingClass.getDeclaredMethod("aTestMethod") - - when: - def linesResolver = new ByteCodeLinesResolver() - def methodLines = linesResolver.getMethodLines(misbehavingMethod) - - then: - !methodLines.isValid() - } - - def "test returns empty method lines when unknown method is attempted to be resolved"() { - setup: - def aTestMethod = NestedClass.getDeclaredMethod("abstractMethod") - def classMethodLines = new ByteCodeLinesResolver.ClassMethodLines() - - when: - def methodLines = classMethodLines.get(aTestMethod) - - then: - !methodLines.isValid() - } - - private static abstract class NestedClass { - static double aTestMethod() { - def random = Math.random() - return random - } - - abstract void abstractMethod() - } -} - diff --git a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.java b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.java new file mode 100644 index 00000000000..7fb99a60cd0 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/ByteCodeLinesResolverTest.java @@ -0,0 +1,115 @@ +package datadog.trace.civisibility.source; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.civisibility.source.LinesResolver.Lines; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Method; +import org.junit.jupiter.api.Test; + +class ByteCodeLinesResolverTest { + + @Test + void testMethodLinesResolution() throws NoSuchMethodException { + Method aTestMethod = NestedClass.class.getDeclaredMethod("aTestMethod"); + + ByteCodeLinesResolver linesResolver = new ByteCodeLinesResolver(); + Lines methodLines = linesResolver.getMethodLines(aTestMethod); + + assertTrue(methodLines.isValid()); + assertTrue(methodLines.getStartLineNumber() > 0); + assertTrue(methodLines.getEndLineNumber() > methodLines.getStartLineNumber()); + } + + @Test + void testAlwaysInvalidClassLinesResolution() { + ByteCodeLinesResolver linesResolver = new ByteCodeLinesResolver(); + Lines classLines = linesResolver.getClassLines(NestedClass.class); + + assertFalse(classLines.isValid()); + } + + @Test + void testInvalidMethodLinesResolution() throws NoSuchMethodException { + Method abstractMethod = NestedClass.class.getDeclaredMethod("abstractMethod"); + + ByteCodeLinesResolver linesResolver = new ByteCodeLinesResolver(); + Lines methodLines = linesResolver.getMethodLines(abstractMethod); + + assertFalse(methodLines.isValid()); + } + + @Test + void testReturnsEmptyMethodLinesWhenClassCannotBeLoaded() + throws IOException, ClassNotFoundException, NoSuchMethodException { + MisbehavingClassLoader misbehavingClassLoader = new MisbehavingClassLoader(); + + try (InputStream stream = Utils.getClassStream(NestedClass.class)) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = stream.read(buffer)) != -1) { + baos.write(buffer, 0, bytesRead); + } + misbehavingClassLoader.putClass(NestedClass.class.getName(), baos.toByteArray()); + } + + Class misbehavingClass = misbehavingClassLoader.loadClass(NestedClass.class.getName()); + Method misbehavingMethod = misbehavingClass.getDeclaredMethod("aTestMethod"); + + ByteCodeLinesResolver linesResolver = new ByteCodeLinesResolver(); + Lines methodLines = linesResolver.getMethodLines(misbehavingMethod); + + assertFalse(methodLines.isValid()); + } + + @Test + void testReturnsEmptyMethodLinesWhenClassResourceIsMissing() + throws IOException, ClassNotFoundException { + // regression test: Utils.getClassStream() returns null (rather than throwing) for + // classes whose bytecode resource cannot be located (e.g. certain generated/proxy classes). + // Calls ClassMethodLines.parse() directly rather than going through + // ByteCodeLinesResolver.getMethodLines() -- that outer method already catches any exception + // and returns Lines.EMPTY, so it would pass even without the null-stream guard under test. + NullResourceClassLoader nullResourceClassLoader = new NullResourceClassLoader(); + + try (InputStream stream = Utils.getClassStream(NestedClass.class)) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = stream.read(buffer)) != -1) { + baos.write(buffer, 0, bytesRead); + } + nullResourceClassLoader.putClass(NestedClass.class.getName(), baos.toByteArray()); + } + + Class unresolvableClass = nullResourceClassLoader.loadClass(NestedClass.class.getName()); + + assertDoesNotThrow(() -> ByteCodeLinesResolver.ClassMethodLines.parse(unresolvableClass)); + } + + @Test + void testReturnsEmptyMethodLinesWhenUnknownMethodIsAttemptedToBeResolved() + throws NoSuchMethodException { + Method abstractMethod = NestedClass.class.getDeclaredMethod("abstractMethod"); + ByteCodeLinesResolver.ClassMethodLines classMethodLines = + new ByteCodeLinesResolver.ClassMethodLines(); + + Lines methodLines = classMethodLines.get(abstractMethod); + + assertFalse(methodLines.isValid()); + } + + private abstract static class NestedClass { + static double aTestMethod() { + double random = Math.random(); + return random; + } + + abstract void abstractMethod(); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/NullResourceClassLoader.java b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/NullResourceClassLoader.java new file mode 100644 index 00000000000..8b7bc588a05 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/java/datadog/trace/civisibility/source/NullResourceClassLoader.java @@ -0,0 +1,36 @@ +package datadog.trace.civisibility.source; + +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * A {@link ClassLoader} that defines classes from an in-memory map and returns {@code null} from + * resource lookups (rather than throwing), used to exercise {@code ByteCodeLinesResolver} when a + * class's bytecode resource cannot be located. + * + *

Kept in Java rather than the Java 8 test suite's Groovy counterpart on purpose: see {@link + * MisbehavingClassLoader} for why. + */ +final class NullResourceClassLoader extends ClassLoader { + + private final Map classes = new HashMap<>(); + + @Override + public InputStream getResourceAsStream(String name) { + return null; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + byte[] bytes = classes.get(name); + if (bytes != null) { + return defineClass(name, bytes, 0, bytes.length); + } + return super.loadClass(name); + } + + void putClass(String name, byte[] bytes) { + classes.put(name, bytes); + } +} From 89f324c35bf4256bc2eeedf3d5cc9fefc70f234e Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Fri, 11 Sep 2026 11:29:50 +0200 Subject: [PATCH 17/30] Use ConcurrentHashtable for telemetry log (#12367) chore: improve some ConcurrentHashtable methods javadoc and arg names perf: reduce LogCollector allocations fix: count duplicate telemetry logs at capacity chore: align vocabulary in tests chore: improve benchmark robustness fix: retry telemetry log lookup at capacity Co-authored-by: brice.dutheil --- internal-api/build.gradle.kts | 1 + .../api/telemetry/LogCollectorBenchmark.java | 27 +- .../trace/api/telemetry/LogCollector.java | 200 +++++++-- .../trace/util/ConcurrentHashtable.java | 170 +++++--- .../api/telemetry/LogCollectorTest.groovy | 66 --- .../trace/api/telemetry/LogCollectorTest.java | 387 ++++++++++++++++++ 6 files changed, 694 insertions(+), 157 deletions(-) delete mode 100644 internal-api/src/test/groovy/datadog/trace/api/telemetry/LogCollectorTest.groovy create mode 100644 internal-api/src/test/java/datadog/trace/api/telemetry/LogCollectorTest.java diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 806f36d0299..966b916572b 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -282,6 +282,7 @@ dependencies { testImplementation("org.snakeyaml:snakeyaml-engine:2.9") testImplementation(project(":utils:test-utils")) testImplementation(libs.bundles.junit5) + testImplementation(libs.assertj.core) testImplementation("org.junit.vintage:junit-vintage-engine:${libs.versions.junit5.get()}") testImplementation(libs.commons.math) testImplementation(libs.bundles.mockito) diff --git a/internal-api/src/jmh/java/datadog/trace/api/telemetry/LogCollectorBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/telemetry/LogCollectorBenchmark.java index d7c836b8621..81d2a9180b6 100644 --- a/internal-api/src/jmh/java/datadog/trace/api/telemetry/LogCollectorBenchmark.java +++ b/internal-api/src/jmh/java/datadog/trace/api/telemetry/LogCollectorBenchmark.java @@ -2,7 +2,11 @@ import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; @@ -11,15 +15,26 @@ @Measurement(iterations = 5) @Threads(8) public class LogCollectorBenchmark { + @State(Scope.Benchmark) + public static class CollectorState { + LogCollector collector; + + @Setup(Level.Iteration) + public void setup() { + collector = new LogCollector(4); + collector.addLogMessage("error", "ugh!", null); + } + } + @Benchmark - public void noException_before() { - LogCollector.get().addLogMessage("error", "ugh!", null); + public void duplicateWithoutException(CollectorState state) { + state.collector.addLogMessage("error", "ugh!", null); } static final Object NULL = null; @Benchmark - public void nullPointerException() { + public void nullPointerException(CollectorState state) { // Represents the fast throw case where the JVM switches to using // a single Exception instance to handle a hot throw location // of NullPointerException, ArrayIndexOutOfBoundsException, etc. @@ -27,18 +42,18 @@ public void nullPointerException() { try { NULL.hashCode(); } catch (Throwable t) { - LogCollector.get().addLogMessage("error", "npe", t); + state.collector.addLogMessage("error", "npe", t); } } @Benchmark - public void unsupportedOperationException() { + public void unsupportedOperationException(CollectorState state) { // Represents the common case where stack trace is preserved // despite hot throw try { unsupportedOperation(); } catch (Throwable t) { - LogCollector.get().addLogMessage("error", "unsupported", t); + state.collector.addLogMessage("error", "unsupported", t); } } diff --git a/internal-api/src/main/java/datadog/trace/api/telemetry/LogCollector.java b/internal-api/src/main/java/datadog/trace/api/telemetry/LogCollector.java index b7ad3cb0eb0..a6d23b6e73e 100644 --- a/internal-api/src/main/java/datadog/trace/api/telemetry/LogCollector.java +++ b/internal-api/src/main/java/datadog/trace/api/telemetry/LogCollector.java @@ -1,16 +1,22 @@ package datadog.trace.api.telemetry; -import datadog.trace.util.HashingUtils; +import static datadog.trace.util.ConcurrentHashtable.bucketAt; +import static datadog.trace.util.ConcurrentHashtable.bucketIndex; +import static datadog.trace.util.ConcurrentHashtable.estimateSize; +import static datadog.trace.util.ConcurrentHashtable.getTableWriteLock; +import static datadog.trace.util.ConcurrentHashtable.insertReserved; +import static datadog.trace.util.ConcurrentHashtable.isFull; +import static datadog.trace.util.LongHashingUtils.hash; + +import datadog.trace.api.internal.VisibleForTesting; +import datadog.trace.util.ConcurrentHashtable; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import javax.annotation.Nullable; import org.slf4j.Marker; import org.slf4j.MarkerFactory; @@ -20,8 +26,7 @@ public class LogCollector { public static final Marker EXCLUDE_TELEMETRY = MarkerFactory.getMarker("EXCLUDE_TELEMETRY"); private static final int DEFAULT_MAX_CAPACITY = 10; private static final LogCollector INSTANCE = new LogCollector(); - private final Map rawLogMessages; - private final int maxCapacity; + @VisibleForTesting final ConcurrentHashtable.State rawLogMessages; public static LogCollector get() { return INSTANCE; @@ -35,8 +40,7 @@ private LogCollector() { value = "SING_SINGLETON_HAS_NONPRIVATE_CONSTRUCTOR", justification = "Usage in tests") LogCollector(int maxCapacity) { - this.maxCapacity = maxCapacity; - this.rawLogMessages = new ConcurrentHashMap<>(maxCapacity); + this.rawLogMessages = ConcurrentHashtable.State.createBounded(RawLogMessage.class, maxCapacity); } public void addLogMessage(String logLevel, String message, @Nullable Throwable throwable) { @@ -54,52 +58,159 @@ public void addLogMessage(String logLevel, String message, @Nullable Throwable t */ public void addLogMessage( String logLevel, String message, @Nullable Throwable throwable, @Nullable String tags) { - if (rawLogMessages.size() >= maxCapacity) { + long keyHash = RawLogMessage.computeHash(logLevel, message, throwable); + int bucketIndex = bucketIndex(rawLogMessages.buckets, keyHash); + // Fast path for duplicates: search the target bucket without locking. + RawLogMessage rawLogMessage = find(bucketIndex, keyHash, logLevel, message, throwable); + if (rawLogMessage != null) { + rawLogMessage.increment(); + return; + } + + // Fast path after a miss for a full table: reject without locking when the target bucket is + // populated. If the bucket is empty, drain() may have detached it before releasing capacity, + // so continue to the locked capacity check. + if (isFull(rawLogMessages) && bucketAt(rawLogMessages, bucketIndex) != null) { + // Mitigate a race where another writer could claim the bucket before the previous find + rawLogMessage = find(bucketIndex, keyHash, logLevel, message, throwable); + if (rawLogMessage != null) { + rawLogMessage.increment(); + return; + } // TODO: We could emit a metric for dropped logs. return; } - RawLogMessage rawLogMessage = - new RawLogMessage(logLevel, message, throwable, tags, System.currentTimeMillis() / 1000); - AtomicInteger count = rawLogMessages.computeIfAbsent(rawLogMessage, k -> new AtomicInteger()); - count.incrementAndGet(); + + // Slow path after a miss: repeat the lookup and capacity checks under the table write lock + // because another writer or drain() may have changed the table. + synchronized (getTableWriteLock(rawLogMessages)) { + rawLogMessage = find(bucketIndex, keyHash, logLevel, message, throwable); + if (rawLogMessage != null) { + rawLogMessage.increment(); + return; + } + // Capacity may have been released by drain() or consumed by another writer while waiting. + if (isFull(rawLogMessages)) { + return; + } + + // Allocate before reserving because a reservation cannot + // be rolled back if construction fails. + rawLogMessage = + new RawLogMessage(logLevel, message, throwable, tags, System.currentTimeMillis() / 1000); + // Reserve before linking so every published entry is included in the capacity count. + if (rawLogMessages.sizeManager.tryReserve()) { + insertReserved(rawLogMessages, keyHash, rawLogMessage); + } + } } + /** + * Removes all available log group from this collector and returns them. + * + *

The count of each returned group is captured during removal. Increments that complete + * after the count is captured are not included. + * + * @return a collection containing the removed log groups + */ public Collection drain() { - if (rawLogMessages.isEmpty()) { + int size = estimateSize(rawLogMessages); + if (size == 0) { return Collections.emptyList(); } - List list = new ArrayList<>(rawLogMessages.size()); - Iterator> iterator = - rawLogMessages.entrySet().iterator(); - - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - RawLogMessage logMessage = entry.getKey(); - // XXX: There might be lost writers to the counters under concurrency if another thread - // increments it - // while we are reading it here. At the moment, we are not overdoing this to prevent some - // counter losses. - logMessage.count = entry.getValue().get(); - iterator.remove(); - list.add(logMessage); - } - + // Note drain takes the table lock + List list = new ArrayList<>(size); + ConcurrentHashtable.drain( + rawLogMessages, + list, + (drained, logMessage) -> { + // Snapshot each log group's count before adding it to the drain result. + logMessage.snapshotCount(); + drained.add(logMessage); + }); return list; } - public static final class RawLogMessage { + /** + * Finds a log group with the same level, message, and + * throwable in the selected bucket. + * + *

Note, throwables are matched by identity or by class and stack trace. + * + *

The bucket chain supports lock-free reads. A caller that inserts after a miss must repeat + * the search under the table write lock. + * + * @param bucketIndex bucket selected for {@code keyHash} + * @param keyHash precomputed hash of the level, message, and throwable class + * @param logLevel log level to match + * @param message message to match + * @param throwable optional throwable to match + * @return the matching log group, or {@code null} if none is present + */ + @Nullable + private RawLogMessage find( + int bucketIndex, + long keyHash, + String logLevel, + String message, + @Nullable Throwable throwable) { + // Start searching from given bucket, and follow the entry next links + StackTraceElement[] stackTrace = null; + for (RawLogMessage entry = bucketAt(rawLogMessages, bucketIndex); + entry != null; + entry = entry.next()) { + if (entry.keyHash != keyHash + || !Objects.equals(logLevel, entry.logLevel) + || !Objects.equals(message, entry.message)) { + continue; + } + // throwables are more costly to compare, check first the identity + if (throwable == entry.throwable) { + return entry; + } + if (throwable != null + && entry.throwable != null + && throwable.getClass().equals(entry.throwable.getClass())) { + if (stackTrace == null) { + stackTrace = throwable.getStackTrace(); + } + if (Objects.deepEquals(stackTrace, entry.stackTrace())) { + return entry; + } + } + } + return null; + } + + /** + * Groups equivalent log messages for a telemetry flush. + * + *

Messages are equivalent when their log level, message, and throwable type and stack trace + * match. The first message supplies the tags and timestamp; later messages only increment the + * occurrence count. + */ + public static final class RawLogMessage extends ConcurrentHashtable.Entry { + private static final AtomicIntegerFieldUpdater LIVE_OCCURRENCE_COUNT_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(RawLogMessage.class, "liveOccurrenceCount"); + public final String message; public final String logLevel; public final Throwable throwable; public final String tags; public final long timestamp; + + /** Number of equivalent log messages captured when this group was drained. */ public int count; - private StackTraceElement[] cachedStackTrace = null; + /** Live counter equivalent log messages accumulated in this group. */ + private volatile int liveOccurrenceCount = 1; + + private volatile StackTraceElement[] cachedStackTrace = null; public RawLogMessage( String logLevel, String message, Throwable throwable, String tags, long timestamp) { + super(computeHash(logLevel, message, throwable)); this.logLevel = logLevel; this.message = message; this.throwable = throwable; @@ -110,11 +221,8 @@ public RawLogMessage( public StackTraceElement[] stackTrace() { if (throwable == null) return null; - // DQH - getStackTrace makes a defensive copy, so getStackTrace can become a significant - // source of allocation - // In the worst case of a hot exception, we'll constantly call hashCode & equals to - // check against the key stored in the map, so avoiding repeated allocation on each - // comparison does provide a measurable gain + // getStackTrace() makes a defensive copy. Cache one safely published copy for concurrent + // comparisons against equivalent throwables. StackTraceElement[] stackTrace = cachedStackTrace; if (stackTrace != null) return stackTrace; @@ -122,6 +230,15 @@ public StackTraceElement[] stackTrace() { return stackTrace; } + private void increment() { + LIVE_OCCURRENCE_COUNT_UPDATER.incrementAndGet(this); + } + + /** Snapshot this log's live occurrence count */ + private void snapshotCount() { + count = LIVE_OCCURRENCE_COUNT_UPDATER.get(this); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -149,7 +266,12 @@ public boolean equals(Object o) { @Override public int hashCode() { - return HashingUtils.hash(logLevel, message, throwable == null ? null : throwable.getClass()); + return (int) keyHash; + } + + private static long computeHash( + String logLevel, String message, @Nullable Throwable throwable) { + return hash(logLevel, message, throwable == null ? null : throwable.getClass()); } } } diff --git a/internal-api/src/main/java/datadog/trace/util/ConcurrentHashtable.java b/internal-api/src/main/java/datadog/trace/util/ConcurrentHashtable.java index 693ca2a1b16..cc0b53ee10e 100644 --- a/internal-api/src/main/java/datadog/trace/util/ConcurrentHashtable.java +++ b/internal-api/src/main/java/datadog/trace/util/ConcurrentHashtable.java @@ -310,23 +310,38 @@ public boolean removeIf(@Nonnull Predicate predicate) { } /** - * Removes all entries and passes each one to {@code sink} while holding the table write lock. - * The sink should be quick and must not throw. If it throws, the partial drain is not rolled - * back and the size is not adjusted. + * Removes all entries and invokes {@code drainedEntryConsumer} for each one. + * + *

The drain holds the table write lock while detaching buckets and invoking the consumer. + * For each removed entry, the consumer is invoked synchronously after its bucket is detached. + * Capacity is released only after all invocations return. The consumer should be quick and must + * not throw; failures are not rolled back. * *

Use {@link #drain(Object, BiConsumer)} to avoid a capturing lambda. + * + * @param drainedEntryConsumer action invoked for each removed entry */ - public void drain(@Nonnull Consumer sink) { - ConcurrentHashtable.drain(state, sink); + public void drain(@Nonnull Consumer drainedEntryConsumer) { + ConcurrentHashtable.drain(state, drainedEntryConsumer); } /** - * Context-passing {@link #drain(Consumer)}. Pass a non-capturing {@link BiConsumer} (typically - * a {@code static final}) plus the accumulator as {@code context} (e.g. the target list or - * event builder) to avoid a capturing-lambda allocation. + * Context-passing {@link #drain(Consumer)}. The drain holds the table write lock while + * detaching buckets and invoking {@code drainedEntryConsumer}. For each removed entry, the + * consumer is invoked synchronously after its bucket is detached. Capacity is released only + * after all invocations return. + * + *

Pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus the + * accumulator as {@code context} (e.g. the target list or event builder) to avoid a + * capturing-lambda allocation. + * + * @param context type + * @param context context passed to each invocation of {@code drainedEntryConsumer} + * @param drainedEntryConsumer action invoked with the context and each removed entry */ - public void drain(C context, @Nonnull BiConsumer sink) { - ConcurrentHashtable.drain(state, context, sink); + public void drain( + C context, @Nonnull BiConsumer drainedEntryConsumer) { + ConcurrentHashtable.drain(state, context, drainedEntryConsumer); } /** Removes all entries. Lock-free readers mid-walk complete against the entries they hold. */ @@ -582,23 +597,38 @@ public boolean removeIf(@Nonnull Predicate predicate) { } /** - * Removes all entries and passes each one to {@code sink} while holding the table write lock. - * The sink should be quick and must not throw. If it throws, the partial drain is not rolled - * back and the size is not adjusted. + * Removes all entries and invokes {@code drainedEntryConsumer} for each one. + * + *

The drain holds the table write lock while detaching buckets and invoking the consumer. + * For each removed entry, the consumer is invoked synchronously after its bucket is detached. + * Capacity is released only after all invocations return. The consumer should be quick and must + * not throw; failures are not rolled back. * *

Use {@link #drain(Object, BiConsumer)} to avoid a capturing lambda. + * + * @param drainedEntryConsumer action invoked for each removed entry */ - public void drain(@Nonnull Consumer sink) { - ConcurrentHashtable.drain(state, sink); + public void drain(@Nonnull Consumer drainedEntryConsumer) { + ConcurrentHashtable.drain(state, drainedEntryConsumer); } /** - * Context-passing {@link #drain(Consumer)}. Pass a non-capturing {@link BiConsumer} (typically - * a {@code static final}) plus the accumulator as {@code context} (e.g. the target list or - * event builder) to avoid a capturing-lambda allocation. + * Context-passing {@link #drain(Consumer)}. The drain holds the table write lock while + * detaching buckets and invoking {@code drainedEntryConsumer}. For each removed entry, the + * consumer is invoked synchronously after its bucket is detached. Capacity is released only + * after all invocations return. + * + *

Pass a non-capturing {@link BiConsumer} (typically a {@code static final}) plus the + * accumulator as {@code context} (e.g. the target list or event builder) to avoid a + * capturing-lambda allocation. + * + * @param context type + * @param context context passed to each invocation of {@code drainedEntryConsumer} + * @param drainedEntryConsumer action invoked with the context and each removed entry */ - public void drain(C context, @Nonnull BiConsumer sink) { - ConcurrentHashtable.drain(state, context, sink); + public void drain( + C context, @Nonnull BiConsumer drainedEntryConsumer) { + ConcurrentHashtable.drain(state, context, drainedEntryConsumer); } /** Removes all entries. Lock-free readers mid-walk complete against the entries they hold. */ @@ -1168,25 +1198,32 @@ public static boolean removeIf( } /** - * Removes all entries while holding the table write lock. Each bucket head is cleared with a - * volatile write before its detached chain is passed to {@code sink}, so subsequent lock-free - * readers observe an empty bucket while readers already on that chain can continue through its - * retained {@code next} links. This overload does not update size accounting. + * Removes all entries from {@code buckets} and invokes {@code drainedEntryConsumer} for each one. + * + *

The drain holds the table write lock while detaching buckets and invoking the consumer. For + * each removed entry, the consumer is invoked synchronously after its bucket is detached. A + * lock-free reader already traversing the detached chain can continue through its retained links. * - *

The sink must not throw. If it does, the partial drain is not rolled back. + *

This overload does not update size accounting. The consumer should be quick and must not + * throw; failures are not rolled back. + * + * @param entry type + * @param buckets bucket array to drain + * @param drainedEntryConsumer action invoked for each entry after its bucket is detached */ public static void drain( - @Nonnull AtomicReferenceArray buckets, @Nonnull Consumer sink) { - drainCounting(buckets, sink); + @Nonnull AtomicReferenceArray buckets, + @Nonnull Consumer drainedEntryConsumer) { + drainCounting(buckets, drainedEntryConsumer); } /** - * {@link #drain(AtomicReferenceArray, Consumer)} returning how many entries it handed to {@code - * sink}, so a {@link State} form can subtract exactly that from its {@link SizeManager} instead - * of zeroing. The count is free here: the sweep already visits every entry. + * {@link #drain(AtomicReferenceArray, Consumer)} returning the number of entries passed to {@code + * drainedEntryConsumer} for size accounting. */ private static int drainCounting( - @Nonnull AtomicReferenceArray buckets, @Nonnull Consumer sink) { + @Nonnull AtomicReferenceArray buckets, + @Nonnull Consumer drainedEntryConsumer) { int removed = 0; synchronized (getTableWriteLock(buckets)) { for (int i = 0; i < buckets.length(); i++) { @@ -1197,26 +1234,40 @@ private static int drainCounting( buckets.set(i, null); for (TEntry e = head; e != null; e = e.next()) { removed++; - sink.accept(e); + drainedEntryConsumer.accept(e); } } } return removed; } - /** Context-passing variant of {@link #drain(AtomicReferenceArray, Consumer)}. Self-locking. */ + /** + * Removes all entries from {@code buckets} and invokes {@code drainedEntryConsumer} with {@code + * context} and each removed entry. + * + *

The drain holds the table write lock while detaching buckets and invoking the consumer. For + * each removed entry, the consumer is invoked synchronously after its bucket is detached. This + * overload does not update size accounting. The consumer should be quick and must not throw; + * failures are not rolled back. + * + * @param context type + * @param entry type + * @param buckets bucket array to drain + * @param context context passed to each invocation of {@code drainedEntryConsumer} + * @param drainedEntryConsumer action invoked with the context and each removed entry + */ public static void drain( @Nonnull AtomicReferenceArray buckets, C context, - @Nonnull BiConsumer sink) { - drainCounting(buckets, context, sink); + @Nonnull BiConsumer drainedEntryConsumer) { + drainCounting(buckets, context, drainedEntryConsumer); } /** {@link #drainCounting(AtomicReferenceArray, Consumer)}, context-passing form. */ private static int drainCounting( @Nonnull AtomicReferenceArray buckets, C context, - @Nonnull BiConsumer sink) { + @Nonnull BiConsumer drainedEntryConsumer) { int removed = 0; synchronized (getTableWriteLock(buckets)) { for (int i = 0; i < buckets.length(); i++) { @@ -1227,7 +1278,7 @@ private static int drainCounting( buckets.set(i, null); for (TEntry e = head; e != null; e = e.next()) { removed++; - sink.accept(context, e); + drainedEntryConsumer.accept(context, e); } } } @@ -1235,25 +1286,52 @@ private static int drainCounting( } /** - * {@link #drain(AtomicReferenceArray, Consumer)} plus the matching bookkeeping: empties {@code - * state} into {@code sink} and gives its {@link SizeManager} back exactly the slots the sweep - * freed. Draining without that leaves the cap permanently consumed, so the two belong in one call - * rather than as a pair the caller has to remember. + * Removes all entries from {@code state}, invokes {@code drainedEntryConsumer} for each one, and + * releases one capacity slot for each removed entry. + * + *

The drain holds the table write lock while detaching buckets and invoking the consumer. For + * each removed entry, the consumer is invoked synchronously after its bucket is detached. + * Capacity for removed entries is released only after all invocations return. Outstanding + * reservations remain counted. + * + *

The consumer should be quick and must not throw; if it throws, removed entries are not + * restored and their capacity is not released. + * + * @param entry type + * @param state table state to drain + * @param drainedEntryConsumer action invoked for each entry after its bucket is detached */ public static void drain( - @Nonnull State state, @Nonnull Consumer sink) { + @Nonnull State state, @Nonnull Consumer drainedEntryConsumer) { synchronized (getTableWriteLock(state)) { - state.sizeManager.release(drainCounting(state.buckets, sink)); + state.sizeManager.release(drainCounting(state.buckets, drainedEntryConsumer)); } } - /** Context-passing form of {@link #drain(State, Consumer)}. */ + /** + * Removes all entries from {@code state}, invokes {@code drainedEntryConsumer} with {@code + * context} and each removed entry, and releases one capacity slot for each removed entry. + * + *

The drain holds the table write lock while detaching buckets and invoking the consumer. For + * each removed entry, the consumer is invoked synchronously after its bucket is detached. + * Capacity for removed entries is released only after all invocations return. Outstanding + * reservations remain counted. + * + *

The consumer should be quick and must not throw; if it throws, removed entries are not + * restored and their capacity is not released. + * + * @param context type + * @param entry type + * @param state table state to drain + * @param context context passed to each invocation of {@code drainedEntryConsumer} + * @param drainedEntryConsumer action invoked with the context and each removed entry + */ public static void drain( @Nonnull State state, C context, - @Nonnull BiConsumer sink) { + @Nonnull BiConsumer drainedEntryConsumer) { synchronized (getTableWriteLock(state)) { - state.sizeManager.release(drainCounting(state.buckets, context, sink)); + state.sizeManager.release(drainCounting(state.buckets, context, drainedEntryConsumer)); } } diff --git a/internal-api/src/test/groovy/datadog/trace/api/telemetry/LogCollectorTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/telemetry/LogCollectorTest.groovy deleted file mode 100644 index 4f798f5bf9f..00000000000 --- a/internal-api/src/test/groovy/datadog/trace/api/telemetry/LogCollectorTest.groovy +++ /dev/null @@ -1,66 +0,0 @@ -package datadog.trace.api.telemetry - -import datadog.trace.test.util.DDSpecification - -class LogCollectorTest extends DDSpecification { - - void "tracer time is set"() { - setup: - def logCollector = new LogCollector(1) - - when: - logCollector.addLogMessage("ERROR", "Message 1", null) - - then: - def log = logCollector.drain().toList().get(0) - def ts = log.timestamp - ts > 0L - // Check tracer time is not in millis - ts < 1706529524286L - } - - void "limit log messages in LogCollector"() { - setup: - def logCollector = new LogCollector(3) - - when: - logCollector.addLogMessage("ERROR", "Message 1", null) - logCollector.addLogMessage("ERROR", "Message 2", null) - logCollector.addLogMessage("ERROR", "Message 3", null) - logCollector.addLogMessage("ERROR", "Message 4", null) - - then: - logCollector.rawLogMessages.size() == 3 - } - - void "grouping messages in LogCollector"() { - when: - LogCollector.get().addLogMessage("ERROR", "First Message", null) - LogCollector.get().addLogMessage("ERROR", "Second Message", null) - LogCollector.get().addLogMessage("ERROR", "Third Message", null) - LogCollector.get().addLogMessage("ERROR", "Forth Message", null) - LogCollector.get().addLogMessage("ERROR", "Second Message", null) - LogCollector.get().addLogMessage("ERROR", "Third Message", null) - LogCollector.get().addLogMessage("ERROR", "Forth Message", null) - LogCollector.get().addLogMessage("ERROR", "Third Message", null) - LogCollector.get().addLogMessage("ERROR", "Forth Message", null) - LogCollector.get().addLogMessage("ERROR", "Forth Message", null) - - then: - def list = LogCollector.get().drain() - list.size() == 4 - listContains(list, 'ERROR', "First Message", null, 1) - listContains(list, 'ERROR', "Second Message", null, 2) - listContains(list, 'ERROR', "Third Message", null,3) - listContains(list, 'ERROR', "Forth Message", null, 4) - } - - boolean listContains(Collection list, String logLevel, String message, Throwable t, int count) { - for (final def logMsg in list) { - if (logMsg.logLevel == logLevel && logMsg.message == message && logMsg.throwable == t && logMsg.count == count) { - return true - } - } - return false - } -} diff --git a/internal-api/src/test/java/datadog/trace/api/telemetry/LogCollectorTest.java b/internal-api/src/test/java/datadog/trace/api/telemetry/LogCollectorTest.java new file mode 100644 index 00000000000..23b65cc523b --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/api/telemetry/LogCollectorTest.java @@ -0,0 +1,387 @@ +package datadog.trace.api.telemetry; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; + +import datadog.trace.test.util.PollingConditions; +import datadog.trace.util.ConcurrentHashtable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import org.junit.jupiter.api.Test; + +class LogCollectorTest { + private static final long TIMEOUT_SECONDS = 10; + + @Test + void recordsLogGroupTimestamp() { + // Given + LogCollector logCollector = new LogCollector(1); + long before = System.currentTimeMillis() / 1000; + + // When + logCollector.addLogMessage("ERROR", "Message 1", null); + + long after = System.currentTimeMillis() / 1000; + LogCollector.RawLogMessage logGroup = singleLogGroup(logCollector.drain()); + + // Then + assertThat(logGroup.timestamp).isBetween(before, after); + } + + @Test + void limitsLogGroupsToCapacity() { + // Given + LogCollector logCollector = new LogCollector(3); + + // When + logCollector.addLogMessage("ERROR", "Message 1", null); + logCollector.addLogMessage("ERROR", "Message 2", null); + logCollector.addLogMessage("ERROR", "Message 3", null); + logCollector.addLogMessage("ERROR", "Message 4", null); + + Collection logGroups = logCollector.drain(); + + // Then + assertThat(logGroups).hasSize(3); + assertContainsLogGroup(logGroups, "Message 1", 1); + assertContainsLogGroup(logGroups, "Message 2", 1); + assertContainsLogGroup(logGroups, "Message 3", 1); + assertThat(logGroups).extracting(logGroup -> logGroup.message).doesNotContain("Message 4"); + } + + @Test + void groupsEquivalentMessages() { + // Given + LogCollector logCollector = new LogCollector(10); + + // When + logCollector.addLogMessage("ERROR", "Foo Message", null); + logCollector.addLogMessage("ERROR", "Bar Message", null); + logCollector.addLogMessage("ERROR", "Baz Message", null); + logCollector.addLogMessage("ERROR", "Qux Message", null); + logCollector.addLogMessage("ERROR", "Bar Message", null); + logCollector.addLogMessage("ERROR", "Baz Message", null); + logCollector.addLogMessage("ERROR", "Qux Message", null); + logCollector.addLogMessage("ERROR", "Baz Message", null); + logCollector.addLogMessage("ERROR", "Qux Message", null); + logCollector.addLogMessage("ERROR", "Qux Message", null); + + Collection logGroups = logCollector.drain(); + + // Then + assertThat(logGroups).hasSize(4); + assertContainsLogGroup(logGroups, "Foo Message", 1); + assertContainsLogGroup(logGroups, "Bar Message", 2); + assertContainsLogGroup(logGroups, "Baz Message", 3); + assertContainsLogGroup(logGroups, "Qux Message", 4); + } + + @Test + void countsEquivalentMessagesWhenTableIsFull() { + // Given + LogCollector logCollector = new LogCollector(1); + logCollector.addLogMessage("ERROR", "Message", null); + + // When + logCollector.addLogMessage("ERROR", "Message", null); + + // Then + assertThat(singleLogGroup(logCollector.drain()).count).isEqualTo(2); + } + + @Test + void dropsNewLogGroupWhenTableIsFull() { + // Given + LogCollector logCollector = new LogCollector(1); + logCollector.addLogMessage("ERROR", "Existing message", null); + + // When + logCollector.addLogMessage("ERROR", "New message", null); + + // Then + assertThat(singleLogGroup(logCollector.drain()).message).isEqualTo("Existing message"); + } + + @Test + void reusesLogGroupCapacityAfterDrain() { + // Given + LogCollector logCollector = new LogCollector(1); + + // When + logCollector.addLogMessage("ERROR", "First", null); + + // Then + assertThat(singleLogGroup(logCollector.drain()).message).isEqualTo("First"); + + // When + logCollector.addLogMessage("ERROR", "Second", null); + + // Then + assertThat(singleLogGroup(logCollector.drain()).message).isEqualTo("Second"); + assertThat(logCollector.drain()).isEmpty(); + } + + @Test + void acceptsNewLogGroupAfterDrainDetachesFullBucket() throws Exception { + // Given + LogCollector logCollector = new LogCollector(1); + logCollector.addLogMessage("ERROR", "First", null); + ConcurrentHashtable.State state = logCollector.rawLogMessages; + CountDownLatch bucketDetached = new CountDownLatch(1); + CountDownLatch releaseDrain = new CountDownLatch(1); + FutureTask> drainTask = + new FutureTask<>( + () -> { + List drainedLogGroups = new ArrayList<>(); + ConcurrentHashtable.drain( + state, + drainedLogGroups, + (logGroups, logGroup) -> { + bucketDetached.countDown(); + await(releaseDrain); + logGroups.add(logGroup); + }); + return drainedLogGroups; + }); + Thread drainThread = new Thread(drainTask, "log-collector-drain"); + FutureTask writerTask = + new FutureTask<>( + () -> { + logCollector.addLogMessage("ERROR", "Second", null); + return null; + }); + Thread writerThread = new Thread(writerTask, "log-collector-writer"); + + // When + try { + drainThread.start(); + await(bucketDetached); + writerThread.start(); + new PollingConditions(TIMEOUT_SECONDS) + .eventually(() -> assertThat(writerThread.getState()).isEqualTo(Thread.State.BLOCKED)); + } finally { + releaseDrain.countDown(); + } + + // Then + Collection firstDrainedLogGroups = await(drainTask); + await(writerTask); + assertThat(singleLogGroup(firstDrainedLogGroups).message).isEqualTo("First"); + assertThat(singleLogGroup(logCollector.drain()).message).isEqualTo("Second"); + } + + @Test + void groupsMessagesWithEquivalentThrowablesAndKeepsFirstMetadata() { + // Given + LogCollector logCollector = new LogCollector(2); + Throwable firstThrowable = throwableWithMethod("run", 10); + Throwable equivalentThrowable = throwableWithMethod("run", 10); + + // When + logCollector.addLogMessage("ERROR", "Message", firstThrowable, "source:first"); + logCollector.addLogMessage("ERROR", "Message", equivalentThrowable, "source:second"); + + LogCollector.RawLogMessage logGroup = singleLogGroup(logCollector.drain()); + + // Then + assertThat(logGroup.count).isEqualTo(2); + assertThat(logGroup.throwable).isSameAs(firstThrowable); + assertThat(logGroup.tags).isEqualTo("source:first"); + } + + @Test + void keepsMessagesWithDifferentStackTracesInSeparateGroups() { + // Given + LogCollector logCollector = new LogCollector(2); + + // When + logCollector.addLogMessage("ERROR", "Message", throwableWithMethod("run", 10)); + logCollector.addLogMessage("ERROR", "Message", throwableWithMethod("run", 20)); + + // Then + assertThat(logCollector.drain()).hasSize(2); + } + + @Test + void rawLogMessageEqualityMatchesGrouping() { + // Given + LogCollector.RawLogMessage first = + new LogCollector.RawLogMessage( + "ERROR", "Message", throwableWithMethod("run", 10), "first", 1); + LogCollector.RawLogMessage equivalent = + new LogCollector.RawLogMessage( + "ERROR", "Message", throwableWithMethod("run", 10), "second", 2); + LogCollector.RawLogMessage different = + new LogCollector.RawLogMessage( + "ERROR", "Message", throwableWithMethod("runIt", 20), "first", 1); + + // Then + assertThat(first).isEqualTo(equivalent); + assertThat(first).hasSameHashCodeAs(equivalent); + assertThat(first).isNotEqualTo(different); + } + + @Test + void countsConcurrentEquivalentMessages() throws Exception { + // Given + int threadCount = 16; + int messagesPerThread = 1_000; + LogCollector logCollector = new LogCollector(2); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + CountDownLatch start = new CountDownLatch(1); + Future[] futures = new Future[threadCount]; + try { + for (int i = 0; i < threadCount; i++) { + futures[i] = + executor.submit( + () -> { + await(start); + for (int occurrence = 0; occurrence < messagesPerThread; occurrence++) { + logCollector.addLogMessage("ERROR", "Message", null); + } + return null; + }); + } + + // When + release(start); + await(futures); + } finally { + shutdown(executor); + } + + // Then + assertThat(singleLogGroup(logCollector.drain()).count) + .isEqualTo(threadCount * messagesPerThread); + } + + @Test + void groupsConcurrentMessagesWithEquivalentThrowables() throws Exception { + // Given + int threadCount = 16; + LogCollector logCollector = new LogCollector(2); + Throwable firstThrowable = throwableWithMethod("run", 10); + logCollector.addLogMessage("ERROR", "Message", firstThrowable); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + CountDownLatch start = new CountDownLatch(1); + Future[] futures = new Future[threadCount]; + try { + for (int i = 0; i < threadCount; i++) { + futures[i] = + executor.submit( + () -> { + await(start); + logCollector.addLogMessage("ERROR", "Message", throwableWithMethod("run", 10)); + return null; + }); + } + + // When + release(start); + await(futures); + } finally { + shutdown(executor); + } + + LogCollector.RawLogMessage logGroup = singleLogGroup(logCollector.drain()); + + // Then + assertThat(logGroup.count).isEqualTo(threadCount + 1); + assertThat(logGroup.throwable).isSameAs(firstThrowable); + } + + @Test + void limitsConcurrentLogGroupsToCapacity() throws Exception { + // Given + int capacity = 3; + int threadCount = 16; + LogCollector logCollector = new LogCollector(capacity); + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + CountDownLatch start = new CountDownLatch(1); + Future[] futures = new Future[threadCount]; + try { + for (int i = 0; i < threadCount; i++) { + String message = "Message " + i; + futures[i] = + executor.submit( + () -> { + await(start); + logCollector.addLogMessage("ERROR", message, null); + return null; + }); + } + + // When + release(start); + await(futures); + } finally { + shutdown(executor); + } + + // Then + assertThat(logCollector.drain()).hasSize(capacity); + } + + private static Throwable throwableWithMethod(String methodName, int lineNumber) { + Throwable throwable = new IllegalStateException("ignored when grouping"); + throwable.setStackTrace( + new StackTraceElement[] { + new StackTraceElement("Example", methodName, "Example.java", lineNumber) + }); + return throwable; + } + + private static void await(CountDownLatch latch) { + try { + assertThat(latch.await(TIMEOUT_SECONDS, SECONDS)).isTrue(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + + private static void release(CountDownLatch latch) { + latch.countDown(); + } + + private static T await(Future future) throws Exception { + return future.get(TIMEOUT_SECONDS, SECONDS); + } + + private static void await(Future[] futures) throws Exception { + for (Future future : futures) { + await(future); + } + } + + private static void shutdown(ExecutorService executor) throws InterruptedException { + executor.shutdownNow(); + assertThat(executor.awaitTermination(TIMEOUT_SECONDS, SECONDS)).isTrue(); + } + + private static LogCollector.RawLogMessage singleLogGroup( + Collection logGroups) { + assertThat(logGroups).hasSize(1); + return logGroups.iterator().next(); + } + + private static void assertContainsLogGroup( + Collection logGroups, String message, int count) { + assertThat(logGroups) + .as("log group for message %s", message) + .filteredOn(candidate -> message.equals(candidate.message)) + .singleElement() + .satisfies( + logGroup -> { + assertThat(logGroup.logLevel).isEqualTo("ERROR"); + assertThat(logGroup.count).isEqualTo(count); + assertThat(logGroup.throwable).isNull(); + }); + } +} From 169d83462df2a95cf2eca292b6dcb68a6e9e2f6f Mon Sep 17 00:00:00 2001 From: Clara Poncet Date: Fri, 11 Sep 2026 11:47:17 +0200 Subject: [PATCH 18/30] Report v1 SDK custom events as sdk_version:v1 in telemetry (#12438) Report v1 SDK custom events as sdk_version:v1 in telemetry Co-Authored-By: Claude Opus 5 Exclude AppSecEventTracker anonymous adapter from internal-api coverage Merge branch 'master' into clara.poncet/fix-v1-sdk-custom-event-telemetry-version Strengthen AppSec event tracker mock verification Co-authored-by: devflow.devflow-routing-intake --- .../AppSecEventTrackerSpecification.groovy | 522 --------------- ...ventTrackerAppSecDisabledForkedTest.groovy | 132 ---- .../appsec/user/AppSecEventTrackerTest.java | 596 ++++++++++++++++++ .../EventTrackerAppSecDisabledForkedTest.java | 137 ++++ internal-api/build.gradle.kts | 2 + .../trace/api/appsec/AppSecEventTracker.java | 80 ++- 6 files changed, 787 insertions(+), 682 deletions(-) delete mode 100644 dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/AppSecEventTrackerSpecification.groovy delete mode 100644 dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.groovy create mode 100644 dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/AppSecEventTrackerTest.java create mode 100644 dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.java diff --git a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/AppSecEventTrackerSpecification.groovy b/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/AppSecEventTrackerSpecification.groovy deleted file mode 100644 index 97106fad055..00000000000 --- a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/AppSecEventTrackerSpecification.groovy +++ /dev/null @@ -1,522 +0,0 @@ -package com.datadog.appsec.user - -import com.datadog.appsec.gateway.NoopFlow -import datadog.appsec.api.blocking.BlockingContentType -import datadog.appsec.api.blocking.BlockingException -import datadog.appsec.api.login.EventTrackerV2 -import datadog.appsec.api.user.User -import datadog.trace.api.EventTracker -import datadog.trace.api.GlobalTracer -import datadog.trace.api.ProductTraceSource -import datadog.trace.api.UserIdCollectionMode -import datadog.trace.api.appsec.AppSecEventTracker -import datadog.trace.api.function.TriFunction -import datadog.trace.api.gateway.CallbackProvider -import datadog.trace.api.gateway.Flow -import datadog.trace.api.gateway.RequestContext -import datadog.trace.api.gateway.RequestContextSlot -import datadog.trace.api.internal.TraceSegment -import datadog.trace.api.telemetry.LoginEvent -import datadog.trace.api.telemetry.LoginVersion -import datadog.trace.api.telemetry.WafMetricCollector -import datadog.trace.bootstrap.ActiveSubsystems -import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI -import datadog.trace.test.util.DDSpecification -import spock.lang.Shared - -import java.util.function.BiFunction - -import static datadog.trace.api.UserIdCollectionMode.ANONYMIZATION -import static datadog.trace.api.UserIdCollectionMode.DISABLED -import static datadog.trace.api.UserIdCollectionMode.IDENTIFICATION -import static datadog.trace.api.UserIdCollectionMode.SDK -import static datadog.trace.api.gateway.Events.EVENTS -import static datadog.trace.api.telemetry.LoginEvent.CUSTOM -import static datadog.trace.api.telemetry.LoginEvent.LOGIN_FAILURE -import static datadog.trace.api.telemetry.LoginEvent.LOGIN_SUCCESS -import static datadog.trace.api.telemetry.LoginEvent.SIGN_UP -import static datadog.appsec.api.user.User.setUser -import static datadog.trace.api.telemetry.LoginVersion.V1 -import static datadog.trace.api.telemetry.LoginVersion.V2 - -class AppSecEventTrackerSpecification extends DDSpecification { - - private static final String USER_LOGIN = 'user' - private static final String ANONYMIZED_USER_LOGIN = 'anon_04f8996da763b7a969b1028ee3007569' - private static final String USER_ID = '1' - private static final String ANONYMIZED_USER_ID = 'anon_6b86b273ff34fce19d6b804eff5a3f57' - - @Shared - private static boolean appSecActiveBefore = ActiveSubsystems.APPSEC_ACTIVE - @Shared - private static EventTracker eventTrackerBefore = GlobalTracer.getEventTracker() - - private AppSecEventTracker tracker - private TraceSegment traceSegment - private TracerAPI tracer - private AgentSpan span - private CallbackProvider provider - private BiFunction> user - private TriFunction> loginEvent - - void setup() { - traceSegment = Mock(TraceSegment) - span = Stub(AgentSpan) - user = Mock(BiFunction) - loginEvent = Mock(TriFunction) - - provider = Stub(CallbackProvider) { - getCallback(EVENTS.user()) >> user - getCallback(EVENTS.loginEvent()) >> loginEvent - } - tracer = Stub(TracerAPI) { - getTraceSegment() >> traceSegment - activeSpan() >> span - getCallbackProvider(RequestContextSlot.APPSEC) >> provider - } - tracker = new AppSecEventTracker() { - @Override - protected TracerAPI tracer() { - return tracer - } - } - GlobalTracer.setEventTracker(tracker) - User.setUserService(tracker) - EventTrackerV2.setEventTrackerService(tracker) - ActiveSubsystems.APPSEC_ACTIVE = true - } - - void cleanupSpec() { - ActiveSubsystems.APPSEC_ACTIVE = appSecActiveBefore - GlobalTracer.setEventTracker(eventTrackerBefore) - } - - def 'test track login success event (SDK)'() { - when: - GlobalTracer.getEventTracker().trackLoginSuccessEvent(USER_ID, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('usr.id', USER_ID) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.usr.login', USER_ID, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.track', true, true) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.success.sdk', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, LOGIN_SUCCESS, USER_ID) >> NoopFlow.INSTANCE - 1 * user.apply(_ as RequestContext, USER_ID) >> NoopFlow.INSTANCE - 0 * _ - - assertAppSecSdkEvent(LOGIN_SUCCESS, V1) - } - - def 'test track login failure event (SDK)'() { - when: - GlobalTracer.getEventTracker().trackLoginFailureEvent(USER_ID, true, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.id', USER_ID, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.login', USER_ID, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.exists', true, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.track', true, true) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.sdk', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, LOGIN_FAILURE, USER_ID) >> NoopFlow.INSTANCE - 1 * user.apply(_ as RequestContext, USER_ID) >> NoopFlow.INSTANCE - 0 * _ - - assertAppSecSdkEvent(LOGIN_FAILURE, V1) - } - - def 'test track custom event (SDK)'() { - when: - GlobalTracer.getEventTracker().trackCustomEvent('myevent', ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('appsec.events.myevent', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.myevent.track', true, true) - 1 * traceSegment.setTagTop('_dd.appsec.events.myevent.sdk', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 0 * _ - - assertAppSecSdkEvent(CUSTOM, V2) - } - - def 'test track login success event V2 (SDK)'() { - when: - EventTrackerV2.trackUserLoginSuccess(USER_LOGIN, USER_ID, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('usr.id', USER_ID) - 1 * traceSegment.setTagTop('usr', ['key1': 'value1', 'key2': 'value2']) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.usr.id', USER_ID, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.usr', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('_dd.appsec.user.collection_mode', 'sdk') - 1 * traceSegment.setTagTop('appsec.events.users.login.success.usr.login', USER_LOGIN, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.track', true, true) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.success.sdk', true, true) - 2 * traceSegment.setTagTop('asm.keep', true) - 2 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, LOGIN_SUCCESS, USER_LOGIN) >> NoopFlow.INSTANCE - 1 * user.apply(_ as RequestContext, USER_ID) >> NoopFlow.INSTANCE - 0 * _ - - assertAppSecSdkEvent(LOGIN_SUCCESS, V2) - } - - def 'test track login failure event V2 (SDK)'() { - when: - EventTrackerV2.trackUserLoginFailure(USER_LOGIN, true, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.login', USER_LOGIN, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.exists', true, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.track', true, true) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.sdk', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, LOGIN_FAILURE, USER_LOGIN) >> NoopFlow.INSTANCE - 0 * _ - - assertAppSecSdkEvent(LOGIN_FAILURE, V2) - } - - def 'test track custom event V2 (SDK)'() { - when: - EventTrackerV2.trackCustomEvent('myevent', ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('appsec.events.myevent', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.myevent.track', true, true) - 1 * traceSegment.setTagTop('_dd.appsec.events.myevent.sdk', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 0 * _ - } - - def 'test track user (SDK)'() { - when: - setUser(USER_ID, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('usr.id', USER_ID) - 1 * traceSegment.setTagTop('usr', ['key1': 'value1', 'key2': 'value2']) - 1 * traceSegment.setTagTop('_dd.appsec.user.collection_mode', SDK.fullName()) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * user.apply(_ as RequestContext, USER_ID) >> NoopFlow.INSTANCE - 0 * _ - } - - def 'test wrong event argument validation (SDK)'() { - when: - GlobalTracer.getEventTracker().trackLoginSuccessEvent(null, null) - - then: - thrown IllegalArgumentException - - when: - GlobalTracer.getEventTracker().trackLoginFailureEvent(null, false, null) - - then: - thrown IllegalArgumentException - - when: - GlobalTracer.getEventTracker().trackCustomEvent(null, null) - - then: - thrown IllegalArgumentException - - when: - GlobalTracer.getEventTracker().trackLoginSuccessEvent('', null) - - then: - thrown IllegalArgumentException - - when: - GlobalTracer.getEventTracker().trackLoginFailureEvent('', false, null) - - then: - thrown IllegalArgumentException - - when: - GlobalTracer.getEventTracker().trackCustomEvent('', null) - - then: - thrown IllegalArgumentException - - when: - setUser(null, null) - - then: - thrown IllegalArgumentException - } - - def "test onSignup (#mode)"() { - setup: - final expectedUserLogin = mode == ANONYMIZATION ? ANONYMIZED_USER_LOGIN : USER_LOGIN - - when: - tracker.onSignupEvent(mode, USER_LOGIN, ['key1': 'value1', 'key2': 'value2']) - - then: - if (mode != DISABLED) { - 1 * traceSegment.getTagTop('_dd.appsec.events.users.signup.sdk') >> null // no SDK event before - 1 * traceSegment.setTagTop('_dd.appsec.usr.login', expectedUserLogin) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.signup.auto.mode', mode.fullName(), true) - 1 * traceSegment.setTagTop('appsec.events.users.signup.usr.login', expectedUserLogin, true) - 1 * traceSegment.setTagTop('appsec.events.users.signup', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.signup.track', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, SIGN_UP, expectedUserLogin) >> NoopFlow.INSTANCE - } - 0 * _ - - where: - mode << [IDENTIFICATION, ANONYMIZATION, DISABLED] - } - - def "test onLoginSuccess (#mode)"() { - setup: - final expectedUserLogin = mode == ANONYMIZATION ? ANONYMIZED_USER_LOGIN : USER_LOGIN - - when: - tracker.onLoginSuccessEvent(mode, USER_LOGIN, ['key1': 'value1', 'key2': 'value2']) - - then: - if (mode != DISABLED) { - 1 * traceSegment.getTagTop('_dd.appsec.events.users.login.success.sdk') >> null // no SDK event before - 1 * traceSegment.setTagTop('_dd.appsec.usr.login', expectedUserLogin) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.success.auto.mode', mode.fullName(), true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.usr.login', expectedUserLogin, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.login.success.track', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, LOGIN_SUCCESS, expectedUserLogin) >> NoopFlow.INSTANCE - } - 0 * _ - - where: - mode << [IDENTIFICATION, ANONYMIZATION, DISABLED] - } - - def "test onLoginFailed (#mode)"() { - setup: - final expectedUserLogin = mode == ANONYMIZATION ? ANONYMIZED_USER_LOGIN : USER_LOGIN - - when: - tracker.onLoginFailureEvent(mode, USER_LOGIN, true, ['key1': 'value1', 'key2': 'value2']) - - then: - if (mode != DISABLED) { - 1 * traceSegment.getTagTop('_dd.appsec.events.users.login.failure.sdk') >> null // no SDK event before - 1 * traceSegment.setTagTop('_dd.appsec.usr.login', expectedUserLogin) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.auto.mode', mode.fullName(), true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.login', expectedUserLogin, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure', ['key1': 'value1', 'key2': 'value2'], true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.exists', true, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.track', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * loginEvent.apply(_ as RequestContext, LOGIN_FAILURE, expectedUserLogin) >> NoopFlow.INSTANCE - } - 0 * _ - - where: - mode << [IDENTIFICATION, ANONYMIZATION, DISABLED] - } - - def "test onUserEvent (#mode)"() { - setup: - final expectedUserId = mode == ANONYMIZATION ? ANONYMIZED_USER_ID : USER_ID - - when: - tracker.onUserEvent(mode, USER_ID, [:]) - - then: - if (mode != DISABLED) { - 1 * traceSegment.setTagTop('_dd.appsec.usr.id', expectedUserId) - 1 * traceSegment.getTagTop('_dd.appsec.user.collection_mode') >> null // no user event before - 1 * traceSegment.setTagTop('_dd.appsec.user.collection_mode', mode.fullName()) - 1 * traceSegment.setTagTop('usr.id', expectedUserId) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - 1 * user.apply(_ as RequestContext, expectedUserId) >> NoopFlow.INSTANCE - } - 0 * _ - - where: - mode << [IDENTIFICATION, ANONYMIZATION, DISABLED] - } - - def "test onUserNotFound (#mode)"() { - when: - tracker.onUserNotFound(mode) - - then: - if (mode != DISABLED) { - 1 * traceSegment.getTagTop('_dd.appsec.events.users.login.failure.sdk') >> null // no SDK event before - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.auto.mode', mode.fullName(), true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.usr.exists', false, true) - 1 * traceSegment.setTagTop('appsec.events.users.login.failure.track', true, true) - 1 * traceSegment.setTagTop('asm.keep', true) - 1 * traceSegment.setTagTop('_dd.p.ts', ProductTraceSource.ASM) - } - 0 * _ - - where: - mode << [IDENTIFICATION, ANONYMIZATION, DISABLED] - } - - def "test isEnabled (appsec = #appsec, tracking = #trackingMode, collection = #collectionMode)"() { - setup: - ActiveSubsystems.APPSEC_ACTIVE = appsec - final mode = UserIdCollectionMode.fromString(collectionMode, trackingMode) - - when: - def enabled = tracker.isEnabled(mode) - - then: - enabled == result - - where: - appsec | collectionMode | trackingMode | result - // disabled states - false | null | null | false - false | null | 'safe' | false - false | null | 'extended' | false - false | null | 'disabled' | false - false | 'ident' | null | false - false | 'ident' | 'safe' | false - false | 'ident' | 'extended' | false - false | 'ident' | 'disabled' | false - false | 'anon' | null | false - false | 'anon' | 'safe' | false - false | 'anon' | 'extended' | false - false | 'anon' | 'disabled' | false - false | 'disabled' | null | false - false | 'disabled' | 'safe' | false - false | 'disabled' | 'extended' | false - false | 'disabled' | 'disabled' | false - true | null | 'disabled' | false - true | 'disabled' | null | false - true | 'disabled' | 'safe' | false - true | 'disabled' | 'extended' | false - true | 'disabled' | 'disabled' | false - - // enabled states - true | null | null | true - true | null | 'safe' | true - true | null | 'extended' | true - true | 'ident' | null | true - true | 'ident' | 'safe' | true - true | 'ident' | 'extended' | true - true | 'ident' | 'disabled' | true - true | 'anon' | null | true - true | 'anon' | 'safe' | true - true | 'anon' | 'extended' | true - true | 'anon' | 'disabled' | true - } - - void 'test blocking on a login'() { - setup: - final action = new Flow.Action.RequestBlockingAction(403, BlockingContentType.AUTO) - loginEvent.apply(_ as RequestContext, LOGIN_SUCCESS, USER_LOGIN) >> new ActionFlow(action: action) - - when: - tracker.onLoginSuccessEvent(SDK, USER_LOGIN, USER_ID, ['key1': 'value1', 'key2': 'value2']) - - then: - thrown(BlockingException) - } - - void 'should not fail on null callback'() { - when: - tracker.onUserEvent(IDENTIFICATION, 'test-user', [:]) - - then: - noExceptionThrown() - provider.getCallback(EVENTS.user()) >> null - } - - void 'test onUserEvent (automated login events should not overwrite SDK)'() { - when: - tracker.onUserEvent(IDENTIFICATION, USER_ID, [:]) - - then: 'SDK data remains untouched' - 1 * traceSegment.getTagTop('_dd.appsec.user.collection_mode') >> SDK.fullName() - 1 * traceSegment.setTagTop('_dd.appsec.usr.id', USER_ID) - 0 * _ - } - - - void 'test onLoginSuccess (automated login events should not overwrite SDK)'() { - when: - tracker.onLoginSuccessEvent(IDENTIFICATION, USER_LOGIN, null, [:]) - - then: - 1 * traceSegment.getTagTop('_dd.appsec.events.users.login.success.sdk') >> true - 1 * traceSegment.setTagTop('_dd.appsec.usr.login', USER_LOGIN) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.success.auto.mode', IDENTIFICATION.fullName(), true) - 0 * _ - } - - void 'test onLoginFailure (automated login events should not overwrite SDK)'() { - when: - tracker.onLoginFailureEvent(IDENTIFICATION, USER_LOGIN, null, [:]) - - then: - 1 * traceSegment.getTagTop('_dd.appsec.events.users.login.failure.sdk') >> true - 1 * traceSegment.setTagTop('_dd.appsec.usr.login', USER_LOGIN) - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.auto.mode', IDENTIFICATION.fullName(), true) - 0 * _ - } - - void 'test onUserNotFound (automated login events should not overwrite SDK)'() { - when: - tracker.onUserNotFound(IDENTIFICATION) - - then: - 1 * traceSegment.getTagTop('_dd.appsec.events.users.login.failure.sdk') >> true - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.auto.mode', IDENTIFICATION.fullName(), true) - 0 * _ - } - - private static void assertAppSecSdkEvent(final LoginEvent event, final LoginVersion version) { - final metrics = WafMetricCollector.get().with { - prepareMetrics() - drain() - } - final expectedTags = ["event_type:${event.getTag()}".toString(), "sdk_version:${version.getTag()}".toString()] - final metric = metrics.find { it.metricName == 'sdk.event'} - assert metric != null - assert metric.namespace == 'appsec' - assert metric.type == 'count' - assert metric.value == 1 - assert metric.tags.size() == 2 - assert metric.tags.containsAll(expectedTags) - } - - private static class ActionFlow implements Flow { - - private Action action - - @Override - Action getAction() { - return action - } - - @Override - Object getResult() { - return null - } - } -} diff --git a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.groovy b/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.groovy deleted file mode 100644 index 6ea611d93e0..00000000000 --- a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.groovy +++ /dev/null @@ -1,132 +0,0 @@ -package com.datadog.appsec.user - -import datadog.appsec.api.login.EventTrackerV2 -import datadog.appsec.api.user.User -import datadog.trace.api.GlobalTracer -import datadog.trace.api.UserIdCollectionMode -import datadog.trace.api.appsec.AppSecEventTracker -import datadog.trace.api.gateway.RequestContextSlot -import datadog.trace.api.internal.TraceSegment -import datadog.trace.bootstrap.ActiveSubsystems -import datadog.trace.bootstrap.instrumentation.api.AgentTracer -import datadog.trace.test.util.DDSpecification - -import static datadog.trace.api.UserIdCollectionMode.IDENTIFICATION - -class EventTrackerAppSecDisabledForkedTest extends DDSpecification { - - TraceSegment traceSegment - - AppSecEventTracker tracker - - void setupSpec() { - injectSysConfig('dd.appsec.enabled', 'false') - ActiveSubsystems.APPSEC_ACTIVE = false - } - - void setup() { - tracker = new AppSecEventTracker() - GlobalTracer.setEventTracker(tracker) - EventTrackerV2.setEventTrackerService(tracker) - User.setUserService(tracker) - traceSegment = Mock(TraceSegment) - final tracer = Stub(AgentTracer.TracerAPI) { - getTraceSegment() >> traceSegment - getCallbackProvider(RequestContextSlot.APPSEC) >> null - } - AgentTracer.forceRegister(tracer) - } - - void 'test track login success event (SDK)'() { - when: - GlobalTracer.getEventTracker().trackLoginSuccessEvent('user', ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.success.sdk', true, true) - } - - void 'test track login failure event (SDK)'() { - when: - GlobalTracer.getEventTracker().trackLoginFailureEvent('user', true, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.sdk', true, true) - } - - void 'test track custom event (SDK)'() { - when: - GlobalTracer.getEventTracker().trackCustomEvent('myevent', ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('_dd.appsec.events.myevent.sdk', true, true) - } - - void 'test track login success event V2 (SDK)'() { - when: - EventTrackerV2.trackUserLoginSuccess('user', 'id', ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.success.sdk', true, true) - } - - void 'test track login failure event V2 (SDK)'() { - when: - EventTrackerV2.trackUserLoginFailure('user', true, ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('_dd.appsec.events.users.login.failure.sdk', true, true) - } - - void 'test track custom event V2 (SDK)'() { - when: - EventTrackerV2.trackCustomEvent('myevent', ['key1': 'value1', 'key2': 'value2']) - - then: - 1 * traceSegment.setTagTop('_dd.appsec.events.myevent.sdk', true, true) - } - - void 'test onSignup'() { - when: - tracker.onSignupEvent(IDENTIFICATION, 'user', ['key1': 'value1', 'key2': 'value2']) - - then: - 0 * _ - } - - void 'test onLoginSuccess'() { - - when: - tracker.onLoginSuccessEvent(IDENTIFICATION, 'user', ['key1': 'value1', 'key2': 'value2']) - - then: - 0 * _ - } - - void 'test onLoginFailed'() { - when: - tracker.onLoginFailureEvent(IDENTIFICATION, 'user', true, ['key1': 'value1', 'key2': 'value2']) - - then: - 0 * _ - - where: - mode << UserIdCollectionMode.values() - } - - def 'test onUserEvent'() { - when: - tracker.onUserEvent(IDENTIFICATION, 'user') - - then: - 0 * _ - } - - def 'test onUserNotFound'() { - when: - tracker.onUserNotFound(IDENTIFICATION) - - then: - 0 * _ - } -} - diff --git a/dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/AppSecEventTrackerTest.java b/dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/AppSecEventTrackerTest.java new file mode 100644 index 00000000000..6f827e5f0b2 --- /dev/null +++ b/dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/AppSecEventTrackerTest.java @@ -0,0 +1,596 @@ +package com.datadog.appsec.user; + +import static datadog.appsec.api.user.User.setUser; +import static datadog.trace.api.ProductTraceSource.ASM; +import static datadog.trace.api.UserIdCollectionMode.ANONYMIZATION; +import static datadog.trace.api.UserIdCollectionMode.DISABLED; +import static datadog.trace.api.UserIdCollectionMode.IDENTIFICATION; +import static datadog.trace.api.UserIdCollectionMode.SDK; +import static datadog.trace.api.gateway.Events.EVENTS; +import static datadog.trace.api.telemetry.LoginEvent.CUSTOM; +import static datadog.trace.api.telemetry.LoginEvent.LOGIN_FAILURE; +import static datadog.trace.api.telemetry.LoginEvent.LOGIN_SUCCESS; +import static datadog.trace.api.telemetry.LoginEvent.SIGN_UP; +import static datadog.trace.api.telemetry.LoginVersion.V1; +import static datadog.trace.api.telemetry.LoginVersion.V2; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.ignoreStubs; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.datadog.appsec.gateway.NoopFlow; +import datadog.appsec.api.blocking.BlockingContentType; +import datadog.appsec.api.blocking.BlockingException; +import datadog.appsec.api.login.EventTrackerService; +import datadog.appsec.api.login.EventTrackerV2; +import datadog.appsec.api.user.User; +import datadog.appsec.api.user.UserService; +import datadog.trace.api.EventTracker; +import datadog.trace.api.GlobalTracer; +import datadog.trace.api.UserIdCollectionMode; +import datadog.trace.api.appsec.AppSecEventTracker; +import datadog.trace.api.function.TriFunction; +import datadog.trace.api.gateway.CallbackProvider; +import datadog.trace.api.gateway.Flow; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.internal.TraceSegment; +import datadog.trace.api.telemetry.LoginEvent; +import datadog.trace.api.telemetry.LoginVersion; +import datadog.trace.api.telemetry.WafMetricCollector; +import datadog.trace.api.telemetry.WafMetricCollector.WafMetric; +import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI; +import datadog.trace.test.util.DDJavaSpecification; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +@SuppressWarnings("deprecation") // exercises the deprecated v1 EventTracker API on purpose +class AppSecEventTrackerTest extends DDJavaSpecification { + + private static final String USER_LOGIN = "user"; + private static final String ANONYMIZED_USER_LOGIN = "anon_04f8996da763b7a969b1028ee3007569"; + private static final String USER_ID = "1"; + private static final String ANONYMIZED_USER_ID = "anon_6b86b273ff34fce19d6b804eff5a3f57"; + private static final Map METADATA = metadata(); + + private static boolean appSecActiveBefore; + private static EventTracker eventTrackerBefore; + + private TestAppSecEventTracker tracker; + private TraceSegment traceSegment; + private RequestContext requestContext; + private AgentSpan span; + private CallbackProvider provider; + private TracerAPI tracer; + private BiFunction> user; + private TriFunction> loginEvent; + + @BeforeAll + static void saveGlobalState() { + appSecActiveBefore = ActiveSubsystems.APPSEC_ACTIVE; + eventTrackerBefore = GlobalTracer.getEventTracker(); + } + + /** + * {@link User} and {@link EventTrackerV2} hold their implementation in a static field with no + * getter, so the pre-test value cannot be captured; resetting them to their no-op defaults at + * least keeps this test's tracker (and its mocks) from leaking into later tests. + */ + @AfterAll + static void restoreGlobalState() { + ActiveSubsystems.APPSEC_ACTIVE = appSecActiveBefore; + GlobalTracer.setEventTracker(eventTrackerBefore); + User.setUserService(UserService.NO_OP); + EventTrackerV2.setEventTrackerService(EventTrackerService.NO_OP); + } + + @SuppressWarnings("unchecked") + @BeforeEach + void setup() { + traceSegment = mock(TraceSegment.class); + requestContext = mock(RequestContext.class); + span = mock(AgentSpan.class); + when(span.getRequestContext()).thenReturn(requestContext); + user = mock(BiFunction.class); + loginEvent = mock(TriFunction.class); + when(user.apply(any(), any())).thenReturn(NoopFlow.INSTANCE); + when(loginEvent.apply(any(), any(), any())).thenReturn(NoopFlow.INSTANCE); + + provider = mock(CallbackProvider.class); + when(provider.getCallback(EVENTS.user())).thenReturn(user); + when(provider.getCallback(EVENTS.loginEvent())).thenReturn(loginEvent); + + tracer = mock(TracerAPI.class); + when(tracer.getTraceSegment()).thenReturn(traceSegment); + when(tracer.activeSpan()).thenReturn(span); + when(tracer.getCallbackProvider(RequestContextSlot.APPSEC)).thenReturn(provider); + + tracker = new TestAppSecEventTracker(tracer); + GlobalTracer.setEventTracker(tracker); + User.setUserService(tracker); + EventTrackerV2.setEventTrackerService(tracker.v2EventTrackerService()); + ActiveSubsystems.APPSEC_ACTIVE = true; + + // discard telemetry produced elsewhere so each assertion only sees its own event + drainSdkEvents(); + } + + @Test + void trackLoginSuccessEvent() { + GlobalTracer.getEventTracker().trackLoginSuccessEvent(USER_ID, METADATA); + + verify(traceSegment).setTagTop("usr.id", USER_ID); + verify(traceSegment).setTagTop("appsec.events.users.login.success.usr.login", USER_ID, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success.track", true, true); + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.success.sdk", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(LOGIN_SUCCESS), eq(USER_ID)); + verify(user).apply(isA(RequestContext.class), eq(USER_ID)); + verifyNoMoreMockInteractions(); + + assertAppSecSdkEvent(LOGIN_SUCCESS, V1); + } + + @Test + void trackLoginFailureEvent() { + GlobalTracer.getEventTracker().trackLoginFailureEvent(USER_ID, true, METADATA); + + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.id", USER_ID, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.login", USER_ID, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.exists", true, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.track", true, true); + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.failure.sdk", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(LOGIN_FAILURE), eq(USER_ID)); + verify(user).apply(isA(RequestContext.class), eq(USER_ID)); + verifyNoMoreMockInteractions(); + + assertAppSecSdkEvent(LOGIN_FAILURE, V1); + } + + @Test + void trackCustomEvent() { + GlobalTracer.getEventTracker().trackCustomEvent("myevent", METADATA); + + verify(traceSegment).setTagTop("appsec.events.myevent", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.myevent.track", true, true); + verify(traceSegment).setTagTop("_dd.appsec.events.myevent.sdk", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verifyNoMoreMockInteractions(); + + assertAppSecSdkEvent(CUSTOM, V1); + } + + @Test + void trackLoginSuccessEventV2() { + EventTrackerV2.trackUserLoginSuccess(USER_LOGIN, USER_ID, METADATA); + + verify(traceSegment).setTagTop("usr.id", USER_ID); + verify(traceSegment).setTagTop("usr", METADATA); + verify(traceSegment).setTagTop("appsec.events.users.login.success.usr.id", USER_ID, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success.usr", METADATA, true); + verify(traceSegment).setTagTop("_dd.appsec.user.collection_mode", "sdk"); + verify(traceSegment).setTagTop("appsec.events.users.login.success.usr.login", USER_LOGIN, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success.track", true, true); + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.success.sdk", true, true); + verify(traceSegment, times(2)).setTagTop("asm.keep", true); + verify(traceSegment, times(2)).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(LOGIN_SUCCESS), eq(USER_LOGIN)); + verify(user).apply(isA(RequestContext.class), eq(USER_ID)); + verifyNoMoreMockInteractions(); + + assertAppSecSdkEvent(LOGIN_SUCCESS, V2); + } + + @Test + void trackLoginFailureEventV2() { + EventTrackerV2.trackUserLoginFailure(USER_LOGIN, true, METADATA); + + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.login", USER_LOGIN, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.exists", true, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.track", true, true); + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.failure.sdk", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(LOGIN_FAILURE), eq(USER_LOGIN)); + verifyNoMoreMockInteractions(); + + assertAppSecSdkEvent(LOGIN_FAILURE, V2); + } + + @Test + void trackCustomEventV2() { + EventTrackerV2.trackCustomEvent("myevent", METADATA); + + verify(traceSegment).setTagTop("appsec.events.myevent", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.myevent.track", true, true); + verify(traceSegment).setTagTop("_dd.appsec.events.myevent.sdk", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verifyNoMoreMockInteractions(); + + assertAppSecSdkEvent(CUSTOM, V2); + } + + @Test + void trackUser() { + setUser(USER_ID, METADATA); + + verify(traceSegment).setTagTop("usr.id", USER_ID); + verify(traceSegment).setTagTop("usr", METADATA); + verify(traceSegment).setTagTop("_dd.appsec.user.collection_mode", SDK.fullName()); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(user).apply(isA(RequestContext.class), eq(USER_ID)); + verifyNoMoreMockInteractions(); + } + + @Test + void wrongEventArgumentValidation() { + EventTracker eventTracker = GlobalTracer.getEventTracker(); + + assertThrows( + IllegalArgumentException.class, () -> eventTracker.trackLoginSuccessEvent(null, null)); + assertThrows( + IllegalArgumentException.class, + () -> eventTracker.trackLoginFailureEvent(null, false, null)); + assertThrows(IllegalArgumentException.class, () -> eventTracker.trackCustomEvent(null, null)); + assertThrows( + IllegalArgumentException.class, () -> eventTracker.trackLoginSuccessEvent("", null)); + assertThrows( + IllegalArgumentException.class, () -> eventTracker.trackLoginFailureEvent("", false, null)); + assertThrows(IllegalArgumentException.class, () -> eventTracker.trackCustomEvent("", null)); + assertThrows(IllegalArgumentException.class, () -> setUser(null, null)); + } + + @TableTest({ + "scenario | mode ", + "identification | IDENTIFICATION", + "anonymization | ANONYMIZATION ", + "disabled | DISABLED " + }) + void onSignup(UserIdCollectionMode mode) { + String expectedUserLogin = mode == ANONYMIZATION ? ANONYMIZED_USER_LOGIN : USER_LOGIN; + + tracker.onSignupEvent(mode, USER_LOGIN, METADATA); + + if (mode != DISABLED) { + verify(traceSegment).getTagTop("_dd.appsec.events.users.signup.sdk"); // no SDK event before + verify(traceSegment).setTagTop("_dd.appsec.usr.login", expectedUserLogin); + verify(traceSegment) + .setTagTop("_dd.appsec.events.users.signup.auto.mode", mode.fullName(), true); + verify(traceSegment) + .setTagTop("appsec.events.users.signup.usr.login", expectedUserLogin, true); + verify(traceSegment).setTagTop("appsec.events.users.signup", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.signup.track", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(SIGN_UP), eq(expectedUserLogin)); + } + verifyNoMoreMockInteractions(); + } + + @TableTest({ + "scenario | mode ", + "identification | IDENTIFICATION", + "anonymization | ANONYMIZATION ", + "disabled | DISABLED " + }) + void onLoginSuccess(UserIdCollectionMode mode) { + String expectedUserLogin = mode == ANONYMIZATION ? ANONYMIZED_USER_LOGIN : USER_LOGIN; + + tracker.onLoginSuccessEvent(mode, USER_LOGIN, METADATA); + + if (mode != DISABLED) { + // no SDK event before + verify(traceSegment).getTagTop("_dd.appsec.events.users.login.success.sdk"); + verify(traceSegment).setTagTop("_dd.appsec.usr.login", expectedUserLogin); + verify(traceSegment) + .setTagTop("_dd.appsec.events.users.login.success.auto.mode", mode.fullName(), true); + verify(traceSegment) + .setTagTop("appsec.events.users.login.success.usr.login", expectedUserLogin, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.login.success.track", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(LOGIN_SUCCESS), eq(expectedUserLogin)); + } + verifyNoMoreMockInteractions(); + } + + @TableTest({ + "scenario | mode ", + "identification | IDENTIFICATION", + "anonymization | ANONYMIZATION ", + "disabled | DISABLED " + }) + void onLoginFailed(UserIdCollectionMode mode) { + String expectedUserLogin = mode == ANONYMIZATION ? ANONYMIZED_USER_LOGIN : USER_LOGIN; + + tracker.onLoginFailureEvent(mode, USER_LOGIN, true, METADATA); + + if (mode != DISABLED) { + // no SDK event before + verify(traceSegment).getTagTop("_dd.appsec.events.users.login.failure.sdk"); + verify(traceSegment).setTagTop("_dd.appsec.usr.login", expectedUserLogin); + verify(traceSegment) + .setTagTop("_dd.appsec.events.users.login.failure.auto.mode", mode.fullName(), true); + verify(traceSegment) + .setTagTop("appsec.events.users.login.failure.usr.login", expectedUserLogin, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure", METADATA, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.exists", true, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.track", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(loginEvent).apply(isA(RequestContext.class), eq(LOGIN_FAILURE), eq(expectedUserLogin)); + } + verifyNoMoreMockInteractions(); + } + + @TableTest({ + "scenario | mode ", + "identification | IDENTIFICATION", + "anonymization | ANONYMIZATION ", + "disabled | DISABLED " + }) + void onUserEvent(UserIdCollectionMode mode) { + String expectedUserId = mode == ANONYMIZATION ? ANONYMIZED_USER_ID : USER_ID; + + tracker.onUserEvent(mode, USER_ID, emptyMap()); + + if (mode != DISABLED) { + verify(traceSegment).setTagTop("_dd.appsec.usr.id", expectedUserId); + verify(traceSegment).getTagTop("_dd.appsec.user.collection_mode"); // no user event before + verify(traceSegment).setTagTop("_dd.appsec.user.collection_mode", mode.fullName()); + verify(traceSegment).setTagTop("usr.id", expectedUserId); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + verify(user).apply(isA(RequestContext.class), eq(expectedUserId)); + } + verifyNoMoreMockInteractions(); + } + + @TableTest({ + "scenario | mode ", + "identification | IDENTIFICATION", + "anonymization | ANONYMIZATION ", + "disabled | DISABLED " + }) + void onUserNotFound(UserIdCollectionMode mode) { + tracker.onUserNotFound(mode); + + if (mode != DISABLED) { + // no SDK event before + verify(traceSegment).getTagTop("_dd.appsec.events.users.login.failure.sdk"); + verify(traceSegment) + .setTagTop("_dd.appsec.events.users.login.failure.auto.mode", mode.fullName(), true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.usr.exists", false, true); + verify(traceSegment).setTagTop("appsec.events.users.login.failure.track", true, true); + verify(traceSegment).setTagTop("asm.keep", true); + verify(traceSegment).setTagTop("_dd.p.ts", ASM); + } + verifyNoMoreMockInteractions(); + } + + // spotless:off + @TableTest({ + "scenario | appsec | collectionMode | trackingMode | result", + // disabled states + "off/none/none | false | | | false ", + "off/none/safe | false | | safe | false ", + "off/none/extended | false | | extended | false ", + "off/none/disabled | false | | disabled | false ", + "off/ident/none | false | ident | | false ", + "off/ident/safe | false | ident | safe | false ", + "off/ident/extended | false | ident | extended | false ", + "off/ident/disabled | false | ident | disabled | false ", + "off/anon/none | false | anon | | false ", + "off/anon/safe | false | anon | safe | false ", + "off/anon/extended | false | anon | extended | false ", + "off/anon/disabled | false | anon | disabled | false ", + "off/disabled/none | false | disabled | | false ", + "off/disabled/safe | false | disabled | safe | false ", + "off/disabled/ext | false | disabled | extended | false ", + "off/disabled/dis | false | disabled | disabled | false ", + "on/none/disabled | true | | disabled | false ", + "on/disabled/none | true | disabled | | false ", + "on/disabled/safe | true | disabled | safe | false ", + "on/disabled/extended| true | disabled | extended | false ", + "on/disabled/disabled| true | disabled | disabled | false ", + // enabled states + "on/none/none | true | | | true ", + "on/none/safe | true | | safe | true ", + "on/none/extended | true | | extended | true ", + "on/ident/none | true | ident | | true ", + "on/ident/safe | true | ident | safe | true ", + "on/ident/extended | true | ident | extended | true ", + "on/ident/disabled | true | ident | disabled | true ", + "on/anon/none | true | anon | | true ", + "on/anon/safe | true | anon | safe | true ", + "on/anon/extended | true | anon | extended | true ", + "on/anon/disabled | true | anon | disabled | true " + }) + // spotless:on + void isEnabled(boolean appsec, String collectionMode, String trackingMode, boolean result) { + ActiveSubsystems.APPSEC_ACTIVE = appsec; + UserIdCollectionMode mode = UserIdCollectionMode.fromString(collectionMode, trackingMode); + + assertEquals(result, tracker.isEnabled(mode)); + } + + @Test + void blockingOnALogin() { + Flow.Action.RequestBlockingAction action = + new Flow.Action.RequestBlockingAction(403, BlockingContentType.AUTO); + when(loginEvent.apply(isA(RequestContext.class), eq(LOGIN_SUCCESS), eq(USER_LOGIN))) + .thenReturn(new ActionFlow<>(action)); + + assertThrows( + BlockingException.class, + () -> tracker.onLoginSuccessEvent(SDK, USER_LOGIN, USER_ID, METADATA)); + } + + @Test + void shouldNotFailOnNullCallback() { + when(provider.getCallback(EVENTS.user())).thenReturn(null); + + assertDoesNotThrow(() -> tracker.onUserEvent(IDENTIFICATION, "test-user", emptyMap())); + } + + @Test + void onUserEventDoesNotOverwriteSdk() { + when(traceSegment.getTagTop("_dd.appsec.user.collection_mode")).thenReturn(SDK.fullName()); + + tracker.onUserEvent(IDENTIFICATION, USER_ID, emptyMap()); + + // SDK data remains untouched + verify(traceSegment).getTagTop("_dd.appsec.user.collection_mode"); + verify(traceSegment).setTagTop("_dd.appsec.usr.id", USER_ID); + verifyNoMoreMockInteractions(); + } + + @Test + void onLoginSuccessDoesNotOverwriteSdk() { + when(traceSegment.getTagTop("_dd.appsec.events.users.login.success.sdk")).thenReturn(true); + + tracker.onLoginSuccessEvent(IDENTIFICATION, USER_LOGIN, null, emptyMap()); + + verify(traceSegment).getTagTop("_dd.appsec.events.users.login.success.sdk"); + verify(traceSegment).setTagTop("_dd.appsec.usr.login", USER_LOGIN); + verify(traceSegment) + .setTagTop( + "_dd.appsec.events.users.login.success.auto.mode", IDENTIFICATION.fullName(), true); + verifyNoMoreMockInteractions(); + } + + @Test + void onLoginFailureDoesNotOverwriteSdk() { + when(traceSegment.getTagTop("_dd.appsec.events.users.login.failure.sdk")).thenReturn(true); + + tracker.onLoginFailureEvent(IDENTIFICATION, USER_LOGIN, null, emptyMap()); + + verify(traceSegment).getTagTop("_dd.appsec.events.users.login.failure.sdk"); + verify(traceSegment).setTagTop("_dd.appsec.usr.login", USER_LOGIN); + verify(traceSegment) + .setTagTop( + "_dd.appsec.events.users.login.failure.auto.mode", IDENTIFICATION.fullName(), true); + verifyNoMoreMockInteractions(); + } + + @Test + void onUserNotFoundDoesNotOverwriteSdk() { + when(traceSegment.getTagTop("_dd.appsec.events.users.login.failure.sdk")).thenReturn(true); + + tracker.onUserNotFound(IDENTIFICATION); + + verify(traceSegment).getTagTop("_dd.appsec.events.users.login.failure.sdk"); + verify(traceSegment) + .setTagTop( + "_dd.appsec.events.users.login.failure.auto.mode", IDENTIFICATION.fullName(), true); + verifyNoMoreMockInteractions(); + } + + private void verifyNoMoreMockInteractions() { + verifyNoMoreInteractions(traceSegment, requestContext, user, loginEvent); + verifyNoMoreInteractions(ignoreStubs(span, provider, tracer)); + } + + private static Map metadata() { + Map metadata = new HashMap<>(); + metadata.put("key1", "value1"); + metadata.put("key2", "value2"); + return metadata; + } + + private static void assertAppSecSdkEvent(LoginEvent event, LoginVersion version) { + List sdkEvents = drainSdkEvents(); + assertEquals(1, sdkEvents.size()); + WafMetric metric = sdkEvents.get(0); + assertEquals("appsec", metric.namespace); + assertEquals("count", metric.type); + assertEquals(1L, metric.value.longValue()); + assertEquals( + asList("event_type:" + event.getTag(), "sdk_version:" + version.getTag()), metric.tags); + } + + private static List drainSdkEvents() { + WafMetricCollector collector = WafMetricCollector.get(); + collector.prepareMetrics(); + List sdkEvents = new ArrayList<>(); + for (WafMetric metric : collector.drain()) { + if ("sdk.event".equals(metric.metricName)) { + sdkEvents.add(metric); + } + } + return sdkEvents; + } + + /** + * Exposes the tracer used by the tracker and widens {@code isEnabled} so the test package can + * call it; both are {@code protected} on {@link AppSecEventTracker}. + */ + private static class TestAppSecEventTracker extends AppSecEventTracker { + + private final TracerAPI tracer; + + TestAppSecEventTracker(TracerAPI tracer) { + this.tracer = tracer; + } + + @Override + protected TracerAPI tracer() { + return tracer; + } + + @Override + public boolean isEnabled(UserIdCollectionMode mode) { + return super.isEnabled(mode); + } + } + + private static class ActionFlow implements Flow { + + private final Action action; + + ActionFlow(Action action) { + this.action = action; + } + + @Override + public Action getAction() { + return action; + } + + @Override + public T getResult() { + return null; + } + } +} diff --git a/dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.java b/dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.java new file mode 100644 index 00000000000..93a88094db4 --- /dev/null +++ b/dd-java-agent/appsec/src/test/java/com/datadog/appsec/user/EventTrackerAppSecDisabledForkedTest.java @@ -0,0 +1,137 @@ +package com.datadog.appsec.user; + +import static datadog.trace.api.UserIdCollectionMode.IDENTIFICATION; +import static datadog.trace.api.config.AppSecConfig.APPSEC_ENABLED; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import datadog.appsec.api.login.EventTrackerV2; +import datadog.appsec.api.user.User; +import datadog.trace.api.GlobalTracer; +import datadog.trace.api.appsec.AppSecEventTracker; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.api.internal.TraceSegment; +import datadog.trace.bootstrap.ActiveSubsystems; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer.TracerAPI; +import datadog.trace.test.junit.utils.config.WithConfig; +import datadog.trace.test.util.DDJavaSpecification; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("deprecation") // exercises the deprecated v1 EventTracker API on purpose +@WithConfig(key = APPSEC_ENABLED, value = "false") +class EventTrackerAppSecDisabledForkedTest extends DDJavaSpecification { + + private static final Map METADATA = metadata(); + + private TraceSegment traceSegment; + private AppSecEventTracker tracker; + + @BeforeAll + static void disableAppSec() { + ActiveSubsystems.APPSEC_ACTIVE = false; + } + + @BeforeEach + void setup() { + tracker = new AppSecEventTracker(); + GlobalTracer.setEventTracker(tracker); + EventTrackerV2.setEventTrackerService(tracker.v2EventTrackerService()); + User.setUserService(tracker); + traceSegment = mock(TraceSegment.class); + TracerAPI tracer = mock(TracerAPI.class); + when(tracer.getTraceSegment()).thenReturn(traceSegment); + when(tracer.getCallbackProvider(RequestContextSlot.APPSEC)).thenReturn(null); + AgentTracer.forceRegister(tracer); + } + + @Test + void trackLoginSuccessEvent() { + GlobalTracer.getEventTracker().trackLoginSuccessEvent("user", METADATA); + + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.success.sdk", true, true); + } + + @Test + void trackLoginFailureEvent() { + GlobalTracer.getEventTracker().trackLoginFailureEvent("user", true, METADATA); + + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.failure.sdk", true, true); + } + + @Test + void trackCustomEvent() { + GlobalTracer.getEventTracker().trackCustomEvent("myevent", METADATA); + + verify(traceSegment).setTagTop("_dd.appsec.events.myevent.sdk", true, true); + } + + @Test + void trackLoginSuccessEventV2() { + EventTrackerV2.trackUserLoginSuccess("user", "id", METADATA); + + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.success.sdk", true, true); + } + + @Test + void trackLoginFailureEventV2() { + EventTrackerV2.trackUserLoginFailure("user", true, METADATA); + + verify(traceSegment).setTagTop("_dd.appsec.events.users.login.failure.sdk", true, true); + } + + @Test + void trackCustomEventV2() { + EventTrackerV2.trackCustomEvent("myevent", METADATA); + + verify(traceSegment).setTagTop("_dd.appsec.events.myevent.sdk", true, true); + } + + @Test + void onSignup() { + tracker.onSignupEvent(IDENTIFICATION, "user", METADATA); + + verifyNoInteractions(traceSegment); + } + + @Test + void onLoginSuccess() { + tracker.onLoginSuccessEvent(IDENTIFICATION, "user", METADATA); + + verifyNoInteractions(traceSegment); + } + + @Test + void onLoginFailed() { + tracker.onLoginFailureEvent(IDENTIFICATION, "user", true, METADATA); + + verifyNoInteractions(traceSegment); + } + + @Test + void onUserEvent() { + tracker.onUserEvent(IDENTIFICATION, "user"); + + verifyNoInteractions(traceSegment); + } + + @Test + void onUserNotFound() { + tracker.onUserNotFound(IDENTIFICATION); + + verifyNoInteractions(traceSegment); + } + + private static Map metadata() { + Map metadata = new HashMap<>(); + metadata.put("key1", "value1"); + metadata.put("key2", "value2"); + return metadata; + } +} diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index 966b916572b..f36aa876e34 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -58,6 +58,8 @@ extra["excludedClassesCoverage"] = listOf( // These are almost fully abstract classes so nothing to test "datadog.trace.api.profiling.RecordingData", "datadog.trace.api.appsec.AppSecEventTracker", + // Anonymous EventTrackerService adapter; covered by AppSecEventTrackerTest in dd-java-agent:appsec + "datadog.trace.api.appsec.AppSecEventTracker.1", // POJOs "datadog.trace.api.appsec.HttpClientPayload", "datadog.trace.api.appsec.HttpClientRequest", diff --git a/internal-api/src/main/java/datadog/trace/api/appsec/AppSecEventTracker.java b/internal-api/src/main/java/datadog/trace/api/appsec/AppSecEventTracker.java index 6b6d44dc512..8fb38b7cebb 100644 --- a/internal-api/src/main/java/datadog/trace/api/appsec/AppSecEventTracker.java +++ b/internal-api/src/main/java/datadog/trace/api/appsec/AppSecEventTracker.java @@ -43,7 +43,7 @@ import java.util.Map; import java.util.function.BiFunction; -public class AppSecEventTracker extends EventTracker implements UserService, EventTrackerService { +public class AppSecEventTracker extends EventTracker implements UserService { private static final int HASH_SIZE_BYTES = 16; // 128 bits private static final String ANON_PREFIX = "anon_"; @@ -66,10 +66,53 @@ public class AppSecEventTracker extends EventTracker implements UserService, Eve public static void install() { final AppSecEventTracker tracker = new AppSecEventTracker(); GlobalTracer.setEventTracker(tracker); - EventTrackerV2.setEventTrackerService(tracker); + EventTrackerV2.setEventTrackerService(tracker.v2Service); User.setUserService(tracker); } + /** + * The v2 SDK surface, kept separate from the v1 {@link EventTracker} surface this class extends. + * {@code trackCustomEvent} has an identical signature on both APIs, so a single implementation + * cannot tell which one the caller used; routing v2 calls through this adapter lets each API + * report its own version to telemetry. + */ + private final EventTrackerService v2Service = + new EventTrackerService() { + @Override + public void trackUserLoginSuccess( + final String login, final String userId, final Map metadata) { + if (login == null || login.isEmpty()) { + throw new IllegalArgumentException("login is null or empty"); + } + WafMetricCollector.get().appSecSdkEvent(LOGIN_SUCCESS, V2); + if (handleLoginEvent(V2, LOGIN_SUCCESS_EVENT, SDK, login, userId, null, metadata)) { + throw new BlockingException("Blocked request (for login success)"); + } + } + + @Override + public void trackUserLoginFailure( + final String login, final boolean exists, final Map metadata) { + if (login == null || login.isEmpty()) { + throw new IllegalArgumentException("login is null or empty"); + } + WafMetricCollector.get().appSecSdkEvent(LOGIN_FAILURE, V2); + if (handleLoginEvent(V2, LOGIN_FAILURE_EVENT, SDK, login, null, exists, metadata)) { + throw new BlockingException("Blocked request (for login failure)"); + } + } + + @Override + public void trackCustomEvent(final String eventName, final Map metadata) { + AppSecEventTracker.this.trackCustomEvent(eventName, metadata, V2); + } + }; + + /** Returns the v2 SDK implementation to register with {@link EventTrackerV2}. */ + public final EventTrackerService v2EventTrackerService() { + return v2Service; + } + @Override public final void trackLoginSuccessEvent(String userId, Map metadata) { if (userId == null || userId.isEmpty()) { @@ -93,38 +136,19 @@ public final void trackLoginFailureEvent( } } - @Override - public void trackUserLoginSuccess( - final String login, final String userId, final Map metadata) { - if (login == null || login.isEmpty()) { - throw new IllegalArgumentException("login is null or empty"); - } - WafMetricCollector.get().appSecSdkEvent(LOGIN_SUCCESS, V2); - if (handleLoginEvent(V2, LOGIN_SUCCESS_EVENT, SDK, login, userId, null, metadata)) { - throw new BlockingException("Blocked request (for login success)"); - } - } - - @Override - public void trackUserLoginFailure( - final String login, final boolean exists, final Map metadata) { - if (login == null || login.isEmpty()) { - throw new IllegalArgumentException("login is null or empty"); - } - WafMetricCollector.get().appSecSdkEvent(LOGIN_FAILURE, V2); - if (handleLoginEvent(V2, LOGIN_FAILURE_EVENT, SDK, login, null, exists, metadata)) { - throw new BlockingException("Blocked request (for login failure)"); - } - } - @SuppressWarnings("deprecation") @Override public final void trackCustomEvent(String eventName, Map metadata) { + trackCustomEvent(eventName, metadata, V1); + } + + private void trackCustomEvent( + final String eventName, final Map metadata, final LoginVersion version) { if (eventName == null || eventName.isEmpty()) { throw new IllegalArgumentException("eventName is null or empty"); } - WafMetricCollector.get().appSecSdkEvent(CUSTOM, V2); - if (handleLoginEvent(V2, eventName, SDK, null, null, null, metadata)) { + WafMetricCollector.get().appSecSdkEvent(CUSTOM, version); + if (handleLoginEvent(version, eventName, SDK, null, null, null, metadata)) { throw new BlockingException("Blocked request (for custom event)"); } } From 4eb59715a1a75ad62095cf3831293a69e8fc5e91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vandon?= Date: Fri, 11 Sep 2026 13:44:07 +0100 Subject: [PATCH 19/30] fix tibco bw instrumentation for 5.16 (#12449) fix tibco bw instrumentation for 5.16 add constraint on method visibility Co-authored-by: devflow.devflow-routing-intake --- .../tibcobw5/JobPoolInstrumentation.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/tibco-businessworks/tibco-businessworks-5.14/src/main/java/datadog/trace/instrumentation/tibcobw5/JobPoolInstrumentation.java b/dd-java-agent/instrumentation/tibco-businessworks/tibco-businessworks-5.14/src/main/java/datadog/trace/instrumentation/tibcobw5/JobPoolInstrumentation.java index 647a0a6e759..c4b2c6edbb9 100644 --- a/dd-java-agent/instrumentation/tibco-businessworks/tibco-businessworks-5.14/src/main/java/datadog/trace/instrumentation/tibcobw5/JobPoolInstrumentation.java +++ b/dd-java-agent/instrumentation/tibco-businessworks/tibco-businessworks-5.14/src/main/java/datadog/trace/instrumentation/tibcobw5/JobPoolInstrumentation.java @@ -1,9 +1,12 @@ package datadog.trace.instrumentation.tibcobw5; +import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.hasInterface; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.instrumentation.tibcobw5.TibcoDecorator.DECORATE; import static datadog.trace.instrumentation.tibcobw5.TibcoDecorator.TIBCO_PROCESS_OPERATION; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import com.google.auto.service.AutoService; import com.tibco.pe.core.DDJobMate; @@ -29,9 +32,15 @@ public String instrumentedType() { @Override public void methodAdvice(MethodTransformer transformer) { - - transformer.applyAdvice(named("addJob"), getClass().getName() + "$JobStartAdvice"); - transformer.applyAdvice(named("removeJob"), getClass().getName() + "$JobEndAdvice"); + transformer.applyAdvice( + named("addJob") + .and(isPublic()) + .and(takesArgument(0, hasInterface(named("com.tibco.pe.plugin.ProcessContext")))), + getClass().getName() + "$JobStartAdvice"); + transformer.applyAdvice( + named("removeJob") // note: removeJob is a protected method (not public) + .and(takesArgument(0, hasInterface(named("com.tibco.pe.plugin.ProcessContext")))), + getClass().getName() + "$JobEndAdvice"); } public static class JobStartAdvice { From 40a0f9d5d31c1e687ab4e46b654c47fc53775ad7 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 11 Sep 2026 08:57:36 -0400 Subject: [PATCH 20/30] Stop AppSecInterceptor from silently retrying failed okhttp requests (#12242) Stop AppSecInterceptor from silently retrying failed okhttp requests chain.proceed(request) was wrapped in the same try/catch that guards the AppSec request/response hooks, so any IOException from the real network call (e.g. ConnectException) was swallowed and the request was silently retried via chain.proceed(chain.request()). This double- executes non-idempotent requests on transient network failures and surfaces the retry's own failure as an unhandled error blamed on the interceptor. Narrow the try/catch to only cover the AppSec hooks so genuine I/O failures propagate normally. Co-Authored-By: Claude Sonnet 5 Add JUnit5 regression test for AppSecInterceptor silent-retry fix Covers both okhttp-2.2 and okhttp-3.0 AppSecInterceptor.intercept(): asserts an IOException from chain.proceed() propagates without being swallowed/retried, using Mockito + AgentTracer.forceRegister instead of Groovy/Spock. Co-Authored-By: Claude Sonnet 5 Merge branch 'master' into dougqh/fix-appsec-interceptor-duplicate-request Merge branch 'master' into dougqh/fix-appsec-interceptor-duplicate-request Merge branch 'master' into dougqh/fix-appsec-interceptor-duplicate-request Guard against NPE reading HTTP_URL tag in AppSecInterceptor span.getTag(Tags.HTTP_URL) can return null, and .toString() on it throws an NPE that silently skips the AppSec request hook for that call. Null-check instead of relying on the catch-all to swallow it. Co-Authored-By: Claude Sonnet 5 Merge branch 'master' into dougqh/fix-appsec-interceptor-duplicate-request Merge branch 'master' into dougqh/fix-appsec-interceptor-duplicate-request Merge branch 'master' into dougqh/fix-appsec-interceptor-duplicate-request Preserve rebuilt response body when the AppSec response hook fails onResponse() reads and closes the original response body, then rebuilds a Response with a fresh body before calling publish(). If publish()'s WAF/ gateway callback throws a non-blocking exception, that exception used to propagate out of onResponse() entirely, and intercept()'s outer catch fell back to the original (already-drained/closed) response instead of the rebuilt one -- handing the caller an empty or closed body. Guard publish() locally so a non-blocking failure there no longer discards the rebuilt response; BlockingException still propagates as before. Co-Authored-By: Claude Sonnet 5 Merge remote-tracking branch 'origin/dougqh/fix-appsec-interceptor-duplicate-request' into dougqh/fix-appsec-interceptor-duplicate-request Co-authored-by: devflow.devflow-routing-intake --- .../okhttp/okhttp-2.2/build.gradle | 3 + .../okhttp2/AppSecInterceptor.java | 43 ++++++++---- .../okhttp2/AppSecInterceptorTest.java | 68 +++++++++++++++++++ .../okhttp/okhttp-3.0/build.gradle | 3 + .../okhttp3/AppSecInterceptor.java | 43 ++++++++---- .../okhttp3/AppSecInterceptorTest.java | 68 +++++++++++++++++++ 6 files changed, 204 insertions(+), 24 deletions(-) create mode 100644 dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptorTest.java create mode 100644 dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptorTest.java diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/build.gradle b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/build.gradle index 9246a56fa71..c45816a4b2f 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/build.gradle +++ b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/build.gradle @@ -41,6 +41,9 @@ dependencies { } testImplementation group: 'com.squareup.okhttp', name: 'okhttp', version: '2.2.0' + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito + testRuntimeOnly(project(':dd-java-agent:instrumentation:datadog:asm:iast-instrumenter')) testRuntimeOnly(project(':dd-java-agent:instrumentation:java:java-net:java-net-1.8')) diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/main/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptor.java b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/main/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptor.java index 7f55cc9a4fa..8a32cfd4578 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/main/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptor.java +++ b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/main/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptor.java @@ -45,23 +45,33 @@ public class AppSecInterceptor implements Interceptor { @Override public Response intercept(final Chain chain) throws IOException { + Request request = chain.request(); + final AgentSpan span = AgentTracer.activeSpan(); + final RequestContext ctx = span == null ? null : span.getRequestContext(); + if (ctx == null) { + return chain.proceed(request); + } + boolean sampled = false; try { - final AgentSpan span = AgentTracer.activeSpan(); - final RequestContext ctx = span == null ? null : span.getRequestContext(); - if (ctx == null) { - return chain.proceed(chain.request()); - } final long requestId = span.getSpanId(); - final boolean sampled = sampleRequest(ctx, requestId); - final String url = span.getTag(Tags.HTTP_URL).toString(); - final Request request = onRequest(span, sampled, url, chain.request()); - final Response response = chain.proceed(request); + sampled = sampleRequest(ctx, requestId); + final Object urlTag = span.getTag(Tags.HTTP_URL); + final String url = urlTag == null ? null : urlTag.toString(); + request = onRequest(span, sampled, url, request); + } catch (final BlockingException e) { + throw e; + } catch (final Exception e) { + LOGGER.debug("Failed to run AppSec request hooks", e); + } + // let real connection/IO failures propagate rather than swallowing and retrying the request + final Response response = chain.proceed(request); + try { return onResponse(span, sampled, response); } catch (final BlockingException e) { throw e; } catch (final Exception e) { - LOGGER.debug("Failed to intercept request", e); - return chain.proceed(chain.request()); + LOGGER.debug("Failed to run AppSec response hooks", e); + return response; } } @@ -142,7 +152,16 @@ public static Response onResponse( } } - publish(ctx, clientResponse, responseCb); + try { + publish(ctx, clientResponse, responseCb); + } catch (final BlockingException e) { + throw e; + } catch (final Exception e) { + // don't let a failure in the response hook discard the rebuilt response above -- + // its body has already been drained/closed, so falling back to the original response + // (as the caller in intercept() does) would hand back an empty/closed body + LOGGER.debug("Failed to publish AppSec response event", e); + } return result; } diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptorTest.java b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptorTest.java new file mode 100644 index 00000000000..3319774af1f --- /dev/null +++ b/dd-java-agent/instrumentation/okhttp/okhttp-2.2/src/test/java/datadog/trace/instrumentation/okhttp2/AppSecInterceptorTest.java @@ -0,0 +1,68 @@ +package datadog.trace.instrumentation.okhttp2; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.squareup.okhttp.Interceptor; +import com.squareup.okhttp.Request; +import datadog.trace.api.gateway.CallbackProvider; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.io.IOException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AppSecInterceptorTest { + + private final AgentTracer.TracerAPI originalTracer = AgentTracer.get(); + + private Interceptor.Chain chain; + private Request request; + private final AppSecInterceptor interceptor = new AppSecInterceptor(); + + @BeforeEach + void setup() { + request = new Request.Builder().url("http://example.com").build(); + + final RequestContext requestContext = mock(RequestContext.class); + + final AgentSpan span = mock(AgentSpan.class); + when(span.getRequestContext()).thenReturn(requestContext); + when(span.getSpanId()).thenReturn(1L); + when(span.getTag(Tags.HTTP_URL)).thenReturn("http://example.com"); + + final AgentTracer.TracerAPI tracer = mock(AgentTracer.TracerAPI.class); + when(tracer.activeSpan()).thenReturn(span); + when(tracer.getCallbackProvider(any(RequestContextSlot.class))) + .thenReturn(CallbackProvider.CallbackProviderNoop.INSTANCE); + AgentTracer.forceRegister(tracer); + + chain = mock(Interceptor.Chain.class); + when(chain.request()).thenReturn(request); + } + + @AfterEach + void tearDown() { + AgentTracer.forceRegister(originalTracer); + } + + @Test + void ioExceptionFromProceedPropagatesWithoutRetry() throws IOException { + final IOException failure = new IOException("boom"); + when(chain.proceed(request)).thenThrow(failure); + + final IOException thrown = assertThrows(IOException.class, () -> interceptor.intercept(chain)); + + assertSame(failure, thrown); + verify(chain, times(1)).proceed(request); + } +} diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/build.gradle b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/build.gradle index 5364ebbff76..8f023f6f868 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/build.gradle +++ b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/build.gradle @@ -38,6 +38,9 @@ dependencies { latestDepTestImplementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '[3.11.0, 4)' latestDepTestImplementation group: 'com.squareup.okio', name: 'okio', version: '1.+' + testImplementation libs.bundles.junit5 + testImplementation libs.bundles.mockito + testRuntimeOnly(project(':dd-java-agent:instrumentation:datadog:asm:iast-instrumenter')) testRuntimeOnly(project(':dd-java-agent:instrumentation:java:java-net:java-net-1.8')) } diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/main/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptor.java b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/main/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptor.java index e61a78003bb..9d78a5f54ce 100644 --- a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/main/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptor.java +++ b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/main/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptor.java @@ -45,23 +45,33 @@ public class AppSecInterceptor implements Interceptor { @Override public Response intercept(final Chain chain) throws IOException { + Request request = chain.request(); + final AgentSpan span = AgentTracer.activeSpan(); + final RequestContext ctx = span == null ? null : span.getRequestContext(); + if (ctx == null) { + return chain.proceed(request); + } + boolean sampled = false; try { - final AgentSpan span = AgentTracer.activeSpan(); - final RequestContext ctx = span == null ? null : span.getRequestContext(); - if (ctx == null) { - return chain.proceed(chain.request()); - } final long requestId = span.getSpanId(); - final boolean sampled = sampleRequest(ctx, requestId); - final String url = span.getTag(Tags.HTTP_URL).toString(); - final Request request = onRequest(span, sampled, url, chain.request()); - final Response response = chain.proceed(request); + sampled = sampleRequest(ctx, requestId); + final Object urlTag = span.getTag(Tags.HTTP_URL); + final String url = urlTag == null ? null : urlTag.toString(); + request = onRequest(span, sampled, url, request); + } catch (final BlockingException e) { + throw e; + } catch (final Exception e) { + LOGGER.debug("Failed to run AppSec request hooks", e); + } + // let real connection/IO failures propagate rather than swallowing and retrying the request + final Response response = chain.proceed(request); + try { return onResponse(span, sampled, response); } catch (final BlockingException e) { throw e; } catch (final Exception e) { - LOGGER.debug("Failed to intercept request", e); - return chain.proceed(chain.request()); + LOGGER.debug("Failed to run AppSec response hooks", e); + return response; } } @@ -142,7 +152,16 @@ public static Response onResponse( } } - publish(ctx, clientResponse, responseCb); + try { + publish(ctx, clientResponse, responseCb); + } catch (final BlockingException e) { + throw e; + } catch (final Exception e) { + // don't let a failure in the response hook discard the rebuilt response above -- + // its body has already been drained/closed, so falling back to the original response + // (as the caller in intercept() does) would hand back an empty/closed body + LOGGER.debug("Failed to publish AppSec response event", e); + } return result; } diff --git a/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptorTest.java b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptorTest.java new file mode 100644 index 00000000000..56b230bb849 --- /dev/null +++ b/dd-java-agent/instrumentation/okhttp/okhttp-3.0/src/test/java/datadog/trace/instrumentation/okhttp3/AppSecInterceptorTest.java @@ -0,0 +1,68 @@ +package datadog.trace.instrumentation.okhttp3; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import datadog.trace.api.gateway.CallbackProvider; +import datadog.trace.api.gateway.RequestContext; +import datadog.trace.api.gateway.RequestContextSlot; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import java.io.IOException; +import okhttp3.Interceptor; +import okhttp3.Request; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AppSecInterceptorTest { + + private final AgentTracer.TracerAPI originalTracer = AgentTracer.get(); + + private Interceptor.Chain chain; + private Request request; + private final AppSecInterceptor interceptor = new AppSecInterceptor(); + + @BeforeEach + void setup() { + request = new Request.Builder().url("http://example.com").build(); + + final RequestContext requestContext = mock(RequestContext.class); + + final AgentSpan span = mock(AgentSpan.class); + when(span.getRequestContext()).thenReturn(requestContext); + when(span.getSpanId()).thenReturn(1L); + when(span.getTag(Tags.HTTP_URL)).thenReturn("http://example.com"); + + final AgentTracer.TracerAPI tracer = mock(AgentTracer.TracerAPI.class); + when(tracer.activeSpan()).thenReturn(span); + when(tracer.getCallbackProvider(any(RequestContextSlot.class))) + .thenReturn(CallbackProvider.CallbackProviderNoop.INSTANCE); + AgentTracer.forceRegister(tracer); + + chain = mock(Interceptor.Chain.class); + when(chain.request()).thenReturn(request); + } + + @AfterEach + void tearDown() { + AgentTracer.forceRegister(originalTracer); + } + + @Test + void ioExceptionFromProceedPropagatesWithoutRetry() throws IOException { + final IOException failure = new IOException("boom"); + when(chain.proceed(request)).thenThrow(failure); + + final IOException thrown = assertThrows(IOException.class, () -> interceptor.intercept(chain)); + + assertSame(failure, thrown); + verify(chain, times(1)).proceed(request); + } +} From 6ac2b95b2b20e9a2b823d0ae6c9c536bb01aaff2 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 11 Sep 2026 09:02:17 -0400 Subject: [PATCH 21/30] Carry the intercepted HTTP status as an int, rendering it on demand (#12395) Carry the intercepted HTTP status as an int, rendering it on demand Metadata held the intercepted HTTP status as a UTF8BytesString, rendered unconditionally at construction. Every span with a status therefore paid a RadixTreeCache probe and carried a string, whether or not its serializer wanted one -- and a serializer that wants the number had nothing to ask for. That is the wrong default now that OTLP is one of the consumers: semantic conventions type http.response.status_code as an integer, so the string is precisely what it cannot use. Carry the int instead, and offer both accessors: getHttpStatusCode() returns the status (UNSET_STATUS when the span carries none, matching DDSpanContext's own convention), getHttpStatusCodeString() renders it through RadixTreeCache.HTTP_STATUSES for the string-typed protocols. Rendering moves from construction to the one call site that needs it, so the presence checks the mappers already perform become int comparisons and a numeric encoder skips the probe entirely. No wire change: every existing consumer asks for the string exactly where it did before. Emitting the status numerically -- and under its OpenTelemetry name -- is the follow-on this unblocks. Merge master, porting httpStatusCode int change onto migrated TraceGenerator master migrated dd-trace-core/src/traceAgentTest/groovy/TraceGenerator.groovy to Java (#12332) after this branch changed the same file's Metadata constructor call. Take master's migrated TraceGenerator.java and port the httpStatusCode null -> 0 change onto it, matching the same edit already applied to the sibling test/java/.../writer/TraceGenerator.java and DDSpanContext.java in this branch. Co-Authored-By: Claude Sonnet 5 Merge branch 'master' into dougqh/http-status-int Co-authored-by: devflow.devflow-routing-intake --- .../writer/ddintake/CiTestCycleMapperV1.java | 8 ++- .../writer/FileBasedPayloadDispatcher.java | 7 +- .../writer/ddagent/TraceMapperV0_4.java | 7 +- .../writer/ddagent/TraceMapperV0_5.java | 7 +- .../common/writer/ddagent/TraceMapperV1.java | 4 +- .../datadog/trace/core/DDSpanContext.java | 3 +- .../java/datadog/trace/core/Metadata.java | 28 +++++++- .../trace/core/otlp/trace/OtlpTraceJson.java | 5 +- .../trace/core/otlp/trace/OtlpTraceProto.java | 5 +- .../FileBasedPayloadDispatcherTest.java | 2 +- .../trace/common/writer/TraceGenerator.java | 2 +- .../java/datadog/trace/core/MetadataTest.java | 67 +++++++++++++++++++ .../traceAgentTest/java/TraceGenerator.java | 2 +- 13 files changed, 123 insertions(+), 24 deletions(-) create mode 100644 dd-trace-core/src/test/java/datadog/trace/core/MetadataTest.java diff --git a/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java index ac110ca2e15..3de19f68088 100644 --- a/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/civisibility/writer/ddintake/CiTestCycleMapperV1.java @@ -3,6 +3,7 @@ import static datadog.communication.http.OkHttpUtils.gzippedMsgpackRequestBodyOf; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; import static datadog.json.JsonMapper.toJson; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import static datadog.trace.api.civisibility.CIConstants.MAX_META_STRING_VALUE_LENGTH; import static datadog.trace.util.Strings.truncate; @@ -333,7 +334,7 @@ public void accept(Metadata metadata) { int metaSize = metadata.getBaggage().size() + tags.size() - + (null == metadata.getHttpStatusCode() ? 0 : 1); + + (UNSET_STATUS == metadata.getHttpStatusCode() ? 0 : 1); int metricsSize = 0; for (Map.Entry tag : tags.entrySet()) { if (tag.getValue() instanceof Number) { @@ -359,9 +360,10 @@ public void accept(Metadata metadata) { writable.writeString(entry.getKey(), null); writable.writeString(truncate(entry.getValue(), MAX_META_STRING_VALUE_LENGTH), null); } - if (null != metadata.getHttpStatusCode()) { + if (UNSET_STATUS != metadata.getHttpStatusCode()) { writable.writeUTF8(HTTP_STATUS); - writable.writeUTF8(truncate(metadata.getHttpStatusCode(), MAX_META_STRING_VALUE_LENGTH)); + writable.writeUTF8( + truncate(metadata.getHttpStatusCodeString(), MAX_META_STRING_VALUE_LENGTH)); } for (Map.Entry entry : tags.entrySet()) { Object value = entry.getValue(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java index 810952292b8..0733d9a9a04 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/FileBasedPayloadDispatcher.java @@ -1,6 +1,7 @@ package datadog.trace.common.writer; import static datadog.json.JsonMapper.toJson; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import static datadog.trace.api.civisibility.CIConstants.MAX_META_STRING_VALUE_LENGTH; import static datadog.trace.util.Strings.truncate; @@ -388,9 +389,11 @@ public void accept(Metadata metadata) { w.name(entry.getKey()).value(truncate(entry.getValue(), MAX_META_STRING_VALUE_LENGTH)); } } - if (metadata.getHttpStatusCode() != null) { + if (metadata.getHttpStatusCode() != UNSET_STATUS) { w.name(Tags.HTTP_STATUS) - .value(truncate(metadata.getHttpStatusCode().toString(), MAX_META_STRING_VALUE_LENGTH)); + .value( + truncate( + metadata.getHttpStatusCodeString().toString(), MAX_META_STRING_VALUE_LENGTH)); } for (Map.Entry entry : tags.entrySet()) { Object value = entry.getValue(); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java index 0473010d3d1..58fd278cf43 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_4.java @@ -1,6 +1,7 @@ package datadog.trace.common.writer.ddagent; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import datadog.communication.serialization.Codec; import datadog.communication.serialization.GenerationalUtf8Cache; @@ -103,7 +104,7 @@ public void accept(Metadata metadata) { int metaSize = metadata.getBaggage().size() + tags.size() - + (null == metadata.getHttpStatusCode() ? 0 : 1) + + (UNSET_STATUS == metadata.getHttpStatusCode() ? 0 : 1) + (null == metadata.getOrigin() ? 0 : 1) + (null == processTags ? 0 : 1) + 1; @@ -193,9 +194,9 @@ public void accept(Metadata metadata) { } writable.writeUTF8(THREAD_NAME); writable.writeUTF8(metadata.getThreadName()); - if (null != metadata.getHttpStatusCode()) { + if (UNSET_STATUS != metadata.getHttpStatusCode()) { writable.writeUTF8(HTTP_STATUS); - writable.writeUTF8(metadata.getHttpStatusCode()); + writable.writeUTF8(metadata.getHttpStatusCodeString()); } if (null != metadata.getOrigin()) { writable.writeUTF8(ORIGIN_KEY); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_5.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_5.java index 0e8644bdee9..60f221402d5 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_5.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV0_5.java @@ -1,6 +1,7 @@ package datadog.trace.common.writer.ddagent; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import datadog.communication.serialization.GrowableBuffer; import datadog.communication.serialization.Mapper; @@ -225,7 +226,7 @@ public void accept(Metadata metadata) { int metaSize = metadata.getBaggage().size() + tags.size() - + (null == metadata.getHttpStatusCode() ? 0 : 1) + + (UNSET_STATUS == metadata.getHttpStatusCode() ? 0 : 1) + (null == metadata.getOrigin() ? 0 : 1) + (null == processTags ? 0 : 1) + 1; @@ -259,9 +260,9 @@ public void accept(Metadata metadata) { } writeDictionaryEncoded(writable, THREAD_NAME); writeDictionaryEncoded(writable, metadata.getThreadName()); - if (null != metadata.getHttpStatusCode()) { + if (UNSET_STATUS != metadata.getHttpStatusCode()) { writeDictionaryEncoded(writable, HTTP_STATUS); - writeDictionaryEncoded(writable, metadata.getHttpStatusCode()); + writeDictionaryEncoded(writable, metadata.getHttpStatusCodeString()); } if (null != metadata.getOrigin()) { writeDictionaryEncoded(writable, ORIGIN_KEY); diff --git a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java index 8e43e43fa67..6a0ac4d7b3b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java +++ b/dd-trace-core/src/main/java/datadog/trace/common/writer/ddagent/TraceMapperV1.java @@ -1,6 +1,7 @@ package datadog.trace.common.writer.ddagent; import static datadog.communication.http.OkHttpUtils.msgpackRequestBodyOf; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -378,8 +379,9 @@ private void encodeSpanAttributes( Writable writable, int fieldId, Metadata meta, Map metaStruct) { TagMap tags = meta.getTags(); Map baggage = meta.getBaggage(); + // Kept as a String: writeAttribute below logs a debug line for any non-String value. String httpStatusCode = - meta.getHttpStatusCode() == null ? null : meta.getHttpStatusCode().toString(); + meta.getHttpStatusCode() == UNSET_STATUS ? null : meta.getHttpStatusCodeString().toString(); boolean writeHttpStatus = httpStatusCode != null && tags.getString(HTTP_STATUS) == null; boolean writeTopLevel = meta.topLevel(); int tagCount = 0; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index adf4cd66156..8520a222424 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -2,7 +2,6 @@ import static datadog.trace.api.DDTags.PARENT_ID; import static datadog.trace.api.DDTags.SPAN_LINKS; -import static datadog.trace.api.cache.RadixTreeCache.HTTP_STATUSES; import static datadog.trace.bootstrap.instrumentation.api.ErrorPriorities.UNSET; import static datadog.trace.bootstrap.instrumentation.api.ServiceNameSources.MANUAL; @@ -1356,7 +1355,7 @@ void processTagsAndBaggage( samplingPriority != PrioritySampling.UNSET ? samplingPriority : getSamplingPriority(), measured, topLevel, - httpStatusCode == 0 ? null : HTTP_STATUSES.get(httpStatusCode), + httpStatusCode, // Get origin from rootSpan.context getOrigin(), longRunningVersion, diff --git a/dd-trace-core/src/main/java/datadog/trace/core/Metadata.java b/dd-trace-core/src/main/java/datadog/trace/core/Metadata.java index b7f9a6b2cc2..d957358d73a 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/Metadata.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/Metadata.java @@ -4,6 +4,7 @@ import static java.util.Collections.emptyList; import datadog.trace.api.TagMap; +import datadog.trace.api.cache.RadixTreeCache; import datadog.trace.bootstrap.instrumentation.api.AgentSpanLink; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import java.util.List; @@ -12,7 +13,7 @@ public final class Metadata { private final long threadId; private final UTF8BytesString threadName; - private final UTF8BytesString httpStatusCode; + private final int httpStatusCode; private final TagMap tags; private final Map baggage; @@ -32,7 +33,7 @@ public Metadata( int samplingPriority, boolean measured, boolean topLevel, - UTF8BytesString httpStatusCode, + int httpStatusCode, CharSequence origin, int longRunningVersion, UTF8BytesString processTags, @@ -51,10 +52,31 @@ public Metadata( this.spanLinks = spanLinks == null ? emptyList() : spanLinks; } - public UTF8BytesString getHttpStatusCode() { + /** + * The intercepted HTTP status, or {@link RadixTreeCache#UNSET_STATUS} when the span carries none. + * + *

Held as an int rather than as its rendering, so a serializer that encodes the status + * numerically -- OTLP, whose semantic conventions type it as an integer -- never pays for a + * string it will not send, and a serializer that needs the string asks for it explicitly. + */ + public int getHttpStatusCode() { return httpStatusCode; } + /** + * The intercepted HTTP status rendered for the string-typed protocols (the Datadog msgpack + * payloads and the CI Visibility intake), or null when the span carries none. + * + *

Backed by {@link RadixTreeCache#HTTP_STATUSES}, so a repeated status costs a lookup rather + * than an allocation. Call it once per span and hold the result: nothing memoizes it here, since + * a Metadata is consumed by exactly one serializer. + */ + public UTF8BytesString getHttpStatusCodeString() { + return httpStatusCode == RadixTreeCache.UNSET_STATUS + ? null + : RadixTreeCache.HTTP_STATUSES.get(httpStatusCode); + } + public CharSequence getOrigin() { return origin; } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java index d9c5e9c3d90..a56498d0a23 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceJson.java @@ -1,5 +1,6 @@ package datadog.trace.core.otlp.trace; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_PARTIAL_VERSION; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_TOP_LEVEL; @@ -207,8 +208,8 @@ public void accept(Metadata metadata) { writeSpanTag(writer, THREAD_ID, metadata.getThreadId()); writeSpanTag(writer, THREAD_NAME, metadata.getThreadName()); - if (metadata.getHttpStatusCode() != null) { - writeSpanTag(writer, HTTP_STATUS, metadata.getHttpStatusCode()); + if (metadata.getHttpStatusCode() != UNSET_STATUS) { + writeSpanTag(writer, HTTP_STATUS, metadata.getHttpStatusCodeString()); } if (metadata.getOrigin() != null) { writeSpanTag(writer, ORIGIN_KEY, metadata.getOrigin()); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java index f97d05c388d..258cfb73669 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/otlp/trace/OtlpTraceProto.java @@ -1,5 +1,6 @@ package datadog.trace.core.otlp.trace; +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_MEASURED; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_PARTIAL_VERSION; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.DD_TOP_LEVEL; @@ -284,8 +285,8 @@ public void accept(Metadata metadata) { writeSpanTag(buf, THREAD_ID, metadata.getThreadId()); writeSpanTag(buf, THREAD_NAME, metadata.getThreadName()); - if (metadata.getHttpStatusCode() != null) { - writeSpanTag(buf, HTTP_STATUS, metadata.getHttpStatusCode()); + if (metadata.getHttpStatusCode() != UNSET_STATUS) { + writeSpanTag(buf, HTTP_STATUS, metadata.getHttpStatusCodeString()); } if (metadata.getOrigin() != null) { writeSpanTag(buf, ORIGIN_KEY, metadata.getOrigin()); diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java index 2040f685fdc..25c26bcd10a 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/FileBasedPayloadDispatcherTest.java @@ -309,7 +309,7 @@ private static CoreSpan mockSpan(CharSequence type, Map tags) 0, false, false, - null, + 0, null, 0, null, diff --git a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceGenerator.java b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceGenerator.java index c8fad4ab0ee..e777235fef3 100644 --- a/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceGenerator.java +++ b/dd-trace-core/src/test/java/datadog/trace/common/writer/TraceGenerator.java @@ -238,7 +238,7 @@ public PojoSpan( samplingPriority, measured, isTopLevel(), - statusCode == 0 ? null : UTF8BytesString.create(Integer.toString(statusCode)), + statusCode, origin, 0, ProcessTags.getTagsForSerialization(), diff --git a/dd-trace-core/src/test/java/datadog/trace/core/MetadataTest.java b/dd-trace-core/src/test/java/datadog/trace/core/MetadataTest.java new file mode 100644 index 00000000000..2815cac6730 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/core/MetadataTest.java @@ -0,0 +1,67 @@ +package datadog.trace.core; + +import static datadog.trace.api.cache.RadixTreeCache.UNSET_STATUS; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +import datadog.trace.api.TagMap; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The HTTP status is carried as an int and rendered only on demand, so that a serializer encoding + * it numerically never pays for a string it will not send. + */ +class MetadataTest { + + @ParameterizedTest + @ValueSource(ints = {200, 201, 404, 500, 599}) + void rendersTheStatusOnlyWhenAsked(int status) { + Metadata metadata = metadataWithStatus(status); + + assertEquals(status, metadata.getHttpStatusCode()); + assertEquals(Integer.toString(status), metadata.getHttpStatusCodeString().toString()); + } + + @Test + void reportsNoStatusAsUnsetRatherThanZeroString() { + Metadata metadata = metadataWithStatus(UNSET_STATUS); + + assertEquals(UNSET_STATUS, metadata.getHttpStatusCode()); + assertNull( + metadata.getHttpStatusCodeString(), + "an absent status must not render as \"0\": the string protocols write the key only when" + + " the span carries a status"); + } + + @Test + void reusesTheRenderedStatusAcrossSpans() { + // The point of routing through RadixTreeCache rather than rendering per span: two spans with + // the same status share one UTF8BytesString instead of allocating one apiece. + UTF8BytesString first = metadataWithStatus(404).getHttpStatusCodeString(); + UTF8BytesString second = metadataWithStatus(404).getHttpStatusCodeString(); + + assertSame(first, second); + } + + private static Metadata metadataWithStatus(int status) { + return new Metadata( + Thread.currentThread().getId(), + UTF8BytesString.create("main"), + TagMap.fromMap(emptyMap()), + emptyMap(), + 0, + false, + false, + status, + null, + 0, + null, + emptyList()); + } +} diff --git a/dd-trace-core/src/traceAgentTest/java/TraceGenerator.java b/dd-trace-core/src/traceAgentTest/java/TraceGenerator.java index b349e937e68..1e1a9c58fe3 100644 --- a/dd-trace-core/src/traceAgentTest/java/TraceGenerator.java +++ b/dd-trace-core/src/traceAgentTest/java/TraceGenerator.java @@ -178,7 +178,7 @@ static class PojoSpan implements CoreSpan { UNSET, measured, isTopLevel(), - null, + 0, null, 0, getTagsForSerialization(), From 75df78b580ddad57074e97642724d95b07b72f04 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Fri, 11 Sep 2026 15:41:39 +0100 Subject: [PATCH 22/30] Return non-null traceConfig from ExtractedSpan (#12413) Return non-null traceConfig from ExtractedSpan Annotate AgentSpan.traceConfig() as non-null, migrate ExtractedSpanTest to JUnit 5 Co-Authored-By: Claude Sonnet 5 Co-authored-by: devflow.devflow-routing-intake --- .../instrumentation/api/AgentSpan.java | 1 + .../instrumentation/api/ExtractedSpan.java | 8 +- .../api/ExtractedSpanTest.groovy | 53 ----------- .../api/ExtractedSpanTest.java | 91 +++++++++++++++++++ 4 files changed, 99 insertions(+), 54 deletions(-) delete mode 100644 internal-api/src/test/groovy/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.groovy create mode 100644 internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.java diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java index aa270020e78..df6b2f2054a 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java @@ -211,6 +211,7 @@ default boolean isValid() { AgentSpan setSamplingPriority(final int newPriority, int samplingMechanism); + @Nonnull TraceConfig traceConfig(); void addLink(AgentSpanLink link); diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpan.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpan.java index 80d2331ce43..d589141b8d2 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpan.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpan.java @@ -140,7 +140,13 @@ public AgentSpanContext spanContext() { @Override public TraceConfig traceConfig() { - return null; + if (this.spanContext instanceof TagContext) { + TraceConfig traceConfig = ((TagContext) this.spanContext).getTraceConfig(); + if (traceConfig != null) { + return traceConfig; + } + } + return AgentTracer.traceConfig(); } @Override diff --git a/internal-api/src/test/groovy/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.groovy b/internal-api/src/test/groovy/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.groovy deleted file mode 100644 index 81bda406fbc..00000000000 --- a/internal-api/src/test/groovy/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.groovy +++ /dev/null @@ -1,53 +0,0 @@ -package datadog.trace.bootstrap.instrumentation.api - -import datadog.trace.api.DDTraceId -import datadog.trace.api.TagMap -import spock.lang.Specification - -class ExtractedSpanTest extends Specification { - def 'test extracted span from partial tracing context'() { - given: - def tags = TagMap.fromMap(['tag-1': 'value-1', 'tag-2': 'value-2']) - def baggage = ['baggage-1': 'value-1', 'baggage-2': 'value-2'] - def traceId = DDTraceId.from(12345) - def context = new TagContext('origin', tags, null, baggage, 0, null, null, traceId) - def extractedSpan = new ExtractedSpan(context) - - expect: - extractedSpan.getTraceId() == traceId - extractedSpan.getSpanId() == context.getSpanId() - extractedSpan.spanContext() == context - extractedSpan.getTags() == tags - extractedSpan.getTag('tag-1') == 'value-1' - extractedSpan.getBaggageItem('baggage-2') == 'value-2' - extractedSpan.isSameTrace(new ExtractedSpan(context)) - extractedSpan.toString() != null - - when: - extractedSpan.setTag('tag-1', 'updated') - extractedSpan.setBaggageItem('baggage-2', 'updated') - - then: - extractedSpan.getTag('tag-1') == 'value-1' - extractedSpan.getBaggageItem('baggage-2') == 'value-2' - } - - def 'test extracted span from custom span context'() { - given: - def context = Mock(AgentSpanContext) - context.getTraceId() >> DDTraceId.from(12345) - context.getSpanId() >> 67890 - context.baggageItems() >> Collections.emptyMap().entrySet() - def extractedSpan = new ExtractedSpan(context) - - expect: - extractedSpan.getTraceId() == context.getTraceId() - extractedSpan.getSpanId() == context.getSpanId() - extractedSpan.spanContext() == context - extractedSpan.getTags().isEmpty() - extractedSpan.getTag('tag-1') == null - extractedSpan.getBaggageItem('baggage-2') == null - extractedSpan.isSameTrace(new ExtractedSpan(context)) - extractedSpan.toString() != null - } -} diff --git a/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.java b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.java new file mode 100644 index 00000000000..852e0142ba2 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/ExtractedSpanTest.java @@ -0,0 +1,91 @@ +package datadog.trace.bootstrap.instrumentation.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.trace.api.DDTraceId; +import datadog.trace.api.TagMap; +import datadog.trace.api.TraceConfig; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ExtractedSpanTest { + + @Test + void extractedSpanFromPartialTracingContext() { + Map tagValues = new HashMap<>(); + tagValues.put("tag-1", "value-1"); + tagValues.put("tag-2", "value-2"); + TagMap tags = TagMap.fromMap(tagValues); + Map baggage = new HashMap<>(); + baggage.put("baggage-1", "value-1"); + baggage.put("baggage-2", "value-2"); + DDTraceId traceId = DDTraceId.from(12345); + TagContext context = new TagContext("origin", tags, null, baggage, 0, null, null, traceId); + ExtractedSpan extractedSpan = new ExtractedSpan(context); + + assertEquals(traceId, extractedSpan.getTraceId()); + assertEquals(context.getSpanId(), extractedSpan.getSpanId()); + assertEquals(context, extractedSpan.spanContext()); + assertEquals(tags, extractedSpan.getTags()); + assertEquals("value-1", extractedSpan.getTag("tag-1")); + assertEquals("value-2", extractedSpan.getBaggageItem("baggage-2")); + assertTrue(extractedSpan.isSameTrace(new ExtractedSpan(context))); + assertNotNull(extractedSpan.toString()); + + extractedSpan.setTag("tag-1", "updated"); + extractedSpan.setBaggageItem("baggage-2", "updated"); + + assertEquals("value-1", extractedSpan.getTag("tag-1")); + assertEquals("value-2", extractedSpan.getBaggageItem("baggage-2")); + } + + @Test + void extractedSpanFromCustomSpanContext() { + AgentSpanContext context = mock(AgentSpanContext.class); + when(context.getTraceId()).thenReturn(DDTraceId.from(12345)); + when(context.getSpanId()).thenReturn(67890L); + when(context.baggageItems()).thenReturn(Collections.emptyMap().entrySet()); + ExtractedSpan extractedSpan = new ExtractedSpan(context); + + assertEquals(context.getTraceId(), extractedSpan.getTraceId()); + assertEquals(context.getSpanId(), extractedSpan.getSpanId()); + assertEquals(context, extractedSpan.spanContext()); + assertTrue(extractedSpan.getTags().isEmpty()); + assertNull(extractedSpan.getTag("tag-1")); + assertNull(extractedSpan.getBaggageItem("baggage-2")); + assertTrue(extractedSpan.isSameTrace(new ExtractedSpan(context))); + assertNotNull(extractedSpan.toString()); + } + + @Test + void traceConfigReturnsExtractedSnapshotWhenPresent() { + TraceConfig snapshot = mock(TraceConfig.class); + TagContext context = new TagContext(null, null, null, null, 0, snapshot, null, DDTraceId.ZERO); + ExtractedSpan extractedSpan = new ExtractedSpan(context); + + assertEquals(snapshot, extractedSpan.traceConfig()); + } + + @Test + void traceConfigFallsBackToCurrentConfigWhenSnapshotAbsent() { + TagContext context = new TagContext(); + ExtractedSpan extractedSpan = new ExtractedSpan(context); + + assertNotNull(extractedSpan.traceConfig()); + } + + @Test + void traceConfigFallsBackToCurrentConfigForCustomSpanContext() { + AgentSpanContext context = mock(AgentSpanContext.class); + ExtractedSpan extractedSpan = new ExtractedSpan(context); + + assertNotNull(extractedSpan.traceConfig()); + } +} From b0ce4bb131293876aec06594a97ebaf4c229fb82 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 11 Sep 2026 18:56:31 +0200 Subject: [PATCH 23/30] Prevent duplicate Kotlin coroutine continuation release (#12466) Prevent duplicate Kotlin coroutine continuation release Co-authored-by: andrea.marziali --- .../coroutines/DatadogThreadContextElement.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/kotlin-coroutines-1.3/src/main/java/datadog/trace/instrumentation/kotlin/coroutines/DatadogThreadContextElement.java b/dd-java-agent/instrumentation/kotlin-coroutines-1.3/src/main/java/datadog/trace/instrumentation/kotlin/coroutines/DatadogThreadContextElement.java index 3e52c58ccb3..55516db89e0 100644 --- a/dd-java-agent/instrumentation/kotlin-coroutines-1.3/src/main/java/datadog/trace/instrumentation/kotlin/coroutines/DatadogThreadContextElement.java +++ b/dd-java-agent/instrumentation/kotlin-coroutines-1.3/src/main/java/datadog/trace/instrumentation/kotlin/coroutines/DatadogThreadContextElement.java @@ -2,6 +2,7 @@ import datadog.context.Context; import datadog.context.ContextContinuation; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import javax.annotation.Nonnull; import javax.annotation.Nullable; import kotlin.coroutines.CoroutineContext; @@ -11,6 +12,11 @@ /** Manages the Datadog context for coroutines, switching contexts as coroutines switch threads. */ public final class DatadogThreadContextElement implements ThreadContextElement { + private static final AtomicReferenceFieldUpdater + CONTINUATION = + AtomicReferenceFieldUpdater.newUpdater( + DatadogThreadContextElement.class, ContextContinuation.class, "continuation"); + private static final CoroutineContext.Key DATADOG_KEY = new CoroutineContext.Key() {}; @@ -22,7 +28,7 @@ public static CoroutineContext addDatadogElement(CoroutineContext coroutineConte } private Context context; - private ContextContinuation continuation; + private volatile ContextContinuation continuation; @Nonnull @Override @@ -42,9 +48,11 @@ public static void captureDatadogContext(@Nonnull AbstractCoroutine coroutine public static void cancelDatadogContext(@Nonnull AbstractCoroutine coroutine) { DatadogThreadContextElement datadog = coroutine.getContext().get(DATADOG_KEY); - if (datadog != null && datadog.continuation != null) { + ContextContinuation continuation = + datadog == null ? null : CONTINUATION.getAndSet(datadog, null); + if (continuation != null) { // release enclosing trace now the coroutine has completed - datadog.continuation.release(); + continuation.release(); } } From 64c7b9f317deb1dc60e91d50cbd8aa6530c34f94 Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Fri, 11 Sep 2026 18:06:24 +0100 Subject: [PATCH 24/30] Use ObjectStore to track context bound to arbitrary objects. (#12463) Use ObjectStore to track context bound to arbitrary objects. Replaces the placeholder synchronized WeakHashMap approach. Co-authored-by: devflow.devflow-routing-intake --- components/context/build.gradle.kts | 4 ++++ .../datadog/context/ContextProviders.java | 2 +- ...der.java => ObjectStoreContextBinder.java} | 19 +++++++++---------- .../datadog/context/TestContextBinder.java | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) rename components/context/src/main/java/datadog/context/{WeakMapContextBinder.java => ObjectStoreContextBinder.java} (55%) diff --git a/components/context/build.gradle.kts b/components/context/build.gradle.kts index f58031d2ef4..8a41745f364 100644 --- a/components/context/build.gradle.kts +++ b/components/context/build.gradle.kts @@ -2,6 +2,10 @@ plugins { id("dd-trace-java.module.platform-component") } +dependencies { + implementation(libs.instrument.java) +} + extra["excludedClassesInstructionCoverage"] = listOf("datadog.context.ContextProviders") // covered by forked test diff --git a/components/context/src/main/java/datadog/context/ContextProviders.java b/components/context/src/main/java/datadog/context/ContextProviders.java index c4421b0a2ab..a7431eaa196 100644 --- a/components/context/src/main/java/datadog/context/ContextProviders.java +++ b/components/context/src/main/java/datadog/context/ContextProviders.java @@ -17,7 +17,7 @@ private static final class ProvidedBinder { static final ContextBinder INSTANCE = null != ContextProviders.customBinder ? ContextProviders.customBinder - : WeakMapContextBinder.INSTANCE; + : ObjectStoreContextBinder.INSTANCE; } static ContextManager manager() { diff --git a/components/context/src/main/java/datadog/context/WeakMapContextBinder.java b/components/context/src/main/java/datadog/context/ObjectStoreContextBinder.java similarity index 55% rename from components/context/src/main/java/datadog/context/WeakMapContextBinder.java rename to components/context/src/main/java/datadog/context/ObjectStoreContextBinder.java index 15e0154f25a..d9af1947033 100644 --- a/components/context/src/main/java/datadog/context/WeakMapContextBinder.java +++ b/components/context/src/main/java/datadog/context/ObjectStoreContextBinder.java @@ -1,22 +1,21 @@ package datadog.context; import static datadog.context.Context.root; -import static java.util.Collections.synchronizedMap; import static java.util.Objects.requireNonNull; -import java.util.Map; -import java.util.WeakHashMap; +import datadog.instrument.fieldinject.ObjectStore; -/** {@link ContextBinder} that uses a global weak map of carriers to contexts. */ -final class WeakMapContextBinder implements ContextBinder { - static final ContextBinder INSTANCE = new WeakMapContextBinder(); +/** {@link ContextBinder} that uses {@link ObjectStore} to track context carriers. */ +final class ObjectStoreContextBinder implements ContextBinder { + static final ContextBinder INSTANCE = new ObjectStoreContextBinder(); - private static final Map TRACKED = synchronizedMap(new WeakHashMap<>()); + private static final ObjectStore CONTEXT_STORE = + ObjectStore.of(Object.class, Context.class); @Override public Context from(Object carrier) { requireNonNull(carrier, "Context carrier cannot be null"); - Context bound = TRACKED.get(carrier); + Context bound = CONTEXT_STORE.get(carrier); return null != bound ? bound : root(); } @@ -24,13 +23,13 @@ public Context from(Object carrier) { public void attachTo(Object carrier, Context context) { requireNonNull(carrier, "Context carrier cannot be null"); requireNonNull(context, "Context cannot be null. Use detachFrom() instead."); - TRACKED.put(carrier, context); + CONTEXT_STORE.put(carrier, context); } @Override public Context detachFrom(Object carrier) { requireNonNull(carrier, "Context key cannot be null"); - Context previous = TRACKED.remove(carrier); + Context previous = CONTEXT_STORE.remove(carrier); return null != previous ? previous : root(); } } diff --git a/components/context/src/main/java/datadog/context/TestContextBinder.java b/components/context/src/main/java/datadog/context/TestContextBinder.java index e1fbbcf0a7f..8717019d9cd 100644 --- a/components/context/src/main/java/datadog/context/TestContextBinder.java +++ b/components/context/src/main/java/datadog/context/TestContextBinder.java @@ -31,7 +31,7 @@ private static ContextBinder delegate() { ContextBinder delegate = ContextProviders.customBinder; if (delegate == TEST_INSTANCE) { // fall back to default context binder - return WeakMapContextBinder.INSTANCE; + return ObjectStoreContextBinder.INSTANCE; } else { return delegate; } From 3110972ccf744f304a7cf59973e240f9ccd1a071 Mon Sep 17 00:00:00 2001 From: Douglas Q Hawkins Date: Fri, 11 Sep 2026 13:23:34 -0400 Subject: [PATCH 25/30] Introduce SpanPrototype: baked-once constant span-tag descriptor (phase 1b) (#11894) Introduce SpanPrototype: baked-once constant span-tag descriptor The builder API (extends_/init*) plus its per-mechanism microbenchmark and a pure-API test, split out from the combined span-prototype work so the abstraction lands independently of the decorator demo. Co-Authored-By: Claude Opus 4.8 Drop null/empty SpanPrototype constants; add TagMap.Entry.isEmptyValue A prototype constant that is null or an empty CharSequence should be "no tag" -- matching AgentSpan.setTag and the decorators' cached-Entry path -- not a baked empty tag. Add TagMap.Entry.isEmptyValue as the single definition of an empty value (both Entry.create overloads now delegate to it), and gate SpanPrototype.Builder.initTag on it via the plain set(key, value) path so no Entry is allocated (the wrong path once tags are stored densely). Co-Authored-By: Claude Opus 4.8 Add SpanPrototype construction path: buildSpan/startSpan(SpanPrototype) Thread a SpanPrototype through span construction: AgentTracer gains buildSpan/startSpan(SpanPrototype, operationName) (defaults seed identity only, correct for the noop tracer, with an explicit NoopTracerAPI.startSpan override). CoreTracer overrides buildSpan to seed the prototype's frozen constant tags in buildSpanContext at the precedence slot just before the builder's own tags (prototype and builder form one precedence atom; explicit builder tags win), and overrides startSpan to seed builder-free via the static CoreSpanBuilder.startSpan path (no MultiSpanBuilder allocation, mirroring startSpan(String,...)). Explicit operationName wins; null falls back to the prototype's. Intercepted constants (e.g. span.kind) seed through the interceptor so their context side-effects still fire. Prototype params @Nonnull. Co-Authored-By: Claude Opus 4.8 Carry integration name in SpanPrototype; make the builder surface uniformly init* BaseDecorator.afterStart sets the integration name as a side effect alongside the component tag (setIntegrationName(component)), which IntegrationAdder later serializes as _dd.integration. A prototype baking only the component tag would drop that. Add initComponentAndIntegration(component): sets the component tag AND records it as the integration name (inherited via extends_), applied via setIntegrationName at construction. Rename the builder setters to a uniform init* surface now that a component sibling exists and to convey "everything here bakes the prototype's initial state": initComponent -> initComponentOnly, instrumentationName -> initInstrumentationName(s), operationName -> initOperationName, spanType -> initSpanType. Accessors are unchanged. Renames are confined to SpanPrototype.Builder and its callers. Co-Authored-By: Claude Opus 4.8 Add span-creation benchmark for the SpanPrototype construction path A dd-trace-core JMH benchmark covering the full create -> (tag) -> finish lifecycle, finished against a no-op DropWriter so -prof gc isolates create/tag/finish allocation from serialization. Pairs baseline shapes (web-server 7 tags, JDBC 9 tags; setTag and builder-withTag) with prototype arms: buildSpan(SpanPrototype).start() and the builder-free startSpan(SpanPrototype). Measured (Threads(8), -f3 -wi5 -i5 -prof gc): prototype construction cuts gc.alloc.rate.norm ~-5% web (-80 B/op) / ~-10% jdbc (-120 B/op) vs baseline -- tracking the number of baked constants (fewer per-span TagMap.Entry allocations). The builder-free startSpan is deterministic (no MultiSpanBuilder); buildSpan's builder is escape-analyzed away in this shallow micro, so startSpan is the EA-independent path for production's deeper/megamorphic call sites. Co-Authored-By: Claude Opus 4.8 Add AgentSpan/DDSpanContext.apply(SpanPrototype) seam; route construction through it Introduce apply(SpanPrototype) as the single seam for stamping a prototype's constant initial state. It applies span type, constant tags, and integration name as fallback defaults -- only where the span has not already set them -- so it never clobbers explicit values, is order-independent, and self-neutralizes once construction has already seeded the same prototype. DDSpanContext.apply is the authoritative implementation (the context owns the tag map and will host the eventual bulk-share fast path + identity short-circuit); DDSpan.apply routes straight to it. The AgentSpan default is the best-effort fallback for non-core spans. The construction path (CoreSpanBuilder) now calls context.apply(prototype) instead of inlining the tag + integration-name seeding. Co-Authored-By: Claude Opus 4.8 Guard integration name in AgentSpan.apply via AgentSpanContext getter Add getIntegrationName() (default null) to AgentSpanContext, symmetric with the existing no-op setIntegrationName, so the default apply() can honor never-clobber like DDSpanContext.apply instead of unconditionally overwriting. Co-Authored-By: Claude Opus 4.8 Override NoopTracerAPI.buildSpan(SpanPrototype) to return null Mirrors the existing buildSpan(String,...) noop contract so the prototype builder path matches. startSpan(SpanPrototype) was already noop-safe. A chainable NoopSpanBuilder to fix the null-vs-NoopSpan asymmetry is left to a separate PR. Co-Authored-By: Claude Opus 4.8 Bulk up SpanPrototype.Builder test coverage Cover every Builder branch and getter so the SpanPrototype.Builder jacoco rule (branch >= 0.7, instr >= 0.8) that failed test_base is satisfied: initInstrumentationNames null/empty/multi, extends_(null) + full copy, initComponentAndIntegration set/empty, initTag(Object)/initTag(EntryReader) null and non-null, and all five getters incl. integrationName(). Co-Authored-By: Claude Opus 4.8 Merge remote-tracking branch 'origin/master' into HEAD # Conflicts: # dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java Expose SpanPrototype starts through the static AgentTracer facade Auto-instrumentation statically imports AgentTracer, but only the instance-level TracerAPI had a startSpan(SpanPrototype, CharSequence) overload. Add the matching static facade method so callers can use it. Co-Authored-By: Claude Sonnet 5 Merge branch 'master' into dougqh/span-prototype-api Co-authored-by: devflow.devflow-routing-intake --- .../trace/core/SpanCreationBenchmark.java | 93 ++++++++- .../java/datadog/trace/core/CoreTracer.java | 75 ++++++++ .../main/java/datadog/trace/core/DDSpan.java | 8 + .../datadog/trace/core/DDSpanContext.java | 57 ++++++ .../core/SpanPrototypeConstructionTest.java | 159 ++++++++++++++++ .../trace/api/SpanPrototypeBenchmark.java | 96 ++++++++++ .../main/java/datadog/trace/api/TagMap.java | 24 +-- .../instrumentation/api/AgentSpan.java | 40 ++++ .../instrumentation/api/AgentSpanContext.java | 9 + .../instrumentation/api/AgentTracer.java | 48 +++++ .../instrumentation/api/SpanPrototype.java | 179 +++++++++++++++++ .../api/SpanPrototypeTest.java | 180 ++++++++++++++++++ 12 files changed, 956 insertions(+), 12 deletions(-) create mode 100644 dd-trace-core/src/test/java/datadog/trace/core/SpanPrototypeConstructionTest.java create mode 100644 internal-api/src/jmh/java/datadog/trace/api/SpanPrototypeBenchmark.java create mode 100644 internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototype.java create mode 100644 internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototypeTest.java diff --git a/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java b/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java index d288463faad..c2bcd4a1cfb 100644 --- a/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java +++ b/dd-trace-core/src/jmh/java/datadog/trace/core/SpanCreationBenchmark.java @@ -3,6 +3,7 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.SpanPrototype; import datadog.trace.bootstrap.instrumentation.api.Tags; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -42,7 +43,10 @@ * five-method {@code Writer} interface (implemented as a no-op {@link DropWriter}). If you add to * it, keep it inside that stable surface or grafting it onto old tags for the historical curve will * stop compiling. (Source rebuilds only reach ~v1.53 — older tags hit dead build-time dependencies; - * deeper history is a published-jar job.) + * deeper history is a published-jar job.) The {@code *ViaPrototype} / {@code + * *ViaPrototypeStartSpan} arms are the exception: they exercise {@link SpanPrototype} (new in this + * PR) and do not graft onto pre-SpanPrototype tags — the historical table above covers the baseline + * arms only. * *

Spans are finished against {@link DropWriter} so the create/tag/finish allocation is isolated * from serialization and agent I/O — those live on a different lever and would otherwise leak into @@ -132,11 +136,33 @@ public class SpanCreationBenchmark { CoreTracer tracer; + // Baked-once prototypes carrying only the type-constant subset each baseline sets individually + // (component + span.kind; jdbc also db.type). The dynamic tags are set per-span in both arms, so + // the *ViaPrototype vs *Span delta isolates the construction-path seeding of just those + // constants. + SpanPrototype webProto; + SpanPrototype jdbcProto; + @Setup public void setup(Blackhole blackhole) { // DropWriter keeps finish() from pulling in serialization / agent I/O, so -prof gc reflects // span creation + tagging + PendingTrace completion only. this.tracer = CoreTracer.builder().writer(new DropWriter(blackhole)).build(); + this.webProto = + SpanPrototype.builder() + .initInstrumentationName(INSTRUMENTATION_NAME) + .initOperationName(SERVER_OPERATION_NAME) + .initComponentOnly(COMPONENT_VALUE) + .initKind(Tags.SPAN_KIND_SERVER) + .build(); + this.jdbcProto = + SpanPrototype.builder() + .initInstrumentationName(INSTRUMENTATION_NAME) + .initOperationName(JDBC_OPERATION_NAME) + .initComponentOnly(DB_COMPONENT_VALUE) + .initKind(Tags.SPAN_KIND_CLIENT) + .initTag(Tags.DB_TYPE, DB_TYPE_VALUE) + .build(); } @TearDown @@ -211,4 +237,69 @@ public void jdbcClientSpan() { span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); span.finish(); } + + /** + * Web-server-shaped span via {@link SpanPrototype}: the type-constants (component, span.kind) + * ride a baked-once prototype seeded at construction; the dynamic http.* / peer.port tags are set + * per-span, as real instrumentation does. Compare against {@link #webServerSpan} (identical tags, + * all set individually) to read the prototype's construction-path win on a full span. + */ + @Benchmark + public void webServerSpanViaPrototype() { + AgentSpan span = tracer.buildSpan(webProto, null).start(); // null -> prototype's operationName + span.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + span.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + span.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + span.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + span.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + span.finish(); + } + + /** + * JDBC/DB-client-shaped span via {@link SpanPrototype}: component, span.kind, and db.type ride + * the prototype; the dynamic db.* / peer.* tags are set per-span. Compare against {@link + * #jdbcClientSpan}. + */ + @Benchmark + public void jdbcClientSpanViaPrototype() { + AgentSpan span = tracer.buildSpan(jdbcProto, null).start(); + span.setTag(Tags.DB_INSTANCE, DB_INSTANCE_VALUE); + span.setTag(Tags.DB_USER, DB_USER_VALUE); + span.setTag(Tags.DB_OPERATION, DB_OPERATION_VALUE); + span.setTag(Tags.DB_STATEMENT, DB_STATEMENT_VALUE); + span.setTag(Tags.PEER_HOSTNAME, DB_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); + span.finish(); + } + + /** + * Web-server-shaped span via {@code startSpan(SpanPrototype, ...)} — the builder-free + * construction entry (no MultiSpanBuilder allocation), the auto-instrumentation path. Compare + * against {@link #webServerSpanViaPrototype} (same prototype, but {@code buildSpan(...).start()} + * allocates a builder) to read the builder-free saving, and against {@link #webServerSpan} for + * the full win. + */ + @Benchmark + public void webServerSpanViaPrototypeStartSpan() { + AgentSpan span = tracer.startSpan(webProto, null); // null -> prototype's operationName + span.setTag(Tags.HTTP_METHOD, HTTP_METHOD_VALUE); + span.setTag(Tags.HTTP_ROUTE, HTTP_ROUTE_VALUE); + span.setTag(Tags.HTTP_URL, HTTP_URL_VALUE); + span.setTag(Tags.HTTP_STATUS, HTTP_STATUS_VALUE); + span.setTag(Tags.PEER_PORT, PEER_PORT_VALUE); + span.finish(); + } + + /** JDBC/DB-client-shaped span via the builder-free {@code startSpan(SpanPrototype, ...)}. */ + @Benchmark + public void jdbcClientSpanViaPrototypeStartSpan() { + AgentSpan span = tracer.startSpan(jdbcProto, null); + span.setTag(Tags.DB_INSTANCE, DB_INSTANCE_VALUE); + span.setTag(Tags.DB_USER, DB_USER_VALUE); + span.setTag(Tags.DB_OPERATION, DB_OPERATION_VALUE); + span.setTag(Tags.DB_STATEMENT, DB_STATEMENT_VALUE); + span.setTag(Tags.PEER_HOSTNAME, DB_PEER_HOSTNAME_VALUE); + span.setTag(Tags.PEER_PORT, DB_PEER_PORT_VALUE); + span.finish(); + } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java index c280000f252..55ee9e2372c 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/CoreTracer.java @@ -76,6 +76,7 @@ import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; import datadog.trace.bootstrap.instrumentation.api.SpanAttributes; import datadog.trace.bootstrap.instrumentation.api.SpanLink; +import datadog.trace.bootstrap.instrumentation.api.SpanPrototype; import datadog.trace.bootstrap.instrumentation.api.TagContext; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.civisibility.interceptor.CiVisibilityApmProtocolInterceptor; @@ -133,6 +134,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; import java.util.zip.ZipOutputStream; +import javax.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1077,6 +1079,27 @@ public CoreSpanBuilder buildSpan( return createMultiSpanBuilder(instrumentationName, operationName); } + /** + * Seeds identity (instrumentation name, operation, span type) and constant tags from a prototype. + * {@code operationName} overrides the prototype's when non-null — the explicit value wins, the + * prototype is the fallback. The prototype's tags are seeded during {@link CoreSpanBuilder} + * construction just before the builder's own tags, so explicit tags override prototype constants. + */ + @Override + public CoreSpanBuilder buildSpan( + @Nonnull final SpanPrototype prototype, CharSequence operationName) { + if (operationName == null) { + operationName = prototype.operationName(); + } + CoreSpanBuilder builder = + createMultiSpanBuilder(prototype.instrumentationName(), operationName); + builder.spanPrototype = prototype; + if (prototype.spanType() != null) { + builder.spanType = prototype.spanType(); + } + return builder; + } + MultiSpanBuilder createMultiSpanBuilder( final String instrumentationName, final CharSequence operationName) { return new MultiSpanBuilder(this, instrumentationName, operationName); @@ -1182,6 +1205,17 @@ public AgentSpan startSpan( this, instrumentationName, spanName, parent, CoreSpanBuilder.IGNORE_SCOPE, startTimeMicros); } + @Override + public AgentSpan startSpan( + @Nonnull final SpanPrototype prototype, final CharSequence operationName) { + return CoreSpanBuilder.startSpan( + this, + prototype, + operationName != null ? operationName : prototype.operationName(), + CoreSpanBuilder.USE_SCOPE, + CoreSpanBuilder.AUTO_ASSIGN_TIMESTAMP); + } + @Override public AgentScope activateSpan(AgentSpan span) { return scopeManager.activateSpan(span); @@ -1622,6 +1656,7 @@ public abstract static class CoreSpanBuilder implements AgentTracer.SpanBuilder // Builder attributes // Make sure any fields added here are also reset properly in ReusableSingleSpanBuilder.reset protected TagMap.Ledger tagLedger; + protected SpanPrototype spanPrototype = SpanPrototype.NONE; protected long timestampMicro; protected AgentSpanContext parent; protected String serviceName; @@ -1660,6 +1695,7 @@ protected static final DDSpan buildSpan( boolean errorFlag, CharSequence spanType, TagMap.Ledger tagLedger, + SpanPrototype spanPrototype, List links, Object builderRequestContextDataAppSec, Object builderRequestContextDataIast, @@ -1679,6 +1715,7 @@ protected static final DDSpan buildSpan( errorFlag, spanType, tagLedger, + spanPrototype, links, builderRequestContextDataAppSec, builderRequestContextDataIast, @@ -1764,6 +1801,7 @@ protected AgentSpan startImpl() { this.errorFlag, this.spanType, this.tagLedger, + this.spanPrototype, this.links, this.builderRequestContextDataAppSec, this.builderRequestContextDataIast, @@ -1790,6 +1828,33 @@ protected static final AgentSpan startSpan( false /* errorFlag */, null /* spanType */, null /* tagLedger */, + SpanPrototype.NONE /* spanPrototype */, + null /* links */, + null /* appSec */, + null /* iast */, + null /* ciViz */); + } + + protected static final AgentSpan startSpan( + final CoreTracer tracer, + final SpanPrototype prototype, + final CharSequence operationName, + final boolean ignoreScope, + final long timestampMicros) { + return startSpan( + tracer, + AUTO_ASSIGN_SPAN_ID, + prototype.instrumentationName(), + timestampMicros, + null /* serviceName */, + operationName, + null /* resourceName */, + null /* specifiedParentSpanContext */, + ignoreScope, + false /* errorFlag */, + prototype.spanType(), + null /* tagLedger */, + prototype, null /* links */, null /* appSec */, null /* iast */, @@ -1809,6 +1874,7 @@ protected static final AgentSpan startSpan( boolean errorFlag, CharSequence spanType, TagMap.Ledger tagLedger, + SpanPrototype spanPrototype, List links, Object builderRequestContextDataAppSec, Object builderRequestContextDataIast, @@ -1871,6 +1937,7 @@ protected static final AgentSpan startSpan( errorFlag, spanType, tagLedger, + spanPrototype, links, builderRequestContextDataAppSec, builderRequestContextDataIast, @@ -2009,6 +2076,7 @@ protected static final DDSpanContext buildSpanContext( boolean errorFlag, CharSequence spanType, TagMap.Ledger tagLedger, + SpanPrototype spanPrototype, List links, Object builderRequestContextDataAppSec, Object builderRequestContextDataIast, @@ -2256,6 +2324,12 @@ protected static final DDSpanContext buildSpanContext( if (mergedTracerTagsNeedsIntercept) { context.setAllTags(mergedTracerTags, true); } + if (spanPrototype != SpanPrototype.NONE) { + // Seed the prototype's constant tags + integration name as fallback defaults (span type was + // already seeded onto the builder). apply never clobbers, so tags set below still win, and + // this is the same seam decorator afterStart uses. + context.apply(spanPrototype); + } context.setAllTags(tagLedger); context.setAllTags(coreTags, coreTagsNeedsIntercept); context.setAllTags(rootSpanTags, rootSpanTagsNeedsIntercept); @@ -2340,6 +2414,7 @@ final boolean reset(String instrumentationName, CharSequence operationName) { this.operationName = operationName; if (this.tagLedger != null) this.tagLedger.reset(); + this.spanPrototype = SpanPrototype.NONE; this.timestampMicro = 0L; this.parent = null; this.serviceName = null; diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java index 5517952358e..57809e76069 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpan.java @@ -31,6 +31,7 @@ import datadog.trace.bootstrap.instrumentation.api.AttachableWrapper; import datadog.trace.bootstrap.instrumentation.api.ErrorPriorities; import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities; +import datadog.trace.bootstrap.instrumentation.api.SpanPrototype; import datadog.trace.bootstrap.instrumentation.api.SpanWrapper; import datadog.trace.core.util.StackTraces; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -671,6 +672,13 @@ public final DDSpan setSpanType(final CharSequence type) { return this; } + @Override + public void apply(@Nonnull final SpanPrototype prototype) { + // Route straight to the context (owner of the tag map + future fast path) rather than through + // the interface default's per-setter delegation. + context.apply(prototype); + } + // Getters @Override diff --git a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java index 8520a222424..1b211b5fae1 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/DDSpanContext.java @@ -29,6 +29,7 @@ import datadog.trace.bootstrap.instrumentation.api.ProfilerContext; import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration; import datadog.trace.bootstrap.instrumentation.api.ResourceNamePriorities; +import datadog.trace.bootstrap.instrumentation.api.SpanPrototype; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.core.propagation.PropagationTags; @@ -600,6 +601,61 @@ public void setSpanType(final CharSequence spanType) { this.spanType = spanType; } + /** + * Applies a {@link SpanPrototype} as fallback defaults: stamps its span type, constant tags, and + * integration name only where the span has not already set them. Prototype values are the lowest + * precedence -- anything explicitly set (a builder {@code withSpanType}, explicit tags, an + * earlier decorator) wins. Because it never clobbers, {@code apply} is order-independent and + * self-neutralizes once construction has already seeded the same prototype. + * + *

This is the shared seam for both the construction path ({@code CoreSpanBuilder}) and + * decorator {@code afterStart} (via {@link DDSpan#apply}). The context owns the tag map, so the + * eventual cheaper bulk-share path (skipping interception for non-intercepted tags) and the + * identity short-circuit will land here -- deferred to the dense-store / tag-registry work, which + * exposes intercept status at the internal-api level. Until then the constant tags route through + * the interceptor, identical to the per-tag calls this replaces. + */ + public void apply(@Nonnull final SpanPrototype prototype) { + if (this.spanType == null) { + final CharSequence spanType = prototype.spanType(); + if (spanType != null) { + setSpanType(spanType); + } + } + seedAbsentTags(prototype.tags()); + if (this.integrationName == null) { + final CharSequence integrationName = prototype.integrationName(); + if (integrationName != null) { + setIntegrationName(integrationName); + } + } + } + + /** + * Seeds tags that are not already present, routed through the interceptor. Mirrors {@link + * #setAllTags(TagMap, boolean)}'s intercepting path but skips any key already set, so explicit + * tags keep precedence over the prototype's constant defaults. + */ + private void seedAbsentTags(final TagMap map) { + if (map == null) { + return; + } + synchronized (unsafeTags) { + map.forEach( + this, + (ctx, tagEntry) -> { + final String tag = tagEntry.tag(); + if (ctx.unsafeTags.containsKey(tag)) { + return; + } + final Object value = tagEntry.objectValue(); + if (!ctx.tagInterceptor.interceptTag(ctx, tag, value)) { + ctx.unsafeTags.set(tagEntry); + } + }); + } + } + /** Forces the local root span sampling decision to keep according manual mechanism. */ public void forceKeep() { forceKeep(SamplingMechanism.MANUAL); @@ -1390,6 +1446,7 @@ public void setIntegrationName(CharSequence integrationName) { this.integrationName = integrationName; } + @Override public CharSequence getIntegrationName() { return integrationName; } diff --git a/dd-trace-core/src/test/java/datadog/trace/core/SpanPrototypeConstructionTest.java b/dd-trace-core/src/test/java/datadog/trace/core/SpanPrototypeConstructionTest.java new file mode 100644 index 00000000000..7a3722d77cf --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/core/SpanPrototypeConstructionTest.java @@ -0,0 +1,159 @@ +package datadog.trace.core; + +import static datadog.trace.bootstrap.instrumentation.api.Tags.COMPONENT; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND; +import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_SERVER; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.trace.bootstrap.instrumentation.api.SpanPrototype; +import datadog.trace.common.writer.ListWriter; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Verifies the SpanPrototype construction path: {@code buildSpan(prototype, operationName)} and + * {@code startSpan(prototype, operationName)} seed the prototype's identity + constant tags, with + * the explicit operationName / builder tags overriding the prototype's (prototype = defaults). + */ +public class SpanPrototypeConstructionTest extends DDCoreJavaSpecification { + + private ListWriter writer; + private CoreTracer tracer; + private SpanPrototype prototype; + + @BeforeEach + void setup() { + writer = new ListWriter(); + tracer = tracerBuilder().writer(writer).build(); + prototype = + SpanPrototype.builder() + .initInstrumentationName("test-instr") + .initOperationName("proto.op") + .initSpanType("web") + .initKind(SPAN_KIND_SERVER) + .initComponentOnly("test-component") + .build(); + } + + @AfterEach + void cleanup() { + tracer.close(); + } + + @Test + void buildSpanSeedsPrototypeAndFallsBackToPrototypeOperationName() { + DDSpan span = (DDSpan) tracer.buildSpan(prototype, null).start(); + try { + assertEquals("proto.op", span.getOperationName().toString()); // null -> prototype's + assertEquals("web", span.getSpanType()); + assertEquals("test-component", span.getTags().get(COMPONENT)); // constant tag seeded + } finally { + span.finish(); + } + } + + @Test + void seedsSpanKindOrdinalAndTag() { + // span.kind is intercepted (its ordinal drives isOutbound). The prototype's tags seed through + // the interceptor, so BOTH the ordinal side-effect and the span.kind tag Entry must land. + DDSpan span = (DDSpan) tracer.buildSpan(prototype, null).start(); + try { + assertEquals(SPAN_KIND_SERVER, span.getSpanKindString()); // ordinal side-effect applied + assertEquals(SPAN_KIND_SERVER, span.getTags().get(SPAN_KIND)); // tag (shared Entry) present + } finally { + span.finish(); + } + } + + @Test + void explicitOperationNameOverridesPrototype() { + DDSpan span = (DDSpan) tracer.buildSpan(prototype, "explicit.op").start(); + try { + assertEquals("explicit.op", span.getOperationName().toString()); // explicit wins + } finally { + span.finish(); + } + } + + @Test + void startSpanSeedsPrototype() { + DDSpan span = (DDSpan) tracer.startSpan(prototype, null); + try { + assertEquals("proto.op", span.getOperationName().toString()); + assertEquals("test-component", span.getTags().get(COMPONENT)); + } finally { + span.finish(); + } + } + + @Test + void explicitBuilderTagOverridesPrototypeConstant() { + // prototype seeds `component` just before the builder's own tags, so the explicit withTag wins + DDSpan span = (DDSpan) tracer.buildSpan(prototype, null).withTag(COMPONENT, "override").start(); + try { + assertEquals("override", span.getTags().get(COMPONENT)); + } finally { + span.finish(); + } + } + + @Test + void initComponentAndIntegrationSetsIntegrationName() { + // Mirrors BaseDecorator.afterStart: the component tag is seeded AND the integration name is set + // on the context, which IntegrationAdder serializes as _dd.integration (field -> tag mapping is + // covered by IntegrationAdderTest). + SpanPrototype proto = + SpanPrototype.builder() + .initInstrumentationName("test-instr") + .initComponentAndIntegration("netty") + .build(); + DDSpan span = (DDSpan) tracer.buildSpan(proto, "op").start(); + try { + assertEquals("netty", span.getTags().get(COMPONENT)); // component tag seeded + assertEquals("netty", ((DDSpanContext) span.spanContext()).getIntegrationName()); + } finally { + span.finish(); + } + } + + @Test + void initComponentOnlyDoesNotSetIntegrationName() { + // initComponentOnly is tag-only: no integration-name side effect, so no _dd.integration. + SpanPrototype proto = + SpanPrototype.builder() + .initInstrumentationName("test-instr") + .initComponentOnly("netty") + .build(); + DDSpan span = (DDSpan) tracer.buildSpan(proto, "op").start(); + try { + assertEquals("netty", span.getTags().get(COMPONENT)); // tag present + assertNull(((DDSpanContext) span.spanContext()).getIntegrationName()); // but no integration + } finally { + span.finish(); + } + } + + @Test + void extendsWithComponentOnlyOverrideLeavesInheritedIntegrationName() { + // Documents a known desync: overriding an inherited initComponentAndIntegration component with + // tag-only initComponentOnly does NOT clear the inherited integration name. Use + // initComponentAndIntegration to override both together. + SpanPrototype base = + SpanPrototype.builder() + .initInstrumentationName("test-instr") + .initComponentAndIntegration("netty") + .build(); + SpanPrototype derived = + SpanPrototype.builder().extends_(base).initComponentOnly("other").build(); + DDSpan span = (DDSpan) tracer.buildSpan(derived, "op").start(); + try { + assertEquals("other", span.getTags().get(COMPONENT)); // component overridden + // integration name stays inherited from the base (the documented desync) + assertEquals("netty", ((DDSpanContext) span.spanContext()).getIntegrationName()); + } finally { + span.finish(); + } + } +} diff --git a/internal-api/src/jmh/java/datadog/trace/api/SpanPrototypeBenchmark.java b/internal-api/src/jmh/java/datadog/trace/api/SpanPrototypeBenchmark.java new file mode 100644 index 00000000000..c75e9eefa94 --- /dev/null +++ b/internal-api/src/jmh/java/datadog/trace/api/SpanPrototypeBenchmark.java @@ -0,0 +1,96 @@ +package datadog.trace.api; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import datadog.trace.bootstrap.instrumentation.api.SpanPrototype; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Per-mechanism benchmark for {@link SpanPrototype}: the constant-tag application a span pays at + * start. Compares the three phases of the mechanism, holding the resulting tag set identical: + * + *

    + *
  • oldPerSpanStamps — a fresh {@code TagMap} filled by N individual {@code set(entry)} + * calls, as {@code BaseDecorator.afterStart} does today (once per span). + *
  • newBulkApply — a fresh map + one {@code putAll} of the baked-once prototype (the + * afterStart-consolidation on-ramp). + *
  • newConstructionSeed — the span's map is born as a {@code copy()} of the + * frozen prototype (clone-at-birth; what increment 2's construction wiring unlocks). + *
+ * + *

Isolates the constant-application only (not span creation or the {@code afterStart} virtual + * chain), so the delta is purely N-stamps vs. bulk-copy. All arms apply tags at the {@code TagMap} + * level and skip the per-tag {@code TagInterceptor} dispatch that the real construction seed still + * incurs (span.kind, analytics-rate, ... are intercepted). That dispatch is a common cost + * on both the old and new production paths, so it cancels in the delta — but it means the absolute + * ops/s and the new/old ratio here are a TagMap-level upper bound, not the end-to-end win. (The + * interceptor-free bulk-share is what TagInterceptor retirement eventually unlocks; the current + * seed intercepts.) Run with {@code -prof gc} — the interesting axes are ops/s and B/op. + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(SECONDS) +@Warmup(iterations = 5, time = 2) +@Measurement(iterations = 5, time = 2) +@Fork(3) +@Threads(8) +public class SpanPrototypeBenchmark { + + // The constant set a typical server span carries, as cached entries (the shared-Entry + // hand-optimization the decorators use today). + private static final TagMap.Entry COMPONENT = TagMap.Entry.create(Tags.COMPONENT, "netty"); + private static final TagMap.Entry KIND = + TagMap.Entry.create(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + private static final TagMap.Entry LANGUAGE = + TagMap.Entry.create(DDTags.LANGUAGE_TAG_KEY, DDTags.LANGUAGE_TAG_VALUE); + private static final TagMap.Entry ANALYTICS = + TagMap.Entry.create(DDTags.ANALYTICS_SAMPLE_RATE, 1.0d); + + private SpanPrototype prototype; + + @Setup(Level.Trial) + public void setUp() { + // Baked once — the same constants, composed through the builder. + prototype = + SpanPrototype.builder() + .initComponentOnly("netty") + .initKind(Tags.SPAN_KIND_SERVER) + .initTag(DDTags.LANGUAGE_TAG_KEY, DDTags.LANGUAGE_TAG_VALUE) + .initTag(ANALYTICS) + .build(); + } + + @Benchmark + public TagMap oldPerSpanStamps() { + TagMap tags = TagMap.create(); + tags.set(COMPONENT); + tags.set(KIND); + tags.set(LANGUAGE); + tags.set(ANALYTICS); + return tags; + } + + @Benchmark + public TagMap newBulkApply() { + TagMap tags = TagMap.create(); + tags.putAll(prototype.tags()); + return tags; + } + + @Benchmark + public TagMap newConstructionSeed() { + return prototype.tags().copy(); + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/TagMap.java b/internal-api/src/main/java/datadog/trace/api/TagMap.java index 39160ae11ff..2f2423d5239 100644 --- a/internal-api/src/main/java/datadog/trace/api/TagMap.java +++ b/internal-api/src/main/java/datadog/trace/api/TagMap.java @@ -208,6 +208,17 @@ public static final class Entry extends EntryChange */ static final byte ANY = 0; + /** + * Whether {@code value} is treated as "no tag" — a null, or an empty {@link CharSequence}. Set + * paths that honor the tag-filtering contract (e.g. {@code AgentSpan.setTag}, {@link + * SpanPrototype}) can gate on this without constructing an Entry — which matters once a dense + * store makes per-tag Entry allocation the wrong path. + */ + public static boolean isEmptyValue(Object value) { + return value == null + || (value instanceof CharSequence && ((CharSequence) value).length() == 0); + } + /** * Entry for {@code (tag, value)}, or null when {@code value} is null or an empty {@code * CharSequence} -- checked by runtime type, so an empty String passed as {@code Object} skips @@ -215,23 +226,14 @@ public static final class Entry extends EntryChange */ @Nullable public static final Entry create(@Nonnull String tag, Object value) { - if (value == null) { - return null; - } - if (value instanceof CharSequence && ((CharSequence) value).length() == 0) { - return null; - } - return TagMap.Entry.newAnyEntry(tag, value); + return isEmptyValue(value) ? null : TagMap.Entry.newAnyEntry(tag, value); } /** If value is non-null, returns a new TagMap.Entry If value is null or empty, returns null */ @Nullable public static final Entry create(@Nonnull String tag, CharSequence value) { // NOTE: From the static typing, we know that value is not a primitive box - - return (value == null || value.length() == 0) - ? null - : TagMap.Entry.newObjectEntry(tag, value); + return isEmptyValue(value) ? null : TagMap.Entry.newObjectEntry(tag, value); } public static final Entry create(@Nonnull String tag, boolean value) { diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java index df6b2f2054a..dd17b8ccc7c 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpan.java @@ -220,6 +220,46 @@ default boolean isValid() { boolean isOutbound(); + /** + * Applies a {@link SpanPrototype} as fallback defaults: stamps its span type, constant tags, and + * integration name only where this span has not already set them. Prototype values are the lowest + * precedence -- anything explicitly set wins -- so {@code apply} never clobbers, is + * order-independent, and self-neutralizes once construction has already seeded the same + * prototype. + * + *

This is the single seam through which a prototype's constant initial state is applied, + * shared by the construction path (buildSpan/startSpan) and decorator {@code afterStart}. Core + * spans override to route straight to the context, which owns the tag map and will host the + * eventual fast path (bulk share / identity short-circuit); this default is the best-effort + * fallback for other span implementations. + */ + default void apply(@Nonnull final SpanPrototype prototype) { + if (getSpanType() == null) { + final CharSequence spanType = prototype.spanType(); + if (spanType != null) { + setSpanType(spanType); + } + } + + // Prototype tags are fallback defaults: only fill keys this span has not already set. + prototype + .tags() + .forEach( + (tag, value) -> { + if (getTag(tag) == null) { + setTag(tag, value); + } + }); + + // Never-clobber: only stamp the integration name when the context reports none. + if (spanContext().getIntegrationName() == null) { + final CharSequence integrationName = prototype.integrationName(); + if (integrationName != null) { + spanContext().setIntegrationName(integrationName); + } + } + } + default AgentSpan asAgentSpan() { return this; } diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index 1dba9438168..6e950925305 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -55,6 +55,15 @@ default void mergePathwayContext(PathwayContext pathwayContext) {} default void setIntegrationName(CharSequence componentName) {} + /** + * The integration name recorded on this context, or {@code null} if none. Default {@code null} + * mirrors {@link #setIntegrationName}'s no-op default: contexts that do not track an integration + * name report absence, which lets never-clobber callers guard before setting. + */ + default CharSequence getIntegrationName() { + return null; + } + /** * Gets whether the span context used is part of the local trace or from another service * diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentTracer.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentTracer.java index 1fce96a2992..a40bf5e7603 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentTracer.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentTracer.java @@ -69,6 +69,14 @@ public static AgentSpan startSpan( return get().startSpan(instrumentationName, spanName, parent, startTimeMicros); } + /** + * @see TracerAPI#startSpan(SpanPrototype, CharSequence) + */ + public static AgentSpan startSpan( + @Nonnull final SpanPrototype prototype, final CharSequence operationName) { + return get().startSpan(prototype, operationName); + } + public static AgentScope activateSpan(final AgentSpan span) { return get().activateSpan(span); } @@ -322,6 +330,30 @@ default AgentSpan blackholeSpan() { */ SpanBuilder buildSpan(String instrumentationName, CharSequence spanName); + /** + * Returns a SpanBuilder seeded from a {@link SpanPrototype}: the prototype supplies the + * instrumentation name, span type, and constant tags. {@code operationName} overrides the + * prototype's when non-null — the explicit value wins, the prototype is the fallback. + * + *

This default seeds identity only; a real tracer should override it to also seed the + * prototype's tags (see {@code CoreTracer}). The no-op tracer discards tags, so identity-only + * is correct there. + */ + default SpanBuilder buildSpan(@Nonnull SpanPrototype prototype, CharSequence operationName) { + return buildSpan( + prototype.instrumentationName(), + operationName != null ? operationName : prototype.operationName()); + } + + /** + * Creates and starts a span seeded from a {@link SpanPrototype}. This is the + * auto-instrumentation entry point mirroring {@link #startSpan(String, CharSequence)}; see + * {@link #buildSpan(SpanPrototype, CharSequence)}. + */ + default AgentSpan startSpan(@Nonnull SpanPrototype prototype, CharSequence operationName) { + return buildSpan(prototype, operationName).start(); + } + /** * Returns a SpanBuilder that can be used to produce one and only one span. By imposing the * single span creation limitation, this method is more efficient than {@link #buildSpan} @@ -425,6 +457,13 @@ public AgentSpan startSpan(final String instrumentationName, final CharSequence return NoopSpan.INSTANCE; } + @Override + public AgentSpan startSpan( + @Nonnull final SpanPrototype prototype, final CharSequence operationName) { + // The default routes through buildSpan(String,...), which is null on the noop tracer -> NPE. + return NoopSpan.INSTANCE; + } + @Override public AgentSpan startSpan( final String instrumentationName, final CharSequence spanName, final long startTimeMicros) { @@ -501,6 +540,15 @@ public SpanBuilder buildSpan(final String instrumentationName, final CharSequenc return null; } + @Override + public SpanBuilder buildSpan( + @Nonnull final SpanPrototype prototype, final CharSequence operationName) { + // Mirrors buildSpan(String,...): the noop tracer returns a null builder. Callers that need a + // noop-safe entry point use startSpan(...), which is overridden above. A chainable + // NoopSpanBuilder would fix the null-vs-NoopSpan asymmetry, but that is a separate PR. + return null; + } + @Override public SpanBuilder singleSpanBuilder( final String instrumentationName, final CharSequence spanName) { diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototype.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototype.java new file mode 100644 index 00000000000..24ccd2b39d8 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototype.java @@ -0,0 +1,179 @@ +package datadog.trace.bootstrap.instrumentation.api; + +import datadog.trace.api.TagMap; + +/** + * A baked-once, frozen descriptor of a span's constant initial state — the per-decorator constants + * (instrumentation name, span type, {@code span.kind}, component, …) that {@code + * BaseDecorator.afterStart} otherwise stamps one entry at a time, per span. + * + *

Composed through {@link #builder()}: authors set identity and constant tags via typed methods + * and never touch {@link TagMap} directly. {@link Builder#extends_(SpanPrototype)} inherits a base + * prototype (e.g. a SpanType base like {@code HttpServer}) so an integration adds only what's + * specific to it. Rides the existing {@code TagMap} API, so it's independent of any deeper TagMap + * rework — the internal seed can get faster without changing this surface. + * + *

v1 carries identity + constant tags. Derivation / canonicalization / lifecycle hooks are + * deliberately out — grown when the work that needs each arrives, not pre-slotted. + */ +public final class SpanPrototype { + /** The empty prototype — for spans created without a decorator-provided prototype. */ + public static final SpanPrototype NONE = builder().build(); + + public static Builder builder() { + return new Builder(); + } + + private final String instrumentationName; + private final CharSequence operationName; + private final CharSequence spanType; + private final CharSequence integrationName; + private final TagMap tags; // frozen + + private SpanPrototype(final Builder builder) { + this.instrumentationName = builder.instrumentationName; + this.operationName = builder.operationName; + this.spanType = builder.spanType; + this.integrationName = builder.integrationName; + this.tags = builder.tags.immutableCopy(); + } + + public String instrumentationName() { + return instrumentationName; + } + + public CharSequence operationName() { + return operationName; + } + + public CharSequence spanType() { + return spanType; + } + + /** + * The integration name to record on the span context ({@code setIntegrationName}), which the + * IntegrationAdder serializer step turns into {@code _dd.integration}. Null unless set via {@link + * Builder#initComponentAndIntegration}. Mirrors {@code BaseDecorator.afterStart}'s side effect. + */ + public CharSequence integrationName() { + return integrationName; + } + + /** The frozen constant tags — the internal seed applied at span construction. */ + public TagMap tags() { + return tags; + } + + public static final class Builder { + private String instrumentationName; + private CharSequence operationName; + private CharSequence spanType; + private CharSequence integrationName; + // Internal accumulator — never exposed; authors compose via the typed methods below. + private final TagMap tags = TagMap.create(); + + private Builder() {} + + /** + * Inherit a base prototype's identity and constant tags (e.g. a SpanType base). Subsequent + * identity / {@code init*} calls on this builder override the inherited values. + */ + public Builder extends_(final SpanPrototype base) { + if (base != null) { + if (base.instrumentationName != null) { + this.instrumentationName = base.instrumentationName; + } + if (base.operationName != null) { + this.operationName = base.operationName; + } + if (base.spanType != null) { + this.spanType = base.spanType; + } + if (base.integrationName != null) { + this.integrationName = base.integrationName; + } + this.tags.putAll(base.tags); + } + return this; + } + + public Builder initInstrumentationNames(final String[] instrumentationNames) { + return (instrumentationNames == null || instrumentationNames.length == 0) + ? this + : initInstrumentationName(instrumentationNames[0]); + } + + public Builder initInstrumentationName(final String instrumentationName) { + this.instrumentationName = instrumentationName; + return this; + } + + public Builder initOperationName(final CharSequence operationName) { + this.operationName = operationName; + return this; + } + + public Builder initSpanType(final CharSequence spanType) { + this.spanType = spanType; + return this; + } + + /** Sets {@code span.kind}. */ + public Builder initKind(final CharSequence kind) { + return initTag(Tags.SPAN_KIND, kind); + } + + /** + * Sets the {@code component} tag only. Does NOT touch the integration name — so overriding a + * component inherited via {@link #initComponentAndIntegration} with this leaves the inherited + * integration name in place (a desync). Use {@link #initComponentAndIntegration} to override + * both together. + */ + public Builder initComponentOnly(final CharSequence component) { + return initTag(Tags.COMPONENT, component); + } + + /** + * Sets the {@code component} tag AND records it as the integration name — the {@code + * BaseDecorator.afterStart} pairing (component tag + {@code setIntegrationName(component)}, + * which the IntegrationAdder serializer turns into {@code _dd.integration}). Null/empty is a + * no-op for both. + */ + public Builder initComponentAndIntegration(final CharSequence component) { + if (!TagMap.Entry.isEmptyValue(component)) { + this.tags.set(Tags.COMPONENT, component); + this.integrationName = component; + } + return this; + } + + public Builder initTag(final String key, final CharSequence value) { + if (!TagMap.Entry.isEmptyValue(value)) { + this.tags.set(key, value); + } + return this; + } + + public Builder initTag(final String key, final Object value) { + if (!TagMap.Entry.isEmptyValue(value)) { + this.tags.set(key, value); + } + return this; + } + + /** + * Advanced/internal: reuse an already-built entry — a decorator's cached constant or a metric + * entry — rather than re-creating it. Authors should prefer the typed {@code init*} methods. + */ + public Builder initTag(final TagMap.EntryReader entry) { + if (entry != null) { + this.tags.set(entry); + } + return this; + } + + public SpanPrototype build() { + return new SpanPrototype(this); + } + } +} diff --git a/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototypeTest.java b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototypeTest.java new file mode 100644 index 00000000000..e01ba8e8591 --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/SpanPrototypeTest.java @@ -0,0 +1,180 @@ +package datadog.trace.bootstrap.instrumentation.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.api.TagMap; +import org.junit.jupiter.api.Test; + +class SpanPrototypeTest { + + @Test + void noneHasNoIdentityAndNoTags() { + assertNull(SpanPrototype.NONE.instrumentationName()); + assertNull(SpanPrototype.NONE.operationName()); + assertNull(SpanPrototype.NONE.spanType()); + assertNull(SpanPrototype.NONE.integrationName()); + assertTrue(SpanPrototype.NONE.tags().isEmpty()); + } + + @Test + void gettersReflectBuilderState() { + final SpanPrototype proto = + SpanPrototype.builder() + .initInstrumentationName("instr") + .initOperationName("op") + .initSpanType("web") + .initComponentAndIntegration("netty") // sets integration name + component tag + .build(); + + assertEquals("instr", proto.instrumentationName()); + assertEquals("op", proto.operationName()); + assertEquals("web", proto.spanType()); + assertEquals("netty", proto.integrationName()); + assertEquals("netty", proto.tags().getString(Tags.COMPONENT)); + } + + @Test + void initInstrumentationNamesTakesFirstElement() { + final SpanPrototype proto = + SpanPrototype.builder().initInstrumentationNames(new String[] {"first", "second"}).build(); + + assertEquals("first", proto.instrumentationName()); + } + + @Test + void initInstrumentationNamesNullArrayLeavesNameUnset() { + final SpanPrototype proto = SpanPrototype.builder().initInstrumentationNames(null).build(); + + assertNull(proto.instrumentationName()); + } + + @Test + void initInstrumentationNamesEmptyArrayLeavesNameUnset() { + final SpanPrototype proto = + SpanPrototype.builder().initInstrumentationNames(new String[0]).build(); + + assertNull(proto.instrumentationName()); + } + + @Test + void extendsNullBaseIsNoOp() { + final SpanPrototype proto = SpanPrototype.builder().extends_(null).build(); + + assertNull(proto.instrumentationName()); + assertNull(proto.operationName()); + assertNull(proto.spanType()); + assertNull(proto.integrationName()); + assertTrue(proto.tags().isEmpty()); + } + + @Test + void extendsCopiesAllIdentityAndTags() { + final SpanPrototype base = + SpanPrototype.builder() + .initInstrumentationName("base") + .initOperationName("base.op") + .initSpanType("base-type") + .initComponentAndIntegration("base-comp") // integration name + component tag + .initKind("server") + .build(); + final SpanPrototype derived = SpanPrototype.builder().extends_(base).build(); + + assertEquals("base", derived.instrumentationName()); + assertEquals("base.op", derived.operationName()); + assertEquals("base-type", derived.spanType()); + assertEquals("base-comp", derived.integrationName()); + assertEquals("base-comp", derived.tags().getString(Tags.COMPONENT)); + assertEquals("server", derived.tags().getString(Tags.SPAN_KIND)); + } + + @Test + void extendsInheritsBaseIdentityAndTagsThenOverrides() { + final SpanPrototype base = + SpanPrototype.builder() + .initInstrumentationName("base") + .initSpanType("base-type") + .initKind("server") + .build(); + final SpanPrototype derived = + SpanPrototype.builder() + .extends_(base) + .initComponentOnly("netty") + .initSpanType("http") + .build(); + + assertEquals("base", derived.instrumentationName()); // inherited + assertEquals("http", derived.spanType()); // overridden + assertEquals("server", derived.tags().getString(Tags.SPAN_KIND)); // inherited tag + assertEquals("netty", derived.tags().getString(Tags.COMPONENT)); // added tag + } + + @Test + void initComponentAndIntegrationSetsBothComponentTagAndIntegrationName() { + final SpanPrototype proto = + SpanPrototype.builder().initComponentAndIntegration("netty").build(); + + assertEquals("netty", proto.tags().getString(Tags.COMPONENT)); + assertEquals("netty", proto.integrationName()); + } + + @Test + void initComponentAndIntegrationEmptyIsNoOpForBoth() { + final SpanPrototype proto = SpanPrototype.builder().initComponentAndIntegration("").build(); + + assertNull(proto.tags().getString(Tags.COMPONENT)); + assertNull(proto.integrationName()); + } + + @Test + void initKindSetsSpanKindTag() { + final SpanPrototype proto = SpanPrototype.builder().initKind("client").build(); + + assertEquals("client", proto.tags().getString(Tags.SPAN_KIND)); + } + + @Test + void emptyOrNullConstantsAreDroppedNotBaked() { + // Match AgentSpan.setTag / the cached-Entry path: a null or empty constant is "no tag", not an + // empty tag. A raw tags.set would otherwise bake a tag that per-span stamping never emits. + final SpanPrototype proto = + SpanPrototype.builder() + .initComponentOnly("") // empty -> dropped + .initKind("") // empty -> dropped + .initTag("empty.cs", "") // empty CharSequence -> dropped + .initTag("null.cs", (CharSequence) null) // null -> dropped + .initTag("null.obj", (Object) null) // null -> dropped + .initTag("kept", "v") // non-empty -> present + .build(); + + assertNull(proto.tags().getString(Tags.COMPONENT)); + assertNull(proto.tags().getString(Tags.SPAN_KIND)); + assertNull(proto.tags().getString("empty.cs")); + assertNull(proto.tags().getString("null.cs")); + assertNull(proto.tags().getString("null.obj")); + assertEquals("v", proto.tags().getString("kept")); // sanity: non-empty still stored + } + + @Test + void initTagObjectStoresNonEmptyValue() { + final SpanPrototype proto = SpanPrototype.builder().initTag("count", (Object) 42).build(); + + assertEquals(42, proto.tags().get("count")); + } + + @Test + void initTagEntryReaderStoresEntry() { + final TagMap.EntryReader entry = TagMap.Entry.create("cached", "value"); + final SpanPrototype proto = SpanPrototype.builder().initTag(entry).build(); + + assertEquals("value", proto.tags().getString("cached")); + } + + @Test + void initTagNullEntryReaderIsNoOp() { + final SpanPrototype proto = SpanPrototype.builder().initTag((TagMap.EntryReader) null).build(); + + assertTrue(proto.tags().isEmpty()); + } +} From 1a58e6997b78a42d32fc864b69397d42dec97f2a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 9 Sep 2026 04:10:43 +0000 Subject: [PATCH 26/30] Harden Feature Flags EVP transport --- .../communication/BackendApiFactory.java | 94 ++++-- .../ddagent/DDAgentFeaturesDiscovery.java | 7 + .../DDAgentFeaturesDiscoveryTest.groovy | 2 + .../communication/BackendApiFactoryTest.java | 168 +++++++++- .../AgentlessFeatureFlagBackendApi.java | 153 +++++++-- .../featureflag/ExposureWriterImpl.java | 2 +- .../FeatureFlagBackendApiFactory.java | 51 +-- .../AgentlessFeatureFlagBackendApiTest.java | 295 ++++++++++++++++-- .../featureflag/ExposureWriterTests.java | 50 ++- .../FeatureFlagBackendApiFactoryTest.java | 57 +++- 10 files changed, 761 insertions(+), 118 deletions(-) diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 5f7417a8f0f..34b56fd8326 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -25,6 +25,7 @@ public class BackendApiFactory { private final Config config; private final SharedCommunicationObjects sharedCommunicationObjects; private final Map requestHeaders; + private final boolean sendOnce; public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommunicationObjects) { this(config, sharedCommunicationObjects, emptyMap()); @@ -34,9 +35,25 @@ public BackendApiFactory( Config config, SharedCommunicationObjects sharedCommunicationObjects, Map requestHeaders) { + this(config, sharedCommunicationObjects, requestHeaders, false); + } + + /** + * Creates a backend factory with per-request headers and optional send-once transport semantics. + * + *

When {@code sendOnce} is true, both the explicit HTTP retry policy and OkHttp's automatic + * connection retry are disabled. This is required for event payloads that do not carry an + * idempotency key. + */ + public BackendApiFactory( + Config config, + SharedCommunicationObjects sharedCommunicationObjects, + Map requestHeaders, + boolean sendOnce) { this.config = config; this.sharedCommunicationObjects = sharedCommunicationObjects; this.requestHeaders = unmodifiableMap(new HashMap<>(requestHeaders)); + this.sendOnce = sendOnce; } public @Nullable BackendApi createBackendApi(Intake intake) { @@ -82,7 +99,7 @@ public BackendApi createDirectIntakeApi( apiKey, traceId, retryPolicyFactory(), - withRequestHeaders( + configureHttpClient( directIntakeHttpClient( sharedCommunicationObjects.getIntakeHttpClient(), followRedirects)), responseCompression); @@ -136,13 +153,38 @@ static HttpUrl buildEventPlatformIntakeUrl(String site) { /** Creates an API client that sends data through a compatible local EVP proxy. */ public @Nullable BackendApi createEvpProxyApi( Intake intake, boolean responseCompression, HttpRetryPolicy.Factory retryPolicyFactory) { + return createEvpProxyApi(intake, responseCompression, retryPolicyFactory, null, false); + } + + /** + * Creates an EVP proxy client, optionally retaining a compatibility endpoint when Agent discovery + * itself is unavailable. + * + *

An authoritative Agent response that omits EVP support never uses the fallback endpoint. The + * {@code forceDiscovery} form is intended for bounded route-recovery probes. + */ + public @Nullable BackendApi createEvpProxyApi( + Intake intake, + boolean responseCompression, + HttpRetryPolicy.Factory retryPolicyFactory, + @Nullable String discoveryFailureFallbackEndpoint, + boolean forceDiscovery) { DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); - featuresDiscovery.discoverIfOutdated(); - if (!featuresDiscovery.supportsEvpProxy()) { - return null; + if (forceDiscovery) { + featuresDiscovery.discover(); + } else { + featuresDiscovery.discoverIfOutdated(); } String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); + if (evpProxyEndpoint == null + && discoveryFailureFallbackEndpoint != null + && !featuresDiscovery.hasValidInfoResponse()) { + evpProxyEndpoint = discoveryFailureFallbackEndpoint; + } + if (evpProxyEndpoint == null) { + return null; + } String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); log.debug( @@ -156,29 +198,35 @@ static HttpUrl buildEventPlatformIntakeUrl(String site) { traceId, evpProxyUrl, subdomain, - retryPolicyFactory, - withRequestHeaders(sharedCommunicationObjects.agentHttpClient), + sendOnce ? HttpRetryPolicy.Factory.NEVER_RETRY : retryPolicyFactory, + configureHttpClient(sharedCommunicationObjects.agentHttpClient), responseCompression); } - private OkHttpClient withRequestHeaders(final OkHttpClient httpClient) { - if (requestHeaders.isEmpty()) { + OkHttpClient configureHttpClient(final OkHttpClient httpClient) { + if (requestHeaders.isEmpty() && !sendOnce) { return httpClient; } - return httpClient - .newBuilder() - .addInterceptor( - chain -> { - final Request.Builder requestBuilder = chain.request().newBuilder(); - for (Map.Entry header : requestHeaders.entrySet()) { - requestBuilder.header(header.getKey(), header.getValue()); - } - return chain.proceed(requestBuilder.build()); - }) - .build(); - } - - private static HttpRetryPolicy.Factory retryPolicyFactory() { - return new HttpRetryPolicy.Factory(5, 100, 2.0, true); + final OkHttpClient.Builder builder = httpClient.newBuilder(); + if (sendOnce) { + builder.retryOnConnectionFailure(false); + } + if (!requestHeaders.isEmpty()) { + builder.addInterceptor( + chain -> { + final Request.Builder requestBuilder = chain.request().newBuilder(); + for (Map.Entry header : requestHeaders.entrySet()) { + requestBuilder.header(header.getKey(), header.getValue()); + } + return chain.proceed(requestBuilder.build()); + }); + } + return builder.build(); + } + + private HttpRetryPolicy.Factory retryPolicyFactory() { + return sendOnce + ? HttpRetryPolicy.Factory.NEVER_RETRY + : new HttpRetryPolicy.Factory(5, 100, 2.0, true); } } diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index 16be2e84b98..ed053806a5c 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -103,6 +103,7 @@ private static class State { Set peerTags = emptySet(); String orgPropagationMarker; long lastTimeDiscovered; + boolean validInfoResponse; } private volatile State discoveryState; @@ -326,6 +327,7 @@ private boolean processInfoResponse(State newState, String response) { log.debug( "Failed to hash trace agent /info response. Will probe {}", newState.traceEndpoint, ex); } + newState.validInfoResponse = true; return true; } catch (Throwable error) { log.debug("Error parsing trace agent /info response", error); @@ -435,6 +437,11 @@ public boolean supportsEvpProxy() { return discoveryState.evpProxyEndpoint != null; } + /** Returns whether the last discovery attempt received a valid Agent info response. */ + public boolean hasValidInfoResponse() { + return discoveryState.validInfoResponse; + } + public boolean supportsContentEncodingHeadersWithEvpProxy() { // content encoding headers are supported in /v4 and above final String evpProxyEndpoint = discoveryState.evpProxyEndpoint; diff --git a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy index 8bcda6eb811..0079e02650c 100644 --- a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy +++ b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy @@ -73,6 +73,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { features.getDebuggerSnapshotEndpoint() == "debugger/v2/input" features.supportsDebuggerDiagnostics() features.supportsEvpProxy() + features.hasValidInfoResponse() features.supportsContentEncodingHeadersWithEvpProxy() features.getEvpProxyEndpoint() == "evp_proxy/v4/" features.getVersion() == "0.99.0" @@ -101,6 +102,7 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { 0 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.5/traces" }) >> { Request request -> success(request) } 1 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.4/traces" }) >> { Request request -> success(request) } features.getTraceEndpoint() == V04_ENDPOINT + !features.hasValidInfoResponse() 0 * _ } diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index a7fb7642a35..f02f3d81d6b 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -2,6 +2,8 @@ import static datadog.communication.EvpProxy.JAVA_TRACING_LIBRARY; import static datadog.communication.EvpProxy.ORIGIN_HEADER; +import static datadog.communication.EvpProxy.ORIGIN_VERSION_HEADER; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V4_EVP_PROXY_ENDPOINT; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL; import static datadog.trace.api.config.GeneralConfig.API_KEY; @@ -20,7 +22,9 @@ import datadog.trace.api.intake.Intake; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; import java.util.Properties; import okhttp3.HttpUrl; import okhttp3.MediaType; @@ -160,7 +164,7 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce } @Test - void evpProxySendsConfiguredRequestHeaders() throws Exception { + void featureFlagProxyRequestCarriesSdkIdentityWithoutDirectCredentials() throws Exception { final MockWebServer agent = new MockWebServer(); agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); agent.start(); @@ -170,19 +174,31 @@ void evpProxySendsConfiguredRequestHeaders() throws Exception { new BackendApiFactory( Config.get(), sharedCommunicationObjects(discovery, agent.url("/")), - singletonMap(ORIGIN_HEADER, JAVA_TRACING_LIBRARY)); - final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, false); + sdkHeaders(), + true); + final BackendApi api = + factory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false); assertNotNull(api); api.post( - "flagevaluation", + "exposures", RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), stream -> null, null, false); final RecordedRequest request = agent.takeRequest(); + assertEquals("/evp_proxy/v4/api/v2/exposures", request.getPath()); + assertEquals("event-platform-intake", request.getHeader("X-Datadog-EVP-Subdomain")); assertEquals(JAVA_TRACING_LIBRARY, request.getHeader(ORIGIN_HEADER)); + assertEquals("test-version", request.getHeader(ORIGIN_VERSION_HEADER)); + assertNull(request.getHeader("DD-API-KEY")); + assertEquals(1, agent.getRequestCount()); } finally { agent.shutdown(); } @@ -225,6 +241,121 @@ void directIntakeSendsConfiguredRequestHeaders() throws Exception { } } + @Test + void featureFlagDirectRequestCarriesCredentialsAndIdentityExactlyOnce() throws Exception { + final MockWebServer intake = new MockWebServer(); + intake.enqueue(new MockResponse().setResponseCode(500).setBody("failed")); + intake.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + intake.start(); + final OkHttpClient sharedClient = new OkHttpClient.Builder().build(); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), + sharedCommunicationObjects(new FakeFeaturesDiscovery(null), null), + sdkHeaders(), + true); + final OkHttpClient directClient = factory.configureHttpClient(sharedClient); + try { + final IntakeApi api = + new IntakeApi( + intake.url("/api/v2/"), + "api-key", + "123", + HttpRetryPolicy.Factory.NEVER_RETRY, + directClient, + false); + + assertThrows( + IOException.class, + () -> + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false)); + + final RecordedRequest request = intake.takeRequest(); + assertEquals("/api/v2/flagevaluation", request.getPath()); + assertEquals("api-key", request.getHeader("DD-API-KEY")); + assertEquals("dd-trace-java", request.getHeader("DD-EVP-ORIGIN")); + assertEquals("test-version", request.getHeader("DD-EVP-ORIGIN-VERSION")); + assertNull(request.getHeader("X-Datadog-EVP-Subdomain")); + assertEquals(1, intake.getRequestCount()); + assertEquals(false, directClient.retryOnConnectionFailure()); + } finally { + directClient.dispatcher().executorService().shutdownNow(); + directClient.connectionPool().evictAll(); + sharedClient.dispatcher().executorService().shutdownNow(); + sharedClient.connectionPool().evictAll(); + intake.shutdown(); + } + } + + @Test + void discoveryFailureRetainsV2CompatibilityRoute() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null, false); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); + + final BackendApi api = + factory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + assertEquals("/evp_proxy/v2/api/v2/flagevaluation", agent.takeRequest().getPath()); + } finally { + agent.shutdown(); + } + } + + @Test + void authoritativeMissingProxyDoesNotUseV2CompatibilityRoute() { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null, true); + final BackendApiFactory factory = + new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); + + assertNull( + factory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false)); + } + + @Test + void recoveryRequestForcesFreshDiscovery() { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null, true); + final BackendApiFactory factory = + new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); + + assertNull( + factory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + true)); + assertEquals(1, discovery.forcedDiscoveries); + assertEquals(0, discovery.outdatedDiscoveries); + } + @Test void explicitNoRetryProxyPolicyDoesNotReplayAmbiguousFailure() throws Exception { final MockWebServer agent = new MockWebServer(); @@ -265,6 +396,13 @@ private static SharedCommunicationObjects sharedCommunicationObjects( return sco; } + private static Map sdkHeaders() { + final Map headers = new HashMap<>(2); + headers.put("DD-EVP-ORIGIN", "dd-trace-java"); + headers.put("DD-EVP-ORIGIN-VERSION", "test-version"); + return headers; + } + private static final class TestSharedCommunicationObjects extends SharedCommunicationObjects { private final DDAgentFeaturesDiscovery discovery; @@ -280,8 +418,15 @@ public DDAgentFeaturesDiscovery featuresDiscovery(final Config config) { private static final class FakeFeaturesDiscovery extends DDAgentFeaturesDiscovery { private final String evpProxyEndpoint; + private final boolean validInfoResponse; + private int forcedDiscoveries; + private int outdatedDiscoveries; private FakeFeaturesDiscovery(final String evpProxyEndpoint) { + this(evpProxyEndpoint, true); + } + + private FakeFeaturesDiscovery(final String evpProxyEndpoint, final boolean validInfoResponse) { super( new OkHttpClient(), Monitoring.DISABLED, @@ -290,10 +435,18 @@ private FakeFeaturesDiscovery(final String evpProxyEndpoint) { true, false); this.evpProxyEndpoint = evpProxyEndpoint; + this.validInfoResponse = validInfoResponse; } @Override - public void discoverIfOutdated() {} + public void discover() { + forcedDiscoveries++; + } + + @Override + public void discoverIfOutdated() { + outdatedDiscoveries++; + } @Override public String getEvpProxyEndpoint() { @@ -304,5 +457,10 @@ public String getEvpProxyEndpoint() { public boolean supportsEvpProxy() { return evpProxyEndpoint != null; } + + @Override + public boolean hasValidInfoResponse() { + return validInfoResponse; + } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java index 769ebfd1dd1..a32ffd7fd6d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java @@ -7,6 +7,8 @@ import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; import java.util.function.Supplier; import javax.annotation.Nullable; import okhttp3.RequestBody; @@ -18,21 +20,56 @@ final class AgentlessFeatureFlagBackendApi implements BackendApi { private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessFeatureFlagBackendApi.class); + private static final long DEFAULT_RECOVERY_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(1); - private final BackendApi proxyApi; + private final Supplier proxyApiSupplier; private final Supplier directApiSupplier; private final String eventType; - private volatile BackendApi activeApi; + private final LongSupplier nanoTime; + private final long recoveryIntervalNanos; + private volatile Route activeRoute; + private volatile BackendApi directApi; private volatile boolean directApiCreationAttempted; + private volatile long nextProxyProbeNanos; AgentlessFeatureFlagBackendApi( - final BackendApi proxyApi, + @Nullable final BackendApi proxyApi, + @Nullable final BackendApi directApi, + final Supplier proxyApiSupplier, final Supplier directApiSupplier, final String eventType) { - this.proxyApi = proxyApi; + this( + proxyApi, + directApi, + proxyApiSupplier, + directApiSupplier, + eventType, + System::nanoTime, + DEFAULT_RECOVERY_INTERVAL_NANOS); + } + + AgentlessFeatureFlagBackendApi( + @Nullable final BackendApi proxyApi, + @Nullable final BackendApi directApi, + final Supplier proxyApiSupplier, + final Supplier directApiSupplier, + final String eventType, + final LongSupplier nanoTime, + final long recoveryIntervalNanos) { + if (proxyApi == null && directApi == null) { + throw new IllegalArgumentException("A Feature Flagging event route is required"); + } + this.proxyApiSupplier = proxyApiSupplier; + this.directApi = directApi; this.directApiSupplier = directApiSupplier; this.eventType = eventType; - this.activeApi = proxyApi; + this.nanoTime = nanoTime; + this.recoveryIntervalNanos = recoveryIntervalNanos; + this.activeRoute = proxyApi != null ? new Route(proxyApi, true) : new Route(directApi, false); + this.directApiCreationAttempted = directApi != null; + if (proxyApi == null) { + scheduleProxyRecovery(); + } } @Override @@ -43,59 +80,121 @@ public T post( @Nullable final OkHttpUtils.CustomListener requestListener, final boolean requestCompression) throws IOException { - final BackendApi selectedApi = activeApi; + final Route selectedRoute = selectRoute(); try { - return selectedApi.post( + return selectedRoute.api.post( uri, requestBody, responseParser, requestListener, requestCompression); } catch (final IOException exception) { - if (selectedApi != proxyApi || !isDefinitiveRejection(exception)) { + if (!selectedRoute.proxy) { throw exception; } - final BackendApi directApi = getOrCreateDirectApi(); - if (directApi == null) { + final BackendApi fallbackApi = switchFutureBatchesToDirect(selectedRoute); + if (fallbackApi == null || !isSafeToReplayDirectly(exception)) { throw exception; } - return directApi.post(uri, requestBody, responseParser, requestListener, requestCompression); + return fallbackApi.post( + uri, requestBody, responseParser, requestListener, requestCompression); } } - @Nullable - private BackendApi getOrCreateDirectApi() { - final BackendApi selectedApi = activeApi; - if (selectedApi != proxyApi) { - return selectedApi; + private Route selectRoute() { + final Route selectedRoute = activeRoute; + if (selectedRoute.proxy || !proxyRecoveryDue()) { + return selectedRoute; } synchronized (this) { - final BackendApi currentApi = activeApi; - if (currentApi != proxyApi) { - return currentApi; + final Route currentRoute = activeRoute; + if (currentRoute.proxy || !proxyRecoveryDue()) { + return currentRoute; } - if (directApiCreationAttempted) { - return null; + // Reserve the next recovery window before performing discovery so concurrent senders keep + // using direct intake instead of blocking or creating a probe stampede. + scheduleProxyRecovery(); + } + + BackendApi recoveredProxyApi = null; + try { + recoveredProxyApi = proxyApiSupplier.get(); + } catch (final RuntimeException exception) { + // Route recovery is best effort. A discovery/configuration failure must not interrupt the + // working direct route and lose the current batch. + LOGGER.debug("Could not recover the local Feature Flagging {} route", eventType, exception); + } + if (recoveredProxyApi != null) { + synchronized (this) { + if (!activeRoute.proxy) { + LOGGER.debug( + "Switching Feature Flagging {} delivery from direct intake to the local EVP proxy", + eventType); + activeRoute = new Route(recoveredProxyApi, true); + } } + } + return activeRoute; + } - final BackendApi directApi = directApiSupplier.get(); - if (directApi != null) { + @Nullable + private BackendApi switchFutureBatchesToDirect(final Route failedProxyRoute) { + final BackendApi fallbackApi = getOrCreateDirectApi(); + if (fallbackApi == null) { + return null; + } + + synchronized (this) { + if (activeRoute == failedProxyRoute) { LOGGER.debug( "Switching Feature Flagging {} delivery from the local EVP proxy to direct intake", eventType); - activeApi = directApi; + activeRoute = new Route(fallbackApi, false); + scheduleProxyRecovery(); + } + } + return fallbackApi; + } + + @Nullable + private BackendApi getOrCreateDirectApi() { + if (directApiCreationAttempted) { + return directApi; + } + + synchronized (this) { + if (!directApiCreationAttempted) { + directApi = directApiSupplier.get(); + directApiCreationAttempted = true; } - directApiCreationAttempted = true; return directApi; } } - private static boolean isDefinitiveRejection(final IOException exception) { + private boolean proxyRecoveryDue() { + return nanoTime.getAsLong() - nextProxyProbeNanos >= 0; + } + + private void scheduleProxyRecovery() { + nextProxyProbeNanos = nanoTime.getAsLong() + recoveryIntervalNanos; + } + + private static boolean isSafeToReplayDirectly(final IOException exception) { if (exception instanceof ConnectException) { return true; } if (exception instanceof HttpResponseException) { final int statusCode = ((HttpResponseException) exception).getStatusCode(); - return statusCode == 403 || statusCode == 404 || statusCode == 405; + return statusCode == 404 || statusCode == 405; } return false; } + + private static final class Route { + private final BackendApi api; + private final boolean proxy; + + private Route(final BackendApi api, final boolean proxy) { + this.api = api; + this.proxy = proxy; + } + } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index fcd50e5dc34..6fec2eb3991 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -200,7 +200,7 @@ protected void flushIfNecessary() { private boolean shouldFlush() { long nanoTime = System.nanoTime(); long ticks = nanoTime - lastTicks; - if (ticks > ticksRequiredToFlush || queue.size() >= FLUSH_THRESHOLD) { + if (ticks > ticksRequiredToFlush || buffer.size() >= FLUSH_THRESHOLD) { lastTicks = nanoTime; return true; } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java index 87ddcaaae16..b1755130e13 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -3,6 +3,7 @@ import static datadog.communication.EvpProxy.JAVA_TRACING_LIBRARY; import static datadog.communication.EvpProxy.ORIGIN_HEADER; import static datadog.communication.EvpProxy.ORIGIN_VERSION_HEADER; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static java.util.Collections.unmodifiableMap; @@ -35,7 +36,7 @@ final class FeatureFlagBackendApiFactory { final FeatureFlagEventType eventType) { this( config, - new BackendApiFactory(config, sharedCommunicationObjects, REQUEST_HEADERS), + new BackendApiFactory(config, sharedCommunicationObjects, REQUEST_HEADERS, true), eventType); } @@ -50,18 +51,14 @@ final class FeatureFlagBackendApiFactory { @Nullable BackendApi create() { - final boolean directFallbackAvailable = - CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource()) - && hasDirectCredentials(); - final BackendApi proxyApi = - directFallbackAvailable - ? backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - eventType.responseCompressionEnabled(), - HttpRetryPolicy.Factory.NEVER_RETRY) - : backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, eventType.responseCompressionEnabled()); - if (!CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource())) { + final boolean agentless = + CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource()); + final boolean directFallbackAvailable = agentless && hasDirectCredentials(); + // Preserve the historical v2 endpoint when initial discovery itself is unavailable. Recovery + // from a working direct route is stricter below: only an advertised endpoint proves that local + // delivery has returned, avoiding an ambiguous failed probe of an assumed v2 endpoint. + final BackendApi proxyApi = createProxyApi(false, true); + if (!agentless) { if (proxyApi == null) { LOGGER.warn( "Feature Flagging {} delivery is disabled because the local Agent does not support the EVP proxy", @@ -70,17 +67,18 @@ BackendApi create() { return proxyApi; } - if (proxyApi != null) { - if (directFallbackAvailable) { - return new AgentlessFeatureFlagBackendApi( - proxyApi, this::createDirectApi, eventType.logName()); - } + if (!directFallbackAvailable) { return proxyApi; } - final BackendApi directApi = createDirectApi(); - if (directApi != null) { - return directApi; + final BackendApi directApi = proxyApi == null ? createDirectApi() : null; + if (proxyApi != null || directApi != null) { + return new AgentlessFeatureFlagBackendApi( + proxyApi, + directApi, + () -> createProxyApi(true, false), + this::createDirectApi, + eventType.logName()); } LOGGER.warn( @@ -89,6 +87,17 @@ BackendApi create() { return null; } + @Nullable + private BackendApi createProxyApi( + final boolean forceDiscovery, final boolean useDiscoveryFailureFallback) { + return backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + eventType.responseCompressionEnabled(), + HttpRetryPolicy.Factory.NEVER_RETRY, + useDiscoveryFailureFallback ? V2_EVP_PROXY_ENDPOINT : null, + forceDiscovery); + } + private static Map requestHeaders() { final Map headers = new HashMap<>(2); headers.put(ORIGIN_HEADER, JAVA_TRACING_LIBRARY); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java index 117c72ba2a1..a5802c00ddc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.communication.BackendApi; import datadog.communication.HttpResponseException; @@ -15,7 +16,11 @@ import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; import javax.annotation.Nullable; import okhttp3.MediaType; @@ -29,7 +34,7 @@ class AgentlessFeatureFlagBackendApiTest { @ParameterizedTest - @ValueSource(ints = {403, 404, 405}) + @ValueSource(ints = {404, 405}) void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throws Exception { final RecordingBackendApi local = new RecordingBackendApi(new HttpResponseException(statusCode, "rejected")); @@ -38,6 +43,8 @@ void replaysRejectedBatchDirectlyAndKeepsDirectRoute(final int statusCode) throw final AgentlessFeatureFlagBackendApi api = new AgentlessFeatureFlagBackendApi( local, + null, + () -> local, () -> { directApiCreations.incrementAndGet(); return direct; @@ -66,7 +73,7 @@ void fallsBackAfterConnectionRefusal(final String route, final String eventType) new RecordingBackendApi(new ConnectException("connection refused")); final RecordingBackendApi direct = new RecordingBackendApi(); final AgentlessFeatureFlagBackendApi api = - new AgentlessFeatureFlagBackendApi(local, () -> direct, eventType); + new AgentlessFeatureFlagBackendApi(local, null, () -> local, () -> direct, eventType); api.post(route, requestBody(eventType), stream -> null, null, false); @@ -75,37 +82,150 @@ void fallsBackAfterConnectionRefusal(final String route, final String eventType) } @Test - void doesNotReturnToLocalRouteAfterSwitchingToDirectIntake() throws Exception { - final RecordingBackendApi local = + void recoversTheLocalRouteAfterTheCooldown() throws Exception { + final AtomicLong clock = new AtomicLong(); + final RecordingBackendApi unavailableLocal = new RecordingBackendApi(new ConnectException("connection refused")); + final RecordingBackendApi recoveredLocal = new RecordingBackendApi(); final RecordingBackendApi direct = new RecordingBackendApi(); final AgentlessFeatureFlagBackendApi api = - new AgentlessFeatureFlagBackendApi(local, () -> direct, "exposure"); + new AgentlessFeatureFlagBackendApi( + unavailableLocal, null, () -> recoveredLocal, () -> direct, "exposure", clock::get, 10); api.post("exposures", requestBody("first"), stream -> null, null, false); - direct.failure = new IOException("direct intake failed"); + clock.set(9); + api.post("exposures", requestBody("second"), stream -> null, null, false); + clock.set(10); + api.post("exposures", requestBody("third"), stream -> null, null, false); - assertThrows( - IOException.class, - () -> api.post("exposures", requestBody("second"), stream -> null, null, false)); - assertEquals(1, local.calls); + assertEquals(1, unavailableLocal.calls); assertEquals(2, direct.calls); + assertEquals(1, recoveredLocal.calls); } @ParameterizedTest - @ValueSource(ints = {429, 500}) - void doesNotReplayAmbiguousHttpFailure(final int statusCode) { - assertNoDirectReplay(new HttpResponseException(statusCode, "ambiguous")); + @ValueSource(ints = {403, 429, 500}) + void doesNotReplayAmbiguousHttpFailureButSwitchesFutureBatches(final int statusCode) + throws Exception { + assertNoSameBatchReplayButUsesDirectForNext(new HttpResponseException(statusCode, "ambiguous")); + } + + @Test + void doesNotReplayTimeoutButSwitchesFutureBatches() throws Exception { + assertNoSameBatchReplayButUsesDirectForNext(new SocketTimeoutException("timed out")); + } + + @Test + void doesNotReplayConnectionResetButSwitchesFutureBatches() throws Exception { + assertNoSameBatchReplayButUsesDirectForNext(new SocketException("connection reset")); + } + + @Test + void startsDirectAndRetriesLocalDiscoveryOnlyAfterTheCooldown() throws Exception { + final AtomicLong clock = new AtomicLong(); + final RecordingBackendApi direct = new RecordingBackendApi(); + final RecordingBackendApi recoveredLocal = new RecordingBackendApi(); + final AtomicInteger proxyApiCreations = new AtomicInteger(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi( + null, + direct, + () -> { + proxyApiCreations.incrementAndGet(); + return recoveredLocal; + }, + () -> direct, + "exposure", + clock::get, + 10); + + api.post("exposures", requestBody("first"), stream -> null, null, false); + clock.set(9); + api.post("exposures", requestBody("second"), stream -> null, null, false); + clock.set(10); + api.post("exposures", requestBody("third"), stream -> null, null, false); + + assertEquals(1, proxyApiCreations.get()); + assertEquals(2, direct.calls); + assertEquals(1, recoveredLocal.calls); } @Test - void doesNotReplayTimeout() { - assertNoDirectReplay(new SocketTimeoutException("timed out")); + void concurrentSendersDoNotBlockOnOrDuplicateARecoveryProbe() throws Exception { + final AtomicLong clock = new AtomicLong(10); + final RecordingBackendApi direct = new RecordingBackendApi(); + final RecordingBackendApi recoveredLocal = new RecordingBackendApi(); + final AtomicInteger proxyApiCreations = new AtomicInteger(); + final CountDownLatch probeStarted = new CountDownLatch(1); + final CountDownLatch releaseProbe = new CountDownLatch(1); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi( + null, + direct, + () -> { + proxyApiCreations.incrementAndGet(); + probeStarted.countDown(); + try { + assertTrue(releaseProbe.await(5, TimeUnit.SECONDS)); + } catch (final InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new AssertionError(exception); + } + return recoveredLocal; + }, + () -> direct, + "flag evaluation", + clock::get, + 10); + + // Construction schedules the first probe for t=20. + clock.set(20); + final CompletableFuture recoveringPost = + CompletableFuture.runAsync( + () -> { + try { + api.post("flagevaluation", requestBody("probe"), stream -> null, null, false); + } catch (final IOException exception) { + throw new AssertionError(exception); + } + }); + assertTrue(probeStarted.await(5, TimeUnit.SECONDS)); + + api.post("flagevaluation", requestBody("parallel"), stream -> null, null, false); + releaseProbe.countDown(); + recoveringPost.get(5, TimeUnit.SECONDS); + + assertEquals(1, proxyApiCreations.get()); + assertEquals(1, direct.calls); + assertEquals(1, recoveredLocal.calls); } @Test - void doesNotReplayConnectionReset() { - assertNoDirectReplay(new SocketException("connection reset")); + void failedRecoveryIsStickyForAnotherCooldown() throws Exception { + final AtomicLong clock = new AtomicLong(); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AtomicInteger proxyApiCreations = new AtomicInteger(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi( + null, + direct, + () -> { + proxyApiCreations.incrementAndGet(); + return null; + }, + () -> direct, + "exposure", + clock::get, + 10); + + clock.set(10); + api.post("exposures", requestBody("first"), stream -> null, null, false); + api.post("exposures", requestBody("second"), stream -> null, null, false); + clock.set(20); + api.post("exposures", requestBody("third"), stream -> null, null, false); + + assertEquals(2, proxyApiCreations.get()); + assertEquals(3, direct.calls); } @Test @@ -116,6 +236,8 @@ void doesNotRetryDirectApiCreationWhenFallbackIsUnavailable() { final AgentlessFeatureFlagBackendApi api = new AgentlessFeatureFlagBackendApi( local, + null, + () -> local, () -> { directApiCreations.incrementAndGet(); return null; @@ -133,13 +255,136 @@ void doesNotRetryDirectApiCreationWhenFallbackIsUnavailable() { assertEquals(1, directApiCreations.get()); } - private static void assertNoDirectReplay(final IOException failure) { + @Test + void requiresAtLeastOneInitialRoute() { + assertThrows( + IllegalArgumentException.class, + () -> + new AgentlessFeatureFlagBackendApi( + null, null, () -> null, () -> null, "flag evaluation")); + } + + @Test + void propagatesDirectFailuresWithoutTryingToFallback() { + final SocketTimeoutException failure = new SocketTimeoutException("direct timeout"); + final RecordingBackendApi direct = new RecordingBackendApi(failure); + final AtomicInteger proxyApiCreations = new AtomicInteger(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi( + null, + direct, + () -> { + proxyApiCreations.incrementAndGet(); + return null; + }, + () -> direct, + "exposure"); + + assertSame( + failure, + assertThrows( + SocketTimeoutException.class, + () -> api.post("exposures", requestBody("direct"), stream -> null, null, false))); + assertEquals(0, proxyApiCreations.get()); + assertEquals(1, direct.calls); + } + + @Test + void concurrentProxyFailuresOnlyTransitionTheMatchingRouteOnce() throws Exception { + final CountDownLatch localCallsStarted = new CountDownLatch(2); + final CountDownLatch releaseFailures = new CountDownLatch(1); + final BackendApi local = + new BackendApi() { + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression) + throws IOException { + localCallsStarted.countDown(); + try { + assertTrue(releaseFailures.await(5, TimeUnit.SECONDS)); + } catch (final InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new AssertionError(exception); + } + throw new HttpResponseException(404, "local route missing"); + } + }; + final AtomicInteger directCalls = new AtomicInteger(); + final BackendApi direct = + new BackendApi() { + @Override + public T post( + final String uri, + final RequestBody requestBody, + final IOThrowingFunction responseParser, + @Nullable final OkHttpUtils.CustomListener requestListener, + final boolean requestCompression) { + directCalls.incrementAndGet(); + return null; + } + }; + final AtomicInteger directApiCreations = new AtomicInteger(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi( + local, + null, + () -> local, + () -> { + directApiCreations.incrementAndGet(); + return direct; + }, + "exposure"); + + final CompletableFuture first = + CompletableFuture.runAsync(() -> postWithoutFailure(api, "first concurrent proxy request")); + final CompletableFuture second = + CompletableFuture.runAsync( + () -> postWithoutFailure(api, "second concurrent proxy request")); + assertTrue(localCallsStarted.await(5, TimeUnit.SECONDS)); + releaseFailures.countDown(); + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + + assertEquals(1, directApiCreations.get()); + assertEquals(2, directCalls.get()); + } + + @Test + void recoveryFailureDoesNotInterruptTheWorkingDirectRoute() throws Exception { + final AtomicLong clock = new AtomicLong(); + final RecordingBackendApi direct = new RecordingBackendApi(); + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi( + null, + direct, + () -> { + throw new IllegalStateException("discovery failed"); + }, + () -> direct, + "exposure", + clock::get, + 10); + + clock.set(10); + api.post("exposures", requestBody("survives recovery failure"), stream -> null, null, false); + + assertEquals(1, direct.calls); + } + + private static void assertNoSameBatchReplayButUsesDirectForNext(final IOException failure) + throws Exception { final RecordingBackendApi local = new RecordingBackendApi(failure); final RecordingBackendApi direct = new RecordingBackendApi(); final AtomicInteger directApiCreations = new AtomicInteger(); final AgentlessFeatureFlagBackendApi api = new AgentlessFeatureFlagBackendApi( local, + null, + () -> local, () -> { directApiCreations.incrementAndGet(); return direct; @@ -149,16 +394,26 @@ private static void assertNoDirectReplay(final IOException failure) { assertThrows( IOException.class, () -> api.post("flagevaluation", requestBody("evaluation"), stream -> null, null, false)); + api.post("flagevaluation", requestBody("next"), stream -> null, null, false); assertEquals(1, local.calls); - assertEquals(0, direct.calls); - assertEquals(0, directApiCreations.get()); + assertEquals(1, direct.calls); + assertEquals(1, directApiCreations.get()); } private static RequestBody requestBody(final String value) { return RequestBody.create(MediaType.parse("application/json"), value); } + private static void postWithoutFailure( + final AgentlessFeatureFlagBackendApi api, final String body) { + try { + api.post("exposures", requestBody(body), stream -> null, null, false); + } catch (final IOException exception) { + throw new AssertionError(exception); + } + } + private static Stream featureFlagRoutes() { return Stream.of( Arguments.of("exposures", "exposure"), Arguments.of("flagevaluation", "flag evaluation")); diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index f70a4ff0fa1..4322eac245a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -1,9 +1,11 @@ package com.datadog.featureflag; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -65,8 +67,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.tabletest.junit.TableTest; @@ -79,6 +79,7 @@ class ExposureWriterTests { private final PollingConditions poll = new PollingConditions(TIMEOUT_SECONDS); private Queue requests; + private Queue requestAttempts; private Set failed; private JavaTestHttpServer server; private SharedCommunicationObjects sharedCommunicationObjects; @@ -86,6 +87,7 @@ class ExposureWriterTests { @BeforeEach void setUp() { requests = new ConcurrentLinkedQueue<>(); + requestAttempts = new ConcurrentLinkedQueue<>(); failed = Collections.newSetFromMap(new ConcurrentHashMap()); JsonAdapter adapter = new Moshi.Builder().build().adapter(ExposuresRequest.class); @@ -114,6 +116,7 @@ private void handleExposureRequest(HandlerApi api, JsonAdapter adapter.fromJson( Okio.buffer(Okio.source(new ByteArrayInputStream(api.getRequest().getBody())))); String serviceName = exposuresRequest.context.get("service"); + requestAttempts.add(serviceName); boolean failForever = "fail-forever".equals(serviceName); boolean fail = serviceName.startsWith("fail") && (failed.add(serviceName) || failForever); if (fail) { @@ -262,10 +265,26 @@ void testHighLoadScenario() throws Exception { } } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testFailuresAreRetried(boolean finallyFail) throws Exception { - String serviceName = finallyFail ? "fail-forever" : "fail-once"; + @Test + void testQueueThresholdFlushesWithoutWaitingForTheInterval() throws Exception { + Config config = mockConfig("threshold-service"); + List exposures = buildExposures(101); + + try (ExposureWriterImpl writer = + new ExposureWriterImpl( + 1 << 8, Long.MAX_VALUE, NANOSECONDS, sharedCommunicationObjects, config)) { + for (ExposureEvent exposure : exposures) { + writer.accept(exposure); + } + writer.init(); + + poll.eventually(() -> assertExposures(allExposures(), exposures)); + } + } + + @Test + void testHttpFailureIsNotRetriedAtTransportLayer() throws Exception { + String serviceName = "fail-once"; Config config = mockConfig(serviceName); try (ExposureWriterImpl writer = @@ -273,13 +292,11 @@ void testFailuresAreRetried(boolean finallyFail) throws Exception { writer.init(); writer.accept(buildExposure()); - MILLISECONDS.sleep(500); // wait for a flush to happen - ExposuresRequest found = findRequest(serviceName); - if (finallyFail) { - assertNull(found, requests.toString()); - } else { - poll.eventually(() -> assertNotNull(findRequest(serviceName), requests.toString())); - } + poll.eventually(() -> assertEquals(1, Collections.frequency(requestAttempts, serviceName))); + MILLISECONDS.sleep(500); + + assertEquals(1, Collections.frequency(requestAttempts, serviceName)); + assertNull(findRequest(serviceName), requests.toString()); } } @@ -309,7 +326,11 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception final BackendApi proxyApi = mock(BackendApi.class); final BackendApi directApi = mock(BackendApi.class); when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, true, HttpRetryPolicy.Factory.NEVER_RETRY)) + Intake.EVENT_PLATFORM, + true, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false)) .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(eq(Intake.EVENT_PLATFORM), eq(true), eq(false))) .thenReturn(directApi); @@ -383,6 +404,7 @@ private static Config mockConfig(String serviceName, String env, String version) private SharedCommunicationObjects sharedCommunicationObjects(boolean evpProxyAvailable) { DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); when(discovery.supportsEvpProxy()).thenReturn(evpProxyAvailable); + when(discovery.hasValidInfoResponse()).thenReturn(true); if (evpProxyAvailable) { when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index b89cf3ce2b1..402f327331f 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -2,8 +2,10 @@ import static com.datadog.featureflag.FeatureFlagEventType.EXPOSURE; import static com.datadog.featureflag.FeatureFlagEventType.FLAG_EVALUATION; +import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -14,6 +16,7 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; +import datadog.communication.ddagent.TracerVersion; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; @@ -21,12 +24,27 @@ class FeatureFlagBackendApiFactoryTest { + @Test + void configuresSdkIdentityHeadersForAllFeatureFlagEventTypes() { + assertEquals( + "dd-trace-java", FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN")); + assertEquals( + TracerVersion.TRACER_VERSION, + FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN-VERSION")); + } + @Test void remoteConfigUsesOnlyLocalEvpProxy() { final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi proxyApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(proxyApi); + when(backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false)) + .thenReturn(proxyApi); final BackendApi selected = new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); @@ -44,7 +62,13 @@ void remoteConfigDisablesDeliveryWhenLocalEvpProxyIsUnavailable() { new FeatureFlagBackendApiFactory(config, backendApiFactory, EXPOSURE).create(); assertNull(selected); - verify(backendApiFactory).createEvpProxyApi(Intake.EVENT_PLATFORM, true); + verify(backendApiFactory) + .createEvpProxyApi( + Intake.EVENT_PLATFORM, + true, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, true, false); } @@ -53,7 +77,11 @@ void agentlessPrefersLocalEvpProxyWithDirectFallback() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY)) + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false)) .thenReturn(mock(BackendApi.class)); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false)) .thenReturn(mock(BackendApi.class)); @@ -63,7 +91,12 @@ void agentlessPrefersLocalEvpProxyWithDirectFallback() { assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); verify(backendApiFactory) - .createEvpProxyApi(Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY); + .createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); } @@ -78,7 +111,7 @@ void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { final BackendApi selected = new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); - assertSame(directApi, selected); + assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); } @Test @@ -86,7 +119,13 @@ void agentlessUsesLocalEvpProxyWhenApiKeyIsUnavailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi proxyApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi(Intake.EVENT_PLATFORM, false)).thenReturn(proxyApi); + when(backendApiFactory.createEvpProxyApi( + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false)) + .thenReturn(proxyApi); final BackendApi selected = new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); @@ -124,7 +163,11 @@ void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi proxyApi = mock(BackendApi.class); when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY)) + Intake.EVENT_PLATFORM, + false, + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT, + false)) .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false)) .thenThrow(new IllegalArgumentException("invalid URL")); From 6b57b414908505ed38a6d97b5f6420c9672d9379 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 9 Sep 2026 05:03:26 +0000 Subject: [PATCH 27/30] Drain Feature Flags exposures on shutdown --- .../featureflag/ExposureWriterImpl.java | 158 ++++++++++++++---- .../featureflag/ExposureWriterTests.java | 158 ++++++++++++++++++ 2 files changed, 285 insertions(+), 31 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 6fec2eb3991..9a1091f0dda 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -7,6 +7,7 @@ import datadog.common.queue.MessagePassingBlockingQueue; import datadog.common.queue.Queues; +import datadog.communication.BackendApi; import datadog.communication.ddagent.SharedCommunicationObjects; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; @@ -17,6 +18,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,9 +31,14 @@ public class ExposureWriterImpl implements ExposureWriter { private static final int DEFAULT_FLUSH_INTERVAL_IN_SECONDS = 1; private static final int FLUSH_THRESHOLD = 100; private static final String EXPOSURES_ROUTE = "exposures"; + static final long SHUTDOWN_TIMEOUT_MILLIS = SECONDS.toMillis(5); private final MessagePassingBlockingQueue queue; + private final ExposureSerializingHandler serializer; private final Thread serializerThread; + private final ReentrantReadWriteLock lifecycleLock = new ReentrantReadWriteLock(); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final long shutdownTimeoutMillis; public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config config) { this(DEFAULT_CAPACITY, DEFAULT_FLUSH_INTERVAL_IN_SECONDS, SECONDS, sco, config); @@ -55,35 +64,94 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con final TimeUnit timeUnit, final FeatureFlagBackendApiFactory backendApiFactory, final Config config) { + this( + capacity, + flushInterval, + timeUnit, + backendApiFactory::create, + config, + SHUTDOWN_TIMEOUT_MILLIS); + } + + ExposureWriterImpl( + final int capacity, + final long flushInterval, + final TimeUnit timeUnit, + final Supplier backendApiSupplier, + final Config config, + final long shutdownTimeoutMillis) { this.queue = Queues.mpscBlockingConsumerArrayQueue(capacity); - final ExposureSerializingHandler serializer = + this.serializer = new ExposureSerializingHandler( - backendApiFactory, + backendApiSupplier, queue, flushInterval, timeUnit, FeatureFlagEvpContext.from(config), this::close); this.serializerThread = newAgentThread(FEATURE_FLAG_EXPOSURE_PROCESSOR, serializer); + this.shutdownTimeoutMillis = shutdownTimeoutMillis; } @Override public void init() { - FeatureFlaggingGateway.addExposureListener(this); - this.serializerThread.start(); + lifecycleLock.writeLock().lock(); + try { + if (closed.get()) { + return; + } + FeatureFlaggingGateway.addExposureListener(this); + this.serializerThread.start(); + } finally { + lifecycleLock.writeLock().unlock(); + } } @Override public void close() { - FeatureFlaggingGateway.removeExposureListener(this); - if (this.serializerThread.isAlive()) { - this.serializerThread.interrupt(); + final boolean workerRunning; + lifecycleLock.writeLock().lock(); + try { + if (!closed.compareAndSet(false, true)) { + return; + } + // Exclude all producers before asking the single consumer to perform its final drain. + FeatureFlaggingGateway.removeExposureListener(this); + workerRunning = this.serializerThread.isAlive(); + if (workerRunning) { + serializer.requestShutdown(); + this.serializerThread.interrupt(); + } + } finally { + lifecycleLock.writeLock().unlock(); + } + + // start() failure invokes close() from the serializer itself. It cannot join itself, and no + // final flush is possible when no backend route was created. + if (!workerRunning || Thread.currentThread() == this.serializerThread) { + return; + } + try { + // Bound application shutdown even if the final best-effort network request does not return. + this.serializerThread.join(shutdownTimeoutMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } } @Override public void accept(final ExposureEvent event) { - queue.offer(event); + // Multiple producers may still enqueue concurrently under the read lock. close() takes the + // write lock, so every accepted event is in the queue before the worker's final drain begins, + // while stale CopyOnWriteArrayList dispatch snapshots are rejected after shutdown starts. + lifecycleLock.readLock().lock(); + try { + if (!closed.get()) { + queue.offer(event); + } + } finally { + lifecycleLock.readLock().unlock(); + } } @VisibleForTesting @@ -107,9 +175,10 @@ private static class ExposureSerializingHandler implements Runnable { private final List buffer = new ArrayList<>(); private final Runnable errorCallback; + private final AtomicBoolean shutdownRequested = new AtomicBoolean(false); ExposureSerializingHandler( - final FeatureFlagBackendApiFactory backendApiFactory, + final Supplier backendApiSupplier, final MessagePassingBlockingQueue queue, final long flushInterval, final TimeUnit timeUnit, @@ -117,8 +186,7 @@ private static class ExposureSerializingHandler implements Runnable { final Runnable errorCallback) { this.queue = queue; this.cache = new LRUExposureCache(queue.capacity()); - this.evpPublisher = - new FeatureFlagEvpPublisher<>(backendApiFactory::create, ExposuresRequest.class); + this.evpPublisher = new FeatureFlagEvpPublisher<>(backendApiSupplier, ExposuresRequest.class); this.context = context; this.lastTicks = System.nanoTime(); @@ -129,6 +197,10 @@ private static class ExposureSerializingHandler implements Runnable { LOGGER.debug("starting exposure serializer"); } + void requestShutdown() { + shutdownRequested.set(true); + } + @Override public void run() { if (!evpPublisher.start()) { @@ -140,13 +212,25 @@ public void run() { runDutyCycle(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + } finally { + // close() interrupts this thread to wake queue.poll(). OkHttp fails fast when the calling + // thread is interrupted, so clear the flag for the final best-effort drain and restore it + // only after the final send-once attempt has completed. + final boolean wasInterrupted = Thread.interrupted(); + try { + drainAndFlush(); + } finally { + if (wasInterrupted) { + Thread.currentThread().interrupt(); + } + } } LOGGER.debug("exposure processor worker exited. submitting exposures stopped."); } private void runDutyCycle() throws InterruptedException { final Thread thread = Thread.currentThread(); - while (!thread.isInterrupted()) { + while (!thread.isInterrupted() && !shutdownRequested.get()) { ExposureEvent event; while ((event = queue.poll(100, TimeUnit.MILLISECONDS)) != null) { if (addToBuffer(event)) { @@ -162,6 +246,14 @@ private void consumeBatch() { queue.drain(this::addToBuffer, queue.size()); } + private void drainAndFlush() { + ExposureEvent event; + while ((event = queue.poll()) != null) { + addToBuffer(event); + } + flush(); + } + /** Adds an element to the buffer taking care of duplicated exposures thanks to the LRU cache */ private boolean addToBuffer(final ExposureEvent event) { if (cache.add(event)) { @@ -172,28 +264,32 @@ private boolean addToBuffer(final ExposureEvent event) { } protected void flushIfNecessary() { + if (!buffer.isEmpty() && shouldFlush()) { + flush(); + } + } + + private void flush() { if (buffer.isEmpty()) { return; } - if (shouldFlush()) { - final byte[] payload; - try { - final ExposuresRequest exposures = new ExposuresRequest(this.context, this.buffer); - payload = evpPublisher.serialize(exposures); - } catch (RuntimeException e) { - LOGGER.error(EXCLUDE_TELEMETRY, "Could not serialize exposures; dropping batch", e); - this.buffer.clear(); - return; - } - try { - evpPublisher.post(EXPOSURES_ROUTE, payload); - } catch (Exception e) { - LOGGER.debug("Could not submit exposures", e); - } finally { - // Best-effort delivery must not retry an ambiguously accepted batch. A later definitive - // proxy rejection could otherwise replay the same exposures through direct intake. - this.buffer.clear(); - } + final byte[] payload; + try { + final ExposuresRequest exposures = new ExposuresRequest(this.context, this.buffer); + payload = evpPublisher.serialize(exposures); + } catch (RuntimeException e) { + LOGGER.error(EXCLUDE_TELEMETRY, "Could not serialize exposures; dropping batch", e); + this.buffer.clear(); + return; + } + try { + evpPublisher.post(EXPOSURES_ROUTE, payload); + } catch (Exception e) { + LOGGER.debug("Could not submit exposures", e); + } finally { + // Best-effort delivery must not retry an ambiguously accepted batch. A later definitive + // proxy rejection could otherwise replay the same exposures through direct intake. + this.buffer.clear(); } } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 4322eac245a..9a6f5589a06 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -59,6 +59,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; import okhttp3.RequestBody; @@ -282,6 +283,163 @@ void testQueueThresholdFlushesWithoutWaitingForTheInterval() throws Exception { } } + @Test + void testCloseDrainsAndFinalFlushesExactlyOnceWithoutWaitingForTheInterval() throws Exception { + Config config = mockConfig("shutdown-service"); + List exposures = buildExposures(5); + ExposureWriterImpl writer = + new ExposureWriterImpl( + 1 << 4, Long.MAX_VALUE, NANOSECONDS, sharedCommunicationObjects, config); + + writer.init(); + for (ExposureEvent exposure : exposures) { + writer.accept(exposure); + } + + writer.close(); + + assertFalse(writer.isSerializerThreadAlive()); + assertEquals(1, requests.size()); + assertExposures(allExposures(), exposures); + + // A repeated close and a stale listener invocation after close must neither replay the batch + // nor leave an event stranded in the queue. + writer.close(); + writer.accept(buildExposure()); + MILLISECONDS.sleep(200); + assertEquals(1, requests.size()); + assertEquals(0, writer.queueSize()); + } + + @Test + void testFinalFlushRunsWithoutTheInterruptFlagSet() throws Exception { + BackendApi backendApi = mock(BackendApi.class); + AtomicBoolean interruptedDuringPost = new AtomicBoolean(true); + when(backendApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + invocation -> { + interruptedDuringPost.set(Thread.currentThread().isInterrupted()); + return null; + }); + ExposureWriterImpl writer = + new ExposureWriterImpl( + 1 << 4, + Long.MAX_VALUE, + NANOSECONDS, + () -> backendApi, + mockConfig("shutdown-service"), + ExposureWriterImpl.SHUTDOWN_TIMEOUT_MILLIS); + + writer.init(); + writer.accept(buildExposure()); + writer.close(); + + verify(backendApi, times(1)) + .post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false)); + assertFalse( + interruptedDuringPost.get(), + "The final HTTP request must not inherit the interrupt used to wake queue polling"); + assertFalse(writer.isSerializerThreadAlive()); + } + + @Test + void testCloseWaitIsBoundedWhenFinalPostDoesNotReturn() throws Exception { + BackendApi backendApi = mock(BackendApi.class); + CountDownLatch postStarted = new CountDownLatch(1); + CountDownLatch releasePost = new CountDownLatch(1); + when(backendApi.post(eq("exposures"), any(RequestBody.class), any(), any(), eq(false))) + .thenAnswer( + invocation -> { + postStarted.countDown(); + releasePost.await(); + return null; + }); + ExposureWriterImpl writer = + new ExposureWriterImpl( + 1 << 4, + Long.MAX_VALUE, + NANOSECONDS, + () -> backendApi, + mockConfig("blocked-shutdown-service"), + 100); + + try { + writer.init(); + writer.accept(buildExposure()); + long start = System.nanoTime(); + + writer.close(); + + long elapsedMillis = NANOSECONDS.toMillis(System.nanoTime() - start); + assertTrue(postStarted.await(1, java.util.concurrent.TimeUnit.SECONDS)); + assertTrue(elapsedMillis < 2000, "close exceeded its configured bounded wait"); + assertTrue(writer.isSerializerThreadAlive()); + } finally { + releasePost.countDown(); + } + poll.eventually(() -> assertFalse(writer.isSerializerThreadAlive())); + } + + @Test + void testStaleGatewayDispatchCannotEnqueueAfterClose() throws Exception { + CountDownLatch dispatchSnapshotTaken = new CountDownLatch(1); + CountDownLatch releaseDispatch = new CountDownLatch(1); + FeatureFlaggingGateway.ExposureListener blocker = + ignored -> { + dispatchSnapshotTaken.countDown(); + try { + releaseDispatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }; + FeatureFlaggingGateway.addExposureListener(blocker); + ExposureWriterImpl writer = + new ExposureWriterImpl( + 1 << 4, + Long.MAX_VALUE, + NANOSECONDS, + sharedCommunicationObjects, + mockConfig("stale-dispatch-service")); + Thread dispatcher = + new Thread(() -> FeatureFlaggingGateway.dispatch(buildExposure()), "exposure-dispatcher"); + + try { + writer.init(); + dispatcher.start(); + assertTrue(dispatchSnapshotTaken.await(5, java.util.concurrent.TimeUnit.SECONDS)); + + writer.close(); + releaseDispatch.countDown(); + dispatcher.join(5000); + + assertFalse(dispatcher.isAlive()); + assertFalse(writer.isSerializerThreadAlive()); + assertEquals(0, writer.queueSize()); + assertTrue(requests.isEmpty()); + } finally { + releaseDispatch.countDown(); + dispatcher.join(5000); + FeatureFlaggingGateway.removeExposureListener(blocker); + writer.close(); + } + } + + @Test + void testCloseBeforeInitPreventsLaterStartAndAccept() { + ExposureWriterImpl writer = + new ExposureWriterImpl(sharedCommunicationObjects, mockConfig("never-started-service")); + + writer.close(); + writer.init(); + writer.accept(buildExposure()); + writer.close(); + + assertFalse(writer.isSerializerThreadAlive()); + assertEquals(0, writer.queueSize()); + assertTrue(requests.isEmpty()); + } + @Test void testHttpFailureIsNotRetriedAtTransportLayer() throws Exception { String serviceName = "fail-once"; From d99255f0cb0754d1ffeee69441c786c2398eca82 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 9 Sep 2026 06:32:01 +0000 Subject: [PATCH 28/30] Keep agent shutdown active when tracing is disabled --- .../java/lang/ShutdownInstrumentation.java | 10 ++++- ...DisabledFeatureFlaggingShutdownTest.groovy | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy diff --git a/dd-java-agent/instrumentation/java/java-lang/java-lang-1.8/src/main/java/datadog/trace/instrumentation/java/lang/ShutdownInstrumentation.java b/dd-java-agent/instrumentation/java/java-lang/java-lang-1.8/src/main/java/datadog/trace/instrumentation/java/lang/ShutdownInstrumentation.java index 9aa879ad705..d51def6fa3d 100644 --- a/dd-java-agent/instrumentation/java/java-lang/java-lang-1.8/src/main/java/datadog/trace/instrumentation/java/lang/ShutdownInstrumentation.java +++ b/dd-java-agent/instrumentation/java/java-lang/java-lang-1.8/src/main/java/datadog/trace/instrumentation/java/lang/ShutdownInstrumentation.java @@ -8,6 +8,7 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.instrumentation.shutdown.ShutdownHelper; +import java.util.Set; import net.bytebuddy.asm.Advice; /** @@ -15,7 +16,7 @@ * before the shutdown hooks are called.
*/ @AutoService(InstrumenterModule.class) -public class ShutdownInstrumentation extends InstrumenterModule.Tracing +public class ShutdownInstrumentation extends InstrumenterModule implements Instrumenter.ForBootstrap, Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { public ShutdownInstrumentation() { @@ -27,6 +28,13 @@ public String instrumentedType() { return "java.lang.Shutdown"; } + @Override + public boolean isApplicable(Set enabledSystems) { + // Agent-owned subsystems such as Feature Flagging can run while tracing is disabled. Their + // bounded final drains still depend on ShutdownHelper running before application hooks. + return true; + } + @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( diff --git a/dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy new file mode 100644 index 00000000000..eb6f4e8e791 --- /dev/null +++ b/dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy @@ -0,0 +1,41 @@ +package datadog.trace.agent + +import datadog.trace.agent.test.IntegrationTestUtils +import jvmbootstraptest.AgentLoadedChecker +import spock.lang.Specification +import spock.lang.Timeout + +@Timeout(30) +class TraceDisabledFeatureFlaggingShutdownTest extends Specification { + + def "feature flagging is stopped by the real agent when tracing is disabled"() { + setup: + def output = new ByteArrayOutputStream() + def printStream = new PrintStream(output, true, "UTF-8") + + when: + def exitCode = IntegrationTestUtils.runOnSeparateJvm(AgentLoadedChecker.getName() + , [ + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Ddd.trace.enabled=false", + "-Ddd.feature.flags.enabled=true", + "-Ddd.feature.flags.configuration.source=agentless", + "-Ddd.jmxfetch.enabled=false", + "-Ddd.profiling.enabled=false", + "-Ddd.remote_config.enabled=false", + "-Ddd.telemetry.enabled=false" + ] + , [] + , [:] + , printStream) + def logs = output.toString("UTF-8") + + then: + exitCode == 0 + logs.contains("Shutting down agent") + logs.contains("Feature Flagging system stopped") + + cleanup: + printStream.close() + } +} From 84e50e34d7304c8d72f031710d7fde8ddb162a0c Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 9 Sep 2026 07:37:15 +0000 Subject: [PATCH 29/30] Validate direct intake DNS site suffixes --- .../communication/BackendApiFactory.java | 39 +++++++++++- .../communication/BackendApiFactoryTest.java | 62 ++++++++++++++++++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index 34b56fd8326..cfd2de845a7 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -21,6 +21,8 @@ public class BackendApiFactory { private static final Logger log = LoggerFactory.getLogger(BackendApiFactory.class); + private static final int MAX_DNS_LABEL_LENGTH = 63; + private static final int MAX_DNS_HOST_LENGTH = 253; private final Config config; private final SharedCommunicationObjects sharedCommunicationObjects; @@ -121,11 +123,14 @@ private static HttpUrl buildDirectIntakeUrl(Intake intake, Config config) { } static HttpUrl buildEventPlatformIntakeUrl(String site) { - if (site == null || site.isEmpty()) { + if (!isValidDnsSuffix(site)) { throw new IllegalArgumentException("Invalid Datadog site"); } String expectedHost = Intake.EVENT_PLATFORM.getUrlPrefix() + "." + site; + if (expectedHost.length() > MAX_DNS_HOST_LENGTH) { + throw new IllegalArgumentException("Invalid Datadog site"); + } HttpUrl url = new HttpUrl.Builder() .scheme("https") @@ -140,6 +145,38 @@ static HttpUrl buildEventPlatformIntakeUrl(String site) { return url; } + private static boolean isValidDnsSuffix(@Nullable String site) { + if (site == null || site.isEmpty()) { + return false; + } + + int labelLength = 0; + for (int i = 0; i < site.length(); i++) { + final char character = site.charAt(i); + if (character == '.') { + if (labelLength == 0 || labelLength > MAX_DNS_LABEL_LENGTH || site.charAt(i - 1) == '-') { + return false; + } + labelLength = 0; + } else { + if ((!isAsciiLetterOrDigit(character) && character != '-') + || (labelLength == 0 && character == '-')) { + return false; + } + labelLength++; + } + } + return labelLength > 0 + && labelLength <= MAX_DNS_LABEL_LENGTH + && site.charAt(site.length() - 1) != '-'; + } + + private static boolean isAsciiLetterOrDigit(final char character) { + return (character >= 'a' && character <= 'z') + || (character >= 'A' && character <= 'Z') + || (character >= '0' && character <= '9'); + } + /** Creates an API client that uses the specified retry policy with a compatible local proxy. */ public @Nullable BackendApi createEvpProxyApi(Intake intake) { return createEvpProxyApi(intake, true); diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index f02f3d81d6b..1696a3f972c 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -22,10 +22,12 @@ import datadog.trace.api.intake.Intake; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Properties; +import java.util.stream.Stream; import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -35,6 +37,7 @@ import okhttp3.mockwebserver.RecordedRequest; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.NullAndEmptySource; import org.junit.jupiter.params.provider.ValueSource; @@ -43,8 +46,18 @@ class BackendApiFactoryTest { private static final MediaType JSON = MediaType.parse("application/json"); @ParameterizedTest - @ValueSource(strings = {"datadoghq.com", "custom.example", "DATADOGHQ.EU"}) + @ValueSource(strings = {"datadoghq.com", "custom.example", "DATADOGHQ.EU", "mock-intake.invalid"}) void eventPlatformDirectIntakeUsesExactHttpsHost(String site) { + assertEventPlatformIntakeUrl(site); + } + + @ParameterizedTest + @MethodSource("boundaryValidSites") + void eventPlatformDirectIntakeAcceptsDnsLengthBoundaries(String site) { + assertEventPlatformIntakeUrl(site); + } + + private static void assertEventPlatformIntakeUrl(final String site) { final HttpUrl url = BackendApiFactory.buildEventPlatformIntakeUrl(site); assertEquals("https", url.scheme()); @@ -73,13 +86,58 @@ void eventPlatformDirectIntakeUsesExactHttpsHost(String site) { "data doghq.com", " datadoghq.com", "datadoghq.com ", - "datadoghq.com\\evil.example" + "datadoghq.com\\evil.example", + "-foo.bar", + "foo-.bar", + "foo.-bar", + "foo.bar-", + "foo_bar.com", + ".foo.bar", + "foo..bar", + "foo.bar." }) void eventPlatformDirectIntakeRejectsUnsafeSite(String site) { assertThrows( IllegalArgumentException.class, () -> BackendApiFactory.buildEventPlatformIntakeUrl(site)); } + @ParameterizedTest + @MethodSource("invalidLengthSites") + void eventPlatformDirectIntakeRejectsDnsLengthOverflow(String site) { + assertThrows( + IllegalArgumentException.class, () -> BackendApiFactory.buildEventPlatformIntakeUrl(site)); + } + + private static Stream boundaryValidSites() { + return Stream.of( + repeatedAsciiLabel(63) + ".invalid", + repeatedAsciiLabel(63) + + "." + + repeatedAsciiLabel(63) + + "." + + repeatedAsciiLabel(63) + + "." + + repeatedAsciiLabel(39)); + } + + private static Stream invalidLengthSites() { + return Stream.of( + repeatedAsciiLabel(64) + ".invalid", + repeatedAsciiLabel(63) + + "." + + repeatedAsciiLabel(63) + + "." + + repeatedAsciiLabel(63) + + "." + + repeatedAsciiLabel(40)); + } + + private static String repeatedAsciiLabel(final int length) { + final char[] label = new char[length]; + Arrays.fill(label, 'a'); + return new String(label); + } + @ParameterizedTest @ValueSource(ints = {301, 302, 307, 308}) void featureFlagDirectIntakeDoesNotFollowRedirects(final int statusCode) throws Exception { From f1a76966057cb7bf0d9921a8731d016a20fe7575 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 12 Sep 2026 03:45:45 +0000 Subject: [PATCH 30/30] Complete Feature Flags EVP route hardening Share Agentless route state across event writers, preserve Agent prefixes, capability-gate local forwarding, and cover deterministic send-once failover behavior. Environment: Datadog workspace --- .../communication/BackendApiFactory.java | 48 ++++-- .../ddagent/DDAgentFeaturesDiscovery.java | 40 ++++- .../DDAgentFeaturesDiscoveryTest.groovy | 2 - .../communication/BackendApiFactoryTest.java | 158 ++++++++++++++---- ...DisabledFeatureFlaggingShutdownTest.groovy | 41 ----- ...ceDisabledFeatureFlaggingShutdownTest.java | 45 +++++ .../featureflag/FeatureFlaggingSystem.java | 6 +- .../AgentlessFeatureFlagBackendApi.java | 146 ++++++++-------- .../featureflag/ExposureWriterImpl.java | 12 ++ .../FeatureFlagBackendApiFactory.java | 74 ++++---- .../featureflag/FeatureFlagRouteSelector.java | 79 +++++++++ .../featureflag/FlagEvaluationWriterImpl.java | 14 ++ .../AgentlessFeatureFlagBackendApiTest.java | 98 ++++++++--- .../featureflag/ExposureWriterTests.java | 23 +-- .../FeatureFlagBackendApiFactoryTest.java | 119 ++----------- 15 files changed, 557 insertions(+), 348 deletions(-) delete mode 100644 dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy create mode 100644 dd-java-agent/src/test/java/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagRouteSelector.java diff --git a/communication/src/main/java/datadog/communication/BackendApiFactory.java b/communication/src/main/java/datadog/communication/BackendApiFactory.java index cfd2de845a7..08c2b08d897 100644 --- a/communication/src/main/java/datadog/communication/BackendApiFactory.java +++ b/communication/src/main/java/datadog/communication/BackendApiFactory.java @@ -190,22 +190,21 @@ private static boolean isAsciiLetterOrDigit(final char character) { /** Creates an API client that sends data through a compatible local EVP proxy. */ public @Nullable BackendApi createEvpProxyApi( Intake intake, boolean responseCompression, HttpRetryPolicy.Factory retryPolicyFactory) { - return createEvpProxyApi(intake, responseCompression, retryPolicyFactory, null, false); + return createEvpProxyApi(intake, responseCompression, retryPolicyFactory, false, false); } /** - * Creates an EVP proxy client, optionally retaining a compatibility endpoint when Agent discovery - * itself is unavailable. + * Creates an EVP proxy client after Agent discovery, optionally forcing a fresh discovery and + * requiring the Agent to advertise every configured request header. * - *

An authoritative Agent response that omits EVP support never uses the fallback endpoint. The - * {@code forceDiscovery} form is intended for bounded route-recovery probes. + *

The {@code forceDiscovery} form is intended for bounded unavailable-route recovery probes. */ public @Nullable BackendApi createEvpProxyApi( Intake intake, boolean responseCompression, HttpRetryPolicy.Factory retryPolicyFactory, - @Nullable String discoveryFailureFallbackEndpoint, - boolean forceDiscovery) { + boolean forceDiscovery, + boolean requireConfiguredRequestHeaders) { DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); if (forceDiscovery) { @@ -214,22 +213,39 @@ private static boolean isAsciiLetterOrDigit(final char character) { featuresDiscovery.discoverIfOutdated(); } String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); - if (evpProxyEndpoint == null - && discoveryFailureFallbackEndpoint != null - && !featuresDiscovery.hasValidInfoResponse()) { - evpProxyEndpoint = discoveryFailureFallbackEndpoint; + if (evpProxyEndpoint != null + && requireConfiguredRequestHeaders + && !featuresDiscovery.supportsEvpProxyHeaders(requestHeaders.keySet())) { + evpProxyEndpoint = null; } if (evpProxyEndpoint == null) { return null; } + return createEvpProxyApi(intake, responseCompression, retryPolicyFactory, evpProxyEndpoint); + } + + /** Creates an EVP proxy client for a fixed compatibility endpoint without Agent discovery. */ + public BackendApi createEvpProxyApiForEndpoint( + Intake intake, + boolean responseCompression, + HttpRetryPolicy.Factory retryPolicyFactory, + String evpProxyEndpoint) { + return createEvpProxyApi(intake, responseCompression, retryPolicyFactory, evpProxyEndpoint); + } + + private BackendApi createEvpProxyApi( + Intake intake, + boolean responseCompression, + HttpRetryPolicy.Factory retryPolicyFactory, + String evpProxyEndpoint) { String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); log.debug( "Creating EVP proxy client for {} using endpoint {} with responseCompression={}", intake, evpProxyEndpoint, responseCompression); - HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); + HttpUrl evpProxyUrl = appendPath(sharedCommunicationObjects.agentUrl, evpProxyEndpoint); String subdomain = intake.getUrlPrefix(); return new EvpProxyApi( traceId, @@ -240,6 +256,14 @@ private static boolean isAsciiLetterOrDigit(final char character) { responseCompression); } + static HttpUrl appendPath(final HttpUrl baseUrl, final String path) { + int firstCharacter = 0; + while (firstCharacter < path.length() && path.charAt(firstCharacter) == '/') { + firstCharacter++; + } + return baseUrl.newBuilder().addPathSegments(path.substring(firstCharacter)).build(); + } + OkHttpClient configureHttpClient(final OkHttpClient httpClient) { if (requestHeaders.isEmpty() && !sendOnce) { return httpClient; diff --git a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java index ed053806a5c..c7fc60f1d1f 100644 --- a/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java +++ b/communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java @@ -24,6 +24,7 @@ import java.nio.ByteBuffer; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import okhttp3.HttpUrl; @@ -98,12 +99,12 @@ private static class State { String debuggerSnapshotEndpoint; String debuggerDiagnosticsEndpoint; String evpProxyEndpoint; + Set evpProxyAllowedHeaders = emptySet(); String version; String telemetryProxyEndpoint; Set peerTags = emptySet(); String orgPropagationMarker; long lastTimeDiscovered; - boolean validInfoResponse; } private volatile State discoveryState; @@ -157,7 +158,7 @@ private void doDiscovery(State newState) { try (Recording recording = discoveryTimer.start()) { boolean fallback = true; final Request request = - prepareRequest(agentBaseUrl.resolve("info"), emptyMap()).get().build(); + prepareRequest(appendPath(agentBaseUrl, "info"), emptyMap()).get().build(); try (Response response = client.newCall(request).execute()) { if (response.isSuccessful()) { processInfoResponseHeaders(response); @@ -207,7 +208,7 @@ private String probeTracesEndpoint(State newState, List endpoints) { try (Response response = client .newCall( - prepareRequest(agentBaseUrl.resolve(candidate), emptyMap()) + prepareRequest(appendPath(agentBaseUrl, candidate), emptyMap()) .put(msgpackRequestBodyOf(singletonList(ByteBuffer.wrap(PROBE_MESSAGE)))) .build()) .execute()) { @@ -290,6 +291,16 @@ private boolean processInfoResponse(State newState, String response) { break; } } + final Object allowedHeadersObj = map.get("evp_proxy_allowed_headers"); + if (allowedHeadersObj instanceof List) { + final Set allowedHeaders = new HashSet<>(); + for (Object header : (List) allowedHeadersObj) { + if (header instanceof String) { + allowedHeaders.add(((String) header).toLowerCase(Locale.ROOT)); + } + } + newState.evpProxyAllowedHeaders = unmodifiableSet(allowedHeaders); + } for (String endpoint : telemetryProxyEndpoints) { if (containsEndpoint(endpoints, endpoint)) { @@ -327,7 +338,6 @@ private boolean processInfoResponse(State newState, String response) { log.debug( "Failed to hash trace agent /info response. Will probe {}", newState.traceEndpoint, ex); } - newState.validInfoResponse = true; return true; } catch (Throwable error) { log.debug("Error parsing trace agent /info response", error); @@ -426,7 +436,7 @@ public String getEvpProxyEndpoint() { } public HttpUrl buildUrl(String endpoint) { - return agentBaseUrl.resolve(endpoint); + return appendPath(agentBaseUrl, endpoint); } public boolean supportsDataStreams() { @@ -437,9 +447,15 @@ public boolean supportsEvpProxy() { return discoveryState.evpProxyEndpoint != null; } - /** Returns whether the last discovery attempt received a valid Agent info response. */ - public boolean hasValidInfoResponse() { - return discoveryState.validInfoResponse; + /** Returns whether the Agent advertises forwarding every required EVP request header. */ + public boolean supportsEvpProxyHeaders(final Iterable requiredHeaders) { + final Set allowedHeaders = discoveryState.evpProxyAllowedHeaders; + for (String requiredHeader : requiredHeaders) { + if (!allowedHeaders.contains(requiredHeader.toLowerCase(Locale.ROOT))) { + return false; + } + } + return true; } public boolean supportsContentEncodingHeadersWithEvpProxy() { @@ -476,4 +492,12 @@ public boolean active() { public boolean supportsTelemetryProxy() { return discoveryState.telemetryProxyEndpoint != null; } + + private static HttpUrl appendPath(final HttpUrl baseUrl, final String path) { + int firstCharacter = 0; + while (firstCharacter < path.length() && path.charAt(firstCharacter) == '/') { + firstCharacter++; + } + return baseUrl.newBuilder().addPathSegments(path.substring(firstCharacter)).build(); + } } diff --git a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy index 0079e02650c..8bcda6eb811 100644 --- a/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy +++ b/communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy @@ -73,7 +73,6 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { features.getDebuggerSnapshotEndpoint() == "debugger/v2/input" features.supportsDebuggerDiagnostics() features.supportsEvpProxy() - features.hasValidInfoResponse() features.supportsContentEncodingHeadersWithEvpProxy() features.getEvpProxyEndpoint() == "evp_proxy/v4/" features.getVersion() == "0.99.0" @@ -102,7 +101,6 @@ class DDAgentFeaturesDiscoveryTest extends DDSpecification { 0 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.5/traces" }) >> { Request request -> success(request) } 1 * client.newCall({ Request request -> request.url().toString() == "http://localhost:8125/v0.4/traces" }) >> { Request request -> success(request) } features.getTraceEndpoint() == V04_ENDPOINT - !features.hasValidInfoResponse() 0 * _ } diff --git a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java index 1696a3f972c..5ead5496322 100644 --- a/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java +++ b/communication/src/test/java/datadog/communication/BackendApiFactoryTest.java @@ -9,9 +9,11 @@ import static datadog.trace.api.config.GeneralConfig.API_KEY; import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.communication.ddagent.DDAgentFeaturesDiscovery; import datadog.communication.ddagent.SharedCommunicationObjects; @@ -221,6 +223,103 @@ void advertisedEvpProxyEndpointSupportsDisabledResponseCompression() throws Exce } } + @Test + void evpProxyPreservesConfiguredAgentBasePath() throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); + agent.start(); + try { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT); + final BackendApiFactory factory = + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, agent.url("/agent/base"))); + final BackendApi api = factory.createBackendApi(Intake.EVENT_PLATFORM, false); + + assertNotNull(api); + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false); + + assertEquals("/agent/base/evp_proxy/v4/api/v2/flagevaluation", agent.takeRequest().getPath()); + } finally { + agent.shutdown(); + } + } + + @Test + void discoveryPreservesAgentBasePathAndAcceptsCaseInsensitiveIdentityCapabilities() + throws Exception { + final MockWebServer agent = new MockWebServer(); + agent.enqueue( + new MockResponse() + .setResponseCode(200) + .setBody( + "{\"endpoints\":[\"v0.5/traces\",\"/evp_proxy/v4/\"]," + + "\"evp_proxy_allowed_headers\":[\"dd-evp-origin\"," + + "\"DD-EVP-ORIGIN-VERSION\"]}")); + agent.start(); + try { + final DDAgentFeaturesDiscovery discovery = + new DDAgentFeaturesDiscovery( + new OkHttpClient(), + Monitoring.DISABLED, + agent.url("/agent/base"), + ProtocolVersion.V0_5, + true, + false); + + discovery.discover(); + + assertEquals("/agent/base/info", agent.takeRequest().getPath()); + assertEquals(V4_EVP_PROXY_ENDPOINT, discovery.getEvpProxyEndpoint()); + assertTrue(discovery.supportsEvpProxyHeaders(sdkHeaders().keySet())); + } finally { + agent.shutdown(); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "{}", + "{\"evp_proxy_allowed_headers\":null}", + "{\"evp_proxy_allowed_headers\":[\"DD-EVP-ORIGIN\"]}", + "{\"evp_proxy_allowed_headers\":[\"DD-EVP-ORIGIN-VERSION\"]}" + }) + void discoveryRejectsMissingOrPartialIdentityCapabilities(final String capabilityJson) + throws Exception { + final MockWebServer agent = new MockWebServer(); + final String fields = capabilityJson.substring(1, capabilityJson.length() - 1); + agent.enqueue( + new MockResponse() + .setResponseCode(200) + .setBody( + "{\"endpoints\":[\"v0.5/traces\",\"evp_proxy/v4/\"]" + + (fields.isEmpty() ? "" : "," + fields) + + "}")); + agent.start(); + try { + final DDAgentFeaturesDiscovery discovery = + new DDAgentFeaturesDiscovery( + new OkHttpClient(), + Monitoring.DISABLED, + agent.url("/"), + ProtocolVersion.V0_5, + true, + false); + + discovery.discover(); + + assertEquals(V4_EVP_PROXY_ENDPOINT, discovery.getEvpProxyEndpoint()); + assertFalse(discovery.supportsEvpProxyHeaders(sdkHeaders().keySet())); + } finally { + agent.shutdown(); + } + } + @Test void featureFlagProxyRequestCarriesSdkIdentityWithoutDirectCredentials() throws Exception { final MockWebServer agent = new MockWebServer(); @@ -236,11 +335,7 @@ void featureFlagProxyRequestCarriesSdkIdentityWithoutDirectCredentials() throws true); final BackendApi api = factory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false); + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, false, true); assertNotNull(api); api.post( @@ -351,25 +446,34 @@ void featureFlagDirectRequestCarriesCredentialsAndIdentityExactlyOnce() throws E } @Test - void discoveryFailureRetainsV2CompatibilityRoute() throws Exception { + void fixedV2CompatibilityRouteDoesNotDependOnDiscovery() throws Exception { final MockWebServer agent = new MockWebServer(); + agent.enqueue(new MockResponse().setResponseCode(500).setBody("ambiguous")); agent.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); agent.start(); try { - final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null, false); + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null); final BackendApiFactory factory = new BackendApiFactory( Config.get(), sharedCommunicationObjects(discovery, agent.url("/"))); final BackendApi api = - factory.createEvpProxyApi( + factory.createEvpProxyApiForEndpoint( Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false); + V2_EVP_PROXY_ENDPOINT); assertNotNull(api); + assertThrows( + HttpResponseException.class, + () -> + api.post( + "flagevaluation", + RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), + stream -> null, + null, + false)); api.post( "flagevaluation", RequestBody.create(JSON, "{}".getBytes(StandardCharsets.UTF_8)), @@ -377,39 +481,34 @@ void discoveryFailureRetainsV2CompatibilityRoute() throws Exception { null, false); assertEquals("/evp_proxy/v2/api/v2/flagevaluation", agent.takeRequest().getPath()); + assertEquals("/evp_proxy/v2/api/v2/flagevaluation", agent.takeRequest().getPath()); + assertEquals(2, agent.getRequestCount()); } finally { agent.shutdown(); } } @Test - void authoritativeMissingProxyDoesNotUseV2CompatibilityRoute() { - final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null, true); + void capabilityGatedProxyRequiresConfiguredRequestHeaders() { + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(V4_EVP_PROXY_ENDPOINT, false); final BackendApiFactory factory = - new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); + new BackendApiFactory( + Config.get(), sharedCommunicationObjects(discovery, null), sdkHeaders(), true); assertNull( factory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false)); + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, false, true)); } @Test void recoveryRequestForcesFreshDiscovery() { - final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null, true); + final FakeFeaturesDiscovery discovery = new FakeFeaturesDiscovery(null); final BackendApiFactory factory = new BackendApiFactory(Config.get(), sharedCommunicationObjects(discovery, null)); assertNull( factory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - true)); + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, true, true)); assertEquals(1, discovery.forcedDiscoveries); assertEquals(0, discovery.outdatedDiscoveries); } @@ -476,7 +575,7 @@ public DDAgentFeaturesDiscovery featuresDiscovery(final Config config) { private static final class FakeFeaturesDiscovery extends DDAgentFeaturesDiscovery { private final String evpProxyEndpoint; - private final boolean validInfoResponse; + private final boolean supportsRequestHeaders; private int forcedDiscoveries; private int outdatedDiscoveries; @@ -484,7 +583,8 @@ private FakeFeaturesDiscovery(final String evpProxyEndpoint) { this(evpProxyEndpoint, true); } - private FakeFeaturesDiscovery(final String evpProxyEndpoint, final boolean validInfoResponse) { + private FakeFeaturesDiscovery( + final String evpProxyEndpoint, final boolean supportsRequestHeaders) { super( new OkHttpClient(), Monitoring.DISABLED, @@ -493,7 +593,7 @@ private FakeFeaturesDiscovery(final String evpProxyEndpoint, final boolean valid true, false); this.evpProxyEndpoint = evpProxyEndpoint; - this.validInfoResponse = validInfoResponse; + this.supportsRequestHeaders = supportsRequestHeaders; } @Override @@ -517,8 +617,8 @@ public boolean supportsEvpProxy() { } @Override - public boolean hasValidInfoResponse() { - return validInfoResponse; + public boolean supportsEvpProxyHeaders(final Iterable requiredHeaders) { + return supportsRequestHeaders; } } } diff --git a/dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy b/dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy deleted file mode 100644 index eb6f4e8e791..00000000000 --- a/dd-java-agent/src/test/groovy/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.groovy +++ /dev/null @@ -1,41 +0,0 @@ -package datadog.trace.agent - -import datadog.trace.agent.test.IntegrationTestUtils -import jvmbootstraptest.AgentLoadedChecker -import spock.lang.Specification -import spock.lang.Timeout - -@Timeout(30) -class TraceDisabledFeatureFlaggingShutdownTest extends Specification { - - def "feature flagging is stopped by the real agent when tracing is disabled"() { - setup: - def output = new ByteArrayOutputStream() - def printStream = new PrintStream(output, true, "UTF-8") - - when: - def exitCode = IntegrationTestUtils.runOnSeparateJvm(AgentLoadedChecker.getName() - , [ - "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", - "-Ddd.trace.enabled=false", - "-Ddd.feature.flags.enabled=true", - "-Ddd.feature.flags.configuration.source=agentless", - "-Ddd.jmxfetch.enabled=false", - "-Ddd.profiling.enabled=false", - "-Ddd.remote_config.enabled=false", - "-Ddd.telemetry.enabled=false" - ] - , [] - , [:] - , printStream) - def logs = output.toString("UTF-8") - - then: - exitCode == 0 - logs.contains("Shutting down agent") - logs.contains("Feature Flagging system stopped") - - cleanup: - printStream.close() - } -} diff --git a/dd-java-agent/src/test/java/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.java b/dd-java-agent/src/test/java/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.java new file mode 100644 index 00000000000..7dc4596c447 --- /dev/null +++ b/dd-java-agent/src/test/java/datadog/trace/agent/TraceDisabledFeatureFlaggingShutdownTest.java @@ -0,0 +1,45 @@ +package datadog.trace.agent; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.agent.test.IntegrationTestUtils; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import jvmbootstraptest.AgentLoadedChecker; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +class TraceDisabledFeatureFlaggingShutdownTest { + @Test + @Timeout(value = 30, unit = TimeUnit.SECONDS) + void featureFlaggingIsStoppedByTheRealAgentWhenTracingIsDisabled() throws Exception { + try (ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream printStream = new PrintStream(output, true, StandardCharsets.UTF_8.name())) { + int exitCode = + IntegrationTestUtils.runOnSeparateJvm( + AgentLoadedChecker.class.getName(), + Arrays.asList( + "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug", + "-Ddd.trace.enabled=false", + "-Ddd.feature.flags.enabled=true", + "-Ddd.feature.flags.configuration.source=agentless", + "-Ddd.jmxfetch.enabled=false", + "-Ddd.profiling.enabled=false", + "-Ddd.remote_config.enabled=false", + "-Ddd.telemetry.enabled=false"), + Collections.emptyList(), + Collections.emptyMap(), + printStream); + + String logs = output.toString(StandardCharsets.UTF_8.name()); + assertEquals(0, exitCode); + assertTrue(logs.contains("Shutting down agent")); + assertTrue(logs.contains("Feature Flagging system stopped")); + } + } +} diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 7f105d00213..038c409cf64 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -94,7 +94,8 @@ private static void initializeSystem(final SharedCommunicationObjects sco, final LOGGER.debug("Feature Flagging system disabled by unsupported configuration source"); return; } - final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); + final FeatureFlagRouteSelector routeSelector = new FeatureFlagRouteSelector(); + final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config, routeSelector); initialize(configService, exposureWriter); final boolean evalCountsEnabled = @@ -103,7 +104,8 @@ private static void initializeSystem(final SharedCommunicationObjects sco, final .getBoolean(FeatureFlaggingConfig.FLAGGING_EVALUATION_COUNTS_ENABLED, true); FeatureFlaggingGateway.setFlagEvaluationEnqueueEnabled(evalCountsEnabled); if (evalCountsEnabled) { - final FlagEvaluationWriterImpl evalWriter = new FlagEvaluationWriterImpl(sco, config); + final FlagEvaluationWriterImpl evalWriter = + new FlagEvaluationWriterImpl(sco, config, routeSelector); // Publish before start() so a failed start is still reachable by the rollback in stop(). FLAG_EVAL_WRITER = evalWriter; evalWriter.start(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java index a32ffd7fd6d..bfb5d42b759 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApi.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; -import java.util.concurrent.TimeUnit; import java.util.function.LongSupplier; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -15,22 +14,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Sends Feature Flag events through a local EVP proxy, with a safe direct intake fallback. */ +/** Sends Feature Flag events through the process-wide Agentless EVP route selector. */ final class AgentlessFeatureFlagBackendApi implements BackendApi { private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessFeatureFlagBackendApi.class); - private static final long DEFAULT_RECOVERY_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(1); + private final FeatureFlagRouteSelector routeSelector; private final Supplier proxyApiSupplier; private final Supplier directApiSupplier; private final String eventType; - private final LongSupplier nanoTime; - private final long recoveryIntervalNanos; - private volatile Route activeRoute; + private volatile BackendApi proxyApi; private volatile BackendApi directApi; private volatile boolean directApiCreationAttempted; - private volatile long nextProxyProbeNanos; AgentlessFeatureFlagBackendApi( @Nullable final BackendApi proxyApi, @@ -44,8 +40,7 @@ final class AgentlessFeatureFlagBackendApi implements BackendApi { proxyApiSupplier, directApiSupplier, eventType, - System::nanoTime, - DEFAULT_RECOVERY_INTERVAL_NANOS); + new FeatureFlagRouteSelector()); } AgentlessFeatureFlagBackendApi( @@ -56,20 +51,30 @@ final class AgentlessFeatureFlagBackendApi implements BackendApi { final String eventType, final LongSupplier nanoTime, final long recoveryIntervalNanos) { - if (proxyApi == null && directApi == null) { - throw new IllegalArgumentException("A Feature Flagging event route is required"); - } - this.proxyApiSupplier = proxyApiSupplier; + this( + proxyApi, + directApi, + proxyApiSupplier, + directApiSupplier, + eventType, + new FeatureFlagRouteSelector(nanoTime, recoveryIntervalNanos)); + } + + AgentlessFeatureFlagBackendApi( + @Nullable final BackendApi proxyApi, + @Nullable final BackendApi directApi, + final Supplier proxyApiSupplier, + final Supplier directApiSupplier, + final String eventType, + final FeatureFlagRouteSelector routeSelector) { + this.proxyApi = proxyApi; this.directApi = directApi; + this.proxyApiSupplier = proxyApiSupplier; this.directApiSupplier = directApiSupplier; this.eventType = eventType; - this.nanoTime = nanoTime; - this.recoveryIntervalNanos = recoveryIntervalNanos; - this.activeRoute = proxyApi != null ? new Route(proxyApi, true) : new Route(directApi, false); + this.routeSelector = routeSelector; this.directApiCreationAttempted = directApi != null; - if (proxyApi == null) { - scheduleProxyRecovery(); - } + routeSelector.initialize(proxyApi != null, directApi != null); } @Override @@ -80,16 +85,17 @@ public T post( @Nullable final OkHttpUtils.CustomListener requestListener, final boolean requestCompression) throws IOException { - final Route selectedRoute = selectRoute(); + final SelectedApi selected = selectApi(); try { - return selectedRoute.api.post( + return selected.api.post( uri, requestBody, responseParser, requestListener, requestCompression); } catch (final IOException exception) { - if (!selectedRoute.proxy) { + if (!selected.local) { throw exception; } - final BackendApi fallbackApi = switchFutureBatchesToDirect(selectedRoute); + final BackendApi fallbackApi = getOrCreateDirectApi(); + routeSelector.localFailure(fallbackApi != null); if (fallbackApi == null || !isSafeToReplayDirectly(exception)) { throw exception; } @@ -98,60 +104,53 @@ public T post( } } - private Route selectRoute() { - final Route selectedRoute = activeRoute; - if (selectedRoute.proxy || !proxyRecoveryDue()) { - return selectedRoute; + private SelectedApi selectApi() throws IOException { + FeatureFlagRouteSelector.Route selectedRoute = routeSelector.current(); + if (selectedRoute == FeatureFlagRouteSelector.Route.UNAVAILABLE + && routeSelector.tryBeginLocalRecovery()) { + final BackendApi recoveredProxyApi = discoverProxyApi(); + if (recoveredProxyApi != null) { + proxyApi = recoveredProxyApi; + routeSelector.localRecovered(); + } + selectedRoute = routeSelector.current(); } - synchronized (this) { - final Route currentRoute = activeRoute; - if (currentRoute.proxy || !proxyRecoveryDue()) { - return currentRoute; + if (selectedRoute == FeatureFlagRouteSelector.Route.LOCAL) { + BackendApi selectedProxyApi = proxyApi; + if (selectedProxyApi == null) { + selectedProxyApi = discoverProxyApi(); + if (selectedProxyApi != null) { + proxyApi = selectedProxyApi; + } else { + final BackendApi selectedDirectApi = getOrCreateDirectApi(); + routeSelector.localFailure(selectedDirectApi != null); + if (selectedDirectApi != null) { + return new SelectedApi(selectedDirectApi, false); + } + throw unavailableRoute(); + } } - // Reserve the next recovery window before performing discovery so concurrent senders keep - // using direct intake instead of blocking or creating a probe stampede. - scheduleProxyRecovery(); + return new SelectedApi(selectedProxyApi, true); } - BackendApi recoveredProxyApi = null; - try { - recoveredProxyApi = proxyApiSupplier.get(); - } catch (final RuntimeException exception) { - // Route recovery is best effort. A discovery/configuration failure must not interrupt the - // working direct route and lose the current batch. - LOGGER.debug("Could not recover the local Feature Flagging {} route", eventType, exception); - } - if (recoveredProxyApi != null) { - synchronized (this) { - if (!activeRoute.proxy) { - LOGGER.debug( - "Switching Feature Flagging {} delivery from direct intake to the local EVP proxy", - eventType); - activeRoute = new Route(recoveredProxyApi, true); - } + if (selectedRoute == FeatureFlagRouteSelector.Route.DIRECT) { + final BackendApi selectedDirectApi = getOrCreateDirectApi(); + if (selectedDirectApi != null) { + return new SelectedApi(selectedDirectApi, false); } } - return activeRoute; + throw unavailableRoute(); } @Nullable - private BackendApi switchFutureBatchesToDirect(final Route failedProxyRoute) { - final BackendApi fallbackApi = getOrCreateDirectApi(); - if (fallbackApi == null) { + private BackendApi discoverProxyApi() { + try { + return proxyApiSupplier.get(); + } catch (final RuntimeException exception) { + LOGGER.debug("Could not discover the local Feature Flagging {} route", eventType, exception); return null; } - - synchronized (this) { - if (activeRoute == failedProxyRoute) { - LOGGER.debug( - "Switching Feature Flagging {} delivery from the local EVP proxy to direct intake", - eventType); - activeRoute = new Route(fallbackApi, false); - scheduleProxyRecovery(); - } - } - return fallbackApi; } @Nullable @@ -159,7 +158,6 @@ private BackendApi getOrCreateDirectApi() { if (directApiCreationAttempted) { return directApi; } - synchronized (this) { if (!directApiCreationAttempted) { directApi = directApiSupplier.get(); @@ -169,12 +167,8 @@ private BackendApi getOrCreateDirectApi() { } } - private boolean proxyRecoveryDue() { - return nanoTime.getAsLong() - nextProxyProbeNanos >= 0; - } - - private void scheduleProxyRecovery() { - nextProxyProbeNanos = nanoTime.getAsLong() + recoveryIntervalNanos; + private IOException unavailableRoute() { + return new IOException("No Feature Flagging " + eventType + " delivery route is available"); } private static boolean isSafeToReplayDirectly(final IOException exception) { @@ -188,13 +182,13 @@ private static boolean isSafeToReplayDirectly(final IOException exception) { return false; } - private static final class Route { + private static final class SelectedApi { private final BackendApi api; - private final boolean proxy; + private final boolean local; - private Route(final BackendApi api, final boolean proxy) { + private SelectedApi(final BackendApi api, final boolean local) { this.api = api; - this.proxy = proxy; + this.local = local; } } } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java index 9a1091f0dda..7ca34d4c6cb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ExposureWriterImpl.java @@ -44,6 +44,18 @@ public ExposureWriterImpl(final SharedCommunicationObjects sco, final Config con this(DEFAULT_CAPACITY, DEFAULT_FLUSH_INTERVAL_IN_SECONDS, SECONDS, sco, config); } + ExposureWriterImpl( + final SharedCommunicationObjects sco, + final Config config, + final FeatureFlagRouteSelector routeSelector) { + this( + DEFAULT_CAPACITY, + DEFAULT_FLUSH_INTERVAL_IN_SECONDS, + SECONDS, + new FeatureFlagBackendApiFactory(config, sco, FeatureFlagEventType.EXPOSURE, routeSelector), + config); + } + ExposureWriterImpl( final int capacity, final long flushInterval, diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java index b1755130e13..7c41efcc551 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagBackendApiFactory.java @@ -29,73 +29,83 @@ final class FeatureFlagBackendApiFactory { private final Config config; private final BackendApiFactory backendApiFactory; private final FeatureFlagEventType eventType; + private final FeatureFlagRouteSelector routeSelector; FeatureFlagBackendApiFactory( final Config config, final SharedCommunicationObjects sharedCommunicationObjects, final FeatureFlagEventType eventType) { + this(config, sharedCommunicationObjects, eventType, new FeatureFlagRouteSelector()); + } + + FeatureFlagBackendApiFactory( + final Config config, + final SharedCommunicationObjects sharedCommunicationObjects, + final FeatureFlagEventType eventType, + final FeatureFlagRouteSelector routeSelector) { this( config, new BackendApiFactory(config, sharedCommunicationObjects, REQUEST_HEADERS, true), - eventType); + eventType, + routeSelector); } FeatureFlagBackendApiFactory( final Config config, final BackendApiFactory backendApiFactory, final FeatureFlagEventType eventType) { + this(config, backendApiFactory, eventType, new FeatureFlagRouteSelector()); + } + + FeatureFlagBackendApiFactory( + final Config config, + final BackendApiFactory backendApiFactory, + final FeatureFlagEventType eventType, + final FeatureFlagRouteSelector routeSelector) { this.config = config; this.backendApiFactory = backendApiFactory; this.eventType = eventType; + this.routeSelector = routeSelector; } @Nullable BackendApi create() { final boolean agentless = CONFIGURATION_SOURCE_AGENTLESS.equals(config.getFeatureFlaggingConfigurationSource()); - final boolean directFallbackAvailable = agentless && hasDirectCredentials(); - // Preserve the historical v2 endpoint when initial discovery itself is unavailable. Recovery - // from a working direct route is stricter below: only an advertised endpoint proves that local - // delivery has returned, avoiding an ambiguous failed probe of an assumed v2 endpoint. - final BackendApi proxyApi = createProxyApi(false, true); if (!agentless) { - if (proxyApi == null) { - LOGGER.warn( - "Feature Flagging {} delivery is disabled because the local Agent does not support the EVP proxy", - eventType.logName()); - } - return proxyApi; + // Preserve the historical Agent-backed client: fixed EVP v2, no /info discovery, no direct + // credentials, and no Agentless route state. + return backendApiFactory.createEvpProxyApiForEndpoint( + Intake.EVENT_PLATFORM, + eventType.responseCompressionEnabled(), + HttpRetryPolicy.Factory.NEVER_RETRY, + V2_EVP_PROXY_ENDPOINT); } - if (!directFallbackAvailable) { - return proxyApi; - } - - final BackendApi directApi = proxyApi == null ? createDirectApi() : null; - if (proxyApi != null || directApi != null) { - return new AgentlessFeatureFlagBackendApi( - proxyApi, - directApi, - () -> createProxyApi(true, false), - this::createDirectApi, + final BackendApi proxyApi = createProxyApi(false); + final BackendApi directApi = createDirectApi(); + if (proxyApi == null && directApi == null) { + LOGGER.warn( + "Feature Flagging {} delivery is waiting for a compatible local EVP proxy because direct intake credentials are unavailable", eventType.logName()); } - - LOGGER.warn( - "Feature Flagging {} delivery is disabled because no compatible local EVP proxy or direct intake credentials are available", - eventType.logName()); - return null; + return new AgentlessFeatureFlagBackendApi( + proxyApi, + directApi, + () -> createProxyApi(true), + this::createDirectApi, + eventType.logName(), + routeSelector); } @Nullable - private BackendApi createProxyApi( - final boolean forceDiscovery, final boolean useDiscoveryFailureFallback) { + private BackendApi createProxyApi(final boolean forceDiscovery) { return backendApiFactory.createEvpProxyApi( Intake.EVENT_PLATFORM, eventType.responseCompressionEnabled(), HttpRetryPolicy.Factory.NEVER_RETRY, - useDiscoveryFailureFallback ? V2_EVP_PROXY_ENDPOINT : null, - forceDiscovery); + forceDiscovery, + true); } private static Map requestHeaders() { diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagRouteSelector.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagRouteSelector.java new file mode 100644 index 00000000000..da3a2da2125 --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FeatureFlagRouteSelector.java @@ -0,0 +1,79 @@ +package com.datadog.featureflag; + +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; + +/** Process-wide route state shared by all Feature Flagging event writers. */ +final class FeatureFlagRouteSelector { + + static final long DEFAULT_RECOVERY_INTERVAL_NANOS = TimeUnit.MINUTES.toNanos(1); + + enum Route { + UNINITIALIZED, + LOCAL, + DIRECT, + UNAVAILABLE + } + + private final LongSupplier nanoTime; + private final long recoveryIntervalNanos; + private volatile Route route = Route.UNINITIALIZED; + private volatile long nextLocalDiscoveryNanos; + + FeatureFlagRouteSelector() { + this(System::nanoTime, DEFAULT_RECOVERY_INTERVAL_NANOS); + } + + FeatureFlagRouteSelector(final LongSupplier nanoTime, final long recoveryIntervalNanos) { + this.nanoTime = nanoTime; + this.recoveryIntervalNanos = recoveryIntervalNanos; + } + + synchronized Route initialize(final boolean localAvailable, final boolean directAvailable) { + if (route == Route.UNINITIALIZED || route == Route.UNAVAILABLE) { + if (localAvailable) { + route = Route.LOCAL; + } else if (directAvailable) { + route = Route.DIRECT; + } else { + route = Route.UNAVAILABLE; + scheduleLocalDiscovery(); + } + } + return route; + } + + Route current() { + return route; + } + + synchronized Route localFailure(final boolean directAvailable) { + if (route == Route.LOCAL) { + if (directAvailable) { + route = Route.DIRECT; + } else { + route = Route.UNAVAILABLE; + scheduleLocalDiscovery(); + } + } + return route; + } + + synchronized boolean tryBeginLocalRecovery() { + if (route != Route.UNAVAILABLE || nanoTime.getAsLong() - nextLocalDiscoveryNanos < 0) { + return false; + } + scheduleLocalDiscovery(); + return true; + } + + synchronized void localRecovered() { + if (route == Route.UNAVAILABLE) { + route = Route.LOCAL; + } + } + + private void scheduleLocalDiscovery() { + nextLocalDiscoveryNanos = nanoTime.getAsLong() + recoveryIntervalNanos; + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java index 2da8ffe0256..4969376771c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/FlagEvaluationWriterImpl.java @@ -108,6 +108,20 @@ public FlagEvaluationWriterImpl(final SharedCommunicationObjects sco, final Conf config); } + FlagEvaluationWriterImpl( + final SharedCommunicationObjects sco, + final Config config, + final FeatureFlagRouteSelector routeSelector) { + this( + DEFAULT_CAPACITY, + FLUSH_INTERVAL_SECONDS, + SECONDS, + new FeatureFlagBackendApiFactory( + config, sco, FeatureFlagEventType.FLAG_EVALUATION, routeSelector) + ::create, + config); + } + FlagEvaluationWriterImpl( final int capacity, final long flushInterval, diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java index a5802c00ddc..012ba26ea01 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessFeatureFlagBackendApiTest.java @@ -82,7 +82,7 @@ void fallsBackAfterConnectionRefusal(final String route, final String eventType) } @Test - void recoversTheLocalRouteAfterTheCooldown() throws Exception { + void keepsDirectRouteStickyAfterLocalFailure() throws Exception { final AtomicLong clock = new AtomicLong(); final RecordingBackendApi unavailableLocal = new RecordingBackendApi(new ConnectException("connection refused")); @@ -99,8 +99,8 @@ void recoversTheLocalRouteAfterTheCooldown() throws Exception { api.post("exposures", requestBody("third"), stream -> null, null, false); assertEquals(1, unavailableLocal.calls); - assertEquals(2, direct.calls); - assertEquals(1, recoveredLocal.calls); + assertEquals(3, direct.calls); + assertEquals(0, recoveredLocal.calls); } @ParameterizedTest @@ -121,7 +121,7 @@ void doesNotReplayConnectionResetButSwitchesFutureBatches() throws Exception { } @Test - void startsDirectAndRetriesLocalDiscoveryOnlyAfterTheCooldown() throws Exception { + void startsDirectAndNeverProbesLocal() throws Exception { final AtomicLong clock = new AtomicLong(); final RecordingBackendApi direct = new RecordingBackendApi(); final RecordingBackendApi recoveredLocal = new RecordingBackendApi(); @@ -145,15 +145,14 @@ void startsDirectAndRetriesLocalDiscoveryOnlyAfterTheCooldown() throws Exception clock.set(10); api.post("exposures", requestBody("third"), stream -> null, null, false); - assertEquals(1, proxyApiCreations.get()); - assertEquals(2, direct.calls); - assertEquals(1, recoveredLocal.calls); + assertEquals(0, proxyApiCreations.get()); + assertEquals(3, direct.calls); + assertEquals(0, recoveredLocal.calls); } @Test void concurrentSendersDoNotBlockOnOrDuplicateARecoveryProbe() throws Exception { final AtomicLong clock = new AtomicLong(10); - final RecordingBackendApi direct = new RecordingBackendApi(); final RecordingBackendApi recoveredLocal = new RecordingBackendApi(); final AtomicInteger proxyApiCreations = new AtomicInteger(); final CountDownLatch probeStarted = new CountDownLatch(1); @@ -161,7 +160,7 @@ void concurrentSendersDoNotBlockOnOrDuplicateARecoveryProbe() throws Exception { final AgentlessFeatureFlagBackendApi api = new AgentlessFeatureFlagBackendApi( null, - direct, + null, () -> { proxyApiCreations.incrementAndGet(); probeStarted.countDown(); @@ -173,7 +172,7 @@ void concurrentSendersDoNotBlockOnOrDuplicateARecoveryProbe() throws Exception { } return recoveredLocal; }, - () -> direct, + () -> null, "flag evaluation", clock::get, 10); @@ -191,41 +190,46 @@ void concurrentSendersDoNotBlockOnOrDuplicateARecoveryProbe() throws Exception { }); assertTrue(probeStarted.await(5, TimeUnit.SECONDS)); - api.post("flagevaluation", requestBody("parallel"), stream -> null, null, false); + assertThrows( + IOException.class, + () -> api.post("flagevaluation", requestBody("parallel"), stream -> null, null, false)); releaseProbe.countDown(); recoveringPost.get(5, TimeUnit.SECONDS); assertEquals(1, proxyApiCreations.get()); - assertEquals(1, direct.calls); assertEquals(1, recoveredLocal.calls); } @Test void failedRecoveryIsStickyForAnotherCooldown() throws Exception { final AtomicLong clock = new AtomicLong(); - final RecordingBackendApi direct = new RecordingBackendApi(); final AtomicInteger proxyApiCreations = new AtomicInteger(); final AgentlessFeatureFlagBackendApi api = new AgentlessFeatureFlagBackendApi( null, - direct, + null, () -> { proxyApiCreations.incrementAndGet(); return null; }, - () -> direct, + () -> null, "exposure", clock::get, 10); clock.set(10); - api.post("exposures", requestBody("first"), stream -> null, null, false); - api.post("exposures", requestBody("second"), stream -> null, null, false); + assertThrows( + IOException.class, + () -> api.post("exposures", requestBody("first"), stream -> null, null, false)); + assertThrows( + IOException.class, + () -> api.post("exposures", requestBody("second"), stream -> null, null, false)); clock.set(20); - api.post("exposures", requestBody("third"), stream -> null, null, false); + assertThrows( + IOException.class, + () -> api.post("exposures", requestBody("third"), stream -> null, null, false)); assertEquals(2, proxyApiCreations.get()); - assertEquals(3, direct.calls); } @Test @@ -248,20 +252,21 @@ void doesNotRetryDirectApiCreationWhenFallbackIsUnavailable() { HttpResponseException.class, () -> api.post("exposures", requestBody("first"), stream -> null, null, false)); assertThrows( - HttpResponseException.class, + IOException.class, () -> api.post("exposures", requestBody("second"), stream -> null, null, false)); - assertEquals(2, local.calls); + assertEquals(1, local.calls); assertEquals(1, directApiCreations.get()); } @Test - void requiresAtLeastOneInitialRoute() { + void permitsUnavailableStartupSoLocalDeliveryCanRecover() { + final AgentlessFeatureFlagBackendApi api = + new AgentlessFeatureFlagBackendApi(null, null, () -> null, () -> null, "flag evaluation"); + assertThrows( - IllegalArgumentException.class, - () -> - new AgentlessFeatureFlagBackendApi( - null, null, () -> null, () -> null, "flag evaluation")); + IOException.class, + () -> api.post("flagevaluation", requestBody("first"), stream -> null, null, false)); } @Test @@ -354,14 +359,52 @@ public T post( } @Test - void recoveryFailureDoesNotInterruptTheWorkingDirectRoute() throws Exception { + void sharesRouteTransitionsAcrossExposureAndFlagEvaluationWriters() throws Exception { + final FeatureFlagRouteSelector routeSelector = new FeatureFlagRouteSelector(); + final RecordingBackendApi exposureLocal = + new RecordingBackendApi(new SocketTimeoutException("ambiguous timeout")); + final RecordingBackendApi exposureDirect = new RecordingBackendApi(); + final RecordingBackendApi evaluationLocal = new RecordingBackendApi(); + final RecordingBackendApi evaluationDirect = new RecordingBackendApi(); + final AgentlessFeatureFlagBackendApi exposureApi = + new AgentlessFeatureFlagBackendApi( + exposureLocal, + exposureDirect, + () -> exposureLocal, + () -> exposureDirect, + "exposure", + routeSelector); + final AgentlessFeatureFlagBackendApi evaluationApi = + new AgentlessFeatureFlagBackendApi( + evaluationLocal, + evaluationDirect, + () -> evaluationLocal, + () -> evaluationDirect, + "flag evaluation", + routeSelector); + + assertThrows( + SocketTimeoutException.class, + () -> exposureApi.post("exposures", requestBody("first"), stream -> null, null, false)); + evaluationApi.post("flagevaluation", requestBody("second"), stream -> null, null, false); + + assertEquals(1, exposureLocal.calls); + assertEquals(0, exposureDirect.calls); + assertEquals(0, evaluationLocal.calls); + assertEquals(1, evaluationDirect.calls); + } + + @Test + void workingDirectRouteDoesNotAttemptRecovery() throws Exception { final AtomicLong clock = new AtomicLong(); final RecordingBackendApi direct = new RecordingBackendApi(); + final AtomicInteger proxyApiCreations = new AtomicInteger(); final AgentlessFeatureFlagBackendApi api = new AgentlessFeatureFlagBackendApi( null, direct, () -> { + proxyApiCreations.incrementAndGet(); throw new IllegalStateException("discovery failed"); }, () -> direct, @@ -373,6 +416,7 @@ void recoveryFailureDoesNotInterruptTheWorkingDirectRoute() throws Exception { api.post("exposures", requestBody("survives recovery failure"), stream -> null, null, false); assertEquals(1, direct.calls); + assertEquals(0, proxyApiCreations.get()); } private static void assertNoSameBatchReplayButUsesDirectForNext(final IOException failure) diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java index 9a6f5589a06..a0c4f7d59ee 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/ExposureWriterTests.java @@ -1,6 +1,5 @@ package com.datadog.featureflag; -import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; @@ -73,7 +72,7 @@ class ExposureWriterTests { - private static final String EXPOSURES_ENDPOINT = "/evp_proxy/api/v2/exposures"; + private static final String EXPOSURES_ENDPOINT = "/evp_proxy/v2/api/v2/exposures"; private static final String DIRECT_EXPOSURES_ENDPOINT = "/api/v2/exposures"; private static final String API_KEY = "test-api-key"; private static final double TIMEOUT_SECONDS = 5; @@ -484,11 +483,7 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception final BackendApi proxyApi = mock(BackendApi.class); final BackendApi directApi = mock(BackendApi.class); when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - true, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false)) + Intake.EVENT_PLATFORM, true, HttpRetryPolicy.Factory.NEVER_RETRY, false, true)) .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(eq(Intake.EVENT_PLATFORM), eq(true), eq(false))) .thenReturn(directApi); @@ -532,17 +527,14 @@ void testAmbiguousExposureBatchIsNotRetriedOrReplayedDirectly() throws Exception } @Test - void testWriterStopsReceivingExposuresIfEvpProxyIsNotAvailable() throws Exception { + void testAgentlessWriterWaitsForUnavailableProxyRecovery() throws Exception { SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(false); + Config config = mockConfig("unavailable-service"); + when(config.getFeatureFlaggingConfigurationSource()).thenReturn(CONFIGURATION_SOURCE_AGENTLESS); - try (ExposureWriterImpl writer = - new ExposureWriterImpl(sharedCommunicationObjects, Config.get())) { + try (ExposureWriterImpl writer = new ExposureWriterImpl(sharedCommunicationObjects, config)) { writer.init(); - poll.eventually(() -> assertFalse(writer.isSerializerThreadAlive())); - - FeatureFlaggingGateway.dispatch(buildExposure()); - - assertEquals(0, writer.queueSize()); + poll.eventually(() -> assertTrue(writer.isSerializerThreadAlive())); } } @@ -562,7 +554,6 @@ private static Config mockConfig(String serviceName, String env, String version) private SharedCommunicationObjects sharedCommunicationObjects(boolean evpProxyAvailable) { DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); when(discovery.supportsEvpProxy()).thenReturn(evpProxyAvailable); - when(discovery.hasValidInfoResponse()).thenReturn(true); if (evpProxyAvailable) { when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java index 402f327331f..58811e0bfb6 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/FeatureFlagBackendApiFactoryTest.java @@ -5,9 +5,7 @@ import static datadog.communication.ddagent.DDAgentFeaturesDiscovery.V2_EVP_PROXY_ENDPOINT; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_AGENTLESS; import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.CONFIGURATION_SOURCE_REMOTE_CONFIG; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -16,7 +14,6 @@ import datadog.communication.BackendApi; import datadog.communication.BackendApiFactory; -import datadog.communication.ddagent.TracerVersion; import datadog.communication.http.HttpRetryPolicy; import datadog.trace.api.Config; import datadog.trace.api.intake.Intake; @@ -25,63 +22,33 @@ class FeatureFlagBackendApiFactoryTest { @Test - void configuresSdkIdentityHeadersForAllFeatureFlagEventTypes() { - assertEquals( - "dd-trace-java", FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN")); - assertEquals( - TracerVersion.TRACER_VERSION, - FeatureFlagBackendApiFactory.REQUEST_HEADERS.get("DD-EVP-ORIGIN-VERSION")); - } - - @Test - void remoteConfigUsesOnlyLocalEvpProxy() { + void remoteConfigUsesFixedV2WithoutDiscoveryOrDirectIntake() { final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi proxyApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi( + when(backendApiFactory.createEvpProxyApiForEndpoint( Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false)) + V2_EVP_PROXY_ENDPOINT)) .thenReturn(proxyApi); final BackendApi selected = new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertSame(proxyApi, selected); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); - } - - @Test - void remoteConfigDisablesDeliveryWhenLocalEvpProxyIsUnavailable() { - final Config config = config(CONFIGURATION_SOURCE_REMOTE_CONFIG, "api-key"); - final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - - final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, EXPOSURE).create(); - - assertNull(selected); - verify(backendApiFactory) + verify(backendApiFactory, never()) .createEvpProxyApi( - Intake.EVENT_PLATFORM, - true, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, true, false); + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, false, true); + verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); } @Test - void agentlessPrefersLocalEvpProxyWithDirectFallback() { + void agentlessPrefersCapabilityGatedLocalRouteWithDirectFallbackReady() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false)) + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, false, true)) .thenReturn(mock(BackendApi.class)); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false)) .thenReturn(mock(BackendApi.class)); @@ -92,21 +59,16 @@ void agentlessPrefersLocalEvpProxyWithDirectFallback() { assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); verify(backendApiFactory) .createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, false, true); + verify(backendApiFactory).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); } @Test - void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { + void agentlessUsesDirectIntakeWhenLocalRouteIsUnavailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi directApi = mock(BackendApi.class); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false)) - .thenReturn(directApi); + .thenReturn(mock(BackendApi.class)); final BackendApi selected = new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); @@ -115,59 +77,24 @@ void agentlessUsesDirectIntakeWhenLocalEvpProxyIsUnavailable() { } @Test - void agentlessUsesLocalEvpProxyWhenApiKeyIsUnavailable() { - final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); - final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi proxyApi = mock(BackendApi.class); - when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false)) - .thenReturn(proxyApi); - - final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); - - assertSame(proxyApi, selected); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); - } - - @Test - void agentlessDisablesDeliveryWhenNoRouteIsAvailable() { + void agentlessKeepsWriterAliveWhileEveryRouteIsUnavailable() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, null); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); - - assertNull(selected); - } - - @Test - void agentlessDisablesDeliveryWhenApiKeyIsEmpty() { - final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, ""); - final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - final BackendApi selected = new FeatureFlagBackendApiFactory(config, backendApiFactory, EXPOSURE).create(); - assertNull(selected); + assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, true, false); } @Test - void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { + void agentlessKeepsCompatibleLocalRouteWhenDirectUrlIsInvalid() { final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); final BackendApi proxyApi = mock(BackendApi.class); when(backendApiFactory.createEvpProxyApi( - Intake.EVENT_PLATFORM, - false, - HttpRetryPolicy.Factory.NEVER_RETRY, - V2_EVP_PROXY_ENDPOINT, - false)) + Intake.EVENT_PLATFORM, false, HttpRetryPolicy.Factory.NEVER_RETRY, false, true)) .thenReturn(proxyApi); when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false)) .thenThrow(new IllegalArgumentException("invalid URL")); @@ -176,20 +103,6 @@ void agentlessDoesNotValidateDirectUrlWhileLocalRouteIsAvailable() { new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); assertInstanceOf(AgentlessFeatureFlagBackendApi.class, selected); - verify(backendApiFactory, never()).createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false); - } - - @Test - void agentlessDisablesDeliveryWhenDirectUrlIsInvalidAndLocalRouteIsUnavailable() { - final Config config = config(CONFIGURATION_SOURCE_AGENTLESS, "api-key"); - final BackendApiFactory backendApiFactory = mock(BackendApiFactory.class); - when(backendApiFactory.createDirectIntakeApi(Intake.EVENT_PLATFORM, false, false)) - .thenThrow(new IllegalArgumentException("invalid URL")); - - final BackendApi selected = - new FeatureFlagBackendApiFactory(config, backendApiFactory, FLAG_EVALUATION).create(); - - assertNull(selected); } private static Config config(final String source, final String apiKey) {