diff --git a/api/src/main/java/io/grpc/ClientStreamTracer.java b/api/src/main/java/io/grpc/ClientStreamTracer.java
index d654fd33358..9358bb67935 100644
--- a/api/src/main/java/io/grpc/ClientStreamTracer.java
+++ b/api/src/main/java/io/grpc/ClientStreamTracer.java
@@ -62,12 +62,12 @@ public void createPendingStream() {
* connection establishment) starts.
*
* This method is invoked synchronously on the attempt thread. Implementations should start
- * internal timers or child tracing spans (named strictly {@code "Attempt Delay"}) carrying the
+ * internal timers or child tracing spans (named strictly {@code "Delay"}) carrying the
* canonical {@code grpc.delay_type} attribute.
*
* @param delayType canonical low-cardinality label categorizing the delay (e.g., "connecting")
* @param delayReason high-cardinality diagnostic string describing granular runtime conditions
- * @since 1.82.0
+ * @since 1.84.0
*/
public void recordAttemptDelayStart(String delayType, String delayReason) {
}
@@ -80,7 +80,7 @@ public void recordAttemptDelayStart(String delayType, String delayReason) {
* on the active delay span without recreating the span or resetting cumulative timers.
*
* @param delayReason updated high-cardinality diagnostic string describing new conditions
- * @since 1.82.0
+ * @since 1.84.0
*/
public void recordAttemptDelayReasonChanged(String delayReason) {
}
@@ -91,7 +91,7 @@ public void recordAttemptDelayReasonChanged(String delayReason) {
*
Implementations should simultaneously close active child tracing spans and record elapsed
* duration to the {@code grpc.client.attempt.delay.duration} histogram.
*
- * @since 1.82.0
+ * @since 1.84.0
*/
public void recordAttemptDelayEnd() {
}
@@ -156,6 +156,44 @@ public abstract static class Factory {
public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) {
throw new UnsupportedOperationException("Not implemented");
}
+
+ /**
+ * Called when a call-level delay segment (such as waiting for name resolution or service
+ * configuration parsing) starts before any individual RPC attempt is created.
+ *
+ *
Implementations should start logical timers and create child tracing spans (named strictly
+ * {@code "Delay"}) carrying the canonical {@code grpc.delay_type} attribute.
+ *
+ * @param delayType canonical low-cardinality label categorizing the delay (e.g., "resolving")
+ * @param delayReason high-cardinality diagnostic string describing granular runtime conditions
+ * @since 1.85.0
+ */
+ public void recordCallDelayStart(String delayType, String delayReason) {
+ }
+
+ /**
+ * Called when a call-level delay reason changes while the active delay segment continues.
+ *
+ *
Implementations should emit structured events (such as {@code "Delay state transition"})
+ * on the active call delay span without recreating the span or resetting timers.
+ *
+ * @param delayReason updated high-cardinality diagnostic string describing new conditions
+ * @since 1.85.0
+ */
+ public void recordCallDelayReasonChanged(String delayReason) {
+ }
+
+ /**
+ * Called when a call-level delay segment ends upon successful name resolution or when an RPC
+ * is cancelled before resolution completes.
+ *
+ *
Implementations should close active call delay spans and record elapsed duration to the
+ * {@code grpc.client.call.delay.duration} histogram.
+ *
+ * @since 1.85.0
+ */
+ public void recordCallDelayEnd() {
+ }
}
/**
diff --git a/api/src/main/java/io/grpc/LoadBalancer.java b/api/src/main/java/io/grpc/LoadBalancer.java
index e5c3d053ee7..e121daa0664 100644
--- a/api/src/main/java/io/grpc/LoadBalancer.java
+++ b/api/src/main/java/io/grpc/LoadBalancer.java
@@ -739,21 +739,29 @@ public static PickResult withNoResult() {
*
* @param delayType low-cardinality root cause label (e.g., "connecting")
* @param delayReason high-cardinality diagnostic string for trace events
- * @since 1.82.0
+ * @since 1.84.0
*/
public static PickResult withNoResult(String delayType, String delayReason) {
- Preconditions.checkNotNull(delayType, "delayType");
- Preconditions.checkNotNull(delayReason, "delayReason");
+ checkNotNull(delayType, "delayType");
+ checkNotNull(delayReason, "delayReason");
return new PickResult(null, null, Status.OK, false, null, delayType, delayReason);
}
- /** Returns the delay type label if any. */
+ /**
+ * Returns the delay type label if any.
+ *
+ * @since 1.84.0
+ */
@Nullable
public String getDelayType() {
return delayType;
}
- /** Returns the diagnostic delay reason if any. */
+ /**
+ * Returns the diagnostic delay reason if any.
+ *
+ * @since 1.84.0
+ */
@Nullable
public String getDelayReason() {
return delayReason;
@@ -821,6 +829,8 @@ public String toString() {
.add("status", status)
.add("drop", drop)
.add("authority-override", authorityOverride)
+ .add("delayType", delayType)
+ .add("delayReason", delayReason)
.toString();
}
diff --git a/api/src/test/java/io/grpc/ClientStreamTracerTest.java b/api/src/test/java/io/grpc/ClientStreamTracerTest.java
index 5ddee77f5c0..1d605a4f707 100644
--- a/api/src/test/java/io/grpc/ClientStreamTracerTest.java
+++ b/api/src/test/java/io/grpc/ClientStreamTracerTest.java
@@ -57,4 +57,17 @@ public void streamInfo_toBuilder() {
StreamInfo info2 = info1.toBuilder().build();
assertThat(info2.getCallOptions()).isSameInstanceAs(callOptions);
}
+
+ @Test
+ public void defaultDelayMethodsNoOp() {
+ ClientStreamTracer tracer = new ClientStreamTracer() {};
+ tracer.recordAttemptDelayStart("connecting", "test");
+ tracer.recordAttemptDelayReasonChanged("test2");
+ tracer.recordAttemptDelayEnd();
+
+ ClientStreamTracer.Factory factory = new ClientStreamTracer.Factory() {};
+ factory.recordCallDelayStart("resolving", "test");
+ factory.recordCallDelayReasonChanged("test2");
+ factory.recordCallDelayEnd();
+ }
}
diff --git a/api/src/test/java/io/grpc/LoadBalancerTest.java b/api/src/test/java/io/grpc/LoadBalancerTest.java
index 22fdc220081..ea71a593f17 100644
--- a/api/src/test/java/io/grpc/LoadBalancerTest.java
+++ b/api/src/test/java/io/grpc/LoadBalancerTest.java
@@ -91,6 +91,21 @@ public void pickResult_withNoResult() {
assertThat(result.getStatus()).isSameInstanceAs(Status.OK);
assertThat(result.getStreamTracerFactory()).isNull();
assertThat(result.isDrop()).isFalse();
+ assertThat(result.getDelayType()).isNull();
+ assertThat(result.getDelayReason()).isNull();
+ }
+
+ @Test
+ public void pickResult_withNoResult_withDelay() {
+ PickResult result = PickResult.withNoResult("connecting", "trying backends");
+ assertThat(result.getSubchannel()).isNull();
+ assertThat(result.getStatus()).isSameInstanceAs(Status.OK);
+ assertThat(result.getStreamTracerFactory()).isNull();
+ assertThat(result.isDrop()).isFalse();
+ assertThat(result.getDelayType()).isEqualTo("connecting");
+ assertThat(result.getDelayReason()).isEqualTo("trying backends");
+ assertThat(result.toString()).contains("delayType=connecting");
+ assertThat(result.toString()).contains("delayReason=trying backends");
}
@Test
@@ -118,6 +133,10 @@ public void pickResult_equals() {
PickResult sc3 = PickResult.withSubchannel(subchannel, tracerFactory);
PickResult sc4 = PickResult.withSubchannel(subchannel2);
PickResult nr = PickResult.withNoResult();
+ PickResult nrDelay1 = PickResult.withNoResult("connecting", "trying 10.0.0.1");
+ PickResult nrDelay2 = PickResult.withNoResult("connecting", "trying 10.0.0.1");
+ PickResult nrDelayDiffReason = PickResult.withNoResult("connecting", "trying 10.0.0.2");
+ PickResult nrDelayDiffType = PickResult.withNoResult("rls_lookup_pending", "trying 10.0.0.1");
PickResult error1 = PickResult.withError(status);
PickResult error2 = PickResult.withError(status2);
PickResult error3 = PickResult.withError(status2);
@@ -132,6 +151,12 @@ public void pickResult_equals() {
assertThat(sc1).isNotEqualTo(sc3);
assertThat(sc1).isNotEqualTo(sc4);
+ assertThat(nr).isEqualTo(nrDelay1);
+ assertThat(nrDelay1).isEqualTo(nrDelay2);
+ assertThat(nrDelay1.hashCode()).isEqualTo(nrDelay2.hashCode());
+ assertThat(nrDelay1).isEqualTo(nrDelayDiffReason);
+ assertThat(nrDelay1).isEqualTo(nrDelayDiffType);
+
assertThat(error1).isNotEqualTo(error2);
assertThat(error2).isEqualTo(error3);
diff --git a/core/src/main/java/io/grpc/internal/DelayedClientTransport.java b/core/src/main/java/io/grpc/internal/DelayedClientTransport.java
index aa1d820a570..0cb57b025e4 100644
--- a/core/src/main/java/io/grpc/internal/DelayedClientTransport.java
+++ b/core/src/main/java/io/grpc/internal/DelayedClientTransport.java
@@ -405,6 +405,8 @@ private class PendingStream extends DelayedStream {
@Nullable private String activeDelayType;
@GuardedBy("this")
@Nullable private String activeDelayReason;
+ @GuardedBy("this")
+ private boolean delayEnded;
private PendingStream(PickSubchannelArgs args, ClientStreamTracer[] tracers,
@Nullable String initialType, @Nullable String initialReason) {
@@ -428,26 +430,27 @@ private PendingStream(PickSubchannelArgs args, ClientStreamTracer[] tracers,
* structured transition event is appended to the active span without span re-creation.
*/
synchronized void updateDelay(@Nullable String newType, @Nullable String newReason) {
- if (getRealStream() != null) {
+ if (getRealStream() != null || delayEnded) {
return;
}
+ // Delay type changed (e.g., from RLS lookup to connecting). End the previous delay.
if (!Objects.equals(activeDelayType, newType)) {
- // Delay type changed (e.g., from RLS lookup to connecting). End the previous delay.
if (activeDelayType != null) {
for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayEnd();
}
}
activeDelayType = newType;
- activeDelayReason = null;
+ activeDelayReason = newReason;
if (newType != null) {
for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayStart(newType, newReason != null ? newReason : "");
}
}
}
- if (newType != null && newReason != null && !Objects.equals(activeDelayReason, newReason)) {
- // Delay type is unchanged, but the reason changed (e.g., priority failover).
+ // Delay type is unchanged, but the reason changed (e.g., priority failover).
+ else if (newType != null && newReason != null
+ && !Objects.equals(activeDelayReason, newReason)) {
activeDelayReason = newReason;
for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayReasonChanged(newReason);
@@ -459,6 +462,10 @@ synchronized void updateDelay(@Nullable String newType, @Nullable String newReas
* Ends active attempt delay segment telemetry upon stream creation or stream cancellation.
*/
synchronized void endDelay() {
+ if (delayEnded) {
+ return;
+ }
+ delayEnded = true;
if (activeDelayType != null) {
for (ClientStreamTracer tracer : tracers) {
tracer.recordAttemptDelayEnd();
@@ -469,8 +476,9 @@ synchronized void endDelay() {
}
Runnable setStreamAndEndDelay(ClientStream stream) {
+ Runnable runnable = setStream(stream);
endDelay();
- return setStream(stream);
+ return runnable;
}
/** Runnable may be null. */
diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
index 00df05a0c00..b0f337f890a 100644
--- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
+++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
@@ -918,6 +918,7 @@ public void run() {
inUseStateAggregator.updateObjectInUse(pendingCallsInUseObject, true);
}
pendingCalls.add(pendingCall);
+ pendingCall.notifyQueuedForNameResolution();
} else {
pendingCall.reprocess();
}
@@ -997,6 +998,10 @@ private final class PendingCall extends DelayedClientCall method;
final CallOptions callOptions;
private final long callCreationTime;
+ @GuardedBy("this")
+ private boolean queuedForResolution;
+ @GuardedBy("this")
+ private boolean delayEnded;
PendingCall(Context context, MethodDescriptor method, CallOptions callOptions) {
super(
@@ -1010,8 +1015,35 @@ private final class PendingCall extends DelayedClientCall realCall;
Context previous = context.attach();
try {
@@ -1037,6 +1069,7 @@ public void run() {
@Override
protected void callCancelled() {
+ endDelayIfNeeded();
super.callCancelled();
syncContext.execute(new PendingCallRemoval());
}
diff --git a/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java b/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java
index afcce806f2b..8d5e2074c8f 100644
--- a/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java
+++ b/core/src/test/java/io/grpc/internal/DelayedClientTransportTest.java
@@ -42,6 +42,7 @@
import io.grpc.IntegerMarshaller;
import io.grpc.LoadBalancer.PickResult;
import io.grpc.LoadBalancer.PickSubchannelArgs;
+import io.grpc.LoadBalancer.Subchannel;
import io.grpc.LoadBalancer.SubchannelPicker;
import io.grpc.Metadata;
import io.grpc.MethodDescriptor;
@@ -876,8 +877,8 @@ public void streamDelayMetrics_channelFallback_subchannelStateMismatch() {
FakeStreamTracer fakeTracer = new FakeStreamTracer();
ClientStreamTracer[] customTracers = new ClientStreamTracer[] { fakeTracer };
- io.grpc.LoadBalancer.Subchannel disconnectedSubchannel =
- mock(io.grpc.LoadBalancer.Subchannel.class);
+ Subchannel disconnectedSubchannel =
+ mock(Subchannel.class);
when(disconnectedSubchannel.getInternalSubchannel())
.thenReturn(newTransportProvider(null));
diff --git a/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java b/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java
index 5eb5b49fa19..87b5b8f87b4 100644
--- a/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java
+++ b/core/src/test/java/io/grpc/internal/ForwardingClientStreamTracerTest.java
@@ -17,6 +17,7 @@
package io.grpc.internal;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import io.grpc.ClientStreamTracer;
import io.grpc.ForwardingTestUtil;
@@ -40,6 +41,19 @@ public void allMethodsForwarded() throws Exception {
Collections.emptyList());
}
+ @Test
+ public void attemptDelayMethodsForwarded() {
+ TestClientStreamTracer tracer = new TestClientStreamTracer();
+ tracer.recordAttemptDelayStart("connecting", "test");
+ verify(mockDelegate).recordAttemptDelayStart("connecting", "test");
+
+ tracer.recordAttemptDelayReasonChanged("test2");
+ verify(mockDelegate).recordAttemptDelayReasonChanged("test2");
+
+ tracer.recordAttemptDelayEnd();
+ verify(mockDelegate).recordAttemptDelayEnd();
+ }
+
private final class TestClientStreamTracer extends ForwardingClientStreamTracer {
@Override
protected ClientStreamTracer delegate() {
diff --git a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java
index 42fb374a68b..569dd5651dc 100644
--- a/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java
+++ b/core/src/test/java/io/grpc/internal/ManagedChannelImplTest.java
@@ -40,6 +40,7 @@
import static org.junit.Assert.fail;
import static org.mockito.AdditionalAnswers.delegatesTo;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.ArgumentMatchers.same;
@@ -81,6 +82,7 @@
import io.grpc.ConnectivityState;
import io.grpc.ConnectivityStateInfo;
import io.grpc.Context;
+import io.grpc.Deadline;
import io.grpc.EquivalentAddressGroup;
import io.grpc.InsecureChannelCredentials;
import io.grpc.IntegerMarshaller;
@@ -4823,4 +4825,210 @@ private static ManagedChannelServiceConfig createManagedChannelServiceConfig(
return ManagedChannelServiceConfig
.fromServiceConfig(rawServiceConfig, true, 3, 4, policySelection);
}
+
+ @Test
+ public void callDelay_normalDeferredResolution() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory);
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(mockCallListener, new Metadata());
+
+ verify(mockTracerFactory).recordCallDelayStart(
+ eq("resolving"), eq("waiting for name resolution or service config"));
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+
+ nsFactory.allResolved();
+
+ verify(mockTracerFactory).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_immediateResolution() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(true).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory);
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(mockCallListener, new Metadata());
+
+ channel.syncContext.execute(new Runnable() {
+ @Override
+ public void run() {}
+ });
+
+ verify(mockTracerFactory, never()).recordCallDelayStart(anyString(), anyString());
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_cancellationWhileQueued() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory);
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(mockCallListener, new Metadata());
+
+ verify(mockTracerFactory).recordCallDelayStart(
+ eq("resolving"), eq("waiting for name resolution or service config"));
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+
+ call.cancel("Cancelled while queued", null);
+
+ verify(mockTracerFactory).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_deadlineExpirationWhileQueued() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT
+ .withStreamTracerFactory(mockTracerFactory)
+ .withDeadline(Deadline.after(100, TimeUnit.MILLISECONDS, timer.getDeadlineTicker()));
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(mockCallListener, new Metadata());
+
+ verify(mockTracerFactory).recordCallDelayStart(
+ eq("resolving"), eq("waiting for name resolution or service config"));
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+
+ timer.forwardTime(101, TimeUnit.MILLISECONDS);
+
+ verify(mockTracerFactory).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_resolutionFailure() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false)
+ .setError(Status.UNAVAILABLE.withDescription("Simulated resolver failure"))
+ .build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory);
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(mockCallListener, new Metadata());
+
+ verify(mockTracerFactory).recordCallDelayStart(
+ eq("resolving"), eq("waiting for name resolution or service config"));
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+
+ nsFactory.allResolved();
+
+ verify(mockTracerFactory).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_forcefulShutdown() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory);
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(mockCallListener, new Metadata());
+
+ verify(mockTracerFactory).recordCallDelayStart(
+ eq("resolving"), eq("waiting for name resolution or service config"));
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+
+ channel.shutdownNow();
+
+ verify(mockTracerFactory).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_cancelledBeforeQueuedOnSyncContext() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ ClientStreamTracer.Factory mockTracerFactory = mock(ClientStreamTracer.Factory.class);
+ when(mockTracerFactory.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+ CallOptions callOptions = CallOptions.DEFAULT.withStreamTracerFactory(mockTracerFactory);
+
+ channel.syncContext.execute(() -> {
+ ClientCall call = channel.newCall(method, callOptions);
+ call.cancel("Cancelled before syncContext drains", null);
+ });
+
+ verify(mockTracerFactory, never()).recordCallDelayStart(anyString(), anyString());
+ verify(mockTracerFactory, never()).recordCallDelayEnd();
+ executor.runDueTasks();
+ }
+
+ @Test
+ public void callDelay_callCancelledDuringTracerIteration_abortsLoop() {
+ FakeNameResolverFactory nsFactory = new FakeNameResolverFactory.Builder(expectedUri)
+ .setResolvedAtStart(false).build();
+ channelBuilder.nameResolverFactory(nsFactory);
+ createChannel();
+
+ final AtomicReference> callRef = new AtomicReference<>();
+ ClientStreamTracer.Factory tracer1 = new ClientStreamTracer.Factory() {
+ @Override
+ public ClientStreamTracer newClientStreamTracer(StreamInfo info, Metadata headers) {
+ return new ClientStreamTracer() {};
+ }
+
+ @Override
+ public void recordCallDelayStart(String delayType, String delayReason) {
+ callRef.get().cancel("Cancel inside first tracer start", null);
+ }
+ };
+ ClientStreamTracer.Factory tracer2 = mock(ClientStreamTracer.Factory.class);
+ when(tracer2.newClientStreamTracer(any(StreamInfo.class), any(Metadata.class)))
+ .thenReturn(new ClientStreamTracer() {});
+
+ CallOptions callOptions = CallOptions.DEFAULT
+ .withStreamTracerFactory(tracer1)
+ .withStreamTracerFactory(tracer2);
+
+ channel.syncContext.execute(() -> {
+ ClientCall call = channel.newCall(method, callOptions);
+ callRef.set(call);
+ });
+
+ verify(tracer2, never()).recordCallDelayStart(anyString(), anyString());
+ executor.runDueTasks();
+ }
}
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java
index 1243e0fff59..c94b7811ccd 100644
--- a/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java
+++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/GrpcOpenTelemetry.java
@@ -238,11 +238,25 @@ && isMetricEnabled("grpc.client.attempt.delay.duration", enableMetrics, disableD
meter.histogramBuilder(
"grpc.client.attempt.delay.duration")
.setUnit("s")
- .setDescription("Time taken before a client call attempt starts")
+ .setDescription(
+ "EXPERIMENTAL. Time an RPC attempt spent waiting for a load balancing pick"
+ + " or connection establishment.")
.setExplicitBucketBoundariesAdvice(LATENCY_BUCKETS)
.build());
}
+ if (isDelayObservabilityEnabled()
+ && isMetricEnabled("grpc.client.call.delay.duration", enableMetrics, disableDefault)) {
+ builder.clientCallDelayCounter(
+ meter.histogramBuilder(
+ "grpc.client.call.delay.duration")
+ .setUnit("s")
+ .setDescription(
+ "EXPERIMENTAL. Time an RPC spent waiting at the call level before an attempt was"
+ + " initiated, such as waiting for name resolution.")
+ .setExplicitBucketBoundariesAdvice(LATENCY_BUCKETS)
+ .build());
+ }
if (isMetricEnabled("grpc.client.attempt.sent_total_compressed_message_size", enableMetrics,
disableDefault)) {
builder.clientTotalSentCompressedMessageSizeCounter(
@@ -364,7 +378,7 @@ && isMetricEnabled("grpc.client.attempt.delay.duration", enableMetrics, disableD
* Checks whether experimental client attempt and call delay observability is globally enabled.
*
* Guarded strictly by the {@code GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY} environment
- * variable (defaults to {@code false}). When disabled, delay spans and
+ * variable or JVM system property (defaults to {@code false}). When disabled, delay spans and
* duration histograms are suppressed to avoid runtime overhead.
*/
static boolean isDelayObservabilityEnabled() {
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java
index 008a9754b29..0eec807e66d 100644
--- a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java
+++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsModule.java
@@ -105,6 +105,7 @@ final class OpenTelemetryMetricsModule {
private final boolean localityEnabled;
private final boolean backendServiceEnabled;
private final boolean customLabelEnabled;
+ private final boolean delayObservabilityEnabled;
private final ImmutableList plugins;
@Nullable
private final TargetFilter targetAttributeFilter;
@@ -124,6 +125,7 @@ final class OpenTelemetryMetricsModule {
this.localityEnabled = optionalLabels.contains(LOCALITY_KEY.getKey());
this.backendServiceEnabled = optionalLabels.contains(BACKEND_SERVICE_KEY.getKey());
this.customLabelEnabled = optionalLabels.contains(CUSTOM_LABEL_KEY.getKey());
+ this.delayObservabilityEnabled = GrpcOpenTelemetry.isDelayObservabilityEnabled();
this.plugins = ImmutableList.copyOf(plugins);
this.targetAttributeFilter = targetAttributeFilter;
}
@@ -205,8 +207,12 @@ private static final class ClientTracer extends ClientStreamTracer {
volatile String backendService;
long attemptNanos;
Code statusCode;
- @Nullable private volatile Stopwatch activeDelayStopwatch;
- @Nullable private volatile String activeDelayType;
+ @GuardedBy("this")
+ @Nullable private Stopwatch activeDelayStopwatch;
+ @GuardedBy("this")
+ @Nullable private String activeDelayType;
+ @GuardedBy("this")
+ private boolean streamClosed;
ClientTracer(CallAttemptsTracerFactory attemptsState, OpenTelemetryMetricsModule module,
StreamInfo info, String target, String fullMethodName,
@@ -227,14 +233,21 @@ public void streamCreated(io.grpc.Attributes transportAtts, Metadata headers) {
@Override
public void recordAttemptDelayStart(String delayType, String delayReason) {
- if (!GrpcOpenTelemetry.isDelayObservabilityEnabled()
- || (activeDelayStopwatch != null && Objects.equals(activeDelayType, delayType))) {
- // Do not reset the stopwatch if the delay type is unchanged.
+ if (!module.delayObservabilityEnabled) {
return;
}
- recordAttemptDelayEnd();
- activeDelayType = delayType;
- activeDelayStopwatch = module.stopwatchSupplier.get().start();
+ synchronized (this) {
+ if (streamClosed) {
+ return;
+ }
+ if (activeDelayStopwatch != null && Objects.equals(activeDelayType, delayType)) {
+ // Do not reset the stopwatch if the delay type is unchanged.
+ return;
+ }
+ recordAttemptDelayEnd();
+ activeDelayType = delayType;
+ activeDelayStopwatch = module.stopwatchSupplier.get().start();
+ }
}
@Override
@@ -244,27 +257,26 @@ public void recordAttemptDelayReasonChanged(String delayReason) {
@Override
public void recordAttemptDelayEnd() {
- Stopwatch delayStopwatch = activeDelayStopwatch;
- String delayType = activeDelayType;
- if (delayStopwatch != null && delayType != null) {
- delayStopwatch.stop();
- long delayNanos = delayStopwatch.elapsed(TimeUnit.NANOSECONDS);
- activeDelayStopwatch = null;
- activeDelayType = null;
- if (module.resource.clientAttemptDelayCounter() != null) {
- AttributesBuilder builder = Attributes.builder()
- .put(METHOD_KEY, fullMethodName)
- .put(TARGET_KEY, target)
- .put("grpc.delay_type", delayType);
- if (module.customLabelEnabled) {
- builder.put(
- CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
- }
- for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) {
- plugin.addLabels(builder);
+ if (!module.delayObservabilityEnabled) {
+ return;
+ }
+ synchronized (this) {
+ Stopwatch delayStopwatch = activeDelayStopwatch;
+ String delayType = activeDelayType;
+ if (delayStopwatch != null && delayType != null) {
+ delayStopwatch.stop();
+ long delayNanos = delayStopwatch.elapsed(TimeUnit.NANOSECONDS);
+ activeDelayStopwatch = null;
+ activeDelayType = null;
+ if (module.resource.clientAttemptDelayCounter() != null) {
+ AttributesBuilder builder = Attributes.builder()
+ .put(METHOD_KEY, fullMethodName)
+ .put(TARGET_KEY, target)
+ .put("grpc.delay_type", delayType);
+ addOptionalLabels(builder);
+ module.resource.clientAttemptDelayCounter()
+ .record(delayNanos * SECONDS_PER_NANO, builder.build(), attemptsState.otelContext);
}
- module.resource.clientAttemptDelayCounter()
- .record(delayNanos * SECONDS_PER_NANO, builder.build(), attemptsState.otelContext);
}
}
}
@@ -315,7 +327,15 @@ public void inboundTrailers(Metadata trailers) {
@Override
public void streamClosed(Status status) {
- recordAttemptDelayEnd();
+ if (module.delayObservabilityEnabled) {
+ synchronized (this) {
+ if (streamClosed) {
+ return;
+ }
+ streamClosed = true;
+ recordAttemptDelayEnd();
+ }
+ }
stopwatch.stop();
attemptNanos = stopwatch.elapsed(TimeUnit.NANOSECONDS);
Deadline deadline = info.getCallOptions().getDeadline();
@@ -333,32 +353,12 @@ public void streamClosed(Status status) {
}
void recordFinishedAttempt() {
- AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder()
+ AttributesBuilder builder = Attributes.builder()
.put(METHOD_KEY, fullMethodName)
.put(TARGET_KEY, target)
.put(STATUS_KEY, statusCode.toString());
- if (module.localityEnabled) {
- String savedLocality = locality;
- if (savedLocality == null) {
- savedLocality = "";
- }
- builder.put(LOCALITY_KEY, savedLocality);
- }
- if (module.backendServiceEnabled) {
- String savedBackendService = backendService;
- if (savedBackendService == null) {
- savedBackendService = "";
- }
- builder.put(BACKEND_SERVICE_KEY, savedBackendService);
- }
- if (module.customLabelEnabled) {
- builder.put(
- CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
- }
- for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) {
- plugin.addLabels(builder);
- }
- io.opentelemetry.api.common.Attributes attribute = builder.build();
+ addOptionalLabels(builder);
+ Attributes attribute = builder.build();
if (module.resource.clientAttemptDurationCounter() != null ) {
module.resource.clientAttemptDurationCounter()
@@ -373,6 +373,22 @@ void recordFinishedAttempt() {
.record(inboundWireSize, attribute, attemptsState.otelContext);
}
}
+
+ private void addOptionalLabels(AttributesBuilder builder) {
+ if (module.localityEnabled) {
+ builder.put(LOCALITY_KEY, Objects.toString(locality, ""));
+ }
+ if (module.backendServiceEnabled) {
+ builder.put(BACKEND_SERVICE_KEY, Objects.toString(backendService, ""));
+ }
+ if (module.customLabelEnabled) {
+ builder.put(
+ CUSTOM_LABEL_KEY, info.getCallOptions().getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
+ }
+ for (OpenTelemetryPlugin.ClientStreamPlugin plugin : streamPlugins) {
+ plugin.addLabels(builder);
+ }
+ }
}
@VisibleForTesting
@@ -387,6 +403,11 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory
private final List callPlugins;
private final Context otelContext;
private Status status;
+ @GuardedBy("lock")
+ @Nullable private Stopwatch activeCallDelayStopwatch;
+ @GuardedBy("lock")
+ @Nullable private String activeCallDelayType;
+ private final Attributes callLevelBaseAttributes;
private long retryDelayNanos;
private long callLatencyNanos;
private final Object lock = new Object();
@@ -412,18 +433,18 @@ static final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory
this.attemptDelayStopwatch = module.stopwatchSupplier.get();
this.callStopWatch = module.stopwatchSupplier.get().start();
- AttributesBuilder builder = io.opentelemetry.api.common.Attributes.builder()
+ AttributesBuilder builder = Attributes.builder()
.put(METHOD_KEY, fullMethodName)
.put(TARGET_KEY, target);
if (module.customLabelEnabled) {
builder.put(
CUSTOM_LABEL_KEY, callOptions.getOption(Grpc.CALL_OPTION_CUSTOM_LABEL));
}
- io.opentelemetry.api.common.Attributes attribute = builder.build();
+ this.callLevelBaseAttributes = builder.build();
- // Record here in case mewClientStreamTracer() would never be called.
+ // Record here in case newClientStreamTracer() would never be called.
if (module.resource.clientAttemptCountCounter() != null) {
- module.resource.clientAttemptCountCounter().add(1, attribute, otelContext);
+ module.resource.clientAttemptCountCounter().add(1, callLevelBaseAttributes, otelContext);
}
}
@@ -504,6 +525,9 @@ void callEnded(Status status, CallOptions callOptions) {
return;
}
callEnded = true;
+ if (module.delayObservabilityEnabled) {
+ recordCallDelayEnd();
+ }
if (activeStreams == 0 && !finishedCallToBeRecorded) {
shouldRecordFinishedCall = true;
finishedCallToBeRecorded = true;
@@ -579,6 +603,54 @@ void recordFinishedCall(CallOptions callOptions) {
);
}
}
+
+ @Override
+ public void recordCallDelayStart(String delayType, String delayReason) {
+ if (!module.delayObservabilityEnabled) {
+ return;
+ }
+ synchronized (lock) {
+ if (callEnded) {
+ return;
+ }
+ if (activeCallDelayStopwatch != null && Objects.equals(activeCallDelayType, delayType)) {
+ return;
+ }
+ recordCallDelayEnd();
+ activeCallDelayType = delayType;
+ activeCallDelayStopwatch = module.stopwatchSupplier.get().start();
+ }
+ }
+
+ @Override
+ public void recordCallDelayReasonChanged(String delayReason) {
+ // Reason strings are high-cardinality diagnostics intended for tracing spans.
+ }
+
+ @Override
+ public void recordCallDelayEnd() {
+ if (!module.delayObservabilityEnabled) {
+ return;
+ }
+ synchronized (lock) {
+ Stopwatch delayStopwatch = activeCallDelayStopwatch;
+ String delayType = activeCallDelayType;
+ if (delayStopwatch != null && delayType != null) {
+ delayStopwatch.stop();
+ long delayNanos = delayStopwatch.elapsed(TimeUnit.NANOSECONDS);
+ activeCallDelayStopwatch = null;
+ activeCallDelayType = null;
+ if (module.resource.clientCallDelayCounter() != null) {
+ module.resource.clientCallDelayCounter().record(
+ delayNanos * SECONDS_PER_NANO,
+ callLevelBaseAttributes.toBuilder()
+ .put("grpc.delay_type", delayType)
+ .build(),
+ otelContext);
+ }
+ }
+ }
+ }
}
private static final class ServerTracer extends ServerStreamTracer
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java
index 085498d746e..2771b3eb9af 100644
--- a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java
+++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryMetricsResource.java
@@ -38,6 +38,9 @@ abstract class OpenTelemetryMetricsResource {
@Nullable
abstract DoubleHistogram clientAttemptDelayCounter();
+ @Nullable
+ abstract DoubleHistogram clientCallDelayCounter();
+
@Nullable
abstract LongHistogram clientTotalSentCompressedMessageSizeCounter();
@@ -84,6 +87,9 @@ abstract static class Builder {
abstract Builder clientAttemptDelayCounter(DoubleHistogram counter);
+ abstract Builder clientCallDelayCounter(DoubleHistogram counter);
+
+
abstract Builder clientTotalSentCompressedMessageSizeCounter(LongHistogram counter);
abstract Builder clientTotalReceivedCompressedMessageSizeCounter(
diff --git a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java
index 32aab870f0f..a6b9e4825d4 100644
--- a/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java
+++ b/opentelemetry/src/main/java/io/grpc/opentelemetry/OpenTelemetryTracingModule.java
@@ -22,7 +22,7 @@
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.BAGGAGE_KEY;
import com.google.common.annotations.VisibleForTesting;
-import io.grpc.Attributes;
+import com.google.errorprone.annotations.concurrent.GuardedBy;
import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.ClientCall;
@@ -37,11 +37,13 @@
import io.grpc.ServerCallHandler;
import io.grpc.ServerInterceptor;
import io.grpc.ServerStreamTracer;
+import io.grpc.Status;
import io.grpc.internal.GrpcUtil;
import io.grpc.opentelemetry.internal.OpenTelemetryConstants;
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.baggage.Baggage;
import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.common.AttributesBuilder;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.StatusCode;
@@ -99,6 +101,7 @@ final class OpenTelemetryTracingModule {
private final ServerInterceptor serverSpanPropagationInterceptor =
new TracingServerSpanPropagationInterceptor();
private final ServerTracerFactory serverTracerFactory = new ServerTracerFactory();
+ private final boolean delayObservabilityEnabled;
OpenTelemetryTracingModule(OpenTelemetry openTelemetry) {
this.otelTracer = checkNotNull(openTelemetry.getTracerProvider(), "tracerProvider")
@@ -106,6 +109,7 @@ final class OpenTelemetryTracingModule {
.setInstrumentationVersion(IMPLEMENTATION_VERSION)
.build();
this.contextPropagators = checkNotNull(openTelemetry.getPropagators(), "contextPropagators");
+ this.delayObservabilityEnabled = GrpcOpenTelemetry.isDelayObservabilityEnabled();
}
@VisibleForTesting
@@ -144,6 +148,10 @@ final class CallAttemptsTracerFactory extends ClientStreamTracer.Factory {
volatile int callEnded;
private final Span clientSpan;
private final String fullMethodName;
+ @GuardedBy("this")
+ @Nullable private Span activeCallDelaySpan;
+ @GuardedBy("this")
+ @Nullable private String activeCallDelayType;
CallAttemptsTracerFactory(Span clientSpan, MethodDescriptor method) {
checkNotNull(method, "method");
@@ -168,6 +176,13 @@ public ClientStreamTracer newClientStreamTracer(
return new ClientTracer(attemptSpan, clientSpan);
}
+ private boolean isCallEnded() {
+ if (callEndedUpdater != null) {
+ return callEndedUpdater.get(this) != 0;
+ }
+ return callEnded != 0;
+ }
+
/**
* Record a finished call and mark the current time as the end time.
*
@@ -185,8 +200,65 @@ void callEnded(io.grpc.Status status) {
}
callEnded = 1;
}
+ if (delayObservabilityEnabled) {
+ recordCallDelayEnd();
+ }
endSpanWithStatus(clientSpan, status);
}
+
+ @Override
+ public void recordCallDelayStart(String delayType, String delayReason) {
+ if (!delayObservabilityEnabled || isCallEnded()) {
+ return;
+ }
+ synchronized (this) {
+ if (isCallEnded()) {
+ return;
+ }
+ if (activeCallDelaySpan != null && Objects.equals(activeCallDelayType, delayType)) {
+ recordCallDelayReasonChanged(delayReason);
+ return;
+ }
+ recordCallDelayEnd();
+ activeCallDelayType = delayType;
+ activeCallDelaySpan = otelTracer.spanBuilder("Delay")
+ .setParent(Context.current().with(clientSpan))
+ .setAttribute("grpc.delay_type", delayType)
+ .startSpan();
+ activeCallDelaySpan.addEvent(
+ "Delay state transition",
+ Attributes.of(AttributeKey.stringKey("grpc.delay_reason"), delayReason));
+ }
+ }
+
+ @Override
+ public void recordCallDelayReasonChanged(String delayReason) {
+ if (!delayObservabilityEnabled || isCallEnded()) {
+ return;
+ }
+ synchronized (this) {
+ if (isCallEnded() || activeCallDelaySpan == null) {
+ return;
+ }
+ activeCallDelaySpan.addEvent(
+ "Delay state transition",
+ Attributes.of(AttributeKey.stringKey("grpc.delay_reason"), delayReason));
+ }
+ }
+
+ @Override
+ public void recordCallDelayEnd() {
+ if (!delayObservabilityEnabled) {
+ return;
+ }
+ synchronized (this) {
+ if (activeCallDelaySpan != null) {
+ activeCallDelaySpan.end();
+ activeCallDelaySpan = null;
+ activeCallDelayType = null;
+ }
+ }
+ }
}
private final class ClientTracer extends ClientStreamTracer {
@@ -194,8 +266,12 @@ private final class ClientTracer extends ClientStreamTracer {
private final Span parentSpan;
volatile int seqNo;
boolean isPendingStream;
- @Nullable private volatile Span activeDelaySpan;
- @Nullable private volatile String activeDelayType;
+ @GuardedBy("this")
+ @Nullable private Span activeDelaySpan;
+ @GuardedBy("this")
+ @Nullable private String activeDelayType;
+ @GuardedBy("this")
+ private boolean streamClosed;
ClientTracer(Span span, Span parentSpan) {
this.span = checkNotNull(span, "span");
@@ -203,7 +279,7 @@ private final class ClientTracer extends ClientStreamTracer {
}
@Override
- public void streamCreated(Attributes transportAtts, Metadata headers) {
+ public void streamCreated(io.grpc.Attributes transportAtts, Metadata headers) {
recordAttemptDelayEnd();
contextPropagators.getTextMapPropagator().inject(Context.current().with(span), headers,
metadataSetter);
@@ -219,51 +295,59 @@ public void createPendingStream() {
@Override
public void recordAttemptDelayStart(String delayType, String delayReason) {
- if (!GrpcOpenTelemetry.isDelayObservabilityEnabled()) {
+ if (!delayObservabilityEnabled) {
return;
}
- if (activeDelaySpan != null && Objects.equals(activeDelayType, delayType)) {
- // Do not recreate the span if the delay type is unchanged (e.g., priority failover).
- recordAttemptDelayReasonChanged(delayReason);
- return;
+ synchronized (this) {
+ if (streamClosed) {
+ return;
+ }
+ if (activeDelaySpan != null && Objects.equals(activeDelayType, delayType)) {
+ // Do not recreate the span if the delay type is unchanged (e.g., priority failover).
+ recordAttemptDelayReasonChanged(delayReason);
+ return;
+ }
+ // Close any previous delay segment before starting a new canonical segment.
+ recordAttemptDelayEnd();
+ activeDelayType = delayType;
+ // All attempt queuing segments use the strict child span name "Delay".
+ activeDelaySpan = otelTracer.spanBuilder("Delay")
+ .setParent(Context.current().with(span))
+ .setAttribute("grpc.delay_type", delayType)
+ .startSpan();
+ activeDelaySpan.addEvent(
+ "Delay state transition",
+ Attributes.of(AttributeKey.stringKey("grpc.delay_reason"), delayReason));
}
- // Close any previous delay segment before starting a new canonical segment.
- recordAttemptDelayEnd();
- activeDelayType = delayType;
- // All attempt queuing segments use the strict child span name "Attempt Delay".
- Span delaySpan = otelTracer.spanBuilder("Attempt Delay")
- .setParent(Context.current().with(span))
- .setAttribute("grpc.delay_type", delayType)
- .startSpan();
- activeDelaySpan = delaySpan;
- delaySpan.addEvent(
- "Delay state transition",
- io.opentelemetry.api.common.Attributes.of(
- AttributeKey.stringKey("grpc.delay_type"), delayType,
- AttributeKey.stringKey("grpc.delay_reason"), delayReason));
}
@Override
public void recordAttemptDelayReasonChanged(String delayReason) {
- if (!GrpcOpenTelemetry.isDelayObservabilityEnabled() || activeDelaySpan == null) {
+ if (!delayObservabilityEnabled) {
return;
}
- String type = activeDelayType;
- activeDelaySpan.addEvent(
- "Delay state transition",
- io.opentelemetry.api.common.Attributes.of(
- AttributeKey.stringKey("grpc.delay_type"), type != null ? type : "",
- AttributeKey.stringKey("grpc.delay_reason"), delayReason));
+ synchronized (this) {
+ if (streamClosed || activeDelaySpan == null) {
+ return;
+ }
+ activeDelaySpan.addEvent(
+ "Delay state transition",
+ Attributes.of(AttributeKey.stringKey("grpc.delay_reason"), delayReason));
+ }
}
@Override
public void recordAttemptDelayEnd() {
- Span delaySpan = activeDelaySpan;
- if (delaySpan != null) {
- // End active child span upon pick completion or transport cancellation.
- delaySpan.end();
- activeDelaySpan = null;
- activeDelayType = null;
+ if (!delayObservabilityEnabled) {
+ return;
+ }
+ synchronized (this) {
+ if (activeDelaySpan != null) {
+ // End active child span upon pick completion or transport cancellation.
+ activeDelaySpan.end();
+ activeDelaySpan = null;
+ activeDelayType = null;
+ }
}
}
@@ -292,8 +376,14 @@ public void inboundUncompressedSize(long bytes) {
}
@Override
- public void streamClosed(io.grpc.Status status) {
- recordAttemptDelayEnd();
+ public synchronized void streamClosed(Status status) {
+ if (streamClosed) {
+ return;
+ }
+ streamClosed = true;
+ if (delayObservabilityEnabled) {
+ recordAttemptDelayEnd();
+ }
endSpanWithStatus(span, status);
}
}
@@ -511,7 +601,7 @@ public void onClose(io.grpc.Status status, Metadata trailers) {
// 'message-size' = 7854) ----|
private void recordOutboundMessageSentEvent(Span span,
int seqNo, long optionalWireSize, long optionalUncompressedSize) {
- AttributesBuilder attributesBuilder = io.opentelemetry.api.common.Attributes.builder();
+ AttributesBuilder attributesBuilder = Attributes.builder();
attributesBuilder.put("sequence-number", seqNo);
if (optionalUncompressedSize != -1) {
attributesBuilder.put("message-size", optionalUncompressedSize);
@@ -523,14 +613,14 @@ private void recordOutboundMessageSentEvent(Span span,
}
private void recordInboundCompressedMessage(Span span, int seqNo, long optionalWireSize) {
- AttributesBuilder attributesBuilder = io.opentelemetry.api.common.Attributes.builder();
+ AttributesBuilder attributesBuilder = Attributes.builder();
attributesBuilder.put("sequence-number", seqNo);
attributesBuilder.put("message-size-compressed", optionalWireSize);
span.addEvent("Inbound compressed message", attributesBuilder.build());
}
private void recordInboundMessageSize(Span span, int seqNo, long bytes) {
- AttributesBuilder attributesBuilder = io.opentelemetry.api.common.Attributes.builder();
+ AttributesBuilder attributesBuilder = Attributes.builder();
attributesBuilder.put("sequence-number", seqNo);
attributesBuilder.put("message-size", bytes);
span.addEvent("Inbound message", attributesBuilder.build());
diff --git a/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java b/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java
index 77eadf9ebbb..9a4fb87e58e 100644
--- a/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java
+++ b/opentelemetry/src/test/java/io/grpc/opentelemetry/GrpcOpenTelemetryTest.java
@@ -17,6 +17,9 @@
package io.grpc.opentelemetry;
import static com.google.common.truth.Truth.assertThat;
+import static io.grpc.ClientStreamTracer.NAME_RESOLUTION_DELAYED;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Collections.emptyList;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -24,27 +27,82 @@
import static org.mockito.Mockito.verifyNoMoreInteractions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.io.ByteStreams;
+import io.grpc.CallOptions;
+import io.grpc.ClientCall;
import io.grpc.ClientInterceptor;
+import io.grpc.ClientStreamTracer;
import io.grpc.ForwardingChannelBuilder2;
+import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
import io.grpc.MetricSink;
import io.grpc.ServerBuilder;
+import io.grpc.ServerCall;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerServiceDefinition;
+import io.grpc.Status;
+import io.grpc.inprocess.InProcessChannelBuilder;
+import io.grpc.inprocess.InProcessServerBuilder;
+import io.grpc.internal.FakeClock;
import io.grpc.internal.GrpcUtil;
import io.grpc.opentelemetry.GrpcOpenTelemetry.TargetFilter;
+import io.grpc.testing.GrpcCleanupRule;
import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.data.MetricData;
import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader;
+import io.opentelemetry.sdk.testing.junit4.OpenTelemetryRule;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class GrpcOpenTelemetryTest {
+ @Rule
+ public final OpenTelemetryRule openTelemetryRule = OpenTelemetryRule.create();
+ @Rule
+ public final GrpcCleanupRule grpcCleanupRule = new GrpcCleanupRule();
+
+ private static final MethodDescriptor.Marshaller MARSHALLER =
+ new MethodDescriptor.Marshaller() {
+ @Override
+ public InputStream stream(String value) {
+ return new ByteArrayInputStream(value.getBytes(UTF_8));
+ }
+
+ @Override
+ public String parse(InputStream stream) {
+ try {
+ return new String(ByteStreams.toByteArray(stream), UTF_8);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ };
+
+ private final MethodDescriptor method =
+ MethodDescriptor.newBuilder()
+ .setType(MethodDescriptor.MethodType.UNARY)
+ .setRequestMarshaller(MARSHALLER)
+ .setResponseMarshaller(MARSHALLER)
+ .setFullMethodName("test.service/method")
+ .build();
+
private final InMemoryMetricReader inMemoryMetricReader = InMemoryMetricReader.create();
private final SdkMeterProvider meterProvider =
SdkMeterProvider.builder().registerMetricReader(inMemoryMetricReader).build();
@@ -55,11 +113,13 @@ public class GrpcOpenTelemetryTest {
@Before
public void setup() {
originalEnableOtelTracing = GrpcOpenTelemetry.ENABLE_OTEL_TRACING;
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
}
@After
public void tearDown() {
GrpcOpenTelemetry.ENABLE_OTEL_TRACING = originalEnableOtelTracing;
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
}
@Test
@@ -179,6 +239,308 @@ public void configureChannelBuilder_registersMetricSink() {
assertThat(testBuilder.interceptorFactory).isNotNull();
}
+ @Test
+ public void nameResolutionDelay_endToEndClientServerSimulation() throws Exception {
+ String serverName = InProcessServerBuilder.generateName();
+ ServerServiceDefinition serviceDef = ServerServiceDefinition.builder("test.service")
+ .addMethod(method, new ServerCallHandler() {
+ @Override
+ public ServerCall.Listener startCall(
+ ServerCall call, Metadata headers) {
+ call.sendHeaders(new Metadata());
+ call.sendMessage("response_payload");
+ call.close(Status.OK, new Metadata());
+ return new ServerCall.Listener() {};
+ }
+ })
+ .build();
+
+ grpcCleanupRule.register(
+ InProcessServerBuilder.forName(serverName).directExecutor().addService(serviceDef).build()
+ .start());
+
+ OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry();
+ GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder()
+ .sdk(sdk)
+ .enableMetrics(Arrays.asList(
+ "grpc.client.attempt.delay.duration",
+ "grpc.client.call.delay.duration",
+ "grpc.client.attempt.started"))
+ .enableTracing(true)
+ .addOptionalLabel("grpc.delay_type")
+ .build();
+
+ ManagedChannelBuilder channelBuilder = InProcessChannelBuilder.forName(serverName)
+ .directExecutor();
+ grpcOpenTelemetry.configureChannelBuilder(channelBuilder);
+ ManagedChannel channel = grpcCleanupRule.register(channelBuilder.build());
+
+ // Simulate Name Resolution delay on call options and stream tracer
+ CallOptions callOptions = CallOptions.DEFAULT.withOption(
+ NAME_RESOLUTION_DELAYED, TimeUnit.MILLISECONDS.toNanos(120));
+
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(
+ sdk.getMeterProvider().get("grpc-java"),
+ ImmutableMap.of(
+ "grpc.client.attempt.delay.duration", true,
+ "grpc.client.call.delay.duration", true),
+ false);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList());
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory =
+ new OpenTelemetryMetricsModule.CallAttemptsTracerFactory(
+ module, "target:///", callOptions, method.getFullMethodName(),
+ emptyList(), io.opentelemetry.context.Context.root());
+ ClientStreamTracer delayTracer = factory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(callOptions).build(),
+ new Metadata());
+ delayTracer.recordAttemptDelayStart("connecting", "DNS server unreachable temporarily");
+ delayTracer.recordAttemptDelayEnd();
+ factory.recordCallDelayStart("resolving", "DNS resolution pending");
+ factory.recordCallDelayEnd();
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ ClientCall call = channel.newCall(method, callOptions);
+ call.start(new ClientCall.Listener() {
+ @Override
+ public void onClose(Status status, Metadata trailers) {
+ latch.countDown();
+ }
+ }, new Metadata());
+ call.sendMessage("request_payload");
+ call.halfClose();
+ call.request(1);
+ assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue();
+
+ io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions
+ .assertThat(openTelemetryRule.getMetrics())
+ .anySatisfy(
+ metric -> io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions
+ .assertThat(metric)
+ .hasName("grpc.client.attempt.delay.duration")
+ .hasDescription(
+ "EXPERIMENTAL. Time an RPC attempt spent waiting for a load balancing pick"
+ + " or connection establishment.")
+ .hasUnit("s")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> {
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.target"), "target:///");
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.method"), method.getFullMethodName());
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.delay_type"), "connecting");
+ })));
+ io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions
+ .assertThat(openTelemetryRule.getMetrics())
+ .anySatisfy(
+ metric -> io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions
+ .assertThat(metric)
+ .hasName("grpc.client.call.delay.duration")
+ .hasDescription(
+ "EXPERIMENTAL. Time an RPC spent waiting at the call level before an attempt"
+ + " was initiated, such as waiting for name resolution.")
+ .hasUnit("s")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> {
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.target"), "target:///");
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.method"), method.getFullMethodName());
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.delay_type"), "resolving");
+ })));
+ }
+
+ @Test
+ public void delayObservability_flagEnabled_notOptedIn_instrumentsNullAndNoMetrics() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry();
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(
+ sdk.getMeterProvider().get("grpc-java"),
+ ImmutableMap.of(),
+ false);
+
+ assertThat(resource.clientAttemptDelayCounter()).isNull();
+ assertThat(resource.clientCallDelayCounter()).isNull();
+
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList());
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory =
+ new OpenTelemetryMetricsModule.CallAttemptsTracerFactory(
+ module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(),
+ emptyList(), io.opentelemetry.context.Context.root());
+
+ // Verify call delay methods execute cleanly when counters are null
+ factory.recordCallDelayStart("resolving", "resolving name");
+ factory.recordCallDelayEnd();
+ factory.callEnded(Status.OK, CallOptions.DEFAULT);
+
+ // Verify attempt delay methods execute cleanly when counters are null
+ ClientStreamTracer delayTracer = factory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(),
+ new Metadata());
+ delayTracer.recordAttemptDelayStart("connecting", "connecting to backend");
+ delayTracer.recordAttemptDelayReasonChanged("still connecting");
+ delayTracer.recordAttemptDelayEnd();
+ delayTracer.streamClosed(Status.OK);
+
+ for (MetricData m : openTelemetryRule.getMetrics()) {
+ assertThat(m.getName()).isNotIn(
+ ImmutableList.of(
+ "grpc.client.attempt.delay.duration", "grpc.client.call.delay.duration"));
+ }
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void lbPolicyDelay_endToEndClientServerSimulation() throws Exception {
+ String serverName = InProcessServerBuilder.generateName();
+ ServerServiceDefinition serviceDef = ServerServiceDefinition.builder("test.service")
+ .addMethod(method, new ServerCallHandler() {
+ @Override
+ public ServerCall.Listener startCall(
+ ServerCall call, Metadata headers) {
+ call.sendHeaders(new Metadata());
+ call.sendMessage("response_payload");
+ call.close(Status.OK, new Metadata());
+ return new ServerCall.Listener() {};
+ }
+ })
+ .build();
+
+ grpcCleanupRule.register(
+ InProcessServerBuilder.forName(serverName).directExecutor().addService(serviceDef).build()
+ .start());
+
+ OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry();
+ GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder()
+ .sdk(sdk)
+ .enableMetrics(Arrays.asList(
+ "grpc.client.attempt.delay.duration",
+ "grpc.client.call.delay.duration",
+ "grpc.client.attempt.started"))
+ .enableTracing(true)
+ .addOptionalLabel("grpc.delay_type")
+ .build();
+
+ ManagedChannelBuilder channelBuilder = InProcessChannelBuilder.forName(serverName)
+ .directExecutor();
+ grpcOpenTelemetry.configureChannelBuilder(channelBuilder);
+ ManagedChannel channel = grpcCleanupRule.register(channelBuilder.build());
+
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(
+ sdk.getMeterProvider().get("grpc-java"),
+ ImmutableMap.of("grpc.client.attempt.delay.duration", true),
+ false);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ new FakeClock().getStopwatchSupplier(), resource, emptyList(), emptyList());
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory factory =
+ new OpenTelemetryMetricsModule.CallAttemptsTracerFactory(
+ module, "target:///", CallOptions.DEFAULT, method.getFullMethodName(),
+ emptyList(), io.opentelemetry.context.Context.root());
+ ClientStreamTracer tracer = factory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().setCallOptions(CallOptions.DEFAULT).build(),
+ new Metadata());
+ tracer.recordAttemptDelayStart("rls_lookup_pending", "Route Lookup Service query pending");
+ tracer.recordAttemptDelayEnd();
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ ClientCall call = channel.newCall(method, CallOptions.DEFAULT);
+ call.start(new ClientCall.Listener() {
+ @Override
+ public void onClose(Status status, Metadata trailers) {
+ latch.countDown();
+ }
+ }, new Metadata());
+ call.sendMessage("request_payload");
+ call.halfClose();
+ call.request(1);
+ assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue();
+
+ io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions
+ .assertThat(openTelemetryRule.getMetrics())
+ .anySatisfy(
+ metric -> io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions
+ .assertThat(metric)
+ .hasName("grpc.client.attempt.delay.duration")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> {
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.delay_type"), "rls_lookup_pending");
+ })));
+ }
+
+ @Test
+ public void baselineNoDelay_endToEndClientServerSimulation() throws Exception {
+ String serverName = InProcessServerBuilder.generateName();
+ ServerServiceDefinition serviceDef = ServerServiceDefinition.builder("test.service")
+ .addMethod(method, new ServerCallHandler() {
+ @Override
+ public ServerCall.Listener startCall(
+ ServerCall call, Metadata headers) {
+ call.sendHeaders(new Metadata());
+ call.sendMessage("response_payload");
+ call.close(Status.OK, new Metadata());
+ return new ServerCall.Listener() {};
+ }
+ })
+ .build();
+
+ grpcCleanupRule.register(
+ InProcessServerBuilder.forName(serverName).directExecutor().addService(serviceDef).build()
+ .start());
+
+ OpenTelemetrySdk sdk = (OpenTelemetrySdk) openTelemetryRule.getOpenTelemetry();
+ GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder()
+ .sdk(sdk)
+ .enableMetrics(Arrays.asList(
+ "grpc.client.attempt.delay.duration",
+ "grpc.client.call.delay.duration",
+ "grpc.client.attempt.started"))
+ .enableTracing(true)
+ .build();
+
+ ManagedChannelBuilder channelBuilder = InProcessChannelBuilder.forName(serverName)
+ .directExecutor();
+ grpcOpenTelemetry.configureChannelBuilder(channelBuilder);
+ ManagedChannel channel = grpcCleanupRule.register(channelBuilder.build());
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ ClientCall call = channel.newCall(method, CallOptions.DEFAULT);
+ call.start(new ClientCall.Listener() {
+ @Override
+ public void onClose(Status status, Metadata trailers) {
+ latch.countDown();
+ }
+ }, new Metadata());
+ call.sendMessage("request_payload");
+ call.halfClose();
+ call.request(1);
+ assertThat(latch.await(5, TimeUnit.SECONDS)).isTrue();
+
+ boolean hasAttemptDelay = false;
+ boolean hasCallDelay = false;
+ for (MetricData m : openTelemetryRule.getMetrics()) {
+ if ("grpc.client.attempt.delay.duration".equals(m.getName())
+ && !m.getHistogramData().getPoints().isEmpty()) {
+ hasAttemptDelay = true;
+ }
+ if ("grpc.client.call.delay.duration".equals(m.getName())
+ && !m.getHistogramData().getPoints().isEmpty()) {
+ hasCallDelay = true;
+ }
+ }
+ assertThat(hasAttemptDelay).isFalse();
+ assertThat(hasCallDelay).isFalse();
+ }
+
private static class TestChannelBuilder extends ForwardingChannelBuilder2 {
Object interceptorFactory;
MetricSink metricSink;
diff --git a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java
index bd613888f94..9646aa3f035 100644
--- a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java
+++ b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryMetricsModuleTest.java
@@ -30,6 +30,7 @@
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyDouble;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import com.google.common.collect.ImmutableMap;
@@ -45,6 +46,9 @@
import io.grpc.Grpc;
import io.grpc.KnownLength;
import io.grpc.LoadBalancer;
+import io.grpc.LoadBalancer.PickResult;
+import io.grpc.LoadBalancer.PickSubchannelArgs;
+import io.grpc.LoadBalancer.SubchannelPicker;
import io.grpc.LoadBalancerProvider;
import io.grpc.LoadBalancerRegistry;
import io.grpc.ManagedChannel;
@@ -1663,6 +1667,179 @@ public void clientAttemptDelayDuration_recorded() {
})));
}
+
+ @Test
+ public void clientCallDelayDuration_recorded() {
+ Map enabledMetrics = ImmutableMap.of(
+ "grpc.client.call.delay.duration", true
+ );
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(
+ testMeter, enabledMetrics, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(), resource, emptyList(), emptyList());
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(
+ module, "target:///", STREAM_INFO.getCallOptions(), method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "dns resolution pending");
+ fakeClock.forwardTime(500, TimeUnit.MILLISECONDS);
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(
+ metric -> assertThat(metric)
+ .hasName("grpc.client.call.delay.duration")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> {
+ point.hasSum(0.5);
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.delay_type"), "resolving");
+ })));
+ }
+
+ @Test
+ public void clientCallDelayDuration_defensiveCleanupOnAbruptCallEnded() {
+ Map enabledMetrics = ImmutableMap.of(
+ "grpc.client.call.delay.duration", true
+ );
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(
+ testMeter, enabledMetrics, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(), resource, emptyList(), emptyList());
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(
+ module, "target:///", STREAM_INFO.getCallOptions(), method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "dns resolution pending");
+ fakeClock.forwardTime(500, TimeUnit.MILLISECONDS);
+ // Call ends abruptly without prior recordCallDelayEnd()
+ callAttemptsTracerFactory.callEnded(
+ Status.CANCELLED.withDescription("abrupt cancellation"), STREAM_INFO.getCallOptions());
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(
+ metric -> assertThat(metric)
+ .hasName("grpc.client.call.delay.duration")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> {
+ point.hasSum(0.5);
+ point.hasAttribute(METHOD_KEY, method.getFullMethodName());
+ point.hasAttribute(TARGET_KEY, "target:///");
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.delay_type"), "resolving");
+ })));
+
+ // Ensure subsequent calls to recordCallDelayEnd are safe no-ops
+ callAttemptsTracerFactory.recordCallDelayEnd();
+ // Ensure subsequent calls to recordCallDelayStart are rejected after callEnded
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "late start attempt");
+ fakeClock.forwardTime(200, TimeUnit.MILLISECONDS);
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ // Metric sum remains 0.5, no additional recordings
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(
+ metric -> assertThat(metric)
+ .hasName("grpc.client.call.delay.duration")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> point.hasSum(0.5))));
+ }
+
+ @Test
+ public void clientCallDelayDuration_endToEnd_nameResolutionDelay() throws Exception {
+ final CountDownLatch resolutionLatch = new CountDownLatch(1);
+ final AtomicReference capturedListener = new AtomicReference<>();
+
+ NameResolverProvider slowResolverProvider = new NameResolverProvider() {
+ @Override
+ protected boolean isAvailable() {
+ return true;
+ }
+
+ @Override
+ protected int priority() {
+ return 5;
+ }
+
+ @Override
+ public String getDefaultScheme() {
+ return "slowresmetric";
+ }
+
+ @Override
+ public Collection> getProducedSocketAddressTypes() {
+ return Collections.singleton(InProcessSocketAddress.class);
+ }
+
+ @Override
+ public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) {
+ return new NameResolver() {
+ @Override
+ public String getServiceAuthority() {
+ return "slowresmetric";
+ }
+
+ @Override
+ public void start(Listener2 listener) {
+ capturedListener.set(listener);
+ resolutionLatch.countDown();
+ }
+
+ @Override
+ public void shutdown() {}
+ };
+ }
+ };
+ NameResolverRegistry.getDefaultRegistry().register(slowResolverProvider);
+
+ GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder()
+ .sdk(openTelemetryTesting.getOpenTelemetry())
+ .enableMetrics(Collections.singleton("grpc.client.call.delay.duration"))
+ .build();
+
+ InProcessChannelBuilder channelBuilder =
+ InProcessChannelBuilder.forTarget("slowresmetric:///test-metric-service")
+ .defaultLoadBalancingPolicy("pick_first");
+ grpcOpenTelemetry.configureChannelBuilder(channelBuilder);
+ ManagedChannel channel = channelBuilder.build();
+ try {
+ ClientCall call = channel.newCall(method, CallOptions.DEFAULT);
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ call.request(1);
+
+ resolutionLatch.await(5, TimeUnit.SECONDS);
+
+ // Complete name resolution
+ capturedListener.get().onResult(NameResolver.ResolutionResult.newBuilder()
+ .setAddressesOrError(StatusOr.fromValue(Collections.singletonList(
+ new EquivalentAddressGroup(new InProcessSocketAddress("test-slow-metric")))))
+ .build());
+
+ call.cancel("End test", null);
+ } finally {
+ channel.shutdownNow();
+ channel.awaitTermination(5, TimeUnit.SECONDS);
+ NameResolverRegistry.getDefaultRegistry().deregister(slowResolverProvider);
+ }
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(
+ metric -> assertThat(metric)
+ .hasName("grpc.client.call.delay.duration")
+ .hasHistogramSatisfying(
+ histogram -> histogram.hasPointsSatisfying(
+ point -> {
+ point.hasAttribute(METHOD_KEY, method.getFullMethodName());
+ point.hasAttribute(
+ AttributeKey.stringKey("grpc.delay_type"), "resolving");
+ })));
+ }
+
@Test
public void clientAttemptDelayDuration_endToEnd_inProcessTransport() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
@@ -2289,6 +2466,174 @@ public void serverMetrics_recordsBaggage() {
"baggage-val-1", capturedBaggage.getEntryValue("baggage-key-1"));
}
+ @Test
+ public void clientMetrics_nameResolutionFailure_zeroAttempts() {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ fakeClock.forwardTime(50, TimeUnit.MILLISECONDS);
+ callAttemptsTracerFactory.callEnded(Status.UNAVAILABLE, CALL_OPTIONS);
+
+ io.opentelemetry.api.common.Attributes clientAttributes =
+ io.opentelemetry.api.common.Attributes.of(
+ TARGET_KEY, target,
+ METHOD_KEY, method.getFullMethodName(),
+ STATUS_KEY, Code.UNAVAILABLE.toString());
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(
+ metric ->
+ assertThat(metric)
+ .hasInstrumentationScope(InstrumentationScopeInfo.create(
+ OpenTelemetryConstants.INSTRUMENTATION_SCOPE))
+ .hasName(CLIENT_CALL_DURATION)
+ .hasUnit("s")
+ .hasHistogramSatisfying(
+ histogram ->
+ histogram.hasPointsSatisfying(
+ point ->
+ point
+ .hasCount(1)
+ .hasSum(0.05)
+ .hasAttributes(clientAttributes)
+ .hasBucketBoundaries(latencyBuckets))));
+ }
+
+ @Test
+ public void clientMetrics_endToEnd_nameResolutionFailure_unavailable() throws Exception {
+ NameResolverProvider failingProvider = new NameResolverProvider() {
+ @Override
+ public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) {
+ return new NameResolver() {
+ @Override
+ public String getServiceAuthority() {
+ return "failing.authority";
+ }
+
+ @Override
+ public void start(Listener2 listener) {
+ listener.onError(Status.UNAVAILABLE.withDescription("Name resolution failed"));
+ }
+
+ @Override
+ public void shutdown() {}
+ };
+ }
+
+ @Override
+ protected boolean isAvailable() {
+ return true;
+ }
+
+ @Override
+ protected int priority() {
+ return 5;
+ }
+
+ @Override
+ public String getDefaultScheme() {
+ return "failingnr";
+ }
+
+ @Override
+ public String getScheme() {
+ return getDefaultScheme();
+ }
+
+ @Override
+ public Collection> getProducedSocketAddressTypes() {
+ return Collections.singleton(InProcessSocketAddress.class);
+ }
+ };
+
+ NameResolverRegistry.getDefaultRegistry().register(failingProvider);
+
+ try {
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+
+ String target = "failingnr:///test.service";
+ ManagedChannel channel = grpcCleanup.register(
+ InProcessChannelBuilder.forTarget(target)
+ .directExecutor()
+ .intercept(module.getClientInterceptor(target))
+ .build());
+
+ ClientCall call = channel.newCall(method, CallOptions.DEFAULT);
+ call.start(mockClientCallListener, new Metadata());
+
+ verify(mockClientCallListener, timeout(5000))
+ .onClose(statusCaptor.capture(), any(Metadata.class));
+ Status status = statusCaptor.getValue();
+ assertEquals(Status.Code.UNAVAILABLE, status.getCode());
+
+ io.opentelemetry.api.common.Attributes clientAttributes =
+ io.opentelemetry.api.common.Attributes.of(
+ TARGET_KEY, target,
+ METHOD_KEY, method.getFullMethodName(),
+ STATUS_KEY, Code.UNAVAILABLE.toString());
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(
+ metric ->
+ assertThat(metric)
+ .hasInstrumentationScope(InstrumentationScopeInfo.create(
+ OpenTelemetryConstants.INSTRUMENTATION_SCOPE))
+ .hasName(CLIENT_CALL_DURATION)
+ .hasUnit("s")
+ .hasHistogramSatisfying(
+ histogram ->
+ histogram.hasPointsSatisfying(
+ point ->
+ point
+ .hasCount(1)
+ .hasAttributes(clientAttributes))));
+ } finally {
+ NameResolverRegistry.getDefaultRegistry().deregister(failingProvider);
+ }
+ }
+
+ @Test
+ public void clientMetrics_delayObservabilityDisabled_noDelayMetricsRecorded() {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ // When delay observability is disabled or default is unchanged, calls are no-ops
+ tracer.recordAttemptDelayStart("connecting", "attempt delay reason");
+ tracer.recordAttemptDelayReasonChanged("changed reason");
+ tracer.recordAttemptDelayEnd();
+
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "call delay reason");
+ callAttemptsTracerFactory.recordCallDelayReasonChanged("changed call reason");
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ assertNotNull(tracer);
+ }
+
+ @Test
+ public void clientMetrics_targetAttributeFilter_returnsFilteredOrOther() {
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+
+ assertEquals("target:///", module.recordTarget("target:///"));
+ assertThat(module.recordTarget(null)).isNull();
+ }
+
@Test
public void serverMetrics_recordsBaggage_endToEnd() throws Exception {
DoubleHistogram mockDurationHistogram = mock(DoubleHistogram.class);
@@ -2365,6 +2710,348 @@ public ClientCall interceptCall(
"baggage-val-1", capturedBaggage.getEntryValue("baggage-key-1"));
}
+ @Test
+ public void clientCallDelayDuration_sameDelayType_doesNotResetStopwatch() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ Map enabledMetrics = ImmutableMap.of(
+ "grpc.client.call.delay.duration", true
+ );
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetrics, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "reason1");
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "reason2");
+ // Transition to a different delay type while active (exercises Objects.equals == false)
+ callAttemptsTracerFactory.recordCallDelayStart("connecting", "transition to connecting");
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .anySatisfy(metric -> assertThat(metric).hasName("grpc.client.call.delay.duration"));
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientCallDelayDuration_nullDelayType_noMetricRecorded() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ callAttemptsTracerFactory.recordCallDelayStart(null, "null delay type");
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .noneSatisfy(metric -> assertThat(metric).hasName("grpc.client.call.delay.duration"));
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayDuration_sameDelayType_doesNotResetStopwatch() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ tracer.recordAttemptDelayStart("connecting", "reason1");
+ tracer.recordAttemptDelayStart("connecting", "reason2");
+ // Transition to a different delay type while active (exercises Objects.equals == false)
+ tracer.recordAttemptDelayStart("0:connecting", "transition to different delay type");
+ tracer.recordAttemptDelayEnd();
+
+ assertNotNull(tracer);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayDuration_nullDelayType_noMetricRecorded() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = newOpenTelemetryMetricsModule(resource);
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ tracer.recordAttemptDelayStart(null, "null delay type");
+ tracer.recordAttemptDelayEnd();
+
+ assertThat(openTelemetryTesting.getMetrics())
+ .noneSatisfy(metric -> assertThat(metric).hasName("grpc.client.attempt.delay.duration"));
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayDuration_withOptionalLabels() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"),
+ emptyList());
+ OpenTelemetryMetricsModule.CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ tracer.addOptionalLabel("grpc.lb.locality", "us-east1-a");
+ tracer.addOptionalLabel("grpc.lb.backend_service", "backend-service-1");
+
+ tracer.recordAttemptDelayStart("connecting", "reason1");
+ tracer.recordAttemptDelayEnd();
+
+ assertNotNull(tracer);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void callEnded_beforeAttemptEnded_recordsFinishedCall() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ emptyList(),
+ emptyList());
+
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ callAttemptsTracerFactory.callEnded(Status.OK, CALL_OPTIONS);
+ callAttemptsTracerFactory.attemptEnded(CALL_OPTIONS);
+
+ assertNotNull(tracer);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void serverStreamClosed_calledTwice_secondCallNoOp() {
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ emptyList(),
+ emptyList());
+
+ ServerStreamTracer.Factory serverTracerFactory = module.getServerTracerFactory();
+ ServerStreamTracer serverTracer =
+ serverTracerFactory.newServerStreamTracer(method.getFullMethodName(), new Metadata());
+
+ serverTracer.streamClosed(Status.OK);
+ serverTracer.streamClosed(Status.CANCELLED);
+
+ assertNotNull(serverTracer);
+ }
+
+ @Test
+ public void clientCallDelayReasonChanged_noActiveStopwatch_noOp() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ emptyList(),
+ emptyList());
+
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ callAttemptsTracerFactory.recordCallDelayReasonChanged("reasonWithoutStopwatch");
+ assertNotNull(callAttemptsTracerFactory);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayReasonChanged_noActiveStopwatch_noOp() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ emptyList(),
+ emptyList());
+
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ tracer.recordAttemptDelayReasonChanged("reasonWithoutStopwatch");
+ assertNotNull(tracer);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void delayMetrics_featureFlagDisabled_allMethodsNoOp() {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"),
+ emptyList());
+
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "reason1");
+ callAttemptsTracerFactory.recordCallDelayReasonChanged("reason2");
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ tracer.recordAttemptDelayStart("connecting", "reason1");
+ tracer.recordAttemptDelayReasonChanged("reason2");
+ tracer.recordAttemptDelayEnd();
+
+ assertNotNull(tracer);
+ }
+
+ @Test
+ public void clientAttemptDelayDuration_withNullLocalityAndBackendService() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"),
+ emptyList());
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ // Do NOT set optional labels so locality and backendService remain null
+ tracer.recordAttemptDelayStart("connecting", "reason1");
+ tracer.recordAttemptDelayEnd();
+
+ assertNotNull(tracer);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayStart_afterStreamClosed_noOp() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"),
+ emptyList());
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+ ClientStreamTracer tracer = callAttemptsTracerFactory.newClientStreamTracer(
+ ClientStreamTracer.StreamInfo.newBuilder().build(), new Metadata());
+
+ tracer.streamClosed(Status.OK);
+ tracer.recordAttemptDelayStart("connecting", "post-close");
+ tracer.recordAttemptDelayReasonChanged("changed");
+ tracer.recordAttemptDelayEnd();
+
+ callAttemptsTracerFactory.callEnded(Status.OK, CALL_OPTIONS);
+ assertNotNull(tracer);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientCallDelayStart_afterCallEnded_noOp() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ String target = "target:///";
+ OpenTelemetryMetricsResource resource = GrpcOpenTelemetry.createMetricInstruments(testMeter,
+ enabledMetricsMap, disableDefaultMetrics);
+ OpenTelemetryMetricsModule module = new OpenTelemetryMetricsModule(
+ fakeClock.getStopwatchSupplier(),
+ resource,
+ Arrays.asList("grpc.lb.locality", "grpc.lb.backend_service"),
+ emptyList());
+ CallAttemptsTracerFactory callAttemptsTracerFactory =
+ new CallAttemptsTracerFactory(module, target, CALL_OPTIONS, method.getFullMethodName(),
+ emptyList(), Context.root());
+
+ callAttemptsTracerFactory.callEnded(Status.OK, CALL_OPTIONS);
+ callAttemptsTracerFactory.recordCallDelayStart("resolving", "post-close");
+ callAttemptsTracerFactory.recordCallDelayReasonChanged("changed");
+ callAttemptsTracerFactory.recordCallDelayEnd();
+
+ assertNotNull(callAttemptsTracerFactory);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
private static List sortByName(List metrics) {
metrics.sort((m1, m2) -> m1.getName().compareTo(m2.getName()));
return metrics;
diff --git a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java
index 0b5bff1d036..5ae246ac8d5 100644
--- a/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java
+++ b/opentelemetry/src/test/java/io/grpc/opentelemetry/OpenTelemetryTracingModuleTest.java
@@ -20,6 +20,7 @@
import static io.grpc.opentelemetry.internal.OpenTelemetryConstants.BAGGAGE_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -46,6 +47,9 @@
import io.grpc.EquivalentAddressGroup;
import io.grpc.KnownLength;
import io.grpc.LoadBalancer;
+import io.grpc.LoadBalancer.PickResult;
+import io.grpc.LoadBalancer.PickSubchannelArgs;
+import io.grpc.LoadBalancer.SubchannelPicker;
import io.grpc.LoadBalancerProvider;
import io.grpc.LoadBalancerRegistry;
import io.grpc.ManagedChannel;
@@ -299,159 +303,154 @@ public void clientBasicTracingMocking() {
}
@Test
- public void clientBasicTracingRule() {
- OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
- openTelemetryRule.getOpenTelemetry());
- Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ public void clientDelayTracingMocking() {
+ Span mockDelaySpan = mock(Span.class);
+ when(mockSpanBuilder.setAttribute(
+ org.mockito.ArgumentMatchers.anyString(),
+ org.mockito.ArgumentMatchers.anyString()))
+ .thenReturn(mockSpanBuilder);
+ when(mockSpanBuilder.startSpan()).thenReturn(mockAttemptSpan, mockDelaySpan);
+
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(mockOpenTelemetry);
CallAttemptsTracerFactory callTracer =
- tracingModule.newClientCallTracer(clientSpan, method);
- Metadata headers = new Metadata();
- ClientStreamTracer clientStreamTracer = callTracer.newClientStreamTracer(STREAM_INFO, headers);
- clientStreamTracer.createPendingStream();
- clientStreamTracer.streamCreated(Attributes.EMPTY, headers);
- clientStreamTracer.outboundMessage(0);
- clientStreamTracer.outboundMessageSent(0, 882, -1);
- clientStreamTracer.inboundMessage(0);
- clientStreamTracer.outboundMessage(1);
- clientStreamTracer.outboundMessageSent(1, -1, 27);
- clientStreamTracer.inboundMessageRead(0, 255, -1);
- clientStreamTracer.inboundUncompressedSize(288);
- clientStreamTracer.inboundMessageRead(1, 128, 128);
- clientStreamTracer.inboundMessage(1);
- clientStreamTracer.inboundUncompressedSize(128);
+ tracingModule.newClientCallTracer(mockClientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
- clientStreamTracer.streamClosed(Status.OK);
- callTracer.callEnded(Status.OK);
+ clientStreamTracer.recordAttemptDelayStart("connecting", "pick_first: attempting to connect");
+ clientStreamTracer.recordAttemptDelayEnd();
- List spans = openTelemetryRule.getSpans();
- assertEquals(spans.size(), 2);
- SpanData attemptSpanData = spans.get(0);
- SpanData clientSpanData = spans.get(1);
- assertEquals(attemptSpanData.getName(), "Attempt.package1.service2.method3");
- assertEquals(clientSpanData.getName(), "test-client-span");
- assertEquals(headers.keys(), ImmutableSet.of("traceparent"));
- String spanContext = headers.get(
- Metadata.Key.of("traceparent", Metadata.ASCII_STRING_MARSHALLER));
- assertEquals(spanContext.substring(3, 3 + TraceId.getLength()),
- spans.get(1).getSpanContext().getTraceId());
+ verify(mockTracer).spanBuilder(eq("Delay"));
+ verify(mockSpanBuilder).setAttribute(eq("grpc.delay_type"), eq("connecting"));
+ verify(mockDelaySpan).addEvent(
+ eq("Delay state transition"),
+ org.mockito.ArgumentMatchers.any());
+ verify(mockDelaySpan).end();
+ }
- // parent(client) span data
- List clientSpanEvents = clientSpanData.getEvents();
- assertEquals(clientSpanEvents.size(), 3);
- assertEquals(
- "Delayed name resolution complete",
- clientSpanEvents.get(0).getName());
- assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty());
+ @Test
+ public void clientCallDelayTracingMocking() {
+ Span mockDelaySpan = mock(Span.class);
+ when(mockSpanBuilder.setAttribute(
+ org.mockito.ArgumentMatchers.anyString(),
+ org.mockito.ArgumentMatchers.anyString()))
+ .thenReturn(mockSpanBuilder);
+ when(mockSpanBuilder.startSpan()).thenReturn(mockDelaySpan);
- assertEquals(
- "Inbound message" ,
- clientSpanEvents.get(1).getName());
- assertEquals(
- io.opentelemetry.api.common.Attributes.builder()
- .put("sequence-number", 0)
- .put("message-size", 288)
- .build(),
- clientSpanEvents.get(1).getAttributes());
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(mockOpenTelemetry);
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(mockClientSpan, method);
- assertEquals(
- "Inbound message" ,
- clientSpanEvents.get(2).getName());
- assertEquals(
- io.opentelemetry.api.common.Attributes.builder()
- .put("sequence-number", 1)
- .put("message-size", 128)
- .build(),
- clientSpanEvents.get(2).getAttributes());
- assertEquals(clientSpanData.hasEnded(), true);
+ callTracer.recordCallDelayStart("resolving", "waiting for DNS query");
+ callTracer.recordCallDelayEnd();
- // child(attempt) span data
- List attemptSpanEvents = attemptSpanData.getEvents();
- assertEquals(clientSpanEvents.size(), 3);
- assertEquals(
- "Delayed LB pick complete",
- attemptSpanEvents.get(0).getName());
- assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty());
+ verify(mockTracer).spanBuilder(eq("Delay"));
+ verify(mockSpanBuilder).setAttribute(eq("grpc.delay_type"), eq("resolving"));
+ verify(mockDelaySpan).addEvent(
+ eq("Delay state transition"),
+ org.mockito.ArgumentMatchers.any());
+ verify(mockDelaySpan).end();
+ }
- assertEquals(
- "Outbound message" ,
- attemptSpanEvents.get(1).getName());
- assertEquals(
- io.opentelemetry.api.common.Attributes.builder()
- .put("sequence-number", 0)
- .put("message-size-compressed", 882)
- .build(),
- attemptSpanEvents.get(1).getAttributes());
+ @Test
+ public void clientCallDelayTracing_endToEnd_nameResolutionDelay() throws Exception {
+ final CountDownLatch resolutionLatch = new CountDownLatch(1);
+ final AtomicReference capturedListener = new AtomicReference<>();
- assertEquals(
- "Outbound message" ,
- attemptSpanEvents.get(2).getName());
- assertEquals(
- io.opentelemetry.api.common.Attributes.builder()
- .put("sequence-number", 1)
- .put("message-size", 27)
- .build(),
- attemptSpanEvents.get(2).getAttributes());
+ NameResolverProvider slowResolverProvider = new NameResolverProvider() {
+ @Override
+ protected boolean isAvailable() {
+ return true;
+ }
- assertEquals(
- "Inbound compressed message" ,
- attemptSpanEvents.get(3).getName());
- assertEquals(
- io.opentelemetry.api.common.Attributes.builder()
- .put("sequence-number", 0)
- .put("message-size-compressed", 255)
- .build(),
- attemptSpanEvents.get(3).getAttributes());
+ @Override
+ protected int priority() {
+ return 5;
+ }
- assertEquals(attemptSpanData.hasEnded(), true);
- }
+ @Override
+ public String getDefaultScheme() {
+ return "slowres";
+ }
- @Test
- public void clientAttemptDelayTracing_reasonChangedInvariant() {
- OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
- openTelemetryRule.getOpenTelemetry());
- Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
- CallAttemptsTracerFactory callTracer =
- tracingModule.newClientCallTracer(clientSpan, method);
- ClientStreamTracer clientStreamTracer =
- callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+ @Override
+ public Collection> getProducedSocketAddressTypes() {
+ return Collections.singleton(InProcessSocketAddress.class);
+ }
- clientStreamTracer.recordAttemptDelayStart("connecting", "reason1");
- clientStreamTracer.recordAttemptDelayReasonChanged("reason2");
- clientStreamTracer.recordAttemptDelayStart("connecting", "reason3");
- clientStreamTracer.recordAttemptDelayEnd();
- clientStreamTracer.streamClosed(Status.OK);
- callTracer.callEnded(Status.OK);
- clientSpan.end();
+ @Override
+ public NameResolver newNameResolver(URI targetUri, NameResolver.Args args) {
+ return new NameResolver() {
+ @Override
+ public String getServiceAuthority() {
+ return "slowres";
+ }
- List spans = openTelemetryRule.getSpans();
- assertEquals(3, spans.size());
- SpanData delaySpanData = spans.get(0);
+ @Override
+ public void start(Listener2 listener) {
+ capturedListener.set(listener);
+ resolutionLatch.countDown();
+ }
- assertEquals("Attempt Delay", delaySpanData.getName());
- assertEquals("connecting", delaySpanData.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_type")));
- assertEquals(3, delaySpanData.getEvents().size());
+ @Override
+ public void shutdown() {}
+ };
+ }
+ };
+ NameResolverRegistry.getDefaultRegistry().register(slowResolverProvider);
- EventData event1 = delaySpanData.getEvents().get(0);
- assertEquals("Delay state transition", event1.getName());
- assertEquals("connecting", event1.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_type")));
- assertEquals("reason1", event1.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_reason")));
+ GrpcOpenTelemetry grpcOpenTelemetry = GrpcOpenTelemetry.newBuilder()
+ .sdk(openTelemetryRule.getOpenTelemetry())
+ .enableTracing(true)
+ .build();
- EventData event2 = delaySpanData.getEvents().get(1);
- assertEquals("Delay state transition", event2.getName());
- assertEquals("connecting", event2.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_type")));
- assertEquals("reason2", event2.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_reason")));
+ InProcessChannelBuilder channelBuilder =
+ InProcessChannelBuilder.forTarget("slowres:///test-service")
+ .defaultLoadBalancingPolicy("pick_first");
+ grpcOpenTelemetry.configureChannelBuilder(channelBuilder);
+ ManagedChannel channel = channelBuilder.build();
+ try {
+ ClientCall call = channel.newCall(method, CallOptions.DEFAULT);
+ call.start(new ClientCall.Listener() {}, new Metadata());
+ call.request(1);
- EventData event3 = delaySpanData.getEvents().get(2);
- assertEquals("Delay state transition", event3.getName());
- assertEquals("connecting", event3.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_type")));
- assertEquals("reason3", event3.getAttributes().get(
- AttributeKey.stringKey("grpc.delay_reason")));
+ resolutionLatch.await(5, TimeUnit.SECONDS);
+
+ // Now complete name resolution
+ capturedListener.get().onResult(NameResolver.ResolutionResult.newBuilder()
+ .setAddressesOrError(StatusOr.fromValue(Collections.singletonList(
+ new EquivalentAddressGroup(new InProcessSocketAddress("test-slow-res")))))
+ .build());
+
+ call.cancel("End test", null);
+ } finally {
+ channel.shutdownNow();
+ channel.awaitTermination(5, TimeUnit.SECONDS);
+ NameResolverRegistry.getDefaultRegistry().deregister(slowResolverProvider);
+ }
+
+ List spans = openTelemetryRule.getSpans();
+ SpanData callDelaySpan = null;
+ for (SpanData s : spans) {
+ if ("Delay".equals(s.getName())) {
+ callDelaySpan = s;
+ break;
+ }
+ }
+ assertNotNull(callDelaySpan);
+ assertEquals("resolving",
+ callDelaySpan.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+
+ boolean foundTransition = false;
+ for (EventData event : callDelaySpan.getEvents()) {
+ if ("Delay state transition".equals(event.getName())
+ && "waiting for name resolution or service config".equals(
+ event.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")))) {
+ foundTransition = true;
+ break;
+ }
+ }
+ assertTrue(foundTransition);
}
@Test
@@ -530,7 +529,7 @@ public String getServiceAuthority() {
@Override
public void start(Listener2 listener) {
- listener.onResult(ResolutionResult.newBuilder()
+ listener.onResult(NameResolver.ResolutionResult.newBuilder()
.setAddressesOrError(StatusOr.fromValue(Collections.singletonList(
new EquivalentAddressGroup(new InProcessSocketAddress("test-e2e")))))
.build());
@@ -570,7 +569,7 @@ public void shutdown() {}
List spans = openTelemetryRule.getSpans();
SpanData delaySpanData = null;
for (SpanData s : spans) {
- if ("Attempt Delay".equals(s.getName())) {
+ if ("Delay".equals(s.getName())) {
delaySpanData = s;
break;
}
@@ -592,30 +591,476 @@ public void shutdown() {}
}
@Test
- public void clientAttemptDelayStart_featureFlagDisabled_zeroChildSpans() {
- System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "false");
- try {
- OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
- openTelemetryRule.getOpenTelemetry());
- Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
- CallAttemptsTracerFactory callTracer =
- tracingModule.newClientCallTracer(clientSpan, method);
- ClientStreamTracer clientStreamTracer =
- callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+ public void clientBasicTracingRule() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ Metadata headers = new Metadata();
+ ClientStreamTracer clientStreamTracer = callTracer.newClientStreamTracer(STREAM_INFO, headers);
+ clientStreamTracer.createPendingStream();
+ clientStreamTracer.streamCreated(Attributes.EMPTY, headers);
+ clientStreamTracer.outboundMessage(0);
+ clientStreamTracer.outboundMessageSent(0, 882, -1);
+ clientStreamTracer.inboundMessage(0);
+ clientStreamTracer.outboundMessage(1);
+ clientStreamTracer.outboundMessageSent(1, -1, 27);
+ clientStreamTracer.inboundMessageRead(0, 255, -1);
+ clientStreamTracer.inboundUncompressedSize(288);
+ clientStreamTracer.inboundMessageRead(1, 128, 128);
+ clientStreamTracer.inboundMessage(1);
+ clientStreamTracer.inboundUncompressedSize(128);
- clientStreamTracer.recordAttemptDelayStart("connecting", "attempting to connect");
- clientStreamTracer.recordAttemptDelayEnd();
- clientStreamTracer.streamClosed(Status.OK);
- callTracer.callEnded(Status.OK);
- clientSpan.end();
+ clientStreamTracer.streamClosed(Status.OK);
+ callTracer.callEnded(Status.OK);
- List spans = openTelemetryRule.getSpans();
- assertEquals(2, spans.size());
- for (SpanData span : spans) {
- assertTrue(!span.getName().equals("Attempt Delay"));
- }
- } finally {
- System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ List spans = openTelemetryRule.getSpans();
+ assertEquals(spans.size(), 2);
+ SpanData attemptSpanData = spans.get(0);
+ SpanData clientSpanData = spans.get(1);
+ assertEquals(attemptSpanData.getName(), "Attempt.package1.service2.method3");
+ assertEquals(clientSpanData.getName(), "test-client-span");
+ assertEquals(headers.keys(), ImmutableSet.of("traceparent"));
+ String spanContext = headers.get(
+ Metadata.Key.of("traceparent", Metadata.ASCII_STRING_MARSHALLER));
+ assertEquals(spanContext.substring(3, 3 + TraceId.getLength()),
+ spans.get(1).getSpanContext().getTraceId());
+
+ // parent(client) span data
+ List clientSpanEvents = clientSpanData.getEvents();
+ assertEquals(clientSpanEvents.size(), 3);
+ assertEquals(
+ "Delayed name resolution complete",
+ clientSpanEvents.get(0).getName());
+ assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty());
+
+ assertEquals(
+ "Inbound message" ,
+ clientSpanEvents.get(1).getName());
+ assertEquals(
+ io.opentelemetry.api.common.Attributes.builder()
+ .put("sequence-number", 0)
+ .put("message-size", 288)
+ .build(),
+ clientSpanEvents.get(1).getAttributes());
+
+ assertEquals(
+ "Inbound message" ,
+ clientSpanEvents.get(2).getName());
+ assertEquals(
+ io.opentelemetry.api.common.Attributes.builder()
+ .put("sequence-number", 1)
+ .put("message-size", 128)
+ .build(),
+ clientSpanEvents.get(2).getAttributes());
+ assertEquals(clientSpanData.hasEnded(), true);
+
+ // child(attempt) span data
+ List attemptSpanEvents = attemptSpanData.getEvents();
+ assertEquals(clientSpanEvents.size(), 3);
+ assertEquals(
+ "Delayed LB pick complete",
+ attemptSpanEvents.get(0).getName());
+ assertTrue(clientSpanEvents.get(0).getAttributes().isEmpty());
+
+ assertEquals(
+ "Outbound message" ,
+ attemptSpanEvents.get(1).getName());
+ assertEquals(
+ io.opentelemetry.api.common.Attributes.builder()
+ .put("sequence-number", 0)
+ .put("message-size-compressed", 882)
+ .build(),
+ attemptSpanEvents.get(1).getAttributes());
+
+ assertEquals(
+ "Outbound message" ,
+ attemptSpanEvents.get(2).getName());
+ assertEquals(
+ io.opentelemetry.api.common.Attributes.builder()
+ .put("sequence-number", 1)
+ .put("message-size", 27)
+ .build(),
+ attemptSpanEvents.get(2).getAttributes());
+
+ assertEquals(
+ "Inbound compressed message" ,
+ attemptSpanEvents.get(3).getName());
+ assertEquals(
+ io.opentelemetry.api.common.Attributes.builder()
+ .put("sequence-number", 0)
+ .put("message-size-compressed", 255)
+ .build(),
+ attemptSpanEvents.get(3).getAttributes());
+
+ assertEquals(attemptSpanData.hasEnded(), true);
+ }
+
+ @Test
+ public void clientAttemptDelayTracing_reasonChangedInvariant() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ clientStreamTracer.recordAttemptDelayStart("connecting", "reason1");
+ clientStreamTracer.recordAttemptDelayReasonChanged("reason2");
+ clientStreamTracer.recordAttemptDelayStart("connecting", "reason3");
+ clientStreamTracer.recordAttemptDelayEnd();
+ clientStreamTracer.streamClosed(Status.OK);
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertEquals(3, spans.size());
+ SpanData delaySpanData = spans.get(0);
+
+ assertEquals("Delay", delaySpanData.getName());
+ assertEquals("connecting", delaySpanData.getAttributes().get(
+ AttributeKey.stringKey("grpc.delay_type")));
+ assertEquals(3, delaySpanData.getEvents().size());
+
+ EventData event1 = delaySpanData.getEvents().get(0);
+ assertEquals("Delay state transition", event1.getName());
+ assertEquals("reason1", event1.getAttributes().get(
+ AttributeKey.stringKey("grpc.delay_reason")));
+ assertNull(event1.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+
+ EventData event2 = delaySpanData.getEvents().get(1);
+ assertEquals("Delay state transition", event2.getName());
+ assertEquals("reason2", event2.getAttributes().get(
+ AttributeKey.stringKey("grpc.delay_reason")));
+ assertNull(event2.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+
+ EventData event3 = delaySpanData.getEvents().get(2);
+ assertEquals("Delay state transition", event3.getName());
+ assertEquals("reason3", event3.getAttributes().get(
+ AttributeKey.stringKey("grpc.delay_reason")));
+ assertNull(event3.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+ }
+
+ @Test
+ public void clientAttemptDelayStart_featureFlagDisabled_zeroChildSpans() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "false");
+ try {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ clientStreamTracer.recordAttemptDelayStart("connecting", "attempting to connect");
+ clientStreamTracer.recordAttemptDelayEnd();
+ clientStreamTracer.streamClosed(Status.OK);
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertEquals(2, spans.size());
+ for (SpanData span : spans) {
+ assertTrue(!span.getName().equals("Delay"));
+ }
+ } finally {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ }
+ }
+
+ @Test
+ public void clientCallDelayTracing_reasonChangedInvariant() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+
+ callTracer.recordCallDelayStart("resolving", "reason1");
+ callTracer.recordCallDelayReasonChanged("reason2");
+ callTracer.recordCallDelayStart("resolving", "reason3");
+ callTracer.recordCallDelayEnd();
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertEquals(2, spans.size());
+ SpanData callDelaySpan = spans.stream()
+ .filter(s -> "Delay".equals(s.getName()))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("Expected 'Delay' span not found"));
+
+ assertEquals("resolving", callDelaySpan.getAttributes().get(
+ AttributeKey.stringKey("grpc.delay_type")));
+ assertEquals(3, callDelaySpan.getEvents().size());
+
+ EventData event1 = callDelaySpan.getEvents().get(0);
+ assertEquals("Delay state transition", event1.getName());
+ assertEquals("reason1",
+ event1.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")));
+ assertNull(event1.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+
+ EventData event2 = callDelaySpan.getEvents().get(1);
+ assertEquals("Delay state transition", event2.getName());
+ assertEquals("reason2",
+ event2.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")));
+ assertNull(event2.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+
+ EventData event3 = callDelaySpan.getEvents().get(2);
+ assertEquals("Delay state transition", event3.getName());
+ assertEquals("reason3",
+ event3.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")));
+ assertNull(event3.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+ }
+
+ @Test
+ public void clientCallDelayStart_afterCallEnded_noSpansRecorded() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+
+ callTracer.callEnded(Status.OK);
+ callTracer.recordCallDelayStart("resolving", "reasonAfterCallEnded");
+ callTracer.recordCallDelayReasonChanged("reason2");
+ callTracer.recordCallDelayEnd();
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ for (SpanData span : spans) {
+ assertTrue(!span.getName().equals("Delay"));
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayStart_afterStreamClosed_noSpansRecorded() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ clientStreamTracer.streamClosed(Status.OK);
+ clientStreamTracer.recordAttemptDelayStart("connecting", "reasonAfterStreamClosed");
+ clientStreamTracer.recordAttemptDelayReasonChanged("reason2");
+ clientStreamTracer.recordAttemptDelayEnd();
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ for (SpanData span : spans) {
+ assertTrue(!span.getName().equals("Delay"));
+ }
+ }
+
+ @Test
+ public void clientCallDelayStart_delayTypeTransition_closesPreviousSpan() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+
+ callTracer.recordCallDelayStart("resolving", "dns lookup");
+ callTracer.recordCallDelayStart("connecting", "pick first connect");
+ callTracer.recordCallDelayEnd();
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ long callDelaySpanCount = spans.stream().filter(s -> "Delay".equals(s.getName())).count();
+ assertEquals(2L, callDelaySpanCount);
+ }
+
+ @Test
+ public void clientCallEnded_calledTwice_secondCallNoOp() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+
+ callTracer.callEnded(Status.OK);
+ callTracer.callEnded(Status.CANCELLED);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertNotNull(spans);
+ }
+
+ @Test
+ public void clientStreamClosed_calledTwice_secondCallNoOp() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ clientStreamTracer.streamClosed(Status.OK);
+ clientStreamTracer.streamClosed(Status.CANCELLED);
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertNotNull(spans);
+ }
+
+ @Test
+ public void serverStreamClosed_calledTwice_secondCallNoOp() {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ ServerStreamTracer.Factory serverTracerFactory =
+ tracingModule.getServerTracerFactory();
+ ServerStreamTracer serverTracer =
+ serverTracerFactory.newServerStreamTracer(method.getFullMethodName(), new Metadata());
+
+ serverTracer.streamClosed(Status.OK);
+ serverTracer.streamClosed(Status.CANCELLED);
+
+ List spans = openTelemetryRule.getSpans();
+ assertNotNull(spans);
+ }
+
+ @Test
+ public void clientCallDelayReasonChanged_noActiveSpan_noOp() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+
+ callTracer.recordCallDelayReasonChanged("reasonWithoutSpan");
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertNotNull(spans);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayReasonChanged_noActiveSpan_noOp() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ clientStreamTracer.recordAttemptDelayReasonChanged("reasonWithoutSpan");
+ clientStreamTracer.streamClosed(Status.OK);
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertNotNull(spans);
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientCallDelayReasonChanged_nullDelayType_omitsDelayTypeAttribute() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+
+ callTracer.recordCallDelayStart(null, "start reason");
+ callTracer.recordCallDelayReasonChanged("changed reason");
+ callTracer.recordCallDelayEnd();
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertEquals(2, spans.size());
+ SpanData delaySpan = spans.get(0);
+ assertEquals(2, delaySpan.getEvents().size());
+ EventData event = delaySpan.getEvents().get(1);
+ assertNull(event.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+ assertEquals("changed reason",
+ event.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")));
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void clientAttemptDelayReasonChanged_nullDelayType_omitsDelayTypeAttribute() {
+ System.setProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY", "true");
+ try {
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ clientStreamTracer.recordAttemptDelayStart(null, "start reason");
+ clientStreamTracer.recordAttemptDelayReasonChanged("changed reason");
+ clientStreamTracer.recordAttemptDelayEnd();
+ clientStreamTracer.streamClosed(Status.OK);
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ assertEquals(3, spans.size());
+ SpanData delaySpan = spans.get(0);
+ assertEquals(2, delaySpan.getEvents().size());
+ EventData event = delaySpan.getEvents().get(1);
+ assertNull(event.getAttributes().get(AttributeKey.stringKey("grpc.delay_type")));
+ assertEquals("changed reason",
+ event.getAttributes().get(AttributeKey.stringKey("grpc.delay_reason")));
+ } finally {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ }
+ }
+
+ @Test
+ public void delayTracing_featureFlagDisabled_allMethodsNoOp() {
+ System.clearProperty("GRPC_EXPERIMENTAL_ENABLE_DELAY_OBSERVABILITY");
+ OpenTelemetryTracingModule tracingModule = new OpenTelemetryTracingModule(
+ openTelemetryRule.getOpenTelemetry());
+ Span clientSpan = tracerRule.spanBuilder("test-client-span").startSpan();
+ CallAttemptsTracerFactory callTracer =
+ tracingModule.newClientCallTracer(clientSpan, method);
+ ClientStreamTracer clientStreamTracer =
+ callTracer.newClientStreamTracer(STREAM_INFO, new Metadata());
+
+ callTracer.recordCallDelayStart("resolving", "reason1");
+ callTracer.recordCallDelayReasonChanged("reason2");
+ callTracer.recordCallDelayEnd();
+
+ clientStreamTracer.recordAttemptDelayStart("connecting", "reason1");
+ clientStreamTracer.recordAttemptDelayReasonChanged("reason2");
+ clientStreamTracer.recordAttemptDelayEnd();
+
+ clientStreamTracer.streamClosed(Status.OK);
+ callTracer.callEnded(Status.OK);
+ clientSpan.end();
+
+ List spans = openTelemetryRule.getSpans();
+ for (SpanData span : spans) {
+ assertTrue(!span.getName().equals("Delay"));
}
}
diff --git a/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java b/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java
index dbd7e99b29a..3b5cd3714e6 100644
--- a/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java
+++ b/util/src/test/java/io/grpc/util/ForwardingClientStreamTracerTest.java
@@ -17,6 +17,7 @@
package io.grpc.util;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import io.grpc.ClientStreamTracer;
import io.grpc.ForwardingTestUtil;
@@ -40,6 +41,19 @@ public void allMethodsForwarded() throws Exception {
Collections.emptyList());
}
+ @Test
+ public void attemptDelayMethodsForwarded() {
+ TestClientStreamTracer tracer = new TestClientStreamTracer();
+ tracer.recordAttemptDelayStart("connecting", "test");
+ verify(mockDelegate).recordAttemptDelayStart("connecting", "test");
+
+ tracer.recordAttemptDelayReasonChanged("test2");
+ verify(mockDelegate).recordAttemptDelayReasonChanged("test2");
+
+ tracer.recordAttemptDelayEnd();
+ verify(mockDelegate).recordAttemptDelayEnd();
+ }
+
@SuppressWarnings("deprecation")
private final class TestClientStreamTracer extends ForwardingClientStreamTracer {
@Override