feat(tracing): correlate business spans with obs via dedicated wrapper span

src/agentex/lib/core/tracing/obs_ids.py

@@ -11,11 +11,16 @@
 persisted business span to the Tempo/Datadog trace for the turn that produced it,
 while the business trace still groups the entire run by task id.
 
-Source selection follows SGP_OBS_MODE, matching egp-api-backend:
+Source selection follows SGP_OBS_MODE:
   - unset / "dd_only": ddtrace context (current stack)
-  - "dual":            OTel/LGTM preferred, ddtrace fallback
   - "lgtm":            OTel/LGTM only
 
+("dual" was removed: co-resident ddtrace+OTel can't be bridged in-process --
+you can't run ddtrace-run and the OTel operator's auto-instrumentation in the
+same process, and DD_TRACE_OTEL_ENABLED yields a single tracer with nothing to
+bridge. Two-backend export is a collector fan-out under "lgtm", not a mode here.
+An unrecognized SGP_OBS_MODE -- including a stale "dual" -- degrades to dd_only.)
+
 This never fabricates ids -- if no observability context is active, it returns
 an empty dict and the span is simply not tagged.
 """
@@ -27,10 +32,9 @@
 __all__ = ("get_obs_mode", "obs_correlation")
 
 DD_ONLY = "dd_only"
-DUAL = "dual"
 LGTM = "lgtm"
 _DEFAULT_MODE = DD_ONLY
-_VALID_MODES = (DD_ONLY, DUAL, LGTM)
+_VALID_MODES = (DD_ONLY, LGTM)
 
 
 def get_obs_mode() -> str:
@@ -65,19 +69,22 @@ def _ddtrace_ids() -> Optional[Tuple[str, str]]:
 
 
 def obs_correlation() -> Dict[str, str]:
-    """Return ``{"obs.trace_id": ..., "obs.span_id": ...}`` for the active
+    """Return ``{"obs_trace_id": ..., "obs_span_id": ...}`` for the active
     observability context, or ``{}`` if none is active.
 
+    These land in the business span's ``data`` -> egp ``operation_metadata``
+    (an existing JSONB column, GIN-indexed) -> ClickHouse ``metadata_raw``, so
+    the correlation edge needs no schema migration. Underscored keys (not
+    dotted) keep them addressable via Postgres JSON paths
+    (``operation_metadata->>'obs_trace_id'``).
+
     Never fabricates ids -- this is a correlation tag, not the span's id.
     """
-    mode = get_obs_mode()
-    if mode == LGTM:
-        ids = _lgtm_ids()
-    elif mode == DUAL:
-        ids = _lgtm_ids() or _ddtrace_ids()
-    else:  # dd_only
-        ids = _ddtrace_ids()
+    try:
+        ids = _lgtm_ids() if get_obs_mode() == LGTM else _ddtrace_ids()
+    except Exception:  # obs must never fail an app call
+        return {}
 
     if not ids:
         return {}
-    return {"obs.trace_id": ids[0], "obs.span_id": ids[1]}
+    return {"obs_trace_id": ids[0], "obs_span_id": ids[1]}

src/agentex/lib/core/tracing/obs_span.py

@@ -0,0 +1,191 @@
+"""Dedicated per-business-span observability wrapper span.
+
+Capturing obs ids from "whatever instrumentation span happens to be innermost
+at emit time" is coarse -- it could be an arbitrary httpx-client span, and every
+business span in a request would collapse onto the same request/activity span.
+
+Instead, when the SDK creates a business span we open a **real obs span named
+for that step and make it active**. Then:
+  - ``obs_span_id`` is stable and meaningful (a span named for the business
+    step, not an arbitrary leaf), and
+  - any nested instrumentation (httpx, db, ...) parents under it.
+
+The wrapper's own trace_id/span_id are read directly from its span context, so
+the correlation tag is deterministic regardless of what else is on the stack.
+
+Backend follows ``SGP_OBS_MODE``:
+  - ``lgtm``    -> an OpenTelemetry span (the convergence target).
+  - ``dd_only`` -> a ddtrace span, but ONLY when a ddtrace trace is already
+    active for the request. Opening one unconditionally would emit orphan root
+    traces in un-instrumented (bare-uvicorn, no ddtrace-run) agents, so when
+    nothing is active we return ``None`` and the caller keeps its ambient
+    behavior.
+
+No-op when the relevant tracer isn't importable. Never raises -- observability
+must never break a business span.
+"""
+from __future__ import annotations
+
+from typing import Callable, Dict, Optional
+
+from agentex.lib.core.tracing.obs_ids import LGTM, get_obs_mode
+
+__all__ = ("ObsSpanHandle", "open_obs_span", "close_obs_span")
+
+# Instrumentation scope name so these wrapper spans are identifiable in Tempo/DD.
+_TRACER_NAME = "agentex.business"
+
+# Reverse-tag attribute keys: the business span/trace ids stamped onto the obs
+# span so you can pivot obs -> business (search these in Tempo/DD).
+_ATTR_BUSINESS_SPAN_ID = "agentex.business_span_id"
+_ATTR_BUSINESS_TRACE_ID = "agentex.business_trace_id"
+
+
+class ObsSpanHandle:
+    """Live handle for an open wrapper span: the correlation tag read from it
+    plus a backend-specific closer (detach/end or finish)."""
+
+    __slots__ = ("correlation", "_close")
+
+    def __init__(
+        self,
+        correlation: Dict[str, str],
+        close: Callable[[Optional[Dict[str, str]]], None],
+    ):
+        self.correlation = correlation
+        self._close = close
+
+
+def _hex_ids(trace_id: int, span_id: int) -> Dict[str, str]:
+    """W3C-hex form: 32-hex trace, 16-hex span."""
+    return {
+        "obs_trace_id": format(trace_id, "032x"),
+        "obs_span_id": format(span_id, "016x"),
+    }
+
+
+def _open_otel_span(
+    name: str,
+    business_span_id: Optional[str],
+    business_trace_id: Optional[str],
+) -> Optional[ObsSpanHandle]:
+    try:
+        from opentelemetry import context, trace
+    except ImportError:
+        return None
+    try:
+        span = trace.get_tracer(_TRACER_NAME).start_span(name)
+        if business_span_id:
+            span.set_attribute(_ATTR_BUSINESS_SPAN_ID, business_span_id)
+        if business_trace_id:
+            span.set_attribute(_ATTR_BUSINESS_TRACE_ID, business_trace_id)
+        token = context.attach(trace.set_span_in_context(span))
+        sc = span.get_span_context()
+        correlation = _hex_ids(sc.trace_id, sc.span_id) if (sc and sc.is_valid) else {}
+
+        def _close(error: Optional[Dict[str, str]] = None) -> None:
+            try:
+                if error:
+                    # Reflect the business-step failure on the obs span so it
+                    # isn't a false green when you pivot from a failed span.
+                    span.set_status(
+                        trace.Status(trace.StatusCode.ERROR, error.get("message"))
+                    )
+                    if error.get("type"):
+                        span.set_attribute("error.type", error["type"])
+            finally:
+                try:
+                    context.detach(token)
+                finally:
+                    span.end()
+
+        return ObsSpanHandle(correlation, _close)
+    except Exception:  # pragma: no cover - best-effort; never break the business span
+        return None
+
+
+def _open_ddtrace_span(
+    name: str,
+    business_span_id: Optional[str],
+    business_trace_id: Optional[str],
+) -> Optional[ObsSpanHandle]:
+    try:
+        from ddtrace.trace import tracer
+    except ImportError:
+        return None
+    try:
+        # Only wrap when ddtrace is actually tracing the request; otherwise a
+        # wrapper would be an orphan root trace in an un-instrumented process.
+        ctx = tracer.current_trace_context()
+        if ctx is None:
+            return None
+        # child_of=ctx is load-bearing: ddtrace's start_span does NOT auto-parent
+        # to the active span (unlike OTel), so start_span(name) alone mints a NEW
+        # root trace every call -- scattering a turn's business spans across N
+        # Datadog traces. Parenting to the active request/turn context rolls them
+        # into one trace while obs_span_id stays distinct per step.
+        span = tracer.start_span(name, child_of=ctx, activate=True)
+        if business_span_id:
+            span.set_tag(_ATTR_BUSINESS_SPAN_ID, business_span_id)
+        if business_trace_id:
+            span.set_tag(_ATTR_BUSINESS_TRACE_ID, business_trace_id)
+        correlation = _hex_ids(span.trace_id, span.span_id) if span.trace_id else {}
+
+        def _close(error: Optional[Dict[str, str]] = None) -> None:
+            try:
+                if error:
+                    # Reflect the business-step failure on the obs span.
+                    span.error = 1
+                    if error.get("type"):
+                        span.set_tag("error.type", error["type"])
+                    if error.get("message"):
+                        span.set_tag("error.message", error["message"])
+            finally:
+                span.finish()
+
+        return ObsSpanHandle(correlation, _close)
+    except Exception:  # pragma: no cover - best-effort
+        return None
+
+
+def open_obs_span(
+    name: str,
+    business_span_id: Optional[str] = None,
+    business_trace_id: Optional[str] = None,
+) -> Optional[ObsSpanHandle]:
+    """Open an obs span named ``name`` in the active backend, make it the active
+    span, and return a handle carrying its ``{"obs_trace_id","obs_span_id"}``.
+
+    ``business_span_id`` / ``business_trace_id`` are stamped onto the obs span as
+    the reverse tag (``agentex.business_span_id`` / ``agentex.business_trace_id``)
+    so you can pivot obs -> business by searching them in Tempo/DD.
+
+    Returns ``None`` (so the caller falls back to ambient behavior) when the
+    backend tracer isn't available or, in ``dd_only``, no request trace is
+    active.
+
+    Never raises: a top-level guard backstops anything the backend helpers
+    don't (e.g. a broken tracer install raising on import) so observability can
+    never fail an app call.
+    """
+    try:
+        if get_obs_mode() == LGTM:
+            return _open_otel_span(name, business_span_id, business_trace_id)
+        return _open_ddtrace_span(name, business_span_id, business_trace_id)
+    except Exception:  # pragma: no cover - backstop; obs must never break a call
+        return None
+
+
+def close_obs_span(
+    handle: Optional[ObsSpanHandle],
+    error: Optional[Dict[str, str]] = None,
+) -> None:
+    """Close the wrapper span (detach + end, or finish). When ``error`` is given
+    (the business span failed), mark the obs span errored first so it reflects
+    failure rather than a false green. Safe on ``None``."""
+    if handle is None:
+        return
+    try:
+        handle._close(error)
+    except Exception:  # pragma: no cover - best-effort
+        pass

src/agentex/lib/core/tracing/trace.py

@@ -12,7 +12,12 @@
 from agentex.lib.utils.logging import make_logger
 from agentex.lib.utils.model_utils import recursive_model_dump
 from agentex.lib.core.tracing.obs_ids import obs_correlation
-from agentex.lib.core.tracing.span_error import set_span_error
+from agentex.lib.core.tracing.obs_span import (
+    ObsSpanHandle,
+    close_obs_span,
+    open_obs_span,
+)
+from agentex.lib.core.tracing.span_error import get_span_error, set_span_error
 from agentex.lib.core.tracing.span_queue import (
     SpanEventType,
     AsyncSpanQueue,
@@ -49,6 +54,8 @@ def __init__(
         self.processors = processors
         self.client = client
         self.trace_id = trace_id
+        # Live per-business-span obs wrapper spans, keyed by business span id.
+        self._obs_handles: dict[str, ObsSpanHandle] = {}
 
     def start_span(
         self,
@@ -80,13 +87,19 @@ def start_span(
 
         serialized_input = recursive_model_dump(input) if input else None
         serialized_data = recursive_model_dump(data) if data else None
-        # Tag the business span with the active observability trace_id/span_id
-        # (OTel/ddtrace) so it can be correlated to the per-turn obs trace. The
-        # business trace_id stays the run-level task id -- see obs_ids.py.
-        obs = obs_correlation()
+        # Open a dedicated obs wrapper span named for this step and make it
+        # active, so obs_span_id is stable/meaningful (not an arbitrary innermost
+        # httpx span). It also carries the reverse tag (business span/trace id)
+        # so you can pivot obs -> business in Tempo/DD. Falls back to the ambient
+        # obs context (ddtrace) when not in lgtm mode. Business trace_id stays the
+        # run-level task id.
+        id = str(uuid.uuid4())
+        obs_handle = open_obs_span(
+            name, business_span_id=id, business_trace_id=self.trace_id
+        )
+        obs = obs_handle.correlation if obs_handle is not None else obs_correlation()
         if obs:
             serialized_data = {**(serialized_data or {}), **obs}
-        id = str(uuid.uuid4())
 
         span = Span(
             id=id,
@@ -98,6 +111,8 @@ def start_span(
             data=serialized_data,
             task_id=task_id,
         )
+        if obs_handle is not None:
+            self._obs_handles[span.id] = obs_handle
 
         for processor in self.processors:
             processor.on_span_start(span)
@@ -120,6 +135,10 @@ def end_span(
         if span.end_time is None:
             span.end_time = datetime.now(UTC)
 
+        # Close the dedicated obs wrapper span; propagate the business-span error
+        # (if any) so the obs span reflects failure, not a false green.
+        close_obs_span(self._obs_handles.pop(span.id, None), error=get_span_error(span))
+
         span.input = recursive_model_dump(span.input) if span.input else None
         span.output = recursive_model_dump(span.output) if span.output else None
         span.data = recursive_model_dump(span.data) if span.data else None
@@ -206,6 +225,8 @@ def __init__(
         self.client = client
         self.trace_id = trace_id
         self._span_queue = span_queue or get_default_span_queue()
+        # Live per-business-span obs wrapper spans, keyed by business span id.
+        self._obs_handles: dict[str, ObsSpanHandle] = {}
 
     async def start_span(
         self,
@@ -236,13 +257,19 @@ async def start_span(
 
         serialized_input = recursive_model_dump(input) if input else None
         serialized_data = recursive_model_dump(data) if data else None
-        # Tag the business span with the active observability trace_id/span_id
-        # (OTel/ddtrace) so it can be correlated to the per-turn obs trace. The
-        # business trace_id stays the run-level task id -- see obs_ids.py.
-        obs = obs_correlation()
+        # Open a dedicated obs wrapper span named for this step and make it
+        # active, so obs_span_id is stable/meaningful (not an arbitrary innermost
+        # httpx span). It also carries the reverse tag (business span/trace id)
+        # so you can pivot obs -> business in Tempo/DD. Falls back to the ambient
+        # obs context (ddtrace) when not in lgtm mode. Business trace_id stays the
+        # run-level task id.
+        id = str(uuid.uuid4())
+        obs_handle = open_obs_span(
+            name, business_span_id=id, business_trace_id=self.trace_id
+        )
+        obs = obs_handle.correlation if obs_handle is not None else obs_correlation()
         if obs:
             serialized_data = {**(serialized_data or {}), **obs}
-        id = str(uuid.uuid4())
 
         span = Span(
             id=id,
@@ -254,6 +281,8 @@ async def start_span(
             data=serialized_data,
             task_id=task_id,
         )
+        if obs_handle is not None:
+            self._obs_handles[span.id] = obs_handle
 
         if self.processors:
             self._span_queue.enqueue(SpanEventType.START, span.model_copy(deep=True), self.processors)
@@ -276,6 +305,10 @@ async def end_span(
         if span.end_time is None:
             span.end_time = datetime.now(UTC)
 
+        # Close the dedicated obs wrapper span; propagate the business-span error
+        # (if any) so the obs span reflects failure, not a false green.
+        close_obs_span(self._obs_handles.pop(span.id, None), error=get_span_error(span))
+
         span.input = recursive_model_dump(span.input) if span.input else None
         span.output = recursive_model_dump(span.output) if span.output else None
         span.data = recursive_model_dump(span.data) if span.data else None