From e570dfd2e19c22bccde2d5631bf4a2981c5db233 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:20:03 +0200 Subject: [PATCH 01/12] feat(boto3): improve boto3 integration - wrapper owns a single botocore client-call lifecycle. - introduce service-extension architecture. - add generic response, retry, and error attributes. - add common attributes that are the same across services. --- sentry_sdk/consts.py | 60 ++ sentry_sdk/integrations/boto3.py | 269 ------ sentry_sdk/integrations/boto3/__init__.py | 23 + sentry_sdk/integrations/boto3/_client.py | 124 +++ sentry_sdk/integrations/boto3/_context.py | 42 + .../integrations/boto3/_instrumentation.py | 516 ++++++++++ .../integrations/boto3/_services/__init__.py | 0 .../integrations/boto3/_services/base.py | 47 + .../integrations/boto3/_services/registry.py | 36 + tests/integrations/boto3/test_client.py | 880 ++++++++++++++++++ 10 files changed, 1728 insertions(+), 269 deletions(-) delete mode 100644 sentry_sdk/integrations/boto3.py create mode 100644 sentry_sdk/integrations/boto3/__init__.py create mode 100644 sentry_sdk/integrations/boto3/_client.py create mode 100644 sentry_sdk/integrations/boto3/_context.py create mode 100644 sentry_sdk/integrations/boto3/_instrumentation.py create mode 100644 sentry_sdk/integrations/boto3/_services/__init__.py create mode 100644 sentry_sdk/integrations/boto3/_services/base.py create mode 100644 sentry_sdk/integrations/boto3/_services/registry.py create mode 100644 tests/integrations/boto3/test_client.py diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index b1f4a0c5f6..bd5b3cce7b 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -389,6 +389,17 @@ class SPANDATA: Warning messages generated during model execution. Example: ["Token limit exceeded"] """ + AWS_EXTENDED_REQUEST_ID = "aws.extended_request_id" + """ + The AWS extended request ID as returned in the response headers. + Example: "wzHcyEWfmOGDIE5QOhTAqFDoDWP3y8IUvpNINCwL9N4TEHbUw0/gZJ+VZTmCNCWR7fezEN3eCiQ=" + """ + + AWS_REQUEST_ID = "aws.request_id" + """ + The AWS request ID as returned in the response headers. + Example: "79b9da39-b7ae-508a-a6bc-864b2829c622" + """ CACHE_HIT = "cache.hit" """ @@ -414,6 +425,12 @@ class SPANDATA: Example: "10.1.2.80" """ + CLOUD_REGION = "cloud.region" + """ + The geographical region the resource is running. + Example: "us-east-1" + """ + CODE_FILEPATH = "code.filepath" """ .. deprecated:: @@ -541,6 +558,12 @@ class SPANDATA: Example: my_user """ + ERROR_TYPE = "error.type" + """ + Describes a class of error the operation ended with. + Example: "timeout" + """ + GEN_AI_AGENT_NAME = "gen_ai.agent.name" """ The name of the agent being used. @@ -880,6 +903,19 @@ class SPANDATA: Example: GET """ + HTTP_REQUEST_RESEND_COUNT = "http.request.resend_count" + """ + The ordinal number of request resending attempt (for any reason, including redirects). + + Example: 2 + """ + + HTTP_RESPONSE_BODY_SIZE = "http.response.body.size" + """ + The encoded body size of the response (in bytes). + Example: 123 + """ + HTTP_ROUTE = "http.route" """ The matched route, that is, the path template used to match the request. @@ -977,12 +1013,24 @@ class SPANDATA: Example: "com.example.ExampleService/exampleMethod" """ + RPC_SERVICE = "rpc.service" + """ + The full (logical) name of the service being called, including its package name, if applicable. + Example: "myService.BestService" + """ + RPC_RESPONSE_STATUS_CODE = "rpc.response.status_code" """ Status code of the RPC returned by the RPC server or generated by the client. Example: "DEADLINE_EXCEEDED" """ + RPC_SYSTEM_NAME = "rpc.system.name" + """ + A string identifying the remoting system. + Example: "aws-api" + """ + SERVER_ADDRESS = "server.address" """ Name of the database host. @@ -1164,6 +1212,18 @@ class SPANDATA: Example: "prod" """ + SENTRY_OP = "sentry.op" + """ + The operation of a span. + Example: "http.client" + """ + + SENTRY_ORIGIN = "sentry.origin" + """ + The origin of the instrumentation (e.g. span, log, etc.) + Example: "auto.http.otel.fastify" + """ + SENTRY_RELEASE = "sentry.release" """ The Sentry release. diff --git a/sentry_sdk/integrations/boto3.py b/sentry_sdk/integrations/boto3.py deleted file mode 100644 index 18d7accf6d..0000000000 --- a/sentry_sdk/integrations/boto3.py +++ /dev/null @@ -1,269 +0,0 @@ -from functools import partial -from typing import TYPE_CHECKING - -import sentry_sdk -from sentry_sdk.consts import OP, SPANDATA -from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version -from sentry_sdk.traces import StreamedSpan -from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span -from sentry_sdk.tracing_utils import ( - add_http_breadcrumb, - add_sentry_baggage_to_headers, - get_url_attributes, - has_span_streaming_enabled, - should_propagate_trace, -) -from sentry_sdk.utils import ( - capture_internal_exceptions, - parse_url, - parse_version, -) - -if TYPE_CHECKING: - from typing import Any, Dict, Optional, Type, Union - - from botocore.model import ServiceId - - -try: - from botocore import __version__ as BOTOCORE_VERSION - from botocore.awsrequest import AWSRequest - from botocore.client import BaseClient - from botocore.response import StreamingBody -except ImportError: - raise DidNotEnable("botocore is not installed") - - -class Boto3Integration(Integration): - identifier = "boto3" - origin = f"auto.http.{identifier}" - - @staticmethod - def setup_once() -> None: - version = parse_version(BOTOCORE_VERSION) - _check_minimum_version(Boto3Integration, version, "botocore") - - orig_init = BaseClient.__init__ - - def sentry_patched_init( - self: "BaseClient", *args: "Any", **kwargs: "Any" - ) -> None: - orig_init(self, *args, **kwargs) - meta = self.meta - service_id = meta.service_model.service_id - meta.events.register( - "request-created", - partial(_sentry_request_created, service_id=service_id), - ) - # run after other `before-sign` handlers, allowing it to see and preserve existing baggage. - meta.events.register_last("before-sign", _sentry_before_sign) - meta.events.register("after-call", _sentry_after_call) - meta.events.register("after-call-error", _sentry_after_call_error) - - BaseClient.__init__ = sentry_patched_init # type: ignore - - -def _sentry_request_created( - service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any" -) -> None: - description = "aws.%s.%s" % (service_id.hyphenize(), operation_name) - - client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: - return - - parsed_url = None - if request.url is not None: - with capture_internal_exceptions(): - parsed_url = parse_url(request.url, sanitize=False) - - breadcrumb: "dict[str, Any]" = {} - - is_span_streaming_enabled = has_span_streaming_enabled(client.options) - span: "Union[Span, StreamedSpan, None]" = None - if is_span_streaming_enabled: - url_attributes = get_url_attributes(client, parsed_url) - breadcrumb.update(url_attributes) - - if request.method is not None: - breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = request.method - - if sentry_sdk.traces.get_current_span() is not None: - span = sentry_sdk.traces.start_span( - name=description, - attributes={ - "sentry.op": OP.HTTP_CLIENT, - "sentry.origin": Boto3Integration.origin, - SPANDATA.RPC_METHOD: f"{service_id}/{operation_name}", - }, - ) - span.set_attributes(url_attributes) - - if request.method is not None: - span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) - else: - span = sentry_sdk.start_span( - op=OP.HTTP_CLIENT, - name=description, - origin=Boto3Integration.origin, - ) - - if parsed_url: - span.set_data("aws.request.url", parsed_url.url) - span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) - span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) - breadcrumb.update( - { - "aws.request.url": parsed_url.url, - SPANDATA.HTTP_QUERY: parsed_url.query, - SPANDATA.HTTP_FRAGMENT: parsed_url.fragment, - } - ) - - span.set_tag("aws.service_id", service_id.hyphenize()) - span.set_tag("aws.operation_name", operation_name) - if request.method is not None: - span.set_data(SPANDATA.HTTP_METHOD, request.method) - breadcrumb[SPANDATA.HTTP_METHOD] = request.method - - # We do it in order for subsequent http calls/retries be - # attached to this span. - span.__enter__() - - add_http_breadcrumb(None, breadcrumb) - - if span is not None: - # request.context is an open-ended data-structure - # where we can add anything useful in request life cycle. - request.context["_sentrysdk_span"] = span - - -def _sentry_before_sign( - request: "AWSRequest", signature_version: "Any", **kwargs: "Any" -) -> None: - client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: - return - - with capture_internal_exceptions(): - # presigned requests are executed later by another caller. Adding propagation - # headers here would make those headers part of the signature, requiring the caller to reproduce the same values. - if isinstance(signature_version, str) and signature_version.endswith( - ("-query", "-presign-post") - ): - return - - if request.url is None or not should_propagate_trace(client, request.url): - return - - def _replace_header(request: "AWSRequest", key: str, value: str) -> None: - """ - Botocore's `HTTPHeaders` inherits from `email.message.Message`, where: - headers["foo"] = "old" - headers["foo"] = "new" - produces two fields: {"foo": "old", "foo": "new"}. So delete existing - fields before assigning replacement. - """ - if key in request.headers: - del request.headers[key] - request.headers[key] = value - - # use span associated with this botocore request - span = request.context.get("_sentrysdk_span") - - headers = sentry_sdk.get_current_scope().iter_trace_propagation_headers( - span=span - ) - for header_name, header_value in headers: - if header_name != BAGGAGE_HEADER_NAME: - # normal headers (e.g. `sentry-trace`) are non-shared, so replace stale values - _replace_header(request, header_name, header_value) - continue - - # merge existing `baggage` values under single header - existing_values = request.headers.get_all(BAGGAGE_HEADER_NAME, []) - combined_baggage = { - BAGGAGE_HEADER_NAME: ",".join(str(value) for value in existing_values) - } - # preserve third-party baggage, replace stale `sentry-*` values - add_sentry_baggage_to_headers(combined_baggage, header_value) - _replace_header( - request, BAGGAGE_HEADER_NAME, combined_baggage[BAGGAGE_HEADER_NAME] - ) - - -def _sentry_after_call( - context: "Dict[str, Any]", parsed: "Dict[str, Any]", **kwargs: "Any" -) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) - - # Span could be absent if the integration is disabled. - if span is None: - return - - span.__exit__(None, None, None) - - body = parsed.get("Body") - if not isinstance(body, StreamingBody): - return - - streaming_span: "Union[Span, StreamedSpan]" - if isinstance(span, StreamedSpan): - streaming_span = sentry_sdk.traces.start_span( - name=span.name, - parent_span=span, - attributes={ - "sentry.op": OP.HTTP_CLIENT_STREAM, - "sentry.origin": Boto3Integration.origin, - }, - ) - else: - streaming_span = span.start_child( - op=OP.HTTP_CLIENT_STREAM, - name=span.description, - origin=Boto3Integration.origin, - ) - - orig_read = body.read - orig_close = body.close - - def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: - try: - ret = orig_read(*args, **kwargs) - if ret: - return ret - - if isinstance(streaming_span, StreamedSpan): - streaming_span.end() - else: - streaming_span.finish() - return ret - except Exception: - if isinstance(streaming_span, StreamedSpan): - streaming_span.end() - else: - streaming_span.finish() - raise - - body.read = sentry_streaming_body_read # type: ignore - - def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: - if isinstance(streaming_span, StreamedSpan): - streaming_span.end() - else: - streaming_span.finish() - orig_close(*args, **kwargs) - - body.close = sentry_streaming_body_close # type: ignore - - -def _sentry_after_call_error( - context: "Dict[str, Any]", exception: "Type[BaseException]", **kwargs: "Any" -) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) - - # Span could be absent if the integration is disabled. - if span is None: - return - - span.__exit__(type(exception), exception, None) diff --git a/sentry_sdk/integrations/boto3/__init__.py b/sentry_sdk/integrations/boto3/__init__.py new file mode 100644 index 0000000000..2e6e8712a4 --- /dev/null +++ b/sentry_sdk/integrations/boto3/__init__.py @@ -0,0 +1,23 @@ +from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version +from sentry_sdk.utils import parse_version + +try: + from botocore import __version__ as BOTOCORE_VERSION +except ImportError: + raise DidNotEnable("botocore is not installed") + +_SPAN_ORIGIN = "auto.http.boto3" + + +class Boto3Integration(Integration): + origin = _SPAN_ORIGIN + + @staticmethod + def setup_once() -> None: + version = parse_version(BOTOCORE_VERSION) + _check_minimum_version(Boto3Integration, version, "botocore") + + # local import to avoid import cycle + from sentry_sdk.integrations.boto3._client import _patch_botocore_client + + _patch_botocore_client() diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py new file mode 100644 index 0000000000..aa91439764 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_client.py @@ -0,0 +1,124 @@ +from typing import TYPE_CHECKING + +from botocore.client import BaseClient + +import sentry_sdk +from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3._context import AwsCallContext +from sentry_sdk.integrations.boto3._instrumentation import ( + _finish_client_span, + _finish_client_span_with_error, + _sentry_before_sign, + _sentry_request_created, + _start_client_span, +) +from sentry_sdk.integrations.boto3._services.registry import ( + _resolve_service_extension, +) +from sentry_sdk.traces import NoOpStreamedSpan +from sentry_sdk.tracing import NoOpSpan +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Any, Optional, Union + + from sentry_sdk.integrations.boto3._services.base import _ServiceExtension + from sentry_sdk.traces import StreamedSpan + from sentry_sdk.tracing import Span + + +def _patch_botocore_client() -> None: + orig_init = BaseClient.__init__ + orig_make_api_call = BaseClient._make_api_call # type: ignore + + def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None: + orig_init(self, *args, **kwargs) + meta = self.meta + meta.events.register("request-created", _sentry_request_created) + # run after other `before-sign` handlers so existing baggage is preserved. + meta.events.register_last("before-sign", _sentry_before_sign) + + def sentry_patched_make_api_call( + self: "BaseClient", operation_name: str, api_params: "Any" + ) -> "Any": + """ + Own the span lifecycle for one `_make_api_call()` invocation, including + all retries performed by botocore. + + Botocore's ``after-call-error`` event only surrounds ``_make_request``. + Wrapping ``_make_api_call`` also closes the span when parameter building, + serialization, or endpoint resolution fails before the request starts: + https://github.com/boto/botocore/blob/develop/botocore/client.py + + https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span + """ + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return orig_make_api_call(self, operation_name, api_params) + + ctx: "Optional[AwsCallContext]" = None + service_extension: "Optional[_ServiceExtension]" = None + span: "Optional[Union[Span, StreamedSpan]]" = None + + with capture_internal_exceptions(): + ctx = AwsCallContext(self, operation_name, api_params) + + if ctx is not None: + # The resolver contains its own fail-open import boundary. + service_extension = _resolve_service_extension(ctx.service_name) + + with capture_internal_exceptions(): + span = _start_client_span(ctx, service_extension) + if span is not None: + span.__enter__() + + instrumented_api_params = api_params + if ( + ctx is not None + and service_extension is not None + and span is not None + and not isinstance(span, (NoOpSpan, NoOpStreamedSpan)) + and client.options.get("propagate_traces") + and isinstance(api_params, dict) + ): + with capture_internal_exceptions(): + # propagation must use current scope + headers = dict( + sentry_sdk.get_current_scope().iter_trace_propagation_headers( + span=span + ) + ) + propagated_params = service_extension.inject_trace_context( + ctx, + headers, + ) + # Only pass a service extension's replacement when it is a mapping. + # Otherwise preserve the caller's parameters and Botocore behavior. + if isinstance(propagated_params, dict): + instrumented_api_params = propagated_params + + try: + parsed = orig_make_api_call(self, operation_name, instrumented_api_params) + except BaseException as exc: + if span is not None: + with capture_internal_exceptions(): + _finish_client_span_with_error( + span, + exc, + ctx, + service_extension, + ) + raise + + if span is not None: + with capture_internal_exceptions(): + _finish_client_span( + span, + parsed, + ctx, + service_extension, + ) + return parsed + + BaseClient.__init__ = sentry_patched_init # type: ignore + BaseClient._make_api_call = sentry_patched_make_api_call # type: ignore diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py new file mode 100644 index 0000000000..6282ebcbb0 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_context.py @@ -0,0 +1,42 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Dict, Optional + + from botocore.client import BaseClient + + +class AwsCallContext: + __slots__ = ( + "client", + "service_name", + "service_id", + "service_id_hyphenized", + "operation_name", + "region_name", + "endpoint_url", + "api_version", + "params", + ) + + def __init__( + self, + client: "BaseClient", + operation_name: str, + params: "Any", + ) -> None: + client_meta = client.meta + service_model = client_meta.service_model + service_id = service_model.service_id + + self.client: "BaseClient" = client + # botocore's internal identifier, e.g. `apigateway`. + self.service_name: str = service_model.service_name + # modeled AWS service identity used in span names, e.g. `API Gateway`. + self.service_id: str = str(service_id) + self.service_id_hyphenized: str = service_id.hyphenize() + self.operation_name: str = operation_name + self.region_name: "Optional[str]" = getattr(client_meta, "region_name", None) + self.endpoint_url: "Optional[str]" = getattr(client_meta, "endpoint_url", None) + self.api_version: str = service_model.api_version + self.params: "Dict[str, Any]" = dict(params) if isinstance(params, dict) else {} diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py new file mode 100644 index 0000000000..743a902907 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -0,0 +1,516 @@ +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +from botocore.awsrequest import AWSRequest +from botocore.exceptions import ClientError +from botocore.response import StreamingBody + +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS +from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.traces import StreamedSpan +from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span +from sentry_sdk.tracing_utils import ( + add_http_breadcrumb, + add_sentry_baggage_to_headers, + get_url_attributes, + has_span_streaming_enabled, + should_propagate_trace, +) +from sentry_sdk.utils import ( + capture_internal_exceptions, + parse_url, +) + +if TYPE_CHECKING: + from typing import Any, Dict, Optional, Union + + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext + from sentry_sdk.integrations.boto3._services.base import _ServiceExtension + +_AWS_RPC_SYSTEM_NAME = "aws-api" + + +def _set_span_attributes( + span: "Union[Span, StreamedSpan]", attributes: "Attributes" +) -> None: + # streamed and legacy spans expose different attribute APIs. + if isinstance(span, StreamedSpan): + span.set_attributes(attributes) + return + + for key, value in attributes.items(): + span.set_data(key, value) + + +def _get_server_attributes(endpoint_url: "Optional[str]") -> "Attributes": + if not endpoint_url: + return {} + + default_ports = { + "http": 80, + "https": 443, + } + + try: + parsed_url = urlsplit(endpoint_url) + if parsed_url.scheme not in default_ports or not parsed_url.hostname: + return {} + + # `server.port` is only defined together with `server.address`. Infer the + # effective port when the configured HTTP(S) endpoint omits it. + # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ + return { + SPANDATA.SERVER_ADDRESS: parsed_url.hostname, + SPANDATA.SERVER_PORT: parsed_url.port or default_ports[parsed_url.scheme], + } + + except (TypeError, UnicodeError, ValueError): + # Invalid client metadata must not prevent the AWS call from running. + return {} + + +def _get_client_attributes( + ctx: "AwsCallContext", +) -> "Attributes": + # The AWS SDK conventions define `rpc.service` as the modeled AWS service ID + # and `rpc.method` as the modeled operation name. Although the general RPC + # conventions now deprecate `rpc.service`, the AWS-specific convention still + # recommends both attributes and defines the span name as `Service.Operation`. + # https://opentelemetry.io/docs/specs/semconv/cloud-providers/aws-sdk/#aws-sdk-spans + attributes: "Attributes" = { + SPANDATA.RPC_METHOD: ctx.operation_name, + SPANDATA.RPC_SERVICE: ctx.service_id, + SPANDATA.RPC_SYSTEM_NAME: _AWS_RPC_SYSTEM_NAME, + } + + if ctx.region_name: + attributes[SPANDATA.CLOUD_REGION] = ctx.region_name + + attributes.update(_get_server_attributes(ctx.endpoint_url)) + return attributes + + +def _merge_service_attributes( + attributes: "Attributes", + service_attributes: "Any", +) -> None: + if not isinstance(service_attributes, dict): + return + + for key, value in service_attributes.items(): + # Generic attributes are added first and remain authoritative. A service + # extension may only fill attributes that generic instrumentation did + # not already produce. + attributes.setdefault(key, value) + + +def _get_response_attributes(response: "Any") -> "Attributes": + if not isinstance(response, dict): + return {} + + metadata = response.get("ResponseMetadata") + if not isinstance(metadata, dict): + return {} + + attributes: "Attributes" = {} + + status_code = metadata.get("HTTPStatusCode") + # botocore injects HTTP status into `ResponseMetadata` after parsing. + # https://github.com/boto/botocore/blob/develop/botocore/parsers.py#L273-L284 + if ( + isinstance(status_code, int) + and not isinstance(status_code, bool) + and 100 <= status_code <= 599 + ): + attributes[SPANDATA.HTTP_STATUS_CODE] = status_code + + retry_attempts = metadata.get("RetryAttempts") + # botocore represents retries as `attempts - 1`; omit zero. + # https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L221-L229 + # https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span + if ( + isinstance(retry_attempts, int) + and not isinstance(retry_attempts, bool) + and retry_attempts > 0 + ): + attributes["http.request.resend_count"] = retry_attempts + + headers = metadata.get("HTTPHeaders") + if not isinstance(headers, dict): + headers = {} + + request_id = metadata.get("RequestId") + if not isinstance(request_id, str) or not request_id: + request_id = next( + ( + value + for value in ( + headers.get("x-amzn-requestid"), + headers.get("x-amzn-request-id"), + headers.get("x-amz-request-id"), + ) + if isinstance(value, str) and value + ), + None, + ) + if isinstance(request_id, str) and request_id: + attributes[SPANDATA.AWS_REQUEST_ID] = request_id + + # S3's `HostId` is the extended request ID returned in `x-amz-id-2`. + # https://docs.aws.amazon.com/AmazonS3/latest/developerguide/get-request-ids.html + extended_request_id = metadata.get("HostId") + if not isinstance(extended_request_id, str) or not extended_request_id: + extended_request_id = headers.get("x-amz-id-2") + if isinstance(extended_request_id, str) and extended_request_id: + attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] = extended_request_id + + return attributes + + +def _get_error_type(exception: "BaseException") -> str: + if isinstance(exception, ClientError): + # botocore wraps all AWS service errors in `ClientError`; `Error.Code` + # identifies actual service-specific error, e.g. `AccessDenied`. + # https://docs.aws.amazon.com/boto3/latest/guide/error-handling.html + error = exception.response.get("Error") + if isinstance(error, dict): + error_code = error.get("Code") + if isinstance(error_code, str) and error_code: + return error_code + + # failures before a service response, have no error code. Use exception type + # instead. https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ + exception_type = type(exception) + exception_name = exception_type.__qualname__ + exception_module = exception_type.__module__ + if exception_module not in ("builtins", "__builtins__"): + return "%s.%s" % (exception_module, exception_name) + return exception_name + + +def _get_error_attributes(exception: "BaseException") -> "Attributes": + attributes: "Attributes" = {} + if isinstance(exception, ClientError): + attributes.update(_get_response_attributes(exception.response)) + + attributes[SPANDATA.ERROR_TYPE] = _get_error_type(exception) + return attributes + + +def _start_client_span( + ctx: "AwsCallContext", + service_extension: "Optional[_ServiceExtension]" = None, +) -> "Optional[Union[Span, StreamedSpan]]": + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return None + + # AWS client spans use `Service.Operation`, e.g. `DynamoDB.GetItem`. + # https://opentelemetry.io/docs/specs/semconv/cloud-providers/aws-sdk/#aws-sdk-spans + span_name = "%s.%s" % (ctx.service_id, ctx.operation_name) + attributes = _get_client_attributes(ctx) + span_op = OP.HTTP_CLIENT + span_origin = Boto3Integration.origin + + # enrich with service-specific attributes + if service_extension is not None: + service_span_config = None + with capture_internal_exceptions(): + service_span_config = service_extension.get_span_config(ctx) + + with capture_internal_exceptions(): + if service_span_config is not None: + service_op, service_origin = service_span_config + if service_op and isinstance(service_op, str): + span_op = service_op + if service_origin and isinstance(service_origin, str): + span_origin = service_origin + + # Request enrichment is independent from the span configuration. This + # lets HTTP-based services such as S3 return only their attribute delta. + with capture_internal_exceptions(): + _merge_service_attributes( + attributes, + service_extension.get_request_attributes(ctx), + ) + + if has_span_streaming_enabled(client.options): + if sentry_sdk.traces.get_current_span() is None: + return None + + # `start_span()` evaluates `ignore_spans` against the initial attributes. + # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span + attributes.update( + { + SPANDATA.SENTRY_OP: span_op, + SPANDATA.SENTRY_ORIGIN: span_origin, + } + ) + return sentry_sdk.traces.start_span( + name=span_name, + attributes=attributes, + ) + + span = sentry_sdk.start_span( + name=span_name, + op=span_op, + origin=span_origin, + ) + _set_span_attributes(span, attributes) + span.set_tag("aws.service_id", ctx.service_id_hyphenized) + span.set_tag("aws.operation_name", ctx.operation_name) + return span + + +def _finish_client_span( + span: "Union[Span, StreamedSpan]", + parsed: "Dict[str, Any]", + ctx: "Optional[AwsCallContext]" = None, + service_extension: "Optional[_ServiceExtension]" = None, +) -> None: + # response metadata is only available after the call. Keep enrichment + # isolated so failure cannot prevent `__exit__()` below. + attributes: "Attributes" = {} + with capture_internal_exceptions(): + attributes = _get_response_attributes(parsed) + + if ctx is not None and service_extension is not None: + with capture_internal_exceptions(): + _merge_service_attributes( + attributes, + service_extension.get_response_attributes(ctx, parsed), + ) + + with capture_internal_exceptions(): + _set_span_attributes(span, attributes) + span.__exit__(None, None, None) + + body = parsed.get("Body") + if not isinstance(body, StreamingBody): + return + + streaming_span: "Union[Span, StreamedSpan]" + if isinstance(span, StreamedSpan): + streaming_span = sentry_sdk.traces.start_span( + name=span.name, + parent_span=span, + attributes={ + SPANDATA.SENTRY_OP: OP.HTTP_CLIENT_STREAM, + SPANDATA.SENTRY_ORIGIN: Boto3Integration.origin, + }, + ) + else: + streaming_span = span.start_child( + op=OP.HTTP_CLIENT_STREAM, + name=span.description, + origin=Boto3Integration.origin, + ) + + orig_read = body.read + orig_close = body.close + + def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: + try: + ret = orig_read(*args, **kwargs) + if ret: + return ret + + if isinstance(streaming_span, StreamedSpan): + streaming_span.end() + else: + streaming_span.finish() + return ret + except Exception as exc: + # enrichment must not replace exception raised by `orig_read()`. + # finish span with error, then re-raise. + with capture_internal_exceptions(): + _set_span_attributes(streaming_span, _get_error_attributes(exc)) + + with capture_internal_exceptions(): + if isinstance(streaming_span, StreamedSpan): + streaming_span.__exit__(type(exc), exc, exc.__traceback__) + else: + streaming_span.set_status(SPANSTATUS.INTERNAL_ERROR) + streaming_span.finish() + raise + + body.read = sentry_streaming_body_read # type: ignore + + def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: + if isinstance(streaming_span, StreamedSpan): + streaming_span.end() + else: + streaming_span.finish() + orig_close(*args, **kwargs) + + body.close = sentry_streaming_body_close # type: ignore + + +def _finish_client_span_with_error( + span: "Union[Span, StreamedSpan]", + exception: "BaseException", + ctx: "Optional[AwsCallContext]" = None, + service_extension: "Optional[_ServiceExtension]" = None, +) -> None: + attributes: "Attributes" = {} + with capture_internal_exceptions(): + attributes = _get_error_attributes(exception) + + # ClientError.response is the parsed AWS error response, so the same response + # hook can enrich successful and failed service responses. A separate error + # hook is unnecessary until a service needs exception-only information. + # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/error-handling.html#catching-botocore-exceptions + if ( + ctx is not None + and service_extension is not None + and isinstance(exception, ClientError) + ): + with capture_internal_exceptions(): + _merge_service_attributes( + attributes, + service_extension.get_response_attributes(ctx, exception.response), + ) + + with capture_internal_exceptions(): + _set_span_attributes(span, attributes) + span.__exit__(type(exception), exception, exception.__traceback__) + + +def _set_request_attributes( + span: "Union[Span, StreamedSpan]", + request: "AWSRequest", +) -> None: + client = sentry_sdk.get_client() + + parsed_url = None + if request.url is not None: + with capture_internal_exceptions(): + parsed_url = parse_url(request.url, sanitize=False) + + if isinstance(span, StreamedSpan): + span.set_attributes(get_url_attributes(client, parsed_url)) + + if request.method is not None: + span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) + + return + + if parsed_url is not None: + span.set_data("aws.request.url", parsed_url.url) + span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) + span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) + + if request.method is not None: + span.set_data(SPANDATA.HTTP_METHOD, request.method) + + +def _add_request_breadcrumb(request: "AWSRequest") -> None: + client = sentry_sdk.get_client() + + parsed_url = None + if request.url is not None: + with capture_internal_exceptions(): + parsed_url = parse_url(request.url, sanitize=False) + + breadcrumb: "dict[str, Any]" = {} + + if has_span_streaming_enabled(client.options): + breadcrumb.update(get_url_attributes(client, parsed_url)) + if request.method is not None: + breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = request.method + else: + if parsed_url is not None: + breadcrumb.update( + { + "aws.request.url": parsed_url.url, + SPANDATA.HTTP_QUERY: parsed_url.query, + SPANDATA.HTTP_FRAGMENT: parsed_url.fragment, + } + ) + + if request.method is not None: + breadcrumb[SPANDATA.HTTP_METHOD] = request.method + + add_http_breadcrumb(None, breadcrumb) + + +def _sentry_request_created( + request: "AWSRequest", operation_name: str, **kwargs: "Any" +) -> None: + """ + Enrich a single `AWSRequest` attempt. Botocore creates a + fresh `AWSRequest` on every retry. + https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 + """ + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return + + with capture_internal_exceptions(): + _add_request_breadcrumb(request) + + if has_span_streaming_enabled(client.options): + span = sentry_sdk.traces.get_current_span() + else: + span = sentry_sdk.get_current_span() + if span is None: + return + + _set_request_attributes(span, request) + # each attempt has a fresh `request.context`; carry the active client span. + request.context["_sentrysdk_span"] = span + + +def _sentry_before_sign( + request: "AWSRequest", signature_version: "Any", **kwargs: "Any" +) -> None: + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return + + with capture_internal_exceptions(): + # presigned requests are executed later by another caller. Adding propagation + # headers here would make those headers part of the signature, requiring the caller to reproduce the same values. + if isinstance(signature_version, str) and signature_version.endswith( + ("-query", "-presign-post") + ): + return + + if request.url is None or not should_propagate_trace(client, request.url): + return + + def _replace_header(request: "AWSRequest", key: str, value: str) -> None: + """ + Botocore's `HTTPHeaders` inherits from `email.message.Message`, where: + headers["foo"] = "old" + headers["foo"] = "new" + produces two fields: {"foo": "old", "foo": "new"}. So delete existing + fields before assigning replacement. + """ + if key in request.headers: + del request.headers[key] + request.headers[key] = value + + # use span associated with this botocore request + span = request.context.get("_sentrysdk_span") + + headers = sentry_sdk.get_current_scope().iter_trace_propagation_headers( + span=span + ) + for header_name, header_value in headers: + if header_name != BAGGAGE_HEADER_NAME: + # normal headers (e.g. `sentry-trace`) are non-shared, so replace stale values + _replace_header(request, header_name, header_value) + continue + + # merge existing `baggage` values under single header + existing_values = request.headers.get_all(BAGGAGE_HEADER_NAME, []) + combined_baggage = { + BAGGAGE_HEADER_NAME: ",".join(str(value) for value in existing_values) + } + add_sentry_baggage_to_headers(combined_baggage, header_value) + _replace_header( + request, BAGGAGE_HEADER_NAME, combined_baggage[BAGGAGE_HEADER_NAME] + ) diff --git a/sentry_sdk/integrations/boto3/_services/__init__.py b/sentry_sdk/integrations/boto3/_services/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py new file mode 100644 index 0000000000..ba30905509 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -0,0 +1,47 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Dict, Optional, Tuple + + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext + + +class _ServiceExtension: + """ + Specialize generic botocore client-call instrumentation for a specific + AWS service, e.g. SQS. + + ``_resolve_service_extension()`` caches and shares instances across calls, + so extensions must remain stateless and receive the per-call context + explicitly. Boto3 clients are generally thread-safe and may therefore use + the same extension concurrently: + https://docs.aws.amazon.com/boto3/latest/guide/clients.html#multithreading-or-multiprocessing-with-clients + """ + + # avoid arbitrary attributes by disabling per-instance `__dict__`. + __slots__ = () + + def get_span_config( + self, ctx: "AwsCallContext" + ) -> "Optional[Tuple[Optional[str], Optional[str]]]": + """Return an optional `(op, origin)` override; None defaults to `(HTTP_CLIENT, Boto3Integration.origin)`.""" + return None + + def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": + """Return service-specific attributes derived before the call.""" + return {} + + def get_response_attributes( + self, ctx: "AwsCallContext", response: "Any" + ) -> "Attributes": + """Return service-specific attributes derived from an AWS response.""" + return {} + + def inject_trace_context( + self, + ctx: "AwsCallContext", + trace_context: "Dict[str, str]", + ) -> "Optional[Dict[str, Any]]": + """Put the trace context somewhere the consumer of this call can find it.""" + return None diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py new file mode 100644 index 0000000000..d7e41e226e --- /dev/null +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -0,0 +1,36 @@ +from functools import lru_cache +from importlib import import_module +from typing import TYPE_CHECKING + +from sentry_sdk.integrations.boto3._services.base import _ServiceExtension +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Dict, Optional, Tuple + + +# Store import targets instead of importing service modules eagerly. This keeps +# a broken service extension from disabling generic AWS instrumentation. +_SERVICE_EXTENSIONS: "Dict[str, Tuple[str, str]]" = { +} + + +@lru_cache(maxsize=None) +def _resolve_service_extension( + service_name: str, +) -> "Optional[_ServiceExtension]": + """Resolve a shared extension for a botocore service name.""" + target = _SERVICE_EXTENSIONS.get(service_name) + if target is None: + return None + + extension = None + with capture_internal_exceptions(): + module_name, class_name = target + extension_class = getattr(import_module(module_name), class_name) + candidate = extension_class() + if isinstance(candidate, _ServiceExtension): + extension = candidate + + # An unknown or broken service extension falls back to generic instrumentation. + return extension diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py new file mode 100644 index 0000000000..9310e12ecc --- /dev/null +++ b/tests/integrations/boto3/test_client.py @@ -0,0 +1,880 @@ +import boto3 +import pytest +from botocore.awsrequest import AWSResponse +from botocore.config import Config +from botocore.exceptions import ClientError, EndpointConnectionError +from botocore.stub import Stubber + +import sentry_sdk +from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3._instrumentation import ( + _get_error_attributes, + _get_response_attributes, + _get_server_attributes, + _merge_service_attributes, +) +from sentry_sdk.integrations.boto3._services.base import _ServiceExtension +from sentry_sdk.integrations.boto3._services.registry import ( + _SERVICE_EXTENSIONS, + _resolve_service_extension, +) +from tests.integrations.boto3.aws_mock import Body + +session = boto3.Session( # type: ignore[attr-defined] + aws_access_key_id="-", + aws_secret_access_key="-", + region_name="eu-north-1", +) + + +@pytest.mark.parametrize("service_name", sorted(_SERVICE_EXTENSIONS)) +def test_registered_service_extensions_load(service_name): + assert isinstance(_resolve_service_extension(service_name), _ServiceExtension) + + +def test_generic_attributes_take_precedence_over_service_attributes(): + attributes = {SPANDATA.RPC_METHOD: "GetItem"} + + _merge_service_attributes( + attributes, + { + SPANDATA.RPC_METHOD: "overridden", + SPANDATA.DB_SYSTEM_NAME: "aws.dynamodb", + }, + ) + + assert attributes == { + SPANDATA.RPC_METHOD: "GetItem", + SPANDATA.DB_SYSTEM_NAME: "aws.dynamodb", + } + + +def test_public_api(): + assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" + assert Boto3Integration.identifier == "boto3" + + +@pytest.fixture +def client_factory(sentry_init, monkeypatch, span_streaming): + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + trace_lifecycle="stream" if span_streaming else "static", + # avoid SDK's machine hostname being used as server name. + server_name="", + ) + # remove retry delay to speed up tests + monkeypatch.setattr("botocore.endpoint.time.sleep", lambda delay: None) + + def make_client(service_name="s3", attempt_count=1, **client_kwargs): + return session.client( + service_name, + config=Config( + # `total_max_attempts` includes the initial request. + retries={"total_max_attempts": attempt_count, "mode": "standard"} + ), + **client_kwargs, + ) + + return make_client + + +def _mock_responses(client, status_codes): + request_span_ids = [] + + def record_request(request, **kwargs): + span = request.context.get("_sentrysdk_span") + assert span is not None + request_span_ids.append(span.span_id) + + def respond(request, **kwargs): + # `request_created` runs before `before_send`, so use zero-based index for current + # attempt; `min(..., len(status_codes) - 1)` clamps to last status to avoid `IndexError`. + response_index = min(len(request_span_ids) - 1, len(status_codes) - 1) + return AWSResponse(request.url, status_codes[response_index], {}, Body(b"")) + + client.meta.events.register("request-created", record_request) + client.meta.events.register("before-send", respond) + return request_span_ids + + +def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streaming): + items = capture_items() + + if span_streaming: + with sentry_sdk.traces.start_span(name="parent"): # type: ignore[attr-defined] + invoke_client_method() + + sentry_sdk.flush() + spans = [ + item.payload + for item in items + if item.type == "span" + and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) + == Boto3Integration.origin + ] + else: + with sentry_sdk.start_transaction(): + invoke_client_method() + + transaction = next(item.payload for item in items if item.type == "transaction") + spans = [ + span + for span in transaction["spans"] + if span["origin"] == Boto3Integration.origin + ] + + spans_by_op = {} + for span in spans: + op = ( + span["attributes"].get(SPANDATA.SENTRY_OP) if span_streaming else span["op"] + ) + spans_by_op.setdefault(op, []).append(span) + return spans_by_op + + +def _assert_span_finished(span, span_streaming): + finished_timestamp = "end_timestamp" if span_streaming else "timestamp" + assert span[finished_timestamp] is not None + + +def _assert_one_failed_span(spans, span_streaming): + assert len(spans) == 1 + assert spans[0]["status"] in ("error", "internal_error") + attributes = spans[0]["attributes"] if span_streaming else spans[0]["data"] + assert attributes[SPANDATA.ERROR_TYPE] + _assert_span_finished(spans[0], span_streaming) + + +def _capture_stubbed_client_span( + client, + method_name, + api_params, + capture_items, + span_streaming, + response=None, +): + with Stubber(client) as stubber: + stubber.add_response(method_name, response or {}, api_params) + spans_by_op = _capture_boto3_spans_by_op( + lambda: getattr(client, method_name)(**api_params), + capture_items, + span_streaming, + ) + + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + assert len(client_spans) == 1 + return client_spans[0] + + +def _span_attributes(span, span_streaming): + return span["attributes"] if span_streaming else span["data"] + + +@pytest.mark.parametrize( + ("response", "expected"), + [ + (None, {}), + ({}, {}), + ({"ResponseMetadata": None}, {}), + ( + { + "ResponseMetadata": { + "RequestId": "request-id", + "HostId": "extended-request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 0, + } + }, + { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + }, + ), + ( + { + "ResponseMetadata": { + "RequestId": "request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 2, + } + }, + { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + "http.request.resend_count": 2, + }, + ), + ], +) +def test_get_response_attributes(response, expected): + assert _get_response_attributes(response) == expected + + +@pytest.mark.parametrize( + "header_name", + ["x-amzn-requestid", "x-amzn-request-id", "x-amz-request-id"], +) +def test_get_response_attributes_reads_request_id_header(header_name): + response = { + "ResponseMetadata": { + "HTTPHeaders": {header_name: "request-id"}, + } + } + + assert _get_response_attributes(response) == {SPANDATA.AWS_REQUEST_ID: "request-id"} + + +def test_get_response_attributes_reads_extended_request_id_header(): + response = { + "ResponseMetadata": { + "HTTPHeaders": {"x-amz-id-2": "extended-request-id"}, + } + } + + assert _get_response_attributes(response) == { + SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id" + } + + +@pytest.mark.parametrize( + ("field", "value", "attribute"), + [ + ("RequestId", 123, SPANDATA.AWS_REQUEST_ID), + ("RequestId", "", SPANDATA.AWS_REQUEST_ID), + ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), + ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), + ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), + ("RetryAttempts", "2", "http.request.resend_count"), + ("RetryAttempts", False, "http.request.resend_count"), + ("RetryAttempts", -1, "http.request.resend_count"), + ], +) +def test_get_response_attributes_ignores_malformed_field(field, value, attribute): + metadata = { + "RequestId": "request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 2, + } + metadata[field] = value + + attributes = _get_response_attributes({"ResponseMetadata": metadata}) + expected = { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + "http.request.resend_count": 2, + } + expected.pop(attribute) + + # One malformed optional field must not discard other valid metadata. + assert attributes == expected + + +@pytest.mark.parametrize( + "error_response", + [None, {"Code": ""}, {"Code": 123}], +) +def test_get_error_attributes_ignores_malformed_client_error_code(error_response): + error = ClientError( + { + "Error": {"Code": "placeholder"}, + "ResponseMetadata": {"HTTPStatusCode": 400}, + }, + "HeadObject", + ) + # `ClientError` itself expects `Error` to be a dict, so corrupt the stored + # response afterward to exercise defensive handling of arbitrary metadata. + error.response["Error"] = error_response + + assert _get_error_attributes(error) == { + SPANDATA.HTTP_STATUS_CODE: 400, + SPANDATA.ERROR_TYPE: "botocore.exceptions.ClientError", + } + + +@pytest.mark.parametrize( + ( + "service_name", + "method_name", + "api_params", + "span_name", + "rpc_service", + "rpc_method", + "server_address", + ), + [ + ( + "s3", + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + "S3.HeadObject", + "S3", + "HeadObject", + "s3.eu-north-1.amazonaws.com", + ), + ( + "events", + "list_event_buses", + {}, + "EventBridge.ListEventBuses", + "EventBridge", + "ListEventBuses", + "events.eu-north-1.amazonaws.com", + ), + ], +) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_has_common_attributes( + capture_items, + client_factory, + span_streaming, + service_name, + method_name, + api_params, + span_name, + rpc_service, + rpc_method, + server_address, +): + client = client_factory(service_name=service_name) + span = _capture_stubbed_client_span( + client, + method_name, + api_params, + capture_items, + span_streaming, + ) + attributes = _span_attributes(span, span_streaming) + + assert span["name" if span_streaming else "description"] == span_name + assert attributes[SPANDATA.RPC_SERVICE] == rpc_service + assert attributes[SPANDATA.RPC_METHOD] == rpc_method + assert attributes[SPANDATA.RPC_SYSTEM_NAME] == "aws-api" + assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1" + assert attributes[SPANDATA.SERVER_ADDRESS] == server_address + assert attributes[SPANDATA.SERVER_PORT] == 443 + + +def test_client_call_attributes_are_available_at_span_creation( + sentry_init, capture_items +): + # attribute-based filtering happens during span creation, at the same boundary + # where creation attributes are made available for sampling decisions. + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + trace_lifecycle="stream", + ignore_spans=[ + { + "attributes": { + SPANDATA.RPC_METHOD: "HeadObject", + SPANDATA.RPC_SERVICE: "S3", + SPANDATA.RPC_SYSTEM_NAME: "aws-api", + SPANDATA.SERVER_ADDRESS: "s3.eu-north-1.amazonaws.com", + SPANDATA.SERVER_PORT: 443, + } + } + ], + ) + client = session.client("s3") + items = capture_items("span") + + with Stubber(client) as stubber: + stubber.add_response("head_object", {}, {"Bucket": "bucket", "Key": "foo"}) + with sentry_sdk.traces.start_span(name="parent"): + client.head_object(Bucket="bucket", Key="foo") + + sentry_sdk.flush() + client_spans = [ + item.payload + for item in items + if item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) + == Boto3Integration.origin + ] + assert client_spans == [] + + +@pytest.mark.parametrize( + ("endpoint_url", "expected"), + [ + ( + "http://localhost:4566", + { + SPANDATA.SERVER_ADDRESS: "localhost", + SPANDATA.SERVER_PORT: 4566, + }, + ), + ( + "https://aws.example.test:8443", + { + SPANDATA.SERVER_ADDRESS: "aws.example.test", + SPANDATA.SERVER_PORT: 8443, + }, + ), + ( + "https://[2001:db8::1]:9443", + { + SPANDATA.SERVER_ADDRESS: "2001:db8::1", + SPANDATA.SERVER_PORT: 9443, + }, + ), + (None, {}), + ("not-an-endpoint", {}), + ("https://example.com:not-a-port", {}), + ], +) +def test_get_server_attributes(endpoint_url, expected): + assert _get_server_attributes(endpoint_url) == expected + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_omits_missing_region( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + monkeypatch.setattr(client.meta.config, "region_name", None) + + span = _capture_stubbed_client_span( + client, + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + capture_items, + span_streaming, + ) + + assert SPANDATA.CLOUD_REGION not in _span_attributes(span, span_streaming) + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_has_response_attributes( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + span = _capture_stubbed_client_span( + client, + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + capture_items, + span_streaming, + response={ + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "request-id", + "HostId": "extended-request-id", + "RetryAttempts": 0, + } + }, + ) + attributes = _span_attributes(span, span_streaming) + + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" + assert "http.request.resend_count" not in attributes + assert SPANDATA.ERROR_TYPE not in attributes + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_retry_attempts_share_one_client_span( + capture_items, + client_factory, + span_streaming, +): + attempt_count = 3 + client = client_factory(attempt_count=attempt_count) + request_span_ids = _mock_responses(client, [500] * (attempt_count - 1) + [200]) + + spans_by_op = _capture_boto3_spans_by_op( + lambda: client.head_object(Bucket="bucket", Key="foo"), + capture_items, + span_streaming, + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(request_span_ids) == attempt_count + # all `AWSRequest` instances created during retries reference the same client span. + assert len(set(request_span_ids)) == 1 + assert len(client_spans) == 1 + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes["http.request.resend_count"] == attempt_count - 1 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_retries_exhausted_has_one_failed_client_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory(attempt_count=2) + request_span_ids = _mock_responses(client, [500]) + + def attempt_failed_head_object_call(): + with pytest.raises(ClientError): + client.head_object(Bucket="bucket", Key="foo.pdf") + + spans_by_op = _capture_boto3_spans_by_op( + attempt_failed_head_object_call, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(request_span_ids) == 2 + assert len(set(request_span_ids)) == 1 + _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 500 + assert attributes["http.request.resend_count"] == 1 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_error_has_response_attributes_and_is_unchanged( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + original_exception = ClientError( + { + "Error": { + "Code": "AccessDeniedException", + "Message": "must not become a span attribute", + }, + "ResponseMetadata": { + "RequestId": "request-id", + "HTTPStatusCode": 403, + "RetryAttempts": 1, + }, + }, + "HeadObject", + ) + + def raise_client_error(**kwargs): + raise original_exception + + client.meta.events.register("before-parameter-build", raise_client_error) + + def invoke_failing_client_method(): + with pytest.raises(ClientError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 + assert attributes["http.request.resend_count"] == 1 + assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" + assert "Error.Message" not in attributes + assert "exception.message" not in attributes + assert "error.message" not in attributes + + +@pytest.mark.parametrize( + "event_name", + [ + pytest.param("before-parameter-build"), + pytest.param("before-send"), + ], +) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_exception_is_unchanged_and_finishes_span( + capture_items, + client_factory, + span_streaming, + event_name, +): + client = client_factory() + if event_name == "before-send": + original_exception = EndpointConnectionError( + endpoint_url="https://s3.eu-north-1.amazonaws.com" + ) + else: + original_exception = ValueError("parameter processing failed") + + def raise_original_exception(**kwargs): + raise original_exception + + client.meta.events.register(event_name, raise_original_exception) + + def invoke_failing_client_method(): + with pytest.raises(type(original_exception)) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + _assert_one_failed_span(client_spans, span_streaming) + + attributes = _span_attributes(client_spans[0], span_streaming) + expected_error_type = ( + "botocore.exceptions.EndpointConnectionError" + if event_name == "before-send" + else "ValueError" + ) + assert attributes[SPANDATA.ERROR_TYPE] == expected_error_type + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_response_attribute_extraction_failure_does_not_change_response( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + api_params = {"Bucket": "bucket", "Key": "foo"} + original_response = { + "ResponseMetadata": { + "HTTPStatusCode": 200, + } + } + returned_responses = [] + + def fail_attribute_extraction(response): + raise RuntimeError("attribute extraction failed") + + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._instrumentation._get_response_attributes", + fail_attribute_extraction, + ) + + def invoke_client_method(): + returned_responses.append(client.head_object(**api_params)) + + with Stubber(client) as stubber: + stubber.add_response("head_object", original_response, api_params) + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method, capture_items, span_streaming + ) + + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + assert len(returned_responses) == 1 + assert returned_responses[0] is original_response + assert len(client_spans) == 1 + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_error_attribute_extraction_failure_does_not_replace_original_exception( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + original_exception = ValueError("parameter processing failed") + + def raise_original_exception(**kwargs): + raise original_exception + + def fail_attribute_extraction(exception): + raise RuntimeError("attribute extraction failed") + + client.meta.events.register("before-parameter-build", raise_original_exception) + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._instrumentation._get_error_attributes", + fail_attribute_extraction, + ) + + def invoke_failing_client_method(): + with pytest.raises(ValueError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(client_spans) == 1 + assert client_spans[0]["status"] in ("error", "internal_error") + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_service_response_enrichment_failure_preserves_response_and_finishes_span( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + class FailingServiceExtension(_ServiceExtension): + def get_response_attributes(self, ctx, response): + raise RuntimeError("service response enrichment failed") + + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._client._resolve_service_extension", + lambda service_name: FailingServiceExtension(), + ) + client = client_factory() + api_params = {"Bucket": "bucket", "Key": "foo"} + original_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} + returned_responses = [] + + with Stubber(client) as stubber: + stubber.add_response("head_object", original_response, api_params) + spans_by_op = _capture_boto3_spans_by_op( + lambda: returned_responses.append(client.head_object(**api_params)), + capture_items, + span_streaming, + ) + + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + assert returned_responses == [original_response] + assert returned_responses[0] is original_response + assert len(client_spans) == 1 + assert ( + _span_attributes(client_spans[0], span_streaming)[SPANDATA.HTTP_STATUS_CODE] + == 200 + ) + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_service_response_enrichment_failure_on_error_preserves_exception( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + class FailingServiceExtension(_ServiceExtension): + def get_response_attributes(self, ctx, response): + raise RuntimeError("service response enrichment failed") + + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._client._resolve_service_extension", + lambda service_name: FailingServiceExtension(), + ) + client = client_factory() + original_exception = ClientError( + { + "Error": {"Code": "AccessDeniedException"}, + "ResponseMetadata": {"HTTPStatusCode": 403}, + }, + "HeadObject", + ) + + def raise_original_exception(**kwargs): + raise original_exception + + client.meta.events.register("before-parameter-build", raise_original_exception) + + def invoke_failing_client_method(): + with pytest.raises(ClientError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_response_attributes_belong_to_client_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + { + "content-length": "5", + "x-amz-request-id": "request-id", + }, + Body(b"hello"), + ) + + client.meta.events.register("before-send", respond) + + def invoke_client_method_and_read_body(): + body = client.get_object(Bucket="bucket", Key="foo")["Body"] + assert body.read() == b"hello" + assert body.read() == b"" + + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method_and_read_body, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(client_spans) == 1 + assert len(stream_spans) == 1 + client_attributes = _span_attributes(client_spans[0], span_streaming) + stream_attributes = _span_attributes(stream_spans[0], span_streaming) + assert client_attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert client_attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert "http.request.resend_count" not in client_attributes + assert SPANDATA.AWS_REQUEST_ID not in stream_attributes + assert SPANDATA.HTTP_STATUS_CODE not in stream_attributes + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_body_read_failure_finishes_stream_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + original_exception = OSError("stream read failed") + + class _FailingBody(Body): + def __init__(self, exception): + super().__init__(b"") + self._exception = exception + + def read(self, *args, **kwargs): + raise self._exception + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + {"content-length": "1"}, + _FailingBody(original_exception), + ) + + client.meta.events.register("before-send", respond) + + def invoke_client_method_and_read_body(): + body = client.get_object(Bucket="bucket", Key="foo")["Body"] + with pytest.raises(OSError) as exc_info: + body.read() + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method_and_read_body, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(client_spans) == 1 + _assert_one_failed_span(stream_spans, span_streaming) + attributes = _span_attributes(stream_spans[0], span_streaming) + assert attributes[SPANDATA.ERROR_TYPE] == "OSError" From 47997b044aa42ca825e2ec0515ed2543009fd3ca Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:27:10 +0200 Subject: [PATCH 02/12] remove everything related to service-extension --- .../integrations/boto3/_services/__init__.py | 0 .../integrations/boto3/_services/base.py | 47 ------------------- .../integrations/boto3/_services/registry.py | 36 -------------- 3 files changed, 83 deletions(-) delete mode 100644 sentry_sdk/integrations/boto3/_services/__init__.py delete mode 100644 sentry_sdk/integrations/boto3/_services/base.py delete mode 100644 sentry_sdk/integrations/boto3/_services/registry.py diff --git a/sentry_sdk/integrations/boto3/_services/__init__.py b/sentry_sdk/integrations/boto3/_services/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py deleted file mode 100644 index ba30905509..0000000000 --- a/sentry_sdk/integrations/boto3/_services/base.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing import Any, Dict, Optional, Tuple - - from sentry_sdk._types import Attributes - from sentry_sdk.integrations.boto3._context import AwsCallContext - - -class _ServiceExtension: - """ - Specialize generic botocore client-call instrumentation for a specific - AWS service, e.g. SQS. - - ``_resolve_service_extension()`` caches and shares instances across calls, - so extensions must remain stateless and receive the per-call context - explicitly. Boto3 clients are generally thread-safe and may therefore use - the same extension concurrently: - https://docs.aws.amazon.com/boto3/latest/guide/clients.html#multithreading-or-multiprocessing-with-clients - """ - - # avoid arbitrary attributes by disabling per-instance `__dict__`. - __slots__ = () - - def get_span_config( - self, ctx: "AwsCallContext" - ) -> "Optional[Tuple[Optional[str], Optional[str]]]": - """Return an optional `(op, origin)` override; None defaults to `(HTTP_CLIENT, Boto3Integration.origin)`.""" - return None - - def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": - """Return service-specific attributes derived before the call.""" - return {} - - def get_response_attributes( - self, ctx: "AwsCallContext", response: "Any" - ) -> "Attributes": - """Return service-specific attributes derived from an AWS response.""" - return {} - - def inject_trace_context( - self, - ctx: "AwsCallContext", - trace_context: "Dict[str, str]", - ) -> "Optional[Dict[str, Any]]": - """Put the trace context somewhere the consumer of this call can find it.""" - return None diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py deleted file mode 100644 index d7e41e226e..0000000000 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ /dev/null @@ -1,36 +0,0 @@ -from functools import lru_cache -from importlib import import_module -from typing import TYPE_CHECKING - -from sentry_sdk.integrations.boto3._services.base import _ServiceExtension -from sentry_sdk.utils import capture_internal_exceptions - -if TYPE_CHECKING: - from typing import Dict, Optional, Tuple - - -# Store import targets instead of importing service modules eagerly. This keeps -# a broken service extension from disabling generic AWS instrumentation. -_SERVICE_EXTENSIONS: "Dict[str, Tuple[str, str]]" = { -} - - -@lru_cache(maxsize=None) -def _resolve_service_extension( - service_name: str, -) -> "Optional[_ServiceExtension]": - """Resolve a shared extension for a botocore service name.""" - target = _SERVICE_EXTENSIONS.get(service_name) - if target is None: - return None - - extension = None - with capture_internal_exceptions(): - module_name, class_name = target - extension_class = getattr(import_module(module_name), class_name) - candidate = extension_class() - if isinstance(candidate, _ServiceExtension): - extension = candidate - - # An unknown or broken service extension falls back to generic instrumentation. - return extension From c5896e20cad16bffdd7ccd9f9f40311708b6fbe2 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:29:21 +0200 Subject: [PATCH 03/12] remove tests related to service extension --- tests/integrations/boto3/test_client.py | 96 ------------------------- 1 file changed, 96 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 9310e12ecc..6efaf514ef 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -14,11 +14,6 @@ _get_server_attributes, _merge_service_attributes, ) -from sentry_sdk.integrations.boto3._services.base import _ServiceExtension -from sentry_sdk.integrations.boto3._services.registry import ( - _SERVICE_EXTENSIONS, - _resolve_service_extension, -) from tests.integrations.boto3.aws_mock import Body session = boto3.Session( # type: ignore[attr-defined] @@ -28,11 +23,6 @@ ) -@pytest.mark.parametrize("service_name", sorted(_SERVICE_EXTENSIONS)) -def test_registered_service_extensions_load(service_name): - assert isinstance(_resolve_service_extension(service_name), _ServiceExtension) - - def test_generic_attributes_take_precedence_over_service_attributes(): attributes = {SPANDATA.RPC_METHOD: "GetItem"} @@ -706,92 +696,6 @@ def invoke_failing_client_method(): _assert_span_finished(client_spans[0], span_streaming) -@pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_service_response_enrichment_failure_preserves_response_and_finishes_span( - capture_items, - client_factory, - monkeypatch, - span_streaming, -): - class FailingServiceExtension(_ServiceExtension): - def get_response_attributes(self, ctx, response): - raise RuntimeError("service response enrichment failed") - - monkeypatch.setattr( - "sentry_sdk.integrations.boto3._client._resolve_service_extension", - lambda service_name: FailingServiceExtension(), - ) - client = client_factory() - api_params = {"Bucket": "bucket", "Key": "foo"} - original_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} - returned_responses = [] - - with Stubber(client) as stubber: - stubber.add_response("head_object", original_response, api_params) - spans_by_op = _capture_boto3_spans_by_op( - lambda: returned_responses.append(client.head_object(**api_params)), - capture_items, - span_streaming, - ) - - client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) - assert returned_responses == [original_response] - assert returned_responses[0] is original_response - assert len(client_spans) == 1 - assert ( - _span_attributes(client_spans[0], span_streaming)[SPANDATA.HTTP_STATUS_CODE] - == 200 - ) - _assert_span_finished(client_spans[0], span_streaming) - - -@pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_service_response_enrichment_failure_on_error_preserves_exception( - capture_items, - client_factory, - monkeypatch, - span_streaming, -): - class FailingServiceExtension(_ServiceExtension): - def get_response_attributes(self, ctx, response): - raise RuntimeError("service response enrichment failed") - - monkeypatch.setattr( - "sentry_sdk.integrations.boto3._client._resolve_service_extension", - lambda service_name: FailingServiceExtension(), - ) - client = client_factory() - original_exception = ClientError( - { - "Error": {"Code": "AccessDeniedException"}, - "ResponseMetadata": {"HTTPStatusCode": 403}, - }, - "HeadObject", - ) - - def raise_original_exception(**kwargs): - raise original_exception - - client.meta.events.register("before-parameter-build", raise_original_exception) - - def invoke_failing_client_method(): - with pytest.raises(ClientError) as exc_info: - client.head_object(Bucket="bucket", Key="foo") - assert exc_info.value is original_exception - - spans_by_op = _capture_boto3_spans_by_op( - invoke_failing_client_method, capture_items, span_streaming - ) - client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) - - _assert_one_failed_span(client_spans, span_streaming) - attributes = _span_attributes(client_spans[0], span_streaming) - assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" - assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 - - @pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_response_attributes_belong_to_client_span( capture_items, From d91abcac5a8048e81c1b77d90020c4668147d3d5 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:32:40 +0200 Subject: [PATCH 04/12] more removal related to service-enrichment --- sentry_sdk/integrations/boto3/_client.py | 53 ++------------- .../integrations/boto3/_instrumentation.py | 64 ------------------- tests/integrations/boto3/test_client.py | 18 ------ 3 files changed, 4 insertions(+), 131 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index aa91439764..e05f424d39 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -12,17 +12,11 @@ _sentry_request_created, _start_client_span, ) -from sentry_sdk.integrations.boto3._services.registry import ( - _resolve_service_extension, -) -from sentry_sdk.traces import NoOpStreamedSpan -from sentry_sdk.tracing import NoOpSpan from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: from typing import Any, Optional, Union - from sentry_sdk.integrations.boto3._services.base import _ServiceExtension from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span @@ -57,67 +51,28 @@ def sentry_patched_make_api_call( return orig_make_api_call(self, operation_name, api_params) ctx: "Optional[AwsCallContext]" = None - service_extension: "Optional[_ServiceExtension]" = None span: "Optional[Union[Span, StreamedSpan]]" = None with capture_internal_exceptions(): ctx = AwsCallContext(self, operation_name, api_params) if ctx is not None: - # The resolver contains its own fail-open import boundary. - service_extension = _resolve_service_extension(ctx.service_name) - with capture_internal_exceptions(): - span = _start_client_span(ctx, service_extension) + span = _start_client_span(ctx) if span is not None: span.__enter__() - instrumented_api_params = api_params - if ( - ctx is not None - and service_extension is not None - and span is not None - and not isinstance(span, (NoOpSpan, NoOpStreamedSpan)) - and client.options.get("propagate_traces") - and isinstance(api_params, dict) - ): - with capture_internal_exceptions(): - # propagation must use current scope - headers = dict( - sentry_sdk.get_current_scope().iter_trace_propagation_headers( - span=span - ) - ) - propagated_params = service_extension.inject_trace_context( - ctx, - headers, - ) - # Only pass a service extension's replacement when it is a mapping. - # Otherwise preserve the caller's parameters and Botocore behavior. - if isinstance(propagated_params, dict): - instrumented_api_params = propagated_params - try: - parsed = orig_make_api_call(self, operation_name, instrumented_api_params) + parsed = orig_make_api_call(self, operation_name, api_params) except BaseException as exc: if span is not None: with capture_internal_exceptions(): - _finish_client_span_with_error( - span, - exc, - ctx, - service_extension, - ) + _finish_client_span_with_error(span, exc) raise if span is not None: with capture_internal_exceptions(): - _finish_client_span( - span, - parsed, - ctx, - service_extension, - ) + _finish_client_span(span, parsed) return parsed BaseClient.__init__ = sentry_patched_init # type: ignore diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 743a902907..cbd1f44156 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -27,7 +27,6 @@ from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext - from sentry_sdk.integrations.boto3._services.base import _ServiceExtension _AWS_RPC_SYSTEM_NAME = "aws-api" @@ -92,20 +91,6 @@ def _get_client_attributes( return attributes -def _merge_service_attributes( - attributes: "Attributes", - service_attributes: "Any", -) -> None: - if not isinstance(service_attributes, dict): - return - - for key, value in service_attributes.items(): - # Generic attributes are added first and remain authoritative. A service - # extension may only fill attributes that generic instrumentation did - # not already produce. - attributes.setdefault(key, value) - - def _get_response_attributes(response: "Any") -> "Attributes": if not isinstance(response, dict): return {} @@ -201,7 +186,6 @@ def _get_error_attributes(exception: "BaseException") -> "Attributes": def _start_client_span( ctx: "AwsCallContext", - service_extension: "Optional[_ServiceExtension]" = None, ) -> "Optional[Union[Span, StreamedSpan]]": client = sentry_sdk.get_client() if client.get_integration(Boto3Integration) is None: @@ -214,28 +198,6 @@ def _start_client_span( span_op = OP.HTTP_CLIENT span_origin = Boto3Integration.origin - # enrich with service-specific attributes - if service_extension is not None: - service_span_config = None - with capture_internal_exceptions(): - service_span_config = service_extension.get_span_config(ctx) - - with capture_internal_exceptions(): - if service_span_config is not None: - service_op, service_origin = service_span_config - if service_op and isinstance(service_op, str): - span_op = service_op - if service_origin and isinstance(service_origin, str): - span_origin = service_origin - - # Request enrichment is independent from the span configuration. This - # lets HTTP-based services such as S3 return only their attribute delta. - with capture_internal_exceptions(): - _merge_service_attributes( - attributes, - service_extension.get_request_attributes(ctx), - ) - if has_span_streaming_enabled(client.options): if sentry_sdk.traces.get_current_span() is None: return None @@ -267,8 +229,6 @@ def _start_client_span( def _finish_client_span( span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]", - ctx: "Optional[AwsCallContext]" = None, - service_extension: "Optional[_ServiceExtension]" = None, ) -> None: # response metadata is only available after the call. Keep enrichment # isolated so failure cannot prevent `__exit__()` below. @@ -276,13 +236,6 @@ def _finish_client_span( with capture_internal_exceptions(): attributes = _get_response_attributes(parsed) - if ctx is not None and service_extension is not None: - with capture_internal_exceptions(): - _merge_service_attributes( - attributes, - service_extension.get_response_attributes(ctx, parsed), - ) - with capture_internal_exceptions(): _set_span_attributes(span, attributes) span.__exit__(None, None, None) @@ -351,28 +304,11 @@ def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: def _finish_client_span_with_error( span: "Union[Span, StreamedSpan]", exception: "BaseException", - ctx: "Optional[AwsCallContext]" = None, - service_extension: "Optional[_ServiceExtension]" = None, ) -> None: attributes: "Attributes" = {} with capture_internal_exceptions(): attributes = _get_error_attributes(exception) - # ClientError.response is the parsed AWS error response, so the same response - # hook can enrich successful and failed service responses. A separate error - # hook is unnecessary until a service needs exception-only information. - # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/error-handling.html#catching-botocore-exceptions - if ( - ctx is not None - and service_extension is not None - and isinstance(exception, ClientError) - ): - with capture_internal_exceptions(): - _merge_service_attributes( - attributes, - service_extension.get_response_attributes(ctx, exception.response), - ) - with capture_internal_exceptions(): _set_span_attributes(span, attributes) span.__exit__(type(exception), exception, exception.__traceback__) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 6efaf514ef..fd7d2edb0b 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -12,7 +12,6 @@ _get_error_attributes, _get_response_attributes, _get_server_attributes, - _merge_service_attributes, ) from tests.integrations.boto3.aws_mock import Body @@ -23,23 +22,6 @@ ) -def test_generic_attributes_take_precedence_over_service_attributes(): - attributes = {SPANDATA.RPC_METHOD: "GetItem"} - - _merge_service_attributes( - attributes, - { - SPANDATA.RPC_METHOD: "overridden", - SPANDATA.DB_SYSTEM_NAME: "aws.dynamodb", - }, - ) - - assert attributes == { - SPANDATA.RPC_METHOD: "GetItem", - SPANDATA.DB_SYSTEM_NAME: "aws.dynamodb", - } - - def test_public_api(): assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" assert Boto3Integration.identifier == "boto3" From ec6dffe054a54630c1dc0121f0e1e216cd0718db Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:38:56 +0200 Subject: [PATCH 05/12] fix(boto3): add back `origin` --- sentry_sdk/integrations/boto3/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/boto3/__init__.py b/sentry_sdk/integrations/boto3/__init__.py index 2e6e8712a4..745f792369 100644 --- a/sentry_sdk/integrations/boto3/__init__.py +++ b/sentry_sdk/integrations/boto3/__init__.py @@ -6,11 +6,9 @@ except ImportError: raise DidNotEnable("botocore is not installed") -_SPAN_ORIGIN = "auto.http.boto3" - - class Boto3Integration(Integration): - origin = _SPAN_ORIGIN + identifier = "boto3" + origin = f"auto.http.{identifier}" @staticmethod def setup_once() -> None: From 6cee882110765aa86dcf4867a79300442a0f8383 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:39:57 +0200 Subject: [PATCH 06/12] tests(boto3): update `test_s3.py` to match new `Service.Operation` span naming and `rpc.service` and `parse_url` patch target --- tests/integrations/boto3/test_s3.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/integrations/boto3/test_s3.py b/tests/integrations/boto3/test_s3.py index 9e6b296596..c687dcf471 100644 --- a/tests/integrations/boto3/test_s3.py +++ b/tests/integrations/boto3/test_s3.py @@ -53,7 +53,7 @@ def test_basic( assert len(spans) == 2 span = spans[0] assert span["attributes"]["sentry.op"] == "http.client" - assert span["name"] == "aws.s3.ListObjects" + assert span["name"] == "S3.ListObjects" else: events = capture_events() @@ -71,7 +71,7 @@ def test_basic( assert len(event["spans"]) == 1 (span,) = event["spans"] assert span["op"] == "http.client" - assert span["description"] == "aws.s3.ListObjects" + assert span["description"] == "S3.ListObjects" @pytest.mark.parametrize("send_default_pii", [True, False]) @@ -112,11 +112,12 @@ def test_streaming( span1 = spans[0] assert span1["attributes"]["sentry.op"] == "http.client" - assert span1["name"] == "aws.s3.GetObject" + assert span1["name"] == "S3.GetObject" expected_attrs = { "http.request.method": "GET", - "rpc.method": "S3/GetObject", + "rpc.method": "GetObject", + "rpc.service": "S3", "sentry.environment": "production", "sentry.op": "http.client", "sentry.origin": "auto.http.boto3", @@ -140,7 +141,7 @@ def test_streaming( span2 = spans[1] assert span2["attributes"]["sentry.op"] == "http.client.stream" - assert span2["name"] == "aws.s3.GetObject" + assert span2["name"] == "S3.GetObject" assert span2["parent_span_id"] == span1["span_id"] else: events = capture_events() @@ -161,7 +162,7 @@ def test_streaming( span1 = event["spans"][0] assert span1["op"] == "http.client" - assert span1["description"] == "aws.s3.GetObject" + assert span1["description"] == "S3.GetObject" assert span1["data"] == ApproxDict( { "http.method": "GET", @@ -173,7 +174,7 @@ def test_streaming( span2 = event["spans"][1] assert span2["op"] == "http.client.stream" - assert span2["description"] == "aws.s3.GetObject" + assert span2["description"] == "S3.GetObject" assert span2["parent_span_id"] == span1["span_id"] @@ -253,7 +254,7 @@ def test_omit_url_data_if_parsing_fails( items = capture_items("span") with mock.patch( - "sentry_sdk.integrations.boto3.parse_url", + "sentry_sdk.integrations.boto3._instrumentation.parse_url", side_effect=ValueError, ): with sentry_sdk.traces.start_span( @@ -272,7 +273,8 @@ def test_omit_url_data_if_parsing_fails( assert spans[0]["attributes"] == ApproxDict( { "http.request.method": "GET", - "rpc.method": "S3/ListObjects", + "rpc.method": "ListObjects", + "rpc.service": "S3", "sentry.environment": "production", "sentry.op": "http.client", "sentry.origin": "auto.http.boto3", @@ -294,7 +296,7 @@ def test_omit_url_data_if_parsing_fails( events = capture_events() with mock.patch( - "sentry_sdk.integrations.boto3.parse_url", + "sentry_sdk.integrations.boto3._instrumentation.parse_url", side_effect=ValueError, ): with sentry_sdk.start_transaction() as transaction, MockResponse( From 5771275bca9df98e7452a2386a25e838d4ce5865 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:42:32 +0200 Subject: [PATCH 07/12] lint --- sentry_sdk/integrations/boto3/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/boto3/__init__.py b/sentry_sdk/integrations/boto3/__init__.py index 745f792369..66e7d208bd 100644 --- a/sentry_sdk/integrations/boto3/__init__.py +++ b/sentry_sdk/integrations/boto3/__init__.py @@ -6,6 +6,7 @@ except ImportError: raise DidNotEnable("botocore is not installed") + class Boto3Integration(Integration): identifier = "boto3" origin = f"auto.http.{identifier}" From c5712abfd95bb3be259393e6943e27d8a38fde21 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:47:22 +0200 Subject: [PATCH 08/12] remove unused fields from context --- sentry_sdk/consts.py | 6 ------ sentry_sdk/integrations/boto3/_context.py | 4 ---- 2 files changed, 10 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index bd5b3cce7b..390fef2b1c 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -910,12 +910,6 @@ class SPANDATA: Example: 2 """ - HTTP_RESPONSE_BODY_SIZE = "http.response.body.size" - """ - The encoded body size of the response (in bytes). - Example: 123 - """ - HTTP_ROUTE = "http.route" """ The matched route, that is, the path template used to match the request. diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py index 6282ebcbb0..91cfc3db6c 100644 --- a/sentry_sdk/integrations/boto3/_context.py +++ b/sentry_sdk/integrations/boto3/_context.py @@ -8,14 +8,12 @@ class AwsCallContext: __slots__ = ( - "client", "service_name", "service_id", "service_id_hyphenized", "operation_name", "region_name", "endpoint_url", - "api_version", "params", ) @@ -29,7 +27,6 @@ def __init__( service_model = client_meta.service_model service_id = service_model.service_id - self.client: "BaseClient" = client # botocore's internal identifier, e.g. `apigateway`. self.service_name: str = service_model.service_name # modeled AWS service identity used in span names, e.g. `API Gateway`. @@ -38,5 +35,4 @@ def __init__( self.operation_name: str = operation_name self.region_name: "Optional[str]" = getattr(client_meta, "region_name", None) self.endpoint_url: "Optional[str]" = getattr(client_meta, "endpoint_url", None) - self.api_version: str = service_model.api_version self.params: "Dict[str, Any]" = dict(params) if isinstance(params, dict) else {} From 2ba1efbf21c5529c9ca26a73a24ff70007f2c24b Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:48:49 +0200 Subject: [PATCH 09/12] use `SPANDATA.HTTP_REQUEST_RESEND_COUNT` instead of magic `http.request.resend_count` --- .../integrations/boto3/_instrumentation.py | 2 +- tests/integrations/boto3/test_client.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index cbd1f44156..0eadb0702f 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -120,7 +120,7 @@ def _get_response_attributes(response: "Any") -> "Attributes": and not isinstance(retry_attempts, bool) and retry_attempts > 0 ): - attributes["http.request.resend_count"] = retry_attempts + attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] = retry_attempts headers = metadata.get("HTTPHeaders") if not isinstance(headers, dict): diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index fd7d2edb0b..fbe2b04477 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -176,7 +176,7 @@ def _span_attributes(span, span_streaming): { SPANDATA.AWS_REQUEST_ID: "request-id", SPANDATA.HTTP_STATUS_CODE: 200, - "http.request.resend_count": 2, + SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, }, ), ], @@ -219,9 +219,9 @@ def test_get_response_attributes_reads_extended_request_id_header(): ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), - ("RetryAttempts", "2", "http.request.resend_count"), - ("RetryAttempts", False, "http.request.resend_count"), - ("RetryAttempts", -1, "http.request.resend_count"), + ("RetryAttempts", "2", SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ("RetryAttempts", False, SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ("RetryAttempts", -1, SPANDATA.HTTP_REQUEST_RESEND_COUNT), ], ) def test_get_response_attributes_ignores_malformed_field(field, value, attribute): @@ -236,7 +236,7 @@ def test_get_response_attributes_ignores_malformed_field(field, value, attribute expected = { SPANDATA.AWS_REQUEST_ID: "request-id", SPANDATA.HTTP_STATUS_CODE: 200, - "http.request.resend_count": 2, + SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, } expected.pop(attribute) @@ -449,7 +449,7 @@ def test_client_call_has_response_attributes( assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" - assert "http.request.resend_count" not in attributes + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in attributes assert SPANDATA.ERROR_TYPE not in attributes @@ -475,7 +475,7 @@ def test_retry_attempts_share_one_client_span( assert len(set(request_span_ids)) == 1 assert len(client_spans) == 1 attributes = _span_attributes(client_spans[0], span_streaming) - assert attributes["http.request.resend_count"] == attempt_count - 1 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == attempt_count - 1 @pytest.mark.parametrize("span_streaming", [True, False]) @@ -501,7 +501,7 @@ def attempt_failed_head_object_call(): _assert_one_failed_span(client_spans, span_streaming) attributes = _span_attributes(client_spans[0], span_streaming) assert attributes[SPANDATA.HTTP_STATUS_CODE] == 500 - assert attributes["http.request.resend_count"] == 1 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 @pytest.mark.parametrize("span_streaming", [True, False]) @@ -545,7 +545,7 @@ def invoke_failing_client_method(): assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 - assert attributes["http.request.resend_count"] == 1 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" assert "Error.Message" not in attributes assert "exception.message" not in attributes @@ -716,7 +716,7 @@ def invoke_client_method_and_read_body(): stream_attributes = _span_attributes(stream_spans[0], span_streaming) assert client_attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" assert client_attributes[SPANDATA.HTTP_STATUS_CODE] == 200 - assert "http.request.resend_count" not in client_attributes + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in client_attributes assert SPANDATA.AWS_REQUEST_ID not in stream_attributes assert SPANDATA.HTTP_STATUS_CODE not in stream_attributes From 63fee2e28aa98187e4e3efaf644ff2468f89b1db Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:54:42 +0200 Subject: [PATCH 10/12] remove all logic related to #7476 --- sentry_sdk/consts.py | 24 -- .../integrations/boto3/_instrumentation.py | 127 +------ tests/integrations/boto3/test_client.py | 345 +----------------- 3 files changed, 8 insertions(+), 488 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 390fef2b1c..8872453709 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -389,17 +389,6 @@ class SPANDATA: Warning messages generated during model execution. Example: ["Token limit exceeded"] """ - AWS_EXTENDED_REQUEST_ID = "aws.extended_request_id" - """ - The AWS extended request ID as returned in the response headers. - Example: "wzHcyEWfmOGDIE5QOhTAqFDoDWP3y8IUvpNINCwL9N4TEHbUw0/gZJ+VZTmCNCWR7fezEN3eCiQ=" - """ - - AWS_REQUEST_ID = "aws.request_id" - """ - The AWS request ID as returned in the response headers. - Example: "79b9da39-b7ae-508a-a6bc-864b2829c622" - """ CACHE_HIT = "cache.hit" """ @@ -558,12 +547,6 @@ class SPANDATA: Example: my_user """ - ERROR_TYPE = "error.type" - """ - Describes a class of error the operation ended with. - Example: "timeout" - """ - GEN_AI_AGENT_NAME = "gen_ai.agent.name" """ The name of the agent being used. @@ -903,13 +886,6 @@ class SPANDATA: Example: GET """ - HTTP_REQUEST_RESEND_COUNT = "http.request.resend_count" - """ - The ordinal number of request resending attempt (for any reason, including redirects). - - Example: 2 - """ - HTTP_ROUTE = "http.route" """ The matched route, that is, the path template used to match the request. diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 0eadb0702f..261f6feeba 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -2,11 +2,10 @@ from urllib.parse import urlsplit from botocore.awsrequest import AWSRequest -from botocore.exceptions import ClientError from botocore.response import StreamingBody import sentry_sdk -from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS +from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span @@ -91,99 +90,6 @@ def _get_client_attributes( return attributes -def _get_response_attributes(response: "Any") -> "Attributes": - if not isinstance(response, dict): - return {} - - metadata = response.get("ResponseMetadata") - if not isinstance(metadata, dict): - return {} - - attributes: "Attributes" = {} - - status_code = metadata.get("HTTPStatusCode") - # botocore injects HTTP status into `ResponseMetadata` after parsing. - # https://github.com/boto/botocore/blob/develop/botocore/parsers.py#L273-L284 - if ( - isinstance(status_code, int) - and not isinstance(status_code, bool) - and 100 <= status_code <= 599 - ): - attributes[SPANDATA.HTTP_STATUS_CODE] = status_code - - retry_attempts = metadata.get("RetryAttempts") - # botocore represents retries as `attempts - 1`; omit zero. - # https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L221-L229 - # https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span - if ( - isinstance(retry_attempts, int) - and not isinstance(retry_attempts, bool) - and retry_attempts > 0 - ): - attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] = retry_attempts - - headers = metadata.get("HTTPHeaders") - if not isinstance(headers, dict): - headers = {} - - request_id = metadata.get("RequestId") - if not isinstance(request_id, str) or not request_id: - request_id = next( - ( - value - for value in ( - headers.get("x-amzn-requestid"), - headers.get("x-amzn-request-id"), - headers.get("x-amz-request-id"), - ) - if isinstance(value, str) and value - ), - None, - ) - if isinstance(request_id, str) and request_id: - attributes[SPANDATA.AWS_REQUEST_ID] = request_id - - # S3's `HostId` is the extended request ID returned in `x-amz-id-2`. - # https://docs.aws.amazon.com/AmazonS3/latest/developerguide/get-request-ids.html - extended_request_id = metadata.get("HostId") - if not isinstance(extended_request_id, str) or not extended_request_id: - extended_request_id = headers.get("x-amz-id-2") - if isinstance(extended_request_id, str) and extended_request_id: - attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] = extended_request_id - - return attributes - - -def _get_error_type(exception: "BaseException") -> str: - if isinstance(exception, ClientError): - # botocore wraps all AWS service errors in `ClientError`; `Error.Code` - # identifies actual service-specific error, e.g. `AccessDenied`. - # https://docs.aws.amazon.com/boto3/latest/guide/error-handling.html - error = exception.response.get("Error") - if isinstance(error, dict): - error_code = error.get("Code") - if isinstance(error_code, str) and error_code: - return error_code - - # failures before a service response, have no error code. Use exception type - # instead. https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ - exception_type = type(exception) - exception_name = exception_type.__qualname__ - exception_module = exception_type.__module__ - if exception_module not in ("builtins", "__builtins__"): - return "%s.%s" % (exception_module, exception_name) - return exception_name - - -def _get_error_attributes(exception: "BaseException") -> "Attributes": - attributes: "Attributes" = {} - if isinstance(exception, ClientError): - attributes.update(_get_response_attributes(exception.response)) - - attributes[SPANDATA.ERROR_TYPE] = _get_error_type(exception) - return attributes - - def _start_client_span( ctx: "AwsCallContext", ) -> "Optional[Union[Span, StreamedSpan]]": @@ -230,14 +136,6 @@ def _finish_client_span( span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]", ) -> None: - # response metadata is only available after the call. Keep enrichment - # isolated so failure cannot prevent `__exit__()` below. - attributes: "Attributes" = {} - with capture_internal_exceptions(): - attributes = _get_response_attributes(parsed) - - with capture_internal_exceptions(): - _set_span_attributes(span, attributes) span.__exit__(None, None, None) body = parsed.get("Body") @@ -275,18 +173,11 @@ def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: else: streaming_span.finish() return ret - except Exception as exc: - # enrichment must not replace exception raised by `orig_read()`. - # finish span with error, then re-raise. - with capture_internal_exceptions(): - _set_span_attributes(streaming_span, _get_error_attributes(exc)) - - with capture_internal_exceptions(): - if isinstance(streaming_span, StreamedSpan): - streaming_span.__exit__(type(exc), exc, exc.__traceback__) - else: - streaming_span.set_status(SPANSTATUS.INTERNAL_ERROR) - streaming_span.finish() + except Exception: + if isinstance(streaming_span, StreamedSpan): + streaming_span.end() + else: + streaming_span.finish() raise body.read = sentry_streaming_body_read # type: ignore @@ -305,12 +196,6 @@ def _finish_client_span_with_error( span: "Union[Span, StreamedSpan]", exception: "BaseException", ) -> None: - attributes: "Attributes" = {} - with capture_internal_exceptions(): - attributes = _get_error_attributes(exception) - - with capture_internal_exceptions(): - _set_span_attributes(span, attributes) span.__exit__(type(exception), exception, exception.__traceback__) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index fbe2b04477..5d3656c61e 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -9,8 +9,6 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.integrations.boto3._instrumentation import ( - _get_error_attributes, - _get_response_attributes, _get_server_attributes, ) from tests.integrations.boto3.aws_mock import Body @@ -114,8 +112,6 @@ def _assert_span_finished(span, span_streaming): def _assert_one_failed_span(spans, span_streaming): assert len(spans) == 1 assert spans[0]["status"] in ("error", "internal_error") - attributes = spans[0]["attributes"] if span_streaming else spans[0]["data"] - assert attributes[SPANDATA.ERROR_TYPE] _assert_span_finished(spans[0], span_streaming) @@ -144,128 +140,6 @@ def _span_attributes(span, span_streaming): return span["attributes"] if span_streaming else span["data"] -@pytest.mark.parametrize( - ("response", "expected"), - [ - (None, {}), - ({}, {}), - ({"ResponseMetadata": None}, {}), - ( - { - "ResponseMetadata": { - "RequestId": "request-id", - "HostId": "extended-request-id", - "HTTPStatusCode": 200, - "RetryAttempts": 0, - } - }, - { - SPANDATA.AWS_REQUEST_ID: "request-id", - SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id", - SPANDATA.HTTP_STATUS_CODE: 200, - }, - ), - ( - { - "ResponseMetadata": { - "RequestId": "request-id", - "HTTPStatusCode": 200, - "RetryAttempts": 2, - } - }, - { - SPANDATA.AWS_REQUEST_ID: "request-id", - SPANDATA.HTTP_STATUS_CODE: 200, - SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, - }, - ), - ], -) -def test_get_response_attributes(response, expected): - assert _get_response_attributes(response) == expected - - -@pytest.mark.parametrize( - "header_name", - ["x-amzn-requestid", "x-amzn-request-id", "x-amz-request-id"], -) -def test_get_response_attributes_reads_request_id_header(header_name): - response = { - "ResponseMetadata": { - "HTTPHeaders": {header_name: "request-id"}, - } - } - - assert _get_response_attributes(response) == {SPANDATA.AWS_REQUEST_ID: "request-id"} - - -def test_get_response_attributes_reads_extended_request_id_header(): - response = { - "ResponseMetadata": { - "HTTPHeaders": {"x-amz-id-2": "extended-request-id"}, - } - } - - assert _get_response_attributes(response) == { - SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id" - } - - -@pytest.mark.parametrize( - ("field", "value", "attribute"), - [ - ("RequestId", 123, SPANDATA.AWS_REQUEST_ID), - ("RequestId", "", SPANDATA.AWS_REQUEST_ID), - ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), - ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), - ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), - ("RetryAttempts", "2", SPANDATA.HTTP_REQUEST_RESEND_COUNT), - ("RetryAttempts", False, SPANDATA.HTTP_REQUEST_RESEND_COUNT), - ("RetryAttempts", -1, SPANDATA.HTTP_REQUEST_RESEND_COUNT), - ], -) -def test_get_response_attributes_ignores_malformed_field(field, value, attribute): - metadata = { - "RequestId": "request-id", - "HTTPStatusCode": 200, - "RetryAttempts": 2, - } - metadata[field] = value - - attributes = _get_response_attributes({"ResponseMetadata": metadata}) - expected = { - SPANDATA.AWS_REQUEST_ID: "request-id", - SPANDATA.HTTP_STATUS_CODE: 200, - SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, - } - expected.pop(attribute) - - # One malformed optional field must not discard other valid metadata. - assert attributes == expected - - -@pytest.mark.parametrize( - "error_response", - [None, {"Code": ""}, {"Code": 123}], -) -def test_get_error_attributes_ignores_malformed_client_error_code(error_response): - error = ClientError( - { - "Error": {"Code": "placeholder"}, - "ResponseMetadata": {"HTTPStatusCode": 400}, - }, - "HeadObject", - ) - # `ClientError` itself expects `Error` to be a dict, so corrupt the stored - # response afterward to exercise defensive handling of arbitrary metadata. - error.response["Error"] = error_response - - assert _get_error_attributes(error) == { - SPANDATA.HTTP_STATUS_CODE: 400, - SPANDATA.ERROR_TYPE: "botocore.exceptions.ClientError", - } - - @pytest.mark.parametrize( ( "service_name", @@ -422,37 +296,6 @@ def test_client_call_omits_missing_region( assert SPANDATA.CLOUD_REGION not in _span_attributes(span, span_streaming) -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_client_call_has_response_attributes( - capture_items, - client_factory, - span_streaming, -): - client = client_factory() - span = _capture_stubbed_client_span( - client, - "head_object", - {"Bucket": "bucket", "Key": "foo"}, - capture_items, - span_streaming, - response={ - "ResponseMetadata": { - "HTTPStatusCode": 200, - "RequestId": "request-id", - "HostId": "extended-request-id", - "RetryAttempts": 0, - } - }, - ) - attributes = _span_attributes(span, span_streaming) - - assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 - assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" - assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" - assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in attributes - assert SPANDATA.ERROR_TYPE not in attributes - - @pytest.mark.parametrize("span_streaming", [True, False]) def test_retry_attempts_share_one_client_span( capture_items, @@ -474,8 +317,6 @@ def test_retry_attempts_share_one_client_span( # all `AWSRequest` instances created during retries reference the same client span. assert len(set(request_span_ids)) == 1 assert len(client_spans) == 1 - attributes = _span_attributes(client_spans[0], span_streaming) - assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == attempt_count - 1 @pytest.mark.parametrize("span_streaming", [True, False]) @@ -499,57 +340,6 @@ def attempt_failed_head_object_call(): assert len(request_span_ids) == 2 assert len(set(request_span_ids)) == 1 _assert_one_failed_span(client_spans, span_streaming) - attributes = _span_attributes(client_spans[0], span_streaming) - assert attributes[SPANDATA.HTTP_STATUS_CODE] == 500 - assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 - - -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_client_error_has_response_attributes_and_is_unchanged( - capture_items, - client_factory, - span_streaming, -): - client = client_factory() - original_exception = ClientError( - { - "Error": { - "Code": "AccessDeniedException", - "Message": "must not become a span attribute", - }, - "ResponseMetadata": { - "RequestId": "request-id", - "HTTPStatusCode": 403, - "RetryAttempts": 1, - }, - }, - "HeadObject", - ) - - def raise_client_error(**kwargs): - raise original_exception - - client.meta.events.register("before-parameter-build", raise_client_error) - - def invoke_failing_client_method(): - with pytest.raises(ClientError) as exc_info: - client.head_object(Bucket="bucket", Key="foo") - assert exc_info.value is original_exception - - spans_by_op = _capture_boto3_spans_by_op( - invoke_failing_client_method, capture_items, span_streaming - ) - client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) - _assert_one_failed_span(client_spans, span_streaming) - attributes = _span_attributes(client_spans[0], span_streaming) - - assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" - assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 - assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 - assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" - assert "Error.Message" not in attributes - assert "exception.message" not in attributes - assert "error.message" not in attributes @pytest.mark.parametrize( @@ -590,136 +380,6 @@ def invoke_failing_client_method(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) _assert_one_failed_span(client_spans, span_streaming) - attributes = _span_attributes(client_spans[0], span_streaming) - expected_error_type = ( - "botocore.exceptions.EndpointConnectionError" - if event_name == "before-send" - else "ValueError" - ) - assert attributes[SPANDATA.ERROR_TYPE] == expected_error_type - - -@pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_response_attribute_extraction_failure_does_not_change_response( - capture_items, - client_factory, - monkeypatch, - span_streaming, -): - client = client_factory() - api_params = {"Bucket": "bucket", "Key": "foo"} - original_response = { - "ResponseMetadata": { - "HTTPStatusCode": 200, - } - } - returned_responses = [] - - def fail_attribute_extraction(response): - raise RuntimeError("attribute extraction failed") - - monkeypatch.setattr( - "sentry_sdk.integrations.boto3._instrumentation._get_response_attributes", - fail_attribute_extraction, - ) - - def invoke_client_method(): - returned_responses.append(client.head_object(**api_params)) - - with Stubber(client) as stubber: - stubber.add_response("head_object", original_response, api_params) - spans_by_op = _capture_boto3_spans_by_op( - invoke_client_method, capture_items, span_streaming - ) - - client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) - assert len(returned_responses) == 1 - assert returned_responses[0] is original_response - assert len(client_spans) == 1 - _assert_span_finished(client_spans[0], span_streaming) - - -@pytest.mark.tests_internal_exceptions -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_error_attribute_extraction_failure_does_not_replace_original_exception( - capture_items, - client_factory, - monkeypatch, - span_streaming, -): - client = client_factory() - original_exception = ValueError("parameter processing failed") - - def raise_original_exception(**kwargs): - raise original_exception - - def fail_attribute_extraction(exception): - raise RuntimeError("attribute extraction failed") - - client.meta.events.register("before-parameter-build", raise_original_exception) - monkeypatch.setattr( - "sentry_sdk.integrations.boto3._instrumentation._get_error_attributes", - fail_attribute_extraction, - ) - - def invoke_failing_client_method(): - with pytest.raises(ValueError) as exc_info: - client.head_object(Bucket="bucket", Key="foo") - assert exc_info.value is original_exception - - spans_by_op = _capture_boto3_spans_by_op( - invoke_failing_client_method, capture_items, span_streaming - ) - client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) - - assert len(client_spans) == 1 - assert client_spans[0]["status"] in ("error", "internal_error") - _assert_span_finished(client_spans[0], span_streaming) - - -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_streaming_response_attributes_belong_to_client_span( - capture_items, - client_factory, - span_streaming, -): - client = client_factory() - - def respond(request, **kwargs): - return AWSResponse( - request.url, - 200, - { - "content-length": "5", - "x-amz-request-id": "request-id", - }, - Body(b"hello"), - ) - - client.meta.events.register("before-send", respond) - - def invoke_client_method_and_read_body(): - body = client.get_object(Bucket="bucket", Key="foo")["Body"] - assert body.read() == b"hello" - assert body.read() == b"" - - spans_by_op = _capture_boto3_spans_by_op( - invoke_client_method_and_read_body, capture_items, span_streaming - ) - client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) - stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) - - assert len(client_spans) == 1 - assert len(stream_spans) == 1 - client_attributes = _span_attributes(client_spans[0], span_streaming) - stream_attributes = _span_attributes(stream_spans[0], span_streaming) - assert client_attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" - assert client_attributes[SPANDATA.HTTP_STATUS_CODE] == 200 - assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in client_attributes - assert SPANDATA.AWS_REQUEST_ID not in stream_attributes - assert SPANDATA.HTTP_STATUS_CODE not in stream_attributes - @pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_body_read_failure_finishes_stream_span( @@ -761,6 +421,5 @@ def invoke_client_method_and_read_body(): stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) assert len(client_spans) == 1 - _assert_one_failed_span(stream_spans, span_streaming) - attributes = _span_attributes(stream_spans[0], span_streaming) - assert attributes[SPANDATA.ERROR_TYPE] == "OSError" + assert len(stream_spans) == 1 + _assert_span_finished(stream_spans[0], span_streaming) From 93929e64d8bf78b54b2d204be6245f5aefd2fa95 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 16:57:03 +0200 Subject: [PATCH 11/12] fix(tests): simplify response handling --- tests/integrations/boto3/test_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 5d3656c61e..a8e9540799 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -121,10 +121,9 @@ def _capture_stubbed_client_span( api_params, capture_items, span_streaming, - response=None, ): with Stubber(client) as stubber: - stubber.add_response(method_name, response or {}, api_params) + stubber.add_response(method_name, {}, api_params) spans_by_op = _capture_boto3_spans_by_op( lambda: getattr(client, method_name)(**api_params), capture_items, From e384637645c9ca6d2a21ec8be1cc1cdca1e6fb1b Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 11 Sep 2026 17:08:22 +0200 Subject: [PATCH 12/12] refactor(tests): update client tests to include endpoint URL and server attributes; remove direct testing of `_get_server_attributes` --- tests/integrations/boto3/test_client.py | 66 ++++++++----------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index a8e9540799..6ddfd145e5 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -8,9 +8,6 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration -from sentry_sdk.integrations.boto3._instrumentation import ( - _get_server_attributes, -) from tests.integrations.boto3.aws_mock import Body session = boto3.Session( # type: ignore[attr-defined] @@ -147,7 +144,9 @@ def _span_attributes(span, span_streaming): "span_name", "rpc_service", "rpc_method", + "endpoint_url", "server_address", + "server_port", ), [ ( @@ -157,7 +156,9 @@ def _span_attributes(span, span_streaming): "S3.HeadObject", "S3", "HeadObject", - "s3.eu-north-1.amazonaws.com", + "http://localhost:4566", + "localhost", + 4566, ), ( "events", @@ -166,7 +167,9 @@ def _span_attributes(span, span_streaming): "EventBridge.ListEventBuses", "EventBridge", "ListEventBuses", + None, "events.eu-north-1.amazonaws.com", + 443, ), ], ) @@ -181,9 +184,11 @@ def test_client_call_has_common_attributes( span_name, rpc_service, rpc_method, + endpoint_url, server_address, + server_port, ): - client = client_factory(service_name=service_name) + client = client_factory(service_name=service_name, endpoint_url=endpoint_url) span = _capture_stubbed_client_span( client, method_name, @@ -199,7 +204,7 @@ def test_client_call_has_common_attributes( assert attributes[SPANDATA.RPC_SYSTEM_NAME] == "aws-api" assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1" assert attributes[SPANDATA.SERVER_ADDRESS] == server_address - assert attributes[SPANDATA.SERVER_PORT] == 443 + assert attributes[SPANDATA.SERVER_PORT] == server_port def test_client_call_attributes_are_available_at_span_creation( @@ -241,47 +246,18 @@ def test_client_call_attributes_are_available_at_span_creation( assert client_spans == [] -@pytest.mark.parametrize( - ("endpoint_url", "expected"), - [ - ( - "http://localhost:4566", - { - SPANDATA.SERVER_ADDRESS: "localhost", - SPANDATA.SERVER_PORT: 4566, - }, - ), - ( - "https://aws.example.test:8443", - { - SPANDATA.SERVER_ADDRESS: "aws.example.test", - SPANDATA.SERVER_PORT: 8443, - }, - ), - ( - "https://[2001:db8::1]:9443", - { - SPANDATA.SERVER_ADDRESS: "2001:db8::1", - SPANDATA.SERVER_PORT: 9443, - }, - ), - (None, {}), - ("not-an-endpoint", {}), - ("https://example.com:not-a-port", {}), - ], -) -def test_get_server_attributes(endpoint_url, expected): - assert _get_server_attributes(endpoint_url) == expected - - -@pytest.mark.parametrize("span_streaming", [True, False]) def test_client_call_omits_missing_region( + sentry_init, capture_items, - client_factory, monkeypatch, - span_streaming, ): - client = client_factory() + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + trace_lifecycle="stream", + server_name="", + ) + client = session.client("s3") monkeypatch.setattr(client.meta.config, "region_name", None) span = _capture_stubbed_client_span( @@ -289,10 +265,10 @@ def test_client_call_omits_missing_region( "head_object", {"Bucket": "bucket", "Key": "foo"}, capture_items, - span_streaming, + span_streaming=True, ) - assert SPANDATA.CLOUD_REGION not in _span_attributes(span, span_streaming) + assert SPANDATA.CLOUD_REGION not in span["attributes"] @pytest.mark.parametrize("span_streaming", [True, False])