From a033d030710cfa8e5930b3a8f4995c31114c7593 Mon Sep 17 00:00:00 2001 From: Erica Pisani Date: Fri, 31 Jul 2026 15:26:18 -0400 Subject: [PATCH] feat(openai): Make include_prompts override data collection when set Change the default of OpenAIIntegration's include_prompts from True to None (unset). When unset, prompt collection follows the global data_collection.gen_ai.inputs setting; when explicitly set, it takes precedence over the global setting in both directions. The unset default is resolved lazily at call time rather than in __init__, so integrations constructed before sentry_sdk.init() still see the active client's options. Pre-data-collection behavior (defaults to True, subject to send_default_pii) is preserved. Refs PY-2588 --- sentry_sdk/integrations/openai.py | 114 +++++++----- tests/integrations/openai/test_openai.py | 220 ++++++++++++++++++++--- 2 files changed, 271 insertions(+), 63 deletions(-) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 304a3a0899..c080c0db8d 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -114,7 +114,7 @@ class OpenAIIntegration(Integration): def __init__( self: "OpenAIIntegration", - include_prompts: bool = True, + include_prompts: "Optional[bool]" = None, tiktoken_encoding_name: "Optional[str]" = None, ) -> None: self.include_prompts = include_prompts @@ -325,6 +325,18 @@ def _calculate_responses_token_usage( ) +def _set_tool_definitions_responses_api( + set_on_span: "Callable[[str, Any], None]", + kwargs: "dict[str, Any]", +) -> None: + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_responses(tools)), + ) + + def _set_responses_api_input_data( span: "Union[Span, StreamedSpan]", kwargs: "dict[str, Any]", @@ -371,33 +383,24 @@ def _set_responses_api_input_data( client_options = sentry_sdk.get_client().options if has_data_collection_enabled(client_options): - if ( - integration.include_prompts - and client_options["data_collection"]["gen_ai"]["inputs"] - ): - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + if integration.include_prompts is not None: + if integration.include_prompts: + _set_tool_definitions_responses_api(set_on_span, kwargs) + elif client_options["data_collection"]["gen_ai"]["inputs"]: + _set_tool_definitions_responses_api(set_on_span, kwargs) else: # Pre-data collection this was always set, so this needs to be left here for now until # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" # line below - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_responses(tools)), - ) + _set_tool_definitions_responses_api(set_on_span, kwargs) if has_data_collection_enabled(client_options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client_options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client_options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -478,6 +481,18 @@ def _set_responses_api_input_data( ) +def _set_tool_definitions_completions_api( + set_on_span: "Callable[[str, Any], None]", + kwargs: "dict[str, Any]", +) -> None: + tools = kwargs.get("tools") + if tools is not None and _is_given(tools): + set_on_span( + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + json.dumps(_transform_tool_definitions_completions(tools)), + ) + + def _set_completions_api_input_data( span: "Union[Span, StreamedSpan]", kwargs: "dict[str, Any]", @@ -519,27 +534,17 @@ def _set_completions_api_input_data( client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): - if ( - integration.include_prompts - and client.options["data_collection"]["gen_ai"]["inputs"] - ): - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_completions(tools)), - ) + if integration.include_prompts is not None: + if integration.include_prompts: + _set_tool_definitions_completions_api(set_on_span, kwargs) + elif client.options["data_collection"]["gen_ai"]["inputs"]: + _set_tool_definitions_completions_api(set_on_span, kwargs) else: # Pre-data collection this was always set, so this needs to be left here for now until # we deprecate `send_default_pii`. Once we do, this 'else' branch should be removed, # and the above branch placed below the "if not should_send_default_pii() or not integration.include_prompts" # line below - tools = kwargs.get("tools") - if tools is not None and _is_given(tools): - set_on_span( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions_completions(tools)), - ) + _set_tool_definitions_completions_api(set_on_span, kwargs) messages: "Optional[Union[str, Iterable[ChatCompletionMessageParam]]]" = kwargs.get( "messages" @@ -547,9 +552,10 @@ def _set_completions_api_input_data( if has_data_collection_enabled(client.options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client.options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client.options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -628,9 +634,10 @@ def _set_embeddings_input_data( client = sentry_sdk.get_client() if has_data_collection_enabled(client.options): # This takes precedence over the global data collection settings - if not integration.include_prompts: - return - if not client.options["data_collection"]["gen_ai"]["inputs"]: + if integration.include_prompts is not None: + if not integration.include_prompts: + return + elif not client.options["data_collection"]["gen_ai"]["inputs"]: return elif not should_send_default_pii() or not integration.include_prompts: return @@ -763,12 +770,25 @@ def _set_common_output_data( span.__exit__(None, None, None) +def _resolve_include_prompts( + integration: "OpenAIIntegration", options: "Optional[dict[str, Any]]" +) -> None: + if integration.include_prompts is None and not has_data_collection_enabled(options): + # Pre-data-collection behavior: prompts were included by default (subject + # to send_default_pii). Resolve the unset default here, at call time, + # rather than in __init__ so that integrations constructed before + # sentry_sdk.init() still see the active client's options. + integration.include_prompts = True + + def _new_sync_chat_completion(f: "Any", *args: "Any", **kwargs: "Any") -> "Any": client = sentry_sdk.get_client() integration = client.get_integration(OpenAIIntegration) if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + if "messages" not in kwargs: # invalid call (in all versions of openai), let it return error return f(*args, **kwargs) @@ -850,6 +870,8 @@ async def _new_async_chat_completion(f: "Any", *args: "Any", **kwargs: "Any") -> if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + if "messages" not in kwargs: # invalid call (in all versions of openai), let it return error return await f(*args, **kwargs) @@ -1280,6 +1302,8 @@ def _new_sync_embeddings_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") if has_span_streaming_enabled(client.options): @@ -1338,6 +1362,8 @@ async def _new_async_embeddings_create( if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") if has_span_streaming_enabled(client.options): @@ -1418,6 +1444,8 @@ def _new_sync_responses_create(f: "Any", *args: "Any", **kwargs: "Any") -> "Any" if integration is None: return f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 @@ -1488,6 +1516,8 @@ async def _new_async_responses_create(f: "Any", *args: "Any", **kwargs: "Any") - if integration is None: return await f(*args, **kwargs) + _resolve_include_prompts(integration, client.options) + model = kwargs.get("model") # Same bool handling as in https://github.com/openai/openai-python/blob/acd0c54d8a68efeedde0e5b4e6c310eef1ce7867/src/openai/resources/responses/responses.py#L940 diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index e88b67e6d1..a929ed9752 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -158,6 +158,31 @@ async def __call__(self, *args, **kwargs): ] +@pytest.mark.parametrize( + "data_collection_enabled", + [True, False], + ids=["data-collection-enabled", "data-collection-disabled"], +) +@pytest.mark.parametrize("include_prompts", [None, True, False]) +def test_include_prompts_init_stores_passed_value( + sentry_init, data_collection_enabled, include_prompts +): + init_kwargs = { + "traces_sample_rate": 1.0, + "disabled_integrations": [StdlibIntegration], + } + if data_collection_enabled: + init_kwargs["_experiments"] = {"data_collection": {"gen_ai": {"inputs": True}}} + sentry_init(**init_kwargs) + + integration = OpenAIIntegration(include_prompts=include_prompts) + + # The constructor stores the value as passed; the None default is resolved + # lazily at call time based on the active client's options (covered by the + # data_collection parametrizations below). + assert integration.include_prompts is include_prompts + + @pytest.mark.skipif( OPENAI_VERSION <= (2, 10, 0), reason="ChatCompletionCustomToolParam is unavailable before.", @@ -686,13 +711,17 @@ def test_nonstreaming_chat_completion( pytest.param( {"gen_ai": {"inputs": False}}, True, - {}, - [ - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - ], - id="inputs-disabled", + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-enabled-overrides-inputs-disabled", ), pytest.param( {}, @@ -720,6 +749,47 @@ def test_nonstreaming_chat_completion( ], id="include-prompts-disabled-overrides-inputs-enabled", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + None, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + None, + {}, + [ + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + None, + { + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: json.dumps( + [{"type": "text", "content": "You are a helpful assistant."}] + ), + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + [{"role": "user", "content": "hello"}] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + id="include-prompts-default-follows-default-enabled", + ), ], ) def test_completions_api_data_collection( @@ -3026,8 +3096,8 @@ def _collect_embeddings_span_data( {"gen_ai": {"inputs": False}}, True, True, - False, - id="inputs-disabled-overrides-pii-enabled", + True, + id="include-prompts-overrides-inputs-disabled", ), pytest.param( {}, @@ -3040,8 +3110,8 @@ def _collect_embeddings_span_data( {"gen_ai": {"inputs": False}}, False, True, - False, - id="inputs-disabled-and-pii-disabled", + True, + id="include-prompts-overrides-inputs-disabled-and-pii-disabled", ), pytest.param( {"gen_ai": {"inputs": True}}, @@ -3057,6 +3127,27 @@ def _collect_embeddings_span_data( False, id="no-experiment-falls-back-to-pii", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + None, + True, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + None, + False, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + False, + None, + True, + id="include-prompts-default-follows-default-enabled", + ), ], ) def test_embeddings_create_data_collection( @@ -3142,7 +3233,7 @@ def test_embeddings_create_data_collection_inputs_disabled_input_shapes( span_streaming, ): sentry_init( - integrations=[OpenAIIntegration(include_prompts=True)], + integrations=[OpenAIIntegration(include_prompts=False)], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, send_default_pii=True, @@ -3442,8 +3533,8 @@ async def test_embeddings_create_async( {"gen_ai": {"inputs": False}}, True, True, - False, - id="inputs-disabled-overrides-pii-enabled", + True, + id="include-prompts-overrides-inputs-disabled", ), pytest.param( {}, @@ -3466,6 +3557,27 @@ async def test_embeddings_create_async( False, id="no-experiment-falls-back-to-pii", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + False, + None, + True, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + True, + None, + False, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + False, + None, + True, + id="include-prompts-default-follows-default-enabled", + ), ], ) async def test_embeddings_create_async_data_collection( @@ -5293,14 +5405,23 @@ def test_ai_client_span_responses_api( "input": "How do I check if a Python object is an instance of a class?", "tools": EXAMPLE_TOOLS, }, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - ], + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], True, - id="inputs-disabled", + id="include-prompts-enabled-overrides-inputs-disabled", ), pytest.param( {}, @@ -5346,6 +5467,63 @@ def test_ai_client_span_responses_api( False, id="include-prompts-disabled-overrides-inputs-enabled", ), + pytest.param( + {"gen_ai": {"inputs": True}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: safe_serialize( + [ + { + "type": "text", + "content": "You are a coding assistant that talks like a pirate.", + } + ] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [], + None, + id="include-prompts-default-follows-inputs-enabled", + ), + pytest.param( + {"gen_ai": {"inputs": False}}, + { + "instructions": "You are a coding assistant that talks like a pirate.", + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + {}, + [ + SPANDATA.GEN_AI_REQUEST_MESSAGES, + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, + ], + None, + id="include-prompts-default-follows-inputs-disabled", + ), + pytest.param( + {}, + { + "input": "How do I check if a Python object is an instance of a class?", + "tools": EXAMPLE_TOOLS, + }, + { + SPANDATA.GEN_AI_REQUEST_MESSAGES: safe_serialize( + ["How do I check if a Python object is an instance of a class?"] + ), + SPANDATA.GEN_AI_TOOL_DEFINITIONS: safe_serialize(EXAMPLE_TOOLS), + }, + [SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS], + None, + id="include-prompts-default-follows-default-enabled", + ), ], ) @pytest.mark.skipif(SKIP_RESPONSES_TESTS, reason="Responses API not available")