From a74c31c7ae3189344b8b79d2ee27b27cb3af5b84 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 11 Sep 2026 11:48:27 +0200 Subject: [PATCH 1/5] Python: Separate hosted and local shell calls Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../specs/004-python-function-calling-loop.md | 3 + .../agent_framework_openai/_chat_client.py | 62 +++--- .../tests/openai/test_openai_chat_client.py | 178 +++++++++++++++--- 3 files changed, 186 insertions(+), 57 deletions(-) diff --git a/docs/specs/004-python-function-calling-loop.md b/docs/specs/004-python-function-calling-loop.md index 23526ba47c..60cc165ea7 100644 --- a/docs/specs/004-python-function-calling-loop.md +++ b/docs/specs/004-python-function-calling-loop.md @@ -392,6 +392,8 @@ that manually replay messages own the equivalent rule: do not resend an approval Changing arguments after security middleware has processed them fails closed with `MiddlewareFailure`. - Argument-validation failures after middleware retain the established `Argument parsing failed` result contract. Exceptions raised by middleware or the tool body retain the separate `Function failed` contract. +- Provider-hosted shell calls remain informational transcript content. Only a distinct, well-formed local-shell call + with an explicitly configured local executor can enter the local function loop and its approval boundary. ### Reasoning-bound calls @@ -514,6 +516,7 @@ that manually replay messages own the equivalent rule: do not resend an approval | Calls across response messages | Every actionable call is executed once. | `test_base_client_executes_function_calls_across_multiple_response_messages` | | Parallel calls | Results retain the corresponding call ids and execution count. | `test_max_function_calls_limits_parallel_invocations`, `test_streaming_multiple_function_calls_parallel_execution` | | Informational-only call | The call is returned but not executed or approved. | `test_informational_only_function_call_is_not_invoked`, `test_informational_only_function_call_does_not_request_approval`, `test_streaming_informational_only_function_call_is_not_invoked` | +| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell calls can execute, and local execution preserves its configured approval mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | | Declaration-only call | The call is surfaced as user input and is not executed; streaming arguments appear once while finalized request metadata remains available. | `test_declaration_only_tool`, `test_streaming_declaration_only_tool_preserves_metadata_without_duplicate_arguments` | | Function invocation disabled | The client bypasses the invocation loop without losing invocation kwargs. | `test_function_invocation_config_enabled_false`, `test_function_invocation_config_enabled_false_preserves_invocation_kwargs`, `test_streaming_function_invocation_config_enabled_false` | | Runtime tool changes | Added tools become available on the next iteration and retain approval behavior. | `test_add_tools_available_next_iteration`, `test_add_tools_with_approval_required_tool` | diff --git a/python/packages/openai/agent_framework_openai/_chat_client.py b/python/packages/openai/agent_framework_openai/_chat_client.py index 60c6c2aafd..b3e96e4098 100644 --- a/python/packages/openai/agent_framework_openai/_chat_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_client.py @@ -1255,7 +1255,7 @@ def get_shell_tool( dict with explicit hosted container settings. name: Optional local tool name when ``func`` is provided. description: Optional local tool description when ``func`` is provided. - approval_mode: Optional local tool approval mode. + approval_mode: Optional local tool approval mode. Plain callables default to approval required. Returns: A hosted shell declaration or a local shell FunctionTool. @@ -1302,7 +1302,7 @@ def get_shell_tool( func=func, name=name, description=description, - approval_mode=approval_mode, + approval_mode=approval_mode or "always_require", ) if base_tool.func is None: @@ -2175,11 +2175,6 @@ def _to_shell_call_output_payload(content: Content) -> list[dict[str, Any]]: } ] - @staticmethod - def _join_shell_commands(commands: Sequence[str]) -> str: - """Join shell commands into a single executable command string.""" - return "\n".join(command for command in commands if command).strip() - def _shell_item_to_contents(self, item: Any, local_shell_tool_name: str | None) -> list[Content]: """Convert a shell output item into framework ``Content`` objects. @@ -2201,36 +2196,31 @@ def _shell_item_to_contents(self, item: Any, local_shell_tool_name: str | None) shell_commands = list(getattr(action, "commands", []) or []) shell_timeout_ms = getattr(action, "timeout_ms", None) shell_max_output = getattr(action, "max_output_length", None) - if local_shell_tool_name: - command_text = self._join_shell_commands(shell_commands) - contents.append( - Content.from_function_call( - call_id=shell_call_id, - name=local_shell_tool_name, - arguments=json.dumps({"command": command_text}), - additional_properties={ - OPENAI_SHELL_OUTPUT_TYPE_KEY: OPENAI_SHELL_OUTPUT_TYPE_SHELL_CALL, - OPENAI_LOCAL_SHELL_COMMAND_PARTS_KEY: shell_commands, - }, - raw_representation=item, - ) - ) - else: - contents.append( - Content.from_shell_tool_call( - call_id=shell_call_id, - commands=shell_commands, - timeout_ms=shell_timeout_ms, - max_output_length=shell_max_output, - status=getattr(item, "status", None), - raw_representation=item, - ) + contents.append( + Content.from_shell_tool_call( + call_id=shell_call_id, + commands=shell_commands, + timeout_ms=shell_timeout_ms, + max_output_length=shell_max_output, + status=getattr(item, "status", None), + raw_representation=item, ) + ) elif item_type == "local_shell_call": - local_call_id = getattr(item, "call_id", None) or "" - local_command_parts = list(getattr(getattr(item, "action", None), "command", []) or []) + raw_local_call_id = getattr(item, "call_id", None) + local_call_id = raw_local_call_id if isinstance(raw_local_call_id, str) else "" + raw_local_call_item_id = getattr(item, "id", None) + local_call_item_id = raw_local_call_item_id if isinstance(raw_local_call_item_id, str) else "" + raw_local_command_parts = getattr(getattr(item, "action", None), "command", None) + local_command_parts: list[str] = ( + cast("list[str]", raw_local_command_parts) + if isinstance(raw_local_command_parts, list) + and raw_local_command_parts + and all(isinstance(part, str) for part in cast("list[object]", raw_local_command_parts)) + else [] + ) local_command = shlex.join(local_command_parts) if local_command_parts else "" - if local_shell_tool_name: + if local_shell_tool_name and local_call_id and local_call_item_id and local_command_parts: contents.append( Content.from_function_call( call_id=local_call_id, @@ -2238,7 +2228,7 @@ def _shell_item_to_contents(self, item: Any, local_shell_tool_name: str | None) arguments=json.dumps({"command": local_command}), additional_properties={ OPENAI_SHELL_OUTPUT_TYPE_KEY: OPENAI_SHELL_OUTPUT_TYPE_LOCAL_SHELL_CALL, - OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY: getattr(item, "id", None), + OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY: local_call_item_id, OPENAI_LOCAL_SHELL_COMMAND_PARTS_KEY: local_command_parts, }, raw_representation=item, @@ -2849,6 +2839,7 @@ def _parse_response_from_openai( call_id=item.call_id, name=item.name, arguments=item.arguments, + informational_only=item.name == local_shell_tool_name, additional_properties={"fc_id": item.id, "status": item.status}, raw_representation=item, ) @@ -3348,6 +3339,7 @@ def output_text_properties(output: Any) -> dict[str, Any] | None: call_id=call_id, name=name, arguments=event.delta, + informational_only=name == local_shell_tool_name, additional_properties={ "output_index": event.output_index, "fc_id": event.item_id, diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index 58d625fb0e..ffbe578deb 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -2183,6 +2183,15 @@ def test_shell_output_payloads_do_not_expose_exception_diagnostics() -> None: empty_shell_payload = OpenAIChatClient._to_shell_call_output_payload(empty_diagnostic) assert empty_local_payload["exit_code"] == 1 assert empty_shell_payload[0]["outcome"] == {"type": "exit", "exit_code": 1} +def test_get_shell_tool_local_executor_requires_approval_by_default() -> None: + """A plain local shell callable must retain the normal approval boundary.""" + + def local_exec(command: str) -> str: + return command + + local_shell_tool = OpenAIChatClient.get_shell_tool(func=local_exec) + + assert local_shell_tool.approval_mode == "always_require" def test_prepared_local_shell_tool_survives_make_tools() -> None: @@ -2273,9 +2282,36 @@ def local_exec(command: str) -> str: assert call_content.additional_properties[OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY] == "local-shell-item-1" +@pytest.mark.parametrize( + ("item_type", "item_id", "command_parts"), + [ + ("local_shell_call", None, ["echo", "ok"]), + ("local_shell_call", "item-1", "echo ok"), + ("forged_local_shell_call", "item-1", ["echo", "ok"]), + ], +) +def test_malformed_local_shell_call_is_not_executable( + item_type: str, + item_id: str | None, + command_parts: Any, +) -> None: + """Malformed or unknown local shell items must not become function calls.""" + client = OpenAIChatClient(model="test-model", api_key="test-key") + item = MagicMock() + item.type = item_type + item.id = item_id + item.call_id = "local-shell-call-1" + item.action.command = command_parts + item.status = "completed" + + contents = client._shell_item_to_contents(item, "run_shell") + + assert all(content.type != "function_call" for content in contents) + + @pytest.mark.asyncio -async def test_local_shell_tool_is_invoked_in_function_loop() -> None: - """Test local shell call executes executor and sends local_shell_call_output.""" +async def test_local_shell_tool_requires_approval_before_function_loop_execution() -> None: + """An explicit local shell call executes only after approval.""" client = OpenAIChatClient(model="test-model", api_key="test-key") executed_commands: list[str] = [] @@ -2285,7 +2321,7 @@ def local_exec(command: str) -> str: local_shell_tool = OpenAIChatClient.get_shell_tool( func=local_exec, - approval_mode="never_require", + approval_mode="always_require", ) mock_response1 = MagicMock() @@ -2333,11 +2369,22 @@ def local_exec(command: str) -> str: with patch.object( client.client.responses, "create", side_effect=[_as_raw(mock_response1), _as_raw(mock_response2)] ) as mock_create: - await client.get_response( + response = await client.get_response( messages=[Message(role="user", contents=["What Python version is available?"])], options={"tools": [local_shell_tool]}, ) + assert executed_commands == [] + assert mock_create.call_count == 1 + approval_request = next( + content for content in response.messages[0].contents if content.type == "function_approval_request" + ) + approval_response = approval_request.to_function_approval_response(approved=True) + await client.get_response( + messages=[Message(role="user", contents=[approval_response])], + options={"tools": [local_shell_tool]}, + ) + assert executed_commands == ["python --version"] assert mock_create.call_count == 2 second_call_input = mock_create.call_args_list[1].kwargs["input"] @@ -2348,8 +2395,8 @@ def local_exec(command: str) -> str: @pytest.mark.asyncio -async def test_shell_call_is_invoked_as_local_shell_function_loop() -> None: - """Test shell_call maps to local function invocation and returns shell_call_output.""" +async def test_mixed_shell_calls_only_invoke_explicit_local_shell_call() -> None: + """A hosted shell call remains informational alongside an executable local call.""" client = OpenAIChatClient(model="test-model", api_key="test-key") executed_commands: list[str] = [] @@ -2374,7 +2421,7 @@ def local_exec(command: str) -> str: mock_response1.incomplete = None mock_action = MagicMock() - mock_action.commands = ["python --version"] + mock_action.commands = ["pwd"] mock_action.timeout_ms = 30000 mock_action.max_output_length = 4096 @@ -2384,7 +2431,18 @@ def local_exec(command: str) -> str: mock_shell_call.call_id = "shell-call-1" mock_shell_call.action = mock_action mock_shell_call.status = "completed" - mock_response1.output = [mock_shell_call] + + mock_local_action = MagicMock() + mock_local_action.command = ["python", "--version"] + mock_local_action.timeout_ms = 30000 + + mock_local_shell_call = MagicMock() + mock_local_shell_call.type = "local_shell_call" + mock_local_shell_call.id = "local-shell-item-1" + mock_local_shell_call.call_id = "local-shell-call-1" + mock_local_shell_call.action = mock_local_action + mock_local_shell_call.status = "completed" + mock_response1.output = [mock_shell_call, mock_local_shell_call] mock_response2 = MagicMock() mock_response2.output_parsed = None @@ -2416,13 +2474,10 @@ def local_exec(command: str) -> str: assert executed_commands == ["python --version"] assert mock_create.call_count == 2 second_call_input = mock_create.call_args_list[1].kwargs["input"] - shell_outputs = [item for item in second_call_input if item.get("type") == "shell_call_output"] - assert len(shell_outputs) == 1 - assert shell_outputs[0]["call_id"] == "shell-call-1" - assert isinstance(shell_outputs[0]["output"], list) - assert shell_outputs[0]["output"][0]["stdout"] == "Python 3.13.0" + assert all(item.get("type") != "shell_call_output" for item in second_call_input) local_shell_outputs = [item for item in second_call_input if item.get("type") == "local_shell_call_output"] - assert len(local_shell_outputs) == 0 + assert len(local_shell_outputs) == 1 + assert local_shell_outputs[0]["id"] == "local-shell-item-1" async def test_tool_loop_store_false_replays_encrypted_reasoning_group() -> None: @@ -2552,10 +2607,15 @@ async def test_stateless_request_rejects_non_replayable_reasoning_bound_mcp_outp create.assert_not_awaited() -def test_response_content_creation_with_shell_call() -> None: - """Test _parse_response_from_openai with shell_call output.""" +def test_response_content_creation_with_shell_call_remains_hosted_with_local_tool() -> None: + """A hosted shell call remains informational when a local shell tool is configured.""" client = OpenAIChatClient(model="test-model", api_key="test-key") + def local_exec(command: str) -> str: + return command + + local_shell_tool = OpenAIChatClient.get_shell_tool(func=local_exec) + mock_response = MagicMock() mock_response.output_parsed = None mock_response.metadata = {} @@ -2579,7 +2639,7 @@ def test_response_content_creation_with_shell_call() -> None: mock_response.output = [mock_shell_call] - response = client._parse_response_from_openai(mock_response, options={}) # type: ignore + response = client._parse_response_from_openai(mock_response, options={"tools": [local_shell_tool]}) # type: ignore[arg-type] assert len(response.messages[0].contents) == 1 call_content = response.messages[0].contents[0] @@ -2710,6 +2770,39 @@ def test_response_content_creation_with_function_call() -> None: assert function_call.informational_only is False +def test_response_function_call_named_local_shell_is_informational() -> None: + """A generic function call cannot impersonate the configured local shell item.""" + client = OpenAIChatClient(model="test-model", api_key="test-key") + + def local_exec(command: str) -> str: + return command + + local_shell_tool = OpenAIChatClient.get_shell_tool(func=local_exec, approval_mode="never_require") + mock_response = MagicMock() + mock_response.output_parsed = None + mock_response.metadata = {} + mock_response.usage = None + mock_response.id = "test-id" + mock_response.model = "test-model" + mock_response.created_at = 1000000000 + + mock_function_call_item = MagicMock() + mock_function_call_item.type = "function_call" + mock_function_call_item.call_id = "call_123" + mock_function_call_item.name = local_shell_tool.name + mock_function_call_item.arguments = '{"command": "echo blocked"}' + mock_function_call_item.id = "fc_456" + mock_function_call_item.status = "completed" + mock_response.output = [mock_function_call_item] + + response = client._parse_response_from_openai(mock_response, options={"tools": [local_shell_tool]}) # type: ignore[arg-type] + + function_call = response.messages[0].contents[0] + assert function_call.type == "function_call" + assert function_call.name == local_shell_tool.name + assert function_call.informational_only is True + + def test_parse_response_from_openai_with_custom_tool_call_is_informational_only() -> None: """Custom tool calls are hosted Responses items, not local Agent Framework function calls.""" client = OpenAIChatClient(model="test-model", api_key="test-key") @@ -3175,6 +3268,48 @@ def test_parse_chunk_from_openai_function_call_is_actionable() -> None: assert update.contents[0].informational_only is False +def test_parse_chunk_function_call_named_local_shell_is_informational() -> None: + """A streamed generic function call cannot impersonate a local shell item.""" + client = OpenAIChatClient(model="test-model", api_key="test-key") + + def local_exec(command: str) -> str: + return command + + local_shell_tool = OpenAIChatClient.get_shell_tool(func=local_exec, approval_mode="never_require") + chat_options: dict[str, Any] = {"tools": [local_shell_tool]} + function_call_ids: dict[int, tuple[str, str]] = {} + + added_event = MagicMock() + added_event.type = "response.output_item.added" + added_event.output_index = 0 + added_item = MagicMock() + added_item.type = "function_call" + added_item.call_id = "call_123" + added_item.name = local_shell_tool.name + added_event.item = added_item + + delta_event = MagicMock() + delta_event.type = "response.function_call_arguments.delta" + delta_event.output_index = 0 + delta_event.delta = '{"command": "echo blocked"}' + delta_event.item_id = "fc_456" + + client._parse_chunk_from_openai( + added_event, + options=chat_options, + function_call_ids=function_call_ids, + ) + update = client._parse_chunk_from_openai( + delta_event, + options=chat_options, + function_call_ids=function_call_ids, + ) + + assert len(update.contents) == 1 + assert update.contents[0].type == "function_call" + assert update.contents[0].informational_only is True + + def test_parse_chunk_from_openai_custom_tool_call_done_is_informational_only() -> None: client = OpenAIChatClient(model="test-model", api_key="test-key") chat_options: dict[str, Any] = {} @@ -3266,8 +3401,8 @@ def local_exec(command: str) -> str: assert update.contents == [] -def test_parse_chunk_from_openai_shell_call_done_emits_command() -> None: - """A completed shell_call on output_item.done must emit a function call with the real command.""" +def test_parse_chunk_from_openai_shell_call_done_remains_hosted() -> None: + """A completed hosted shell call remains informational in streaming output.""" client = OpenAIChatClient(model="test-model", api_key="test-key") def local_exec(command: str) -> str: @@ -3298,10 +3433,9 @@ def local_exec(command: str) -> str: assert len(update.contents) == 1 call_content = update.contents[0] - assert call_content.type == "function_call" + assert call_content.type == "shell_tool_call" assert call_content.call_id == "shell-call-1" - assert call_content.name == local_shell_tool.name - assert call_content.parse_arguments() == {"command": "ls -la"} + assert call_content.commands == ["ls -la"] def test_parse_chunk_from_openai_local_shell_call_done_emits_command() -> None: From cd36947293bb420d9de8fe79a13df25e79a77f1c Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 11 Sep 2026 16:58:10 +0200 Subject: [PATCH 2/5] Python: Preserve local shell approval default Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../openai/agent_framework_openai/_chat_client.py | 4 ++-- .../openai/tests/openai/test_openai_chat_client.py | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/python/packages/openai/agent_framework_openai/_chat_client.py b/python/packages/openai/agent_framework_openai/_chat_client.py index b3e96e4098..c75beb6de4 100644 --- a/python/packages/openai/agent_framework_openai/_chat_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_client.py @@ -1255,7 +1255,7 @@ def get_shell_tool( dict with explicit hosted container settings. name: Optional local tool name when ``func`` is provided. description: Optional local tool description when ``func`` is provided. - approval_mode: Optional local tool approval mode. Plain callables default to approval required. + approval_mode: Optional local tool approval mode. Returns: A hosted shell declaration or a local shell FunctionTool. @@ -1302,7 +1302,7 @@ def get_shell_tool( func=func, name=name, description=description, - approval_mode=approval_mode or "always_require", + approval_mode=approval_mode, ) if base_tool.func is None: diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index ffbe578deb..5b7435f300 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -2183,16 +2183,6 @@ def test_shell_output_payloads_do_not_expose_exception_diagnostics() -> None: empty_shell_payload = OpenAIChatClient._to_shell_call_output_payload(empty_diagnostic) assert empty_local_payload["exit_code"] == 1 assert empty_shell_payload[0]["outcome"] == {"type": "exit", "exit_code": 1} -def test_get_shell_tool_local_executor_requires_approval_by_default() -> None: - """A plain local shell callable must retain the normal approval boundary.""" - - def local_exec(command: str) -> str: - return command - - local_shell_tool = OpenAIChatClient.get_shell_tool(func=local_exec) - - assert local_shell_tool.approval_mode == "always_require" - def test_prepared_local_shell_tool_survives_make_tools() -> None: """Regression: the prepared shell tool must be a subscriptable dict. From 82ef702025a60227743ddb544cbc3f1debd817c6 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 14 Sep 2026 10:12:18 +0200 Subject: [PATCH 3/5] Python: Distinguish local shell environments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../specs/004-python-function-calling-loop.md | 7 +- .../agent_framework_openai/_chat_client.py | 62 ++++++++++---- .../tests/openai/test_openai_chat_client.py | 83 ++++++++++++++----- 3 files changed, 113 insertions(+), 39 deletions(-) diff --git a/docs/specs/004-python-function-calling-loop.md b/docs/specs/004-python-function-calling-loop.md index 60cc165ea7..727f6a47a7 100644 --- a/docs/specs/004-python-function-calling-loop.md +++ b/docs/specs/004-python-function-calling-loop.md @@ -392,8 +392,9 @@ that manually replay messages own the equivalent rule: do not resend an approval Changing arguments after security middleware has processed them fails closed with `MiddlewareFailure`. - Argument-validation failures after middleware retain the established `Argument parsing failed` result contract. Exceptions raised by middleware or the tool body retain the separate `Function failed` contract. -- Provider-hosted shell calls remain informational transcript content. Only a distinct, well-formed local-shell call - with an explicitly configured local executor can enter the local function loop and its approval boundary. +- Provider-hosted shell calls remain informational transcript content. Only a well-formed explicit `local_shell_call` + or a shell call marked with a local environment, paired with a configured local executor, can enter the local + function loop and its approval boundary. ### Reasoning-bound calls @@ -516,7 +517,7 @@ that manually replay messages own the equivalent rule: do not resend an approval | Calls across response messages | Every actionable call is executed once. | `test_base_client_executes_function_calls_across_multiple_response_messages` | | Parallel calls | Results retain the corresponding call ids and execution count. | `test_max_function_calls_limits_parallel_invocations`, `test_streaming_multiple_function_calls_parallel_execution` | | Informational-only call | The call is returned but not executed or approved. | `test_informational_only_function_call_is_not_invoked`, `test_informational_only_function_call_does_not_request_approval`, `test_streaming_informational_only_function_call_is_not_invoked` | -| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell calls can execute, and local execution preserves its configured approval mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | +| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell items or shell calls marked with a local environment can execute, and local execution preserves its configured approval mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_parse_chunk_from_openai_local_environment_shell_call_done_emits_command`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_malformed_local_environment_shell_call_is_not_executable`, `test_response_content_creation_with_local_shell_call_maps_to_function_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | | Declaration-only call | The call is surfaced as user input and is not executed; streaming arguments appear once while finalized request metadata remains available. | `test_declaration_only_tool`, `test_streaming_declaration_only_tool_preserves_metadata_without_duplicate_arguments` | | Function invocation disabled | The client bypasses the invocation loop without losing invocation kwargs. | `test_function_invocation_config_enabled_false`, `test_function_invocation_config_enabled_false_preserves_invocation_kwargs`, `test_streaming_function_invocation_config_enabled_false` | | Runtime tool changes | Added tools become available on the next iteration and retain approval behavior. | `test_add_tools_available_next_iteration`, `test_add_tools_with_approval_required_tool` | diff --git a/python/packages/openai/agent_framework_openai/_chat_client.py b/python/packages/openai/agent_framework_openai/_chat_client.py index c75beb6de4..51fdffc850 100644 --- a/python/packages/openai/agent_framework_openai/_chat_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_client.py @@ -2188,24 +2188,52 @@ def _shell_item_to_contents(self, item: Any, local_shell_tool_name: str | None) contents: list[Content] = [] item_type = getattr(item, "type", None) if item_type == "shell_call": - shell_call_id = getattr(item, "call_id", None) or "" - shell_commands: list[str] = [] - shell_timeout_ms: int | None = None - shell_max_output: int | None = None - if action := getattr(item, "action", None): - shell_commands = list(getattr(action, "commands", []) or []) - shell_timeout_ms = getattr(action, "timeout_ms", None) - shell_max_output = getattr(action, "max_output_length", None) - contents.append( - Content.from_shell_tool_call( - call_id=shell_call_id, - commands=shell_commands, - timeout_ms=shell_timeout_ms, - max_output_length=shell_max_output, - status=getattr(item, "status", None), - raw_representation=item, - ) + raw_shell_call_id = getattr(item, "call_id", None) + shell_call_id = raw_shell_call_id if isinstance(raw_shell_call_id, str) else "" + raw_shell_call_item_id = getattr(item, "id", None) + shell_call_item_id = raw_shell_call_item_id if isinstance(raw_shell_call_item_id, str) else "" + action = getattr(item, "action", None) + raw_shell_commands = getattr(action, "commands", None) + shell_commands: list[str] = ( + cast("list[str]", raw_shell_commands) + if isinstance(raw_shell_commands, list) + and raw_shell_commands + and all(isinstance(command, str) for command in cast("list[object]", raw_shell_commands)) + else [] ) + shell_timeout_ms = getattr(action, "timeout_ms", None) + shell_max_output = getattr(action, "max_output_length", None) + is_local_environment = getattr(getattr(item, "environment", None), "type", None) == "local" + if ( + local_shell_tool_name + and is_local_environment + and shell_call_id + and shell_call_item_id + and shell_commands + ): + contents.append( + Content.from_function_call( + call_id=shell_call_id, + name=local_shell_tool_name, + arguments=json.dumps({"command": "\n".join(shell_commands).strip()}), + additional_properties={ + OPENAI_SHELL_OUTPUT_TYPE_KEY: OPENAI_SHELL_OUTPUT_TYPE_SHELL_CALL, + OPENAI_LOCAL_SHELL_COMMAND_PARTS_KEY: shell_commands, + }, + raw_representation=item, + ) + ) + else: + contents.append( + Content.from_shell_tool_call( + call_id=shell_call_id, + commands=shell_commands, + timeout_ms=shell_timeout_ms, + max_output_length=shell_max_output, + status=getattr(item, "status", None), + raw_representation=item, + ) + ) elif item_type == "local_shell_call": raw_local_call_id = getattr(item, "call_id", None) local_call_id = raw_local_call_id if isinstance(raw_local_call_id, str) else "" diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index 5b7435f300..aa2b3595f2 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -2299,6 +2299,35 @@ def test_malformed_local_shell_call_is_not_executable( assert all(content.type != "function_call" for content in contents) +@pytest.mark.parametrize( + ("item_id", "call_id", "commands"), + [ + (None, "local-shell-call-1", ["echo ok"]), + ("local-shell-item-1", None, ["echo ok"]), + ("local-shell-item-1", "local-shell-call-1", "echo ok"), + ("local-shell-item-1", "local-shell-call-1", []), + ], +) +def test_malformed_local_environment_shell_call_is_not_executable( + item_id: str | None, + call_id: str | None, + commands: Any, +) -> None: + """Malformed local-environment shell items must not become function calls.""" + client = OpenAIChatClient(model="test-model", api_key="test-key") + item = MagicMock() + item.type = "shell_call" + item.id = item_id + item.call_id = call_id + item.action.commands = commands + item.environment.type = "local" + item.status = "completed" + + contents = client._shell_item_to_contents(item, "run_shell") + + assert all(content.type != "function_call" for content in contents) + + @pytest.mark.asyncio async def test_local_shell_tool_requires_approval_before_function_loop_execution() -> None: """An explicit local shell call executes only after approval.""" @@ -2326,14 +2355,18 @@ def local_exec(command: str) -> str: mock_response1.incomplete = None mock_action = MagicMock() - mock_action.command = ["python", "--version"] + mock_action.commands = ["python --version"] mock_action.timeout_ms = 30000 + mock_environment = MagicMock() + mock_environment.type = "local" + mock_local_shell_call = MagicMock() - mock_local_shell_call.type = "local_shell_call" + mock_local_shell_call.type = "shell_call" mock_local_shell_call.id = "local-shell-item-1" mock_local_shell_call.call_id = "local-shell-call-1" mock_local_shell_call.action = mock_action + mock_local_shell_call.environment = mock_environment mock_local_shell_call.status = "completed" mock_response1.output = [mock_local_shell_call] @@ -2378,10 +2411,10 @@ def local_exec(command: str) -> str: assert executed_commands == ["python --version"] assert mock_create.call_count == 2 second_call_input = mock_create.call_args_list[1].kwargs["input"] - local_shell_outputs = [item for item in second_call_input if item.get("type") == "local_shell_call_output"] + local_shell_outputs = [item for item in second_call_input if item.get("type") == "shell_call_output"] assert len(local_shell_outputs) == 1 - output_payload = json.loads(local_shell_outputs[0]["output"]) - assert output_payload["stdout"] == "Python 3.13.0" + assert local_shell_outputs[0]["call_id"] == "local-shell-call-1" + assert local_shell_outputs[0]["output"][0]["stdout"] == "Python 3.13.0" @pytest.mark.asyncio @@ -2420,17 +2453,24 @@ def local_exec(command: str) -> str: mock_shell_call.id = "sh_test_shell_call_1" mock_shell_call.call_id = "shell-call-1" mock_shell_call.action = mock_action + mock_hosted_environment = MagicMock() + mock_hosted_environment.type = "container_reference" + mock_shell_call.environment = mock_hosted_environment mock_shell_call.status = "completed" mock_local_action = MagicMock() - mock_local_action.command = ["python", "--version"] + mock_local_action.commands = ["python --version"] mock_local_action.timeout_ms = 30000 + mock_local_environment = MagicMock() + mock_local_environment.type = "local" + mock_local_shell_call = MagicMock() - mock_local_shell_call.type = "local_shell_call" + mock_local_shell_call.type = "shell_call" mock_local_shell_call.id = "local-shell-item-1" mock_local_shell_call.call_id = "local-shell-call-1" mock_local_shell_call.action = mock_local_action + mock_local_shell_call.environment = mock_local_environment mock_local_shell_call.status = "completed" mock_response1.output = [mock_shell_call, mock_local_shell_call] @@ -2464,10 +2504,10 @@ def local_exec(command: str) -> str: assert executed_commands == ["python --version"] assert mock_create.call_count == 2 second_call_input = mock_create.call_args_list[1].kwargs["input"] - assert all(item.get("type") != "shell_call_output" for item in second_call_input) - local_shell_outputs = [item for item in second_call_input if item.get("type") == "local_shell_call_output"] + local_shell_outputs = [item for item in second_call_input if item.get("type") == "shell_call_output"] assert len(local_shell_outputs) == 1 - assert local_shell_outputs[0]["id"] == "local-shell-item-1" + assert local_shell_outputs[0]["call_id"] == "local-shell-call-1" + assert all(item.get("type") != "local_shell_call_output" for item in second_call_input) async def test_tool_loop_store_false_replays_encrypted_reasoning_group() -> None: @@ -2625,6 +2665,7 @@ def local_exec(command: str) -> str: mock_shell_call.type = "shell_call" mock_shell_call.call_id = "shell-call-1" mock_shell_call.action = mock_action + mock_shell_call.environment = None mock_shell_call.status = "completed" mock_response.output = [mock_shell_call] @@ -3406,11 +3447,15 @@ def local_exec(command: str) -> str: mock_action.timeout_ms = 30000 mock_action.max_output_length = 4096 + mock_environment = MagicMock() + mock_environment.type = "container_reference" + mock_item = MagicMock() mock_item.type = "shell_call" mock_item.id = "sh_1" mock_item.call_id = "shell-call-1" mock_item.action = mock_action + mock_item.environment = mock_environment mock_item.status = "completed" mock_event = MagicMock() @@ -3428,12 +3473,8 @@ def local_exec(command: str) -> str: assert call_content.commands == ["ls -la"] -def test_parse_chunk_from_openai_local_shell_call_done_emits_command() -> None: - """A completed local_shell_call on output_item.done emits a function call with the command. - - Mirrors the non-streaming local_shell_call mapping: the joined command and the - local-shell metadata (item id) must be present on the completed item. - """ +def test_parse_chunk_from_openai_local_environment_shell_call_done_emits_command() -> None: + """A completed shell call with a local environment emits an executable function call.""" client = OpenAIChatClient(model="test-model", api_key="test-key") def local_exec(command: str) -> str: @@ -3443,14 +3484,18 @@ def local_exec(command: str) -> str: function_call_ids: dict[int, tuple[str, str]] = {} mock_action = MagicMock() - mock_action.command = ["python", "--version"] + mock_action.commands = ["python --version"] mock_action.timeout_ms = 30000 + mock_environment = MagicMock() + mock_environment.type = "local" + mock_item = MagicMock() - mock_item.type = "local_shell_call" + mock_item.type = "shell_call" mock_item.id = "local-shell-item-1" mock_item.call_id = "local-shell-call-1" mock_item.action = mock_action + mock_item.environment = mock_environment mock_item.status = "completed" mock_event = MagicMock() @@ -3467,7 +3512,7 @@ def local_exec(command: str) -> str: assert call_content.call_id == "local-shell-call-1" assert call_content.name == local_shell_tool.name assert call_content.parse_arguments() == {"command": "python --version"} - assert call_content.additional_properties[OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY] == "local-shell-item-1" + assert call_content.additional_properties["openai.responses.shell.output_type"] == "shell_call_output" def test_parse_chunk_from_openai_shell_call_output_added_defers_result() -> None: From 4a51827f904f86d87f37480ae3f72efdebbd0c9b Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 14 Sep 2026 11:02:59 +0200 Subject: [PATCH 4/5] Python: Format shell call tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- python/packages/openai/tests/openai/test_openai_chat_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index aa2b3595f2..eaeec91ee0 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -2184,6 +2184,7 @@ def test_shell_output_payloads_do_not_expose_exception_diagnostics() -> None: assert empty_local_payload["exit_code"] == 1 assert empty_shell_payload[0]["outcome"] == {"type": "exit", "exit_code": 1} + def test_prepared_local_shell_tool_survives_make_tools() -> None: """Regression: the prepared shell tool must be a subscriptable dict. From 28d0d7fb621c6adbe6118d7551a4b7d53b4cc0d5 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 14 Sep 2026 17:46:36 +0200 Subject: [PATCH 5/5] Python: Preserve shell transcript replay Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../specs/004-python-function-calling-loop.md | 6 +- .../agent_framework_openai/_chat_client.py | 71 +++++-- .../tests/openai/test_openai_chat_client.py | 186 +++++++++++++----- 3 files changed, 201 insertions(+), 62 deletions(-) diff --git a/docs/specs/004-python-function-calling-loop.md b/docs/specs/004-python-function-calling-loop.md index 727f6a47a7..bf7d1ed72a 100644 --- a/docs/specs/004-python-function-calling-loop.md +++ b/docs/specs/004-python-function-calling-loop.md @@ -394,7 +394,9 @@ that manually replay messages own the equivalent rule: do not resend an approval Exceptions raised by middleware or the tool body retain the separate `Function failed` contract. - Provider-hosted shell calls remain informational transcript content. Only a well-formed explicit `local_shell_call` or a shell call marked with a local environment, paired with a configured local executor, can enter the local - function loop and its approval boundary. + function loop and its approval boundary. Stateless replay preserves provider shell call/output items in their + original shape and fails explicitly when that provider representation is unavailable or inconsistent. Locally + generated shell outputs are sent in both service-side and stateless continuation modes. ### Reasoning-bound calls @@ -517,7 +519,7 @@ that manually replay messages own the equivalent rule: do not resend an approval | Calls across response messages | Every actionable call is executed once. | `test_base_client_executes_function_calls_across_multiple_response_messages` | | Parallel calls | Results retain the corresponding call ids and execution count. | `test_max_function_calls_limits_parallel_invocations`, `test_streaming_multiple_function_calls_parallel_execution` | | Informational-only call | The call is returned but not executed or approved. | `test_informational_only_function_call_is_not_invoked`, `test_informational_only_function_call_does_not_request_approval`, `test_streaming_informational_only_function_call_is_not_invoked` | -| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell items or shell calls marked with a local environment can execute, and local execution preserves its configured approval mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_parse_chunk_from_openai_local_environment_shell_call_done_emits_command`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_malformed_local_environment_shell_call_is_not_executable`, `test_response_content_creation_with_local_shell_call_maps_to_function_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | +| OpenAI hosted/local shell boundary | Hosted shell calls remain informational in streaming and non-streaming responses even when a local executor is configured; only valid explicit local-shell items or shell calls marked with a local environment can execute, local execution preserves its configured approval mode, stateless loops preserve the complete provider shell transcript or fail explicitly, and locally generated shell outputs are sent in every continuation mode. | `packages/openai/tests/openai/test_openai_chat_client.py::test_response_content_creation_with_shell_call_remains_hosted_with_local_tool`, `test_parse_chunk_from_openai_shell_call_done_remains_hosted`, `test_parse_chunk_from_openai_local_environment_shell_call_done_emits_command`, `test_mixed_shell_calls_only_invoke_explicit_local_shell_call`, `test_stateless_shell_transcript_without_provider_item_fails`, `test_prepare_messages_keeps_local_shell_output_under_storage`, `test_malformed_local_environment_shell_call_is_not_executable`, `test_response_content_creation_with_local_shell_call_maps_to_function_call`, `test_malformed_local_shell_call_is_not_executable`, `test_response_function_call_named_local_shell_is_informational`, `test_parse_chunk_function_call_named_local_shell_is_informational`, `test_local_shell_tool_requires_approval_before_function_loop_execution` | | Declaration-only call | The call is surfaced as user input and is not executed; streaming arguments appear once while finalized request metadata remains available. | `test_declaration_only_tool`, `test_streaming_declaration_only_tool_preserves_metadata_without_duplicate_arguments` | | Function invocation disabled | The client bypasses the invocation loop without losing invocation kwargs. | `test_function_invocation_config_enabled_false`, `test_function_invocation_config_enabled_false_preserves_invocation_kwargs`, `test_streaming_function_invocation_config_enabled_false` | | Runtime tool changes | Added tools become available on the next iteration and retain approval behavior. | `test_add_tools_available_next_iteration`, `test_add_tools_with_approval_required_tool` | diff --git a/python/packages/openai/agent_framework_openai/_chat_client.py b/python/packages/openai/agent_framework_openai/_chat_client.py index 51fdffc850..aa0937658c 100644 --- a/python/packages/openai/agent_framework_openai/_chat_client.py +++ b/python/packages/openai/agent_framework_openai/_chat_client.py @@ -76,6 +76,8 @@ from openai.types.responses import ( FunctionShellToolParam, ResponseCustomToolCall, + ResponseFunctionShellToolCall, + ResponseFunctionShellToolCallOutput, ResponseToolSearchCall, response_create_params, ) @@ -85,6 +87,7 @@ ParsedResponse, ) from openai.types.responses.response import Response as OpenAIResponse +from openai.types.responses.response_input_item_param import LocalShellCall from openai.types.responses.response_stream_event import ( ResponseStreamEvent as OpenAIResponseStreamEvent, ) @@ -96,7 +99,7 @@ Mcp, ) from openai.types.responses.web_search_tool_param import WebSearchToolParam -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter, ValidationError from ._exceptions import OpenAIContentFilterException from ._feature_usage import FeatureIndex @@ -1742,16 +1745,6 @@ def _prepare_message_for_openai( serialized_reasoning_ids.add(content.id) continue case "function_result": - if request_uses_service_side_storage: - props = content.additional_properties or {} - # Local-shell variant serializes as `local_shell_call` carrying a server-issued id; - # plain function_call_output pairs by call_id and is safe under storage. - if props.get( - OPENAI_SHELL_OUTPUT_TYPE_KEY - ) == OPENAI_SHELL_OUTPUT_TYPE_LOCAL_SHELL_CALL and props.get( - OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY - ): - continue new_args: dict[str, Any] = {} new_args.update( self._prepare_content_for_openai( @@ -1771,7 +1764,25 @@ def _prepare_message_for_openai( replays_local_storage=replays_local_storage, ) if function_call: + if function_call.get("type") in {"shell_call", "local_shell_call"} and ( + "content" in args or "tool_calls" in args + ): + all_messages.append(args) + args = {"type": "message", "role": message.role} all_messages.append(function_call) + case "shell_tool_call" | "shell_tool_result": + if request_uses_service_side_storage: + continue + if "content" in args or "tool_calls" in args: + all_messages.append(args) + args = {"type": "message", "role": message.role} + all_messages.append( + self._prepare_content_for_openai( + message.role, + content, + replays_local_storage=replays_local_storage, + ) + ) case "function_approval_request": # Service-stored hosted requests are already present remotely, and local approvals # are resolved in-process; neither should be serialized as an MCP input item. @@ -1984,6 +1995,11 @@ def _prepare_content_for_openai( return _attach_prompt_cache_breakpoint(file_obj, content) return {} case "function_call": + shell_output_type = content.additional_properties.get(OPENAI_SHELL_OUTPUT_TYPE_KEY) + if shell_output_type == OPENAI_SHELL_OUTPUT_TYPE_SHELL_CALL: + return self._prepare_shell_transcript_item_for_openai(content, expected_type="shell_call") + if shell_output_type == OPENAI_SHELL_OUTPUT_TYPE_LOCAL_SHELL_CALL: + return self._prepare_shell_transcript_item_for_openai(content, expected_type="local_shell_call") if not content.call_id: logger.warning(f"FunctionCallContent missing call_id for function '{content.name}'") return {} @@ -2006,6 +2022,10 @@ def _prepare_content_for_openai( if status := content.additional_properties.get("status"): function_call_obj["status"] = status return function_call_obj + case "shell_tool_call": + return self._prepare_shell_transcript_item_for_openai(content, expected_type="shell_call") + case "shell_tool_result": + return self._prepare_shell_transcript_item_for_openai(content, expected_type="shell_call_output") case "function_result": shell_output_type = ( content.additional_properties.get(OPENAI_SHELL_OUTPUT_TYPE_KEY) @@ -2388,6 +2408,35 @@ def _coalesce_pending_mcp_results(items: list[dict[str, Any]]) -> list[dict[str, out.append(item) return out + @classmethod + def _prepare_shell_transcript_item_for_openai( + cls, + content: Content, + *, + expected_type: Literal["shell_call", "shell_call_output", "local_shell_call"], + ) -> dict[str, Any]: + """Restore a provider-issued shell transcript item for stateless replay.""" + payload = cls._serialize_provider_payload(content.raw_representation) + if isinstance(payload, Mapping): + typed_payload = cast("Mapping[str, Any]", payload) + raw_call_id = typed_payload.get("call_id") + try: + if expected_type == "shell_call": + ResponseFunctionShellToolCall.model_validate(typed_payload) + elif expected_type == "shell_call_output": + ResponseFunctionShellToolCallOutput.model_validate(typed_payload) + else: + TypeAdapter(LocalShellCall).validate_python(typed_payload) + except ValidationError: + pass + else: + if isinstance(raw_call_id, str) and raw_call_id and raw_call_id == content.call_id: + return dict(typed_payload) + raise ChatClientInvalidRequestException( + f"Stateless replay cannot reconstruct {expected_type} for shell call {content.call_id!r}. " + "Use service-side continuation or preserve the original provider response item." + ) + @staticmethod def _serialize_provider_payload(value: Any) -> Any: """Convert OpenAI SDK objects into JSON-serializable Python values.""" diff --git a/python/packages/openai/tests/openai/test_openai_chat_client.py b/python/packages/openai/tests/openai/test_openai_chat_client.py index eaeec91ee0..778163c546 100644 --- a/python/packages/openai/tests/openai/test_openai_chat_client.py +++ b/python/packages/openai/tests/openai/test_openai_chat_client.py @@ -42,6 +42,7 @@ SettingNotFoundError, ) from openai import AsyncOpenAI, BadRequestError +from openai.types.responses import ResponseFunctionShellToolCall, ResponseFunctionShellToolCallOutput from openai.types.responses.response_reasoning_item import Summary from openai.types.responses.response_reasoning_summary_text_delta_event import ( ResponseReasoningSummaryTextDeltaEvent, @@ -2420,7 +2421,7 @@ def local_exec(command: str) -> str: @pytest.mark.asyncio async def test_mixed_shell_calls_only_invoke_explicit_local_shell_call() -> None: - """A hosted shell call remains informational alongside an executable local call.""" + """Stateless replay preserves hosted shell transcript while executing only the local call.""" client = OpenAIChatClient(model="test-model", api_key="test-key") executed_commands: list[str] = [] @@ -2432,6 +2433,29 @@ def local_exec(command: str) -> str: func=local_exec, approval_mode="never_require", ) + hosted_shell_call = ResponseFunctionShellToolCall.model_validate({ + "id": "hosted-shell-item-1", + "type": "shell_call", + "call_id": "hosted-shell-call-1", + "action": {"commands": ["pwd"], "timeout_ms": 30000, "max_output_length": 4096}, + "environment": {"type": "container_reference", "container_id": "container-1"}, + "status": "completed", + }) + hosted_shell_output = ResponseFunctionShellToolCallOutput.model_validate({ + "id": "hosted-shell-output-1", + "type": "shell_call_output", + "call_id": "hosted-shell-call-1", + "output": [{"stdout": "/workspace", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}], + "status": "completed", + }) + local_shell_call = ResponseFunctionShellToolCall.model_validate({ + "id": "local-shell-item-1", + "type": "shell_call", + "call_id": "local-shell-call-1", + "action": {"commands": ["python --version"], "timeout_ms": 30000}, + "environment": {"type": "local"}, + "status": "completed", + }) mock_response1 = MagicMock() mock_response1.output_parsed = None @@ -2443,37 +2467,15 @@ def local_exec(command: str) -> str: mock_response1.status = "completed" mock_response1.finish_reason = "tool_calls" mock_response1.incomplete = None - - mock_action = MagicMock() - mock_action.commands = ["pwd"] - mock_action.timeout_ms = 30000 - mock_action.max_output_length = 4096 - - mock_shell_call = MagicMock() - mock_shell_call.type = "shell_call" - mock_shell_call.id = "sh_test_shell_call_1" - mock_shell_call.call_id = "shell-call-1" - mock_shell_call.action = mock_action - mock_hosted_environment = MagicMock() - mock_hosted_environment.type = "container_reference" - mock_shell_call.environment = mock_hosted_environment - mock_shell_call.status = "completed" - - mock_local_action = MagicMock() - mock_local_action.commands = ["python --version"] - mock_local_action.timeout_ms = 30000 - - mock_local_environment = MagicMock() - mock_local_environment.type = "local" - - mock_local_shell_call = MagicMock() - mock_local_shell_call.type = "shell_call" - mock_local_shell_call.id = "local-shell-item-1" - mock_local_shell_call.call_id = "local-shell-call-1" - mock_local_shell_call.action = mock_local_action - mock_local_shell_call.environment = mock_local_environment - mock_local_shell_call.status = "completed" - mock_response1.output = [mock_shell_call, mock_local_shell_call] + prefix_message = MagicMock() + prefix_message.type = "message" + prefix_content = MagicMock() + prefix_content.type = "output_text" + prefix_content.text = "Checking shell environments" + prefix_content.annotations = [] + prefix_content.logprobs = None + prefix_message.content = [prefix_content] + mock_response1.output = [prefix_message, hosted_shell_call, hosted_shell_output, local_shell_call] mock_response2 = MagicMock() mock_response2.output_parsed = None @@ -2499,16 +2501,100 @@ def local_exec(command: str) -> str: ) as mock_create: await client.get_response( messages=[Message(role="user", contents=["What Python version is available?"])], - options={"tools": [local_shell_tool]}, + options={"tools": [local_shell_tool], "store": False}, ) assert executed_commands == ["python --version"] assert mock_create.call_count == 2 second_call_input = mock_create.call_args_list[1].kwargs["input"] - local_shell_outputs = [item for item in second_call_input if item.get("type") == "shell_call_output"] - assert len(local_shell_outputs) == 1 - assert local_shell_outputs[0]["call_id"] == "local-shell-call-1" - assert all(item.get("type") != "local_shell_call_output" for item in second_call_input) + assert [item.get("type") for item in second_call_input] == [ + "message", + "message", + "shell_call", + "shell_call_output", + "shell_call", + "shell_call_output", + ] + assert second_call_input[1]["content"][0]["text"] == "Checking shell environments" + shell_items = [item for item in second_call_input if item.get("type") in {"shell_call", "shell_call_output"}] + assert shell_items == [ + hosted_shell_call.model_dump(mode="json", exclude_none=True), + hosted_shell_output.model_dump(mode="json", exclude_none=True), + local_shell_call.model_dump(mode="json", exclude_none=True), + { + "type": "shell_call_output", + "call_id": "local-shell-call-1", + "output": [ + { + "stdout": "Python 3.13.0", + "stderr": "", + "outcome": {"type": "exit", "exit_code": 0}, + } + ], + }, + ] + assert all(item.get("type") != "function_call" for item in second_call_input) + + +@pytest.mark.parametrize( + ("content", "expected_type"), + [ + (Content.from_shell_tool_call(call_id="shell-call-1", commands=["pwd"]), "shell_call"), + ( + Content.from_shell_tool_result( + call_id="shell-call-1", + outputs=[Content.from_shell_command_output(stdout="/workspace", stderr="", exit_code=0)], + ), + "shell_call_output", + ), + ( + Content.from_shell_tool_call( + call_id="shell-call-1", + commands=["pwd"], + raw_representation={"type": "shell_call", "call_id": "shell-call-1"}, + ), + "shell_call", + ), + ( + Content.from_shell_tool_result( + call_id="shell-call-1", + outputs=[Content.from_shell_command_output(stdout="/workspace", stderr="", exit_code=0)], + raw_representation={"type": "shell_call_output", "call_id": "shell-call-1"}, + ), + "shell_call_output", + ), + ( + Content.from_shell_tool_call( + call_id="shell-call-1", + commands=["pwd"], + raw_representation=ResponseFunctionShellToolCall.model_validate({ + "id": "hosted-shell-item-1", + "type": "shell_call", + "call_id": "different-shell-call", + "action": {"commands": ["pwd"]}, + "environment": {"type": "container_reference", "container_id": "container-1"}, + "status": "completed", + }), + ), + "shell_call", + ), + ], +) +def test_stateless_shell_transcript_without_provider_item_fails( + content: Content, + expected_type: str, +) -> None: + """Stateless replay fails explicitly when provider shell shape is unavailable.""" + client = OpenAIChatClient(model="test-model", api_key="test-key") + + with pytest.raises( + ChatClientInvalidRequestException, + match=rf"cannot reconstruct {expected_type}.*shell-call-1", + ): + client._prepare_messages_for_openai( + [Message(role="assistant", contents=[content])], + request_uses_service_side_storage=False, + ) async def test_tool_loop_store_false_replays_encrypted_reasoning_group() -> None: @@ -8507,9 +8593,8 @@ def test_stateless_history_preserves_pending_hosted_approval_request_until_respo assert resolved_items == [] -def test_prepare_messages_strips_local_shell_call_under_storage() -> None: - """Local-shell-call function_results carry a server-issued local_shell_call_item_id and must - be stripped under storage. Plain function_results (no shell ID) are kept either way (#3295).""" +def test_prepare_messages_keeps_local_shell_output_under_storage() -> None: + """Locally generated shell output must reach the provider in every continuation mode.""" from agent_framework_openai._chat_client import ( OPENAI_LOCAL_SHELL_CALL_ITEM_ID_KEY, OPENAI_SHELL_OUTPUT_TYPE_KEY, @@ -8528,15 +8613,18 @@ def test_prepare_messages_strips_local_shell_call_under_storage() -> None: plain_result = Content.from_function_result(call_id="plain_1", result="plain") message = Message(role="tool", contents=[shell_result, plain_result]) - storage_on = client._prepare_message_for_openai(message, request_uses_service_side_storage=True) - types_on = [item.get("type") for item in storage_on] - assert OPENAI_SHELL_OUTPUT_TYPE_LOCAL_SHELL_CALL not in types_on - assert "function_call_output" in types_on - - storage_off = client._prepare_message_for_openai(message, request_uses_service_side_storage=False) - types_off = [item.get("type") for item in storage_off] - assert OPENAI_SHELL_OUTPUT_TYPE_LOCAL_SHELL_CALL in types_off - assert "function_call_output" in types_off + expected_shell_output = { + "id": "lsh_server_issued", + "type": OPENAI_SHELL_OUTPUT_TYPE_LOCAL_SHELL_CALL, + "output": '{"stdout": "ok", "exit_code": 0}', + } + for request_uses_service_side_storage in (True, False): + prepared = client._prepare_message_for_openai( + message, + request_uses_service_side_storage=request_uses_service_side_storage, + ) + assert expected_shell_output in prepared + assert any(item.get("type") == "function_call_output" for item in prepared) def test_prepare_messages_strips_mcp_items_under_storage() -> None: