diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a80e97b..b8d95b0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-sdk - ref: 7864c1803eb299d136682bcc782f4718c7e69bc6 # @tangle-network/agent-interface@1.1.0 + ref: a3c31b986809689892ec34617df58b007992ef0c # @tangle-network/agent-interface@1.3.0 path: .cohort/agent-sdk persist-credentials: false diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5cab166d..d561547e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -135,7 +135,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: tangle-network/agent-sdk - ref: 7864c1803eb299d136682bcc782f4718c7e69bc6 # @tangle-network/agent-interface@1.1.0 + ref: a3c31b986809689892ec34617df58b007992ef0c # @tangle-network/agent-interface@1.3.0 path: .cohort/agent-sdk persist-credentials: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 016bcb03..248093ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 0.143.0 + +### Sandbox moves to 0.31.0, and its run outcome is the one terminal result + +The Sandbox peer range becomes `>=0.31.0 <0.32.0`, and the catalog requires Sandbox `0.31.0`. +Sandbox 0.31.0 publishes `createAgentRunOutcomeTracker` and `AgentRunOutcome` on its `runtime` subpath, and this package now requires them. +The Interface peer range becomes `^1.3.0` because Sandbox 0.31.0 requires Interface `^1.3.0`, so one consumer install resolves one Interface copy. +Bench moves to 0.8.19 with the same dependency cohort. + +The round-synchronous leaf (`runAgentRounds`), `openSandboxRun`, the steerable sandbox session, and `streamAgentTurn` observe the complete Sandbox event stream with the public outcome tracker and read the terminal result from it. +A `failed` outcome fails the iteration or turn with the Sandbox error. +A `blocked_on_approval` or `awaiting_*` outcome settles as `blocked`. +A `success` outcome completes, so a recovered tool error no longer fails a completed turn. +The outcome rides `Iteration.sandboxOutcome`, `CollectedAgentTurn.sandboxOutcome`, and the final stream event's `metadata.sandboxOutcome`. +A failed or blocked turn keeps its measured token usage and cost. +`SandboxLeafOut` carries `content`, `toolCalls`, and `outcome` beside the raw `events`, so a consumer does not parse raw Sandbox events a second time; `SandboxExecutorToolCall` names one retained tool call. + +`./kernel` drops `assertSandboxEventSucceeded` and `sandboxEventFailure`; the public Sandbox outcome tracker replaces that event-level failure parser. +This is a minor release because two public exports are removed. + ## 0.142.3 ### Sandbox 0.29 and 0.30 compatibility diff --git a/api-surface.json b/api-surface.json index cd95efe5..589c400a 100644 --- a/api-surface.json +++ b/api-surface.json @@ -1313,6 +1313,7 @@ "SandboxClientProviderOptions": "type", "SandboxControlClient": "type", "SandboxEvent": "type", + "SandboxExecutorToolCall": "type", "SandboxInstance": "type", "SandboxLeafOut": "type", "SandboxLineage": "type", @@ -1489,7 +1490,6 @@ "assertCoordinationBinding": "value", "assertModelAllowed": "value", "assertProfileModelsAllowed": "value", - "assertSandboxEventSucceeded": "value", "assertSandboxServedModel": "value", "assertStrategyContract": "value", "assertTraceDerivedFindings": "value", @@ -1684,7 +1684,6 @@ "sampleThenRefine": "value", "sandboxCheckRunner": "value", "sandboxClientAsProvider": "value", - "sandboxEventFailure": "value", "sandboxEventServedBackend": "value", "sandboxSessionTraceSource": "value", "sanitizeMcpToolSchema": "value", diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 765a28bd..382994e5 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 0.8.19 + +- Align the Bench package with Runtime 0.143.0, Interface 1.3.0, and Sandbox 0.31.0. + ## 0.8.18 - Align the Bench package with Runtime 0.142.3 and the Sandbox 0.29.0–0.30.0 compatibility range. diff --git a/bench/package.json b/bench/package.json index c42780ed..47d6870f 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.8.18", + "version": "0.8.19", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index ded4cf2c..d71edd90 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.142.3` and `@tangle-network/agent-eval@0.149.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.143.0` and `@tangle-network/agent-eval@0.149.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -549,7 +549,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 172 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 815 exports. +Import from `@tangle-network/agent-runtime/kernel` — 814 exports. | Symbol | Kind | Summary | |---|---|---| @@ -564,7 +564,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 815 exports. | `assertCoordinationBinding` | function | Fail closed on a non-loopback coordination bind. `serveCoordinationMcp` mounts spawn_agent / | | `assertModelAllowed` | function | Throw a `ConfigError` when `allowed` is set, `model` is defined, and `model` is not a | | `assertProfileModelsAllowed` | function | Check every canonical model-bearing field in a complete profile, including the models a | -| `assertSandboxEventSucceeded` | function | Fail the live execution instead of allowing an in-band failure to become an empty success. | | `assertSandboxServedModel` | function | Fail the execution when the platform reports serving a model other than the exact one asked for. | | `assertStrategyContract` | function | Static CONTRACT lint over an authored strategy module — the module-boundary | | `assessAuthoredProfile` | function | OBSERVE one authored `AgentProfile` and score its richness (no judge verdict is read). The task | @@ -739,7 +738,6 @@ Import from `@tangle-network/agent-runtime/kernel` — 815 exports. | `sampleFromSettled` | function | Build a `ProgressSample` from a scope settlement. The objective is the verdict score and | | `sandboxCheckRunner` | function | Default CheckRunner backend: pipes the check program into `python3` over the sandbox | | `sandboxClientAsProvider` | function | Adapt a `SandboxClient` into the shared `AgentEnvironmentProvider` contract. | -| `sandboxEventFailure` | function | Return the terminal failure carried by one Sandbox event. | | `sandboxEventServedBackend` | function | Read the served execution identity off one Sandbox event. | | `sandboxSessionTraceSource` | function | The SANDBOX / fleet trace source: read a box session's message parts and decode the harness's tool | | `sanitizeMcpToolSchema` | function | Coerce an MCP inputSchema to an OpenAI-tool-valid top-level object schema. | @@ -1025,7 +1023,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 815 exports. | `SandboxClient` | interface | Minimal sandbox client surface the kernel calls. Satisfied structurally by | | `SandboxClientProviderOptions` | interface | Options for wrapping the current Tangle sandbox client as an environment provider. | | `SandboxEvent` | interface | SSE event from sandbox streaming. | -| `SandboxLeafOut` | interface | Parsed output of the sandbox leaf: the iteration's raw event stream. What a | +| `SandboxExecutorToolCall` | interface | One tool call retained in a Sandbox executor artifact. | +| `SandboxLeafOut` | interface | Parsed output of one Sandbox executor turn. | | `SandboxLineage` | interface | Owns box + session handles for one loop run and offers the three | | `SandboxLineageHandle` | interface | A live box plus the session that threads its iterations together. Handed back | | `SandboxSeam` | interface | Sandbox executor seam. The `sandboxClient` the composed `runAgentRounds` creates | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 2cf2d919..3cea7c76 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -852,7 +852,7 @@ One flattened node with the journal tree that owns its records. ###### Inherited from -[`NodeSnapshot`](#nodesnapshot).[`id`](#id-18) +[`NodeSnapshot`](#nodesnapshot).[`id`](#id-19) ##### parent? @@ -7610,6 +7610,50 @@ Sequence for synthesized call ids when an event carries none. *** +### SandboxExecutorToolCall + +One tool call retained in a Sandbox executor artifact. + +#### Properties + +##### id? + +> `optional` **id?**: `string` + +##### name + +> **name**: `string` + +##### arguments + +> **arguments**: `unknown` + +*** + +### SandboxLeafOut + +Parsed output of one Sandbox executor turn. + +#### Properties + +##### events + +> **events**: `SandboxEvent`[] + +##### content + +> **content**: `string` + +##### toolCalls? + +> `optional` **toolCalls?**: [`SandboxExecutorToolCall`](#sandboxexecutortoolcall)[] + +##### outcome? + +> `optional` **outcome?**: `AgentRunOutcome` + +*** + ### SandboxLineageHandle **`Experimental`** @@ -7954,6 +7998,14 @@ nothing" from a transport/FS fault. **`Experimental`** +##### outcome + +> **outcome**: `AgentRunOutcome` + +**`Experimental`** + +Outcome settled by the public Sandbox tracker after the stream drained. + ##### readError? > `optional` **readError?**: `string` @@ -9997,6 +10049,12 @@ Exact underlying transport calls when the Runtime-owned executor reports them. > `optional` **error?**: [`BackendErrorDetail`](index.md#backenderrordetail) +##### sandboxOutcome? + +> `optional` **sandboxOutcome?**: `AgentRunOutcome` + +Public Sandbox outcome, when the turn ran through a Sandbox stream or executor. + *** ### StructuralRolloutPolicy @@ -14170,19 +14228,6 @@ Online observer of each tool step — the seam a `DetectorMonitor` taps to watch *** -### SandboxLeafOut - -Parsed output of the sandbox leaf: the iteration's raw event stream. What a - `SandboxSeam.validator` receives as its `output` argument. - -#### Properties - -##### events - -> **events**: `SandboxEvent`[] - -*** - ### SandboxSteeringOptions Opt-in configuration for the steerable sandbox worker (`SandboxSeam.steering`). Absent, the @@ -14254,11 +14299,11 @@ Drive the worker to settlement. `signal` is the spawn-scoped abort handed to `ex ##### artifact() -> **artifact**(): \{ `outRef`: `string`; `out`: `unknown`; `spent`: [`Spend`](index.md#spend); \} \| `undefined` +> **artifact**(): \{ `outRef`: `string`; `out`: `unknown`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); \} \| `undefined` ###### Returns -\{ `outRef`: `string`; `out`: `unknown`; `spent`: [`Spend`](index.md#spend); \} \| `undefined` +\{ `outRef`: `string`; `out`: `unknown`; `verdict?`: `DefaultVerdict`; `spent`: [`Spend`](index.md#spend); \} \| `undefined` ##### teardown() @@ -18898,6 +18943,12 @@ Stable name of the `AgentRunSpec` that produced this iteration. > `optional` **error?**: `Error` +##### sandboxOutcome? + +> `optional` **sandboxOutcome?**: `AgentRunOutcome` + +Public Sandbox outcome settled after the complete event stream. + ##### events > **events**: `SandboxEvent`[] @@ -24679,54 +24730,6 @@ promise is cached so concurrent fanout branches share one round-trip. *** -### sandboxEventFailure() - -> **sandboxEventFailure**(`event`): `string` \| `undefined` - -Return the terminal failure carried by one Sandbox event. - -Sandbox transports report execution failure in-band: commonly an `error` -event followed by a synthetic `done`. Treating the iterable as successfully -drained therefore turns a provider/configuration failure into a completed -empty artifact. - -The decoder reads a failure from exactly two places, so that a mid-stream -event describing its OWN failure cannot fail the execution: an `error`-typed -event, and a terminal event (`done`/`result`/`final`) whose `success` is -`false` or whose status is a failed one. A failing tool part therefore stays -a tool result, which is what [mapSandboxToolEvent](#mapsandboxtoolevent) already projects it -to. - -#### Parameters - -##### event - -`SandboxEvent` - -#### Returns - -`string` \| `undefined` - -*** - -### assertSandboxEventSucceeded() - -> **assertSandboxEventSucceeded**(`event`): `void` - -Fail the live execution instead of allowing an in-band failure to become an empty success. - -#### Parameters - -##### event - -`SandboxEvent` - -#### Returns - -`void` - -*** - ### sandboxEventServedBackend() > **sandboxEventServedBackend**(`event`): [`SandboxServedBackend`](#sandboxservedbackend) \| `undefined` @@ -24791,8 +24794,8 @@ the event carries usage/cost data. Returns `undefined` for non-cost events so the kernel can iterate the full stream without branching. Pure by contract: it never throws on a failed run. The terminal truth -boundary is [assertSandboxEventSucceeded](#assertsandboxeventsucceeded), applied by the two paths -that SETTLE an execution. Post-hoc readers — [sumSandboxUsage](#sumsandboxusage), the +boundary is the public Sandbox outcome tracker, applied after the complete +stream. Post-hoc readers — [sumSandboxUsage](#sumsandboxusage), the analyst trace store, the chat projection — must stay able to read a failed turn's events, which is when reading them matters most. diff --git a/docs/canonical-api.md b/docs/canonical-api.md index afd5e85a..6141f8e1 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,11 +4,11 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.142.3.** +> **Version 0.143.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.149.0 <0.150.0`. -> `sandbox` must satisfy `>=0.29.0 <0.31.0`. -> Portable profile and tool-part types come from `@tangle-network/agent-interface` `^1.1.0`. +> `sandbox` must satisfy `>=0.31.0 <0.32.0`. +> Portable profile and tool-part types come from `@tangle-network/agent-interface` `^1.3.0`. > > **`./kernel` is the execution kernel**: `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/kernel` lives there — the recursive atom (`Scope`/`Supervisor`), the executor registry, budget conservation, the finalizer seam, analyst wiring, and the round-synchronous loop. > diff --git a/examples/agentic-data-creation/offline-fixtures.ts b/examples/agentic-data-creation/offline-fixtures.ts index 76c60c91..2f5b56aa 100644 --- a/examples/agentic-data-creation/offline-fixtures.ts +++ b/examples/agentic-data-creation/offline-fixtures.ts @@ -121,6 +121,7 @@ export function challengerClient(): SandboxClient { data: { model: 'offline-challenger', tokensIn: 320, tokensOut: 90, costUsd: 0.0006 }, }, { type: 'result', data: { result: example } }, + { type: 'done', data: { outcome: { type: 'completed' } } }, ] }, }) @@ -152,6 +153,7 @@ export function solverClient(strength: 'weak' | 'strong'): SandboxClient { }, }, { type: 'result', data: { result: { answer } } }, + { type: 'done', data: { outcome: { type: 'completed' } } }, ] }, }) diff --git a/package.json b/package.json index b38d295b..a19c757e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.142.3", + "version": "0.143.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -174,8 +174,8 @@ "packageManager": "pnpm@11.17.0", "peerDependencies": { "@tangle-network/agent-eval": ">=0.149.0 <0.150.0", - "@tangle-network/agent-interface": "^1.1.0", - "@tangle-network/sandbox": ">=0.29.0 <0.31.0" + "@tangle-network/agent-interface": "^1.3.0", + "@tangle-network/sandbox": ">=0.31.0 <0.32.0" }, "peerDependenciesMeta": { "@tangle-network/sandbox": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e22a0c23..9a4757e9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -19,8 +19,8 @@ catalogs: specifier: '>=0.149.0 <0.150.0' version: 0.149.0 '@tangle-network/agent-interface': - specifier: ^1.1.0 - version: 1.1.0 + specifier: ^1.3.0 + version: 1.3.0 '@tangle-network/agent-knowledge': specifier: ^8.0.10 version: 8.0.10 @@ -31,8 +31,8 @@ catalogs: specifier: ^1.0.2 version: 1.0.2 '@tangle-network/sandbox': - specifier: '>=0.29.0 <0.31.0' - version: 0.30.0 + specifier: '>=0.31.0 <0.32.0' + version: 0.31.0 '@types/node': specifier: 26.1.1 version: 26.1.1 @@ -58,10 +58,10 @@ importers: version: 0.9.4(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)) '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 8.0.10(@tangle-network/agent-eval@0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)))(@tangle-network/agent-interface@1.1.0) + version: 8.0.10(@tangle-network/agent-eval@0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)))(@tangle-network/agent-interface@1.3.0) '@tangle-network/agent-profile-materialize': specifier: 'catalog:' - version: 0.16.0(@tangle-network/agent-interface@1.1.0) + version: 0.16.0(@tangle-network/agent-interface@1.3.0) '@tangle-network/agent-trace-contract': specifier: 'catalog:' version: 1.0.2 @@ -83,10 +83,10 @@ importers: version: 0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)) '@tangle-network/agent-interface': specifier: 'catalog:' - version: 1.1.0 + version: 1.3.0 '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.30.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.31.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) '@types/node': specifier: 'catalog:' version: 26.1.1 @@ -134,16 +134,16 @@ importers: version: 0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)) '@tangle-network/agent-interface': specifier: 'catalog:' - version: 1.1.0 + version: 1.3.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 8.0.10(@tangle-network/agent-eval@0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)))(@tangle-network/agent-interface@1.1.0) + version: 8.0.10(@tangle-network/agent-eval@0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)))(@tangle-network/agent-interface@1.3.0) '@tangle-network/agent-runtime': specifier: workspace:^ version: link:.. '@tangle-network/sandbox': specifier: 'catalog:' - version: 0.30.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) + version: 0.31.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))(viem@2.54.6(typescript@6.0.3)(zod@4.4.3)) devDependencies: '@arethetypeswrong/cli': specifier: 'catalog:' @@ -1265,8 +1265,8 @@ packages: engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-interface@1.1.0': - resolution: {integrity: sha512-nSQWPTjwfg7ng3JH8cRorVe5sSxuWU0+pmerZeRCmCYLLxCaGDfdl/t3L4Ggmw6XGt3lHwKaEtEnATVGwe92jA==} + '@tangle-network/agent-interface@1.3.0': + resolution: {integrity: sha512-FsuSUkWyEvpGIjp9ENX3yrcXD8G7QhnVIoepknRYaPaaCaDPlHJzF483mBNix1GZ2bTU3aCr2KgHDSUdcjn1kg==} '@tangle-network/agent-knowledge@8.0.10': resolution: {integrity: sha512-QOEIXt1t8iCjSNX7do6wqqUoZ8SQYjg5wIgX+Yi4877tnwEHZAQ41g6ibmtoYfPebBFtos2EU4jctJv7oK6T7w==} @@ -1284,8 +1284,8 @@ packages: '@tangle-network/agent-trace-contract@1.0.2': resolution: {integrity: sha512-v7uMh56jkEp4vckevEU9xKsIatbs5dqzGPp69dFLSSXUVit0RP6VD6EANMXVlTCUk+6wVKBLHJx23XspVCEiIA==} - '@tangle-network/sandbox@0.30.0': - resolution: {integrity: sha512-QDqVoT2AM/Or/okZ86agZN1rlSHc/E1fugnR2k3oEJKDdgQyuwuikfED2WVLZChu2idkc7gcg4JEEVwzeBSoUw==} + '@tangle-network/sandbox@0.31.0': + resolution: {integrity: sha512-3NotVKNH/VbppdQCyRM3EuxttbO/WxBkZfHM8tR1ujtW18sC4beYYG0NYvTsIbqtTGOMhTJI0ji1kETpdrlaDA==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.30.0 @@ -3361,7 +3361,7 @@ snapshots: '@tangle-network/agent-core@0.9.4(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))': dependencies: - '@tangle-network/agent-interface': 1.1.0 + '@tangle-network/agent-interface': 1.3.0 zod: 4.4.3 optionalDependencies: '@modelcontextprotocol/sdk': 1.30.0(supports-color@10.2.2)(zod@4.4.3) @@ -3371,7 +3371,7 @@ snapshots: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.0.12(hono@4.12.32) '@tangle-network/agent-core': 0.9.4(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)) - '@tangle-network/agent-interface': 1.1.0 + '@tangle-network/agent-interface': 1.3.0 '@tangle-network/agent-trace-contract': 1.0.2 hono: 4.12.32 linear-sum-assignment: 1.0.9 @@ -3380,30 +3380,30 @@ snapshots: transitivePeerDependencies: - '@modelcontextprotocol/sdk' - '@tangle-network/agent-interface@1.1.0': + '@tangle-network/agent-interface@1.3.0': dependencies: '@noble/hashes': 1.8.0 spdx-expression-parse: 5.0.0 zod: 4.4.3 - '@tangle-network/agent-knowledge@8.0.10(@tangle-network/agent-eval@0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)))(@tangle-network/agent-interface@1.1.0)': + '@tangle-network/agent-knowledge@8.0.10(@tangle-network/agent-eval@0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)))(@tangle-network/agent-interface@1.3.0)': dependencies: '@tangle-network/agent-eval': 0.149.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)) - '@tangle-network/agent-interface': 1.1.0 + '@tangle-network/agent-interface': 1.3.0 '@types/proper-lockfile': 4.1.4 proper-lockfile: 4.1.2 zod: 4.4.3 - '@tangle-network/agent-profile-materialize@0.16.0(@tangle-network/agent-interface@1.1.0)': + '@tangle-network/agent-profile-materialize@0.16.0(@tangle-network/agent-interface@1.3.0)': dependencies: - '@tangle-network/agent-interface': 1.1.0 + '@tangle-network/agent-interface': 1.3.0 '@tangle-network/agent-trace-contract@1.0.2': {} - '@tangle-network/sandbox@0.30.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.31.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3))(viem@2.54.6(typescript@6.0.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.9.4(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.4.3)) - '@tangle-network/agent-interface': 1.1.0 + '@tangle-network/agent-interface': 1.3.0 zod: 4.4.3 optionalDependencies: '@modelcontextprotocol/sdk': 1.30.0(supports-color@10.2.2)(zod@4.4.3) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 79e9e2d5..5286ee5b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,11 +21,11 @@ catalog: '@tangle-network/agent-core': '>=0.9.4 <0.10.0' '@types/node': 26.1.1 '@tangle-network/agent-eval': '>=0.149.0 <0.150.0' - '@tangle-network/agent-interface': ^1.1.0 + '@tangle-network/agent-interface': ^1.3.0 '@tangle-network/agent-knowledge': ^8.0.10 '@tangle-network/agent-profile-materialize': '>=0.16.0 <0.17.0' '@tangle-network/agent-trace-contract': ^1.0.2 - '@tangle-network/sandbox': '>=0.29.0 <0.31.0' + '@tangle-network/sandbox': '>=0.31.0 <0.32.0' publint: 0.3.22 tsdown: 0.22.14 tsx: 4.23.1 diff --git a/scripts/lib/packed-package-test.mjs b/scripts/lib/packed-package-test.mjs index 1764639d..491fce38 100644 --- a/scripts/lib/packed-package-test.mjs +++ b/scripts/lib/packed-package-test.mjs @@ -1,7 +1,7 @@ const unsupportedDependencyProtocol = /^(?:catalog|file|link|patch|portal|workspace):/ -export const sandboxPeerRange = '>=0.29.0 <0.31.0' -export const sandboxCompatibilityVersions = Object.freeze(['0.29.0', '0.30.0']) +export const sandboxPeerRange = '>=0.31.0 <0.32.0' +export const sandboxCompatibilityVersions = Object.freeze(['0.31.0']) export function assertPublishableDependencySpecs(packageJson) { const packageName = diff --git a/scripts/lib/packed-package-test.test.mjs b/scripts/lib/packed-package-test.test.mjs index b5e53f43..9cf34944 100644 --- a/scripts/lib/packed-package-test.test.mjs +++ b/scripts/lib/packed-package-test.test.mjs @@ -53,9 +53,9 @@ describe('rangeAdmits', () => { it('admits the published Eval and Sandbox cohorts', () => { expect(rangeAdmits('>=0.149.0 <0.150.0', '0.149.0')).toBe(true) expect(rangeAdmits('>=0.149.0 <0.150.0', '0.150.0')).toBe(false) - expect(rangeAdmits(sandboxPeerRange, '0.29.0')).toBe(true) - expect(rangeAdmits(sandboxPeerRange, '0.30.0')).toBe(true) - expect(rangeAdmits(sandboxPeerRange, '0.31.0')).toBe(false) + expect(rangeAdmits(sandboxPeerRange, '0.30.0')).toBe(false) + expect(rangeAdmits(sandboxPeerRange, '0.31.0')).toBe(true) + expect(rangeAdmits(sandboxPeerRange, '0.32.0')).toBe(false) }) it('refuses an exact specifier, which states no range', () => { @@ -108,12 +108,12 @@ describe('assertFirstPartyRangeSpecs', () => { }) describe('compatibility peer ranges', () => { - it('accepts Sandbox 0.29 and 0.30 with a 0.30 development pin', () => { + it('accepts Sandbox 0.31 with a 0.31 development pin', () => { expect(() => assertPeerMatchesDevelopmentDependency( { name: '@tangle-network/agent-runtime', - devDependencies: { '@tangle-network/sandbox': '0.30.0' }, + devDependencies: { '@tangle-network/sandbox': '0.31.0' }, peerDependencies: { '@tangle-network/sandbox': sandboxPeerRange }, }, '@tangle-network/sandbox', @@ -125,20 +125,20 @@ describe('compatibility peer ranges', () => { ).not.toThrow() }) - it('rejects a compatibility range that drops Sandbox 0.29', () => { + it('rejects a compatibility range that drops Sandbox 0.31', () => { expect(() => assertPeerMatchesDevelopmentDependency( { name: '@tangle-network/agent-runtime', - devDependencies: { '@tangle-network/sandbox': '0.30.0' }, - peerDependencies: { '@tangle-network/sandbox': '>=0.30.0 <0.31.0' }, + devDependencies: { '@tangle-network/sandbox': '0.31.0' }, + peerDependencies: { '@tangle-network/sandbox': '>=0.32.0 <0.33.0' }, }, '@tangle-network/sandbox', { - expectedRange: '>=0.30.0 <0.31.0', + expectedRange: '>=0.32.0 <0.33.0', admittedVersions: sandboxCompatibilityVersions, }, ), - ).toThrow(/does not admit 0\.29\.0/) + ).toThrow(/does not admit 0\.31\.0/) }) }) diff --git a/src/runtime/define-leaderboard.test.ts b/src/runtime/define-leaderboard.test.ts index fb9f803d..51fb8af4 100644 --- a/src/runtime/define-leaderboard.test.ts +++ b/src/runtime/define-leaderboard.test.ts @@ -33,6 +33,7 @@ function fakeBackend() { return [ { type: 'llm_call', data: { tokensIn: 12, tokensOut: 6, costUsd: 0.002 } }, { type: 'result', data: { finalText: `final answer=${answer}` } }, + { type: 'done', data: { outcome: { type: 'completed' } } }, ] }, }) @@ -152,6 +153,7 @@ describe('defineLeaderboard', () => { return [ { type: 'llm_call', data: { tokensIn: 12, tokensOut: 6, costUsd: 0.002 } }, { type: 'result', data: { finalText: `final answer=${answer}` } }, + { type: 'done', data: { outcome: { type: 'completed' } } }, ] }, }) diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 0b4e2e60..4f2086f0 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -420,7 +420,6 @@ export { type SandboxCapabilities, } from './sandbox-capabilities' export { - assertSandboxEventSucceeded, assertSandboxServedModel, createSandboxToolPartState, extractLlmCallEvent, @@ -428,7 +427,6 @@ export { mapSandboxToolEvent, type SandboxServedBackend, type SandboxToolPartState, - sandboxEventFailure, sandboxEventServedBackend, sumSandboxUsage, } from './sandbox-events' @@ -831,6 +829,7 @@ export { type ProviderSeam, type RouterSeam, type RouterToolsSeam, + type SandboxExecutorToolCall, type SandboxLeafOut, type SandboxSeam, type ToolSpec, diff --git a/src/runtime/run-loop.provenance.test.ts b/src/runtime/run-loop.provenance.test.ts index 6c02e555..3cac1a2f 100644 --- a/src/runtime/run-loop.provenance.test.ts +++ b/src/runtime/run-loop.provenance.test.ts @@ -11,13 +11,15 @@ function makeBox(id: string, finalText: string): SandboxInstance { status: 'running', async *streamPrompt(_prompt: string): AsyncIterable { yield { type: 'result', data: { finalText } } as SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent }, } as SandboxInstance } const output: OutputAdapter = { parse(events) { - return String(events.at(-1)?.data?.finalText ?? '') + const result = [...events].reverse().find((event) => event.type === 'result') + return String(result?.data?.finalText ?? '') }, } diff --git a/src/runtime/run-loop.test.ts b/src/runtime/run-loop.test.ts index 9ef09baf..19ab5d49 100644 --- a/src/runtime/run-loop.test.ts +++ b/src/runtime/run-loop.test.ts @@ -22,6 +22,7 @@ describe('runAgentRounds sandbox preparation', () => { order.push(`stream:${prompt}`) expect(order).toContain('prepare') yield { type: 'result', data: { finalText: 'done' } } as SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent }, } as SandboxInstance const client: SandboxClient = { @@ -48,7 +49,8 @@ describe('runAgentRounds sandbox preparation', () => { } const output: OutputAdapter = { parse(events) { - return String(events.at(-1)?.data?.finalText ?? '') + const result = [...events].reverse().find((event) => event.type === 'result') + return String(result?.data?.finalText ?? '') }, } @@ -103,6 +105,7 @@ describe('runAgentRounds onSandboxEvent tee', () => { { type: 'token', data: { text: 'hi' } } as SandboxEvent, { type: 'token', data: { text: 'there' } } as SandboxEvent, { type: 'result', data: { finalText: 'done' } } as SandboxEvent, + { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent, ] function runWithObserver(onSandboxEvent: Observer, stream: SandboxEvent[] = STREAM) { @@ -134,7 +137,8 @@ describe('runAgentRounds onSandboxEvent tee', () => { } const output: OutputAdapter = { parse(events) { - return String(events.at(-1)?.data?.finalText ?? '') + const result = [...events].reverse().find((event) => event.type === 'result') + return String(result?.data?.finalText ?? '') }, } return runAgentRounds({ @@ -219,9 +223,13 @@ describe('runAgentRounds onSandboxEvent tee', () => { (event) => { seen.push(event.type) }, - [nonCloneable, { type: 'result', data: { finalText: 'done' } } as SandboxEvent], + [ + nonCloneable, + { type: 'result', data: { finalText: 'done' } } as SandboxEvent, + { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent, + ], ) - expect(seen).toEqual(['tool', 'result']) + expect(seen).toEqual(['tool', 'result', 'done']) expect(result.iterations[0]?.output).toBe('done') }) @@ -248,7 +256,7 @@ describe('runAgentRounds onSandboxEvent tee', () => { d.finalText = 'corrupted' d.usage.inputTokens = 999 }, - [event], + [event, { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent], ) expect(result.iterations[0]?.output).toBe('done') expect(result.tokenUsage.input).toBe(10) @@ -274,7 +282,7 @@ describe('runAgentRounds onSandboxEvent tee', () => { const d = (ev as unknown as { data: { usageAlias: { inputTokens: number } } }).data d.usageAlias.inputTokens = 999 }, - [event], + [event, { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent], ) expect(result.tokenUsage.input).toBe(10) }) @@ -295,7 +303,7 @@ describe('runAgentRounds onSandboxEvent tee', () => { const d = (ev as unknown as { data: { finalText?: string } }).data d.finalText = 'corrupted' }, - [event], + [event, { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent], ) expect(result.iterations[0]?.output).toBe('done') expect(result.tokenUsage.input).toBe(10) diff --git a/src/runtime/run-loop.ts b/src/runtime/run-loop.ts index e872ac55..e714dcd6 100644 --- a/src/runtime/run-loop.ts +++ b/src/runtime/run-loop.ts @@ -25,13 +25,13 @@ */ import type { SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' +import { createAgentRunOutcomeTracker } from '@tangle-network/sandbox/runtime' import { ValidationError } from '../errors' import { notifyRuntimeHookEvent } from '../runtime-hooks' import { acquireSandbox } from './sandbox-acquire' import { buildBackendOptions } from './sandbox-backend' import { probeSandboxCapabilities } from './sandbox-capabilities' import { - assertSandboxEventSucceeded, assertSandboxServedModel, extractLlmCallEvent, notifySandboxEventObserver, @@ -42,6 +42,7 @@ import { type SandboxLineage, type SandboxLineageHandle, } from './sandbox-lineage' +import { projectSandboxOutcome } from './sandbox-outcome' import { assertExecutableAgentProfile, concreteProfileModel } from './supervise/model-policy' import type { AgentRunSpec, @@ -747,8 +748,10 @@ async function executeIteration(args: ExecuteIterationArgs(args: ExecuteIterationArgs(args: ExecuteIterationArgs): unknown return copy } -/** Event types that settle one sandbox execution. A status or `success` flag is read as a - * verdict on the whole execution only here; on any other event it describes that event. */ -const TERMINAL_EVENT_TYPES: ReadonlySet = new Set(['done', 'result', 'final']) - -/** - * Return the terminal failure carried by one Sandbox event. - * - * Sandbox transports report execution failure in-band: commonly an `error` - * event followed by a synthetic `done`. Treating the iterable as successfully - * drained therefore turns a provider/configuration failure into a completed - * empty artifact. - * - * The decoder reads a failure from exactly two places, so that a mid-stream - * event describing its OWN failure cannot fail the execution: an `error`-typed - * event, and a terminal event (`done`/`result`/`final`) whose `success` is - * `false` or whose status is a failed one. A failing tool part therefore stays - * a tool result, which is what {@link mapSandboxToolEvent} already projects it - * to. - */ -export function sandboxEventFailure(event: SandboxEvent): string | undefined { - if (!event || typeof event !== 'object') return undefined - const type = String(event.type ?? '') - const data = - event.data && typeof event.data === 'object' - ? (event.data as Record) - : ({} as Record) - - const outcome = plainRecord(data.outcome) - const status = firstString( - data.status, - outcome?.type, - outcome?.status, - plainRecord(data.result)?.status, - ) - const terminal = TERMINAL_EVENT_TYPES.has(type) - const terminalFailure = - terminal && (data.success === false || (status !== undefined && TERMINAL_FAILURE.test(status))) - if (type !== 'error' && !terminalFailure) return undefined - - return ( - describeSandboxError(data.error) ?? - describeSandboxError(outcome?.error) ?? - describeSandboxError(plainRecord(data.result)?.error) ?? - (typeof data.message === 'string' && data.message.length > 0 ? data.message : undefined) ?? - (status !== undefined - ? `sandbox execution ended with status ${status}` - : 'sandbox execution failed') - ) -} - -/** Fail the live execution instead of allowing an in-band failure to become an empty success. */ -export function assertSandboxEventSucceeded(event: SandboxEvent): void { - const failure = sandboxEventFailure(event) - if (failure !== undefined) throw new Error(`sandbox execution failed: ${failure}`) -} - /** The provider/model the platform reports it actually bound to a turn, when it reports one. * `source` is the platform's own account of where that choice came from — `environment` means * the platform chose, not the request. */ @@ -247,13 +191,6 @@ function stripProviderPrefix(id: string, providers: readonly (string | undefined return out } -function describeSandboxError(value: unknown): string | undefined { - if (typeof value === 'string' && value.length > 0) return value - if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined - const error = value as Record - return firstString(error.message, error.error, error.reason, error.code) -} - function firstString(...values: unknown[]): string | undefined { return values.find((value): value is string => typeof value === 'string' && value.length > 0) } @@ -264,8 +201,8 @@ function firstString(...values: unknown[]): string | undefined { * so the kernel can iterate the full stream without branching. * * Pure by contract: it never throws on a failed run. The terminal truth - * boundary is {@link assertSandboxEventSucceeded}, applied by the two paths - * that SETTLE an execution. Post-hoc readers — {@link sumSandboxUsage}, the + * boundary is the public Sandbox outcome tracker, applied after the complete + * stream. Post-hoc readers — {@link sumSandboxUsage}, the * analyst trace store, the chat projection — must stay able to read a failed * turn's events, which is when reading them matters most. * @@ -603,8 +540,8 @@ export function createSandboxToolPartState(): SandboxToolPartState { return { statusByCall: new Map(), seq: 0 } } -/** Statuses that settle a tool call, or a whole execution, as a failure. One vocabulary for - * both so a status never counts as failed in one projection and settled-fine in the other. */ +/** Statuses that settle one tool call as a failure. A failed tool call stays a tool result: + * the run's terminal state comes from the public Sandbox outcome tracker, never from here. */ const TERMINAL_FAILURE = /^(error|errored|failed|failure|cancelled|canceled|timeout|timed_out)$/i /** diff --git a/src/runtime/sandbox-executor-output.ts b/src/runtime/sandbox-executor-output.ts new file mode 100644 index 00000000..6258c012 --- /dev/null +++ b/src/runtime/sandbox-executor-output.ts @@ -0,0 +1,50 @@ +import { + collectAgentFinalMessageText, + collectAgentResponseText, + type SandboxEvent, +} from '@tangle-network/sandbox' +import type { AgentRunOutcome } from '@tangle-network/sandbox/runtime' +import { createSandboxToolPartState, mapSandboxToolEvent } from './sandbox-events' + +/** One tool call retained in a Sandbox executor artifact. */ +export interface SandboxExecutorToolCall { + id?: string + name: string + arguments: unknown +} + +/** Parsed output of one Sandbox executor turn. */ +export interface SandboxLeafOut { + events: SandboxEvent[] + content: string + toolCalls?: SandboxExecutorToolCall[] + outcome?: AgentRunOutcome +} + +/** Project the complete Sandbox event stream into the standard executor result shape. */ +export function sandboxLeafOutputFromEvents(events: SandboxEvent[]): SandboxLeafOut { + const content = collectAgentFinalMessageText(events) ?? collectAgentResponseText(events) ?? '' + const toolCalls = sandboxToolCalls(events) + return { + events, + content, + ...(toolCalls.length === 0 ? {} : { toolCalls }), + } +} + +function sandboxToolCalls(events: readonly SandboxEvent[]): SandboxExecutorToolCall[] { + const state = createSandboxToolPartState() + return events.flatMap((event) => + mapSandboxToolEvent(event, state).flatMap((projected) => + projected.type === 'tool_call' + ? [ + { + ...(projected.toolCallId === undefined ? {} : { id: projected.toolCallId }), + name: projected.toolName, + arguments: projected.args ?? {}, + }, + ] + : [], + ), + ) +} diff --git a/src/runtime/sandbox-lineage.ts b/src/runtime/sandbox-lineage.ts index aecb800c..f624e647 100644 --- a/src/runtime/sandbox-lineage.ts +++ b/src/runtime/sandbox-lineage.ts @@ -91,14 +91,34 @@ async function* pollPromptEvents( const activeSessionId = dispatched.sessionId const result = await box.session(activeSessionId).result() if (signal.aborted) throwAbort() + const resultData = { + finalText: result.response ?? '', + success: result.success, + ...(result.status ? { status: result.status } : {}), + ...(result.error ? { error: result.error } : {}), + ...(result.errorCode ? { errorCode: result.errorCode } : {}), + ...(result.toolInvocations ? { toolInvocations: result.toolInvocations } : {}), + ...(result.approval ? { approval: result.approval } : {}), + ...(result.question ? { question: result.question } : {}), + ...(result.interaction ? { interaction: result.interaction } : {}), + ...(result.plan ? { plan: result.plan } : {}), + ...(result.usage ? { usage: result.usage } : {}), + ...(result.costUsd !== undefined ? { costUsd: result.costUsd } : {}), + } yield { type: 'result', id: activeSessionId, + data: resultData, + } + yield { + type: 'done', + id: activeSessionId, data: { - finalText: result.response ?? '', - success: result.success, - ...(result.error ? { error: result.error } : {}), - ...(result.usage ? { usage: result.usage } : {}), + ...resultData, + outcome: + result.status === 'awaiting_plan_decision' && result.plan + ? { type: 'awaiting_plan_decision', plan: result.plan } + : { type: 'completed' }, }, } } diff --git a/src/runtime/sandbox-outcome.ts b/src/runtime/sandbox-outcome.ts new file mode 100644 index 00000000..bba2bf48 --- /dev/null +++ b/src/runtime/sandbox-outcome.ts @@ -0,0 +1,51 @@ +import type { AgentRunOutcome } from '@tangle-network/sandbox/runtime' +import type { AgentTaskStatus, BackendErrorDetail } from '../types' +import type { DefaultVerdict } from './supervise/types' + +/** The outcome carried by Runtime-owned Sandbox artifacts. */ +export interface SandboxOutcomeCarrier { + outcome?: AgentRunOutcome +} + +/** Runtime's terminal projection of the public Sandbox outcome contract. */ +export interface SandboxOutcomeProjection { + status: AgentTaskStatus + reason: string + verdict: DefaultVerdict + error?: BackendErrorDetail +} + +/** Map the public Sandbox outcome to Runtime's stream and verdict vocabulary. */ +export function projectSandboxOutcome(outcome: AgentRunOutcome): SandboxOutcomeProjection { + if (outcome.status === 'success') { + return { + status: 'completed', + reason: 'turn completed', + verdict: { valid: true, score: 1 }, + } + } + + const reason = outcome.error ?? 'Sandbox agent run failed' + if (outcome.status === 'failed') { + return { + status: 'failed', + reason, + verdict: { valid: false, score: 0 }, + error: { kind: 'backend', message: reason }, + } + } + + return { + status: 'blocked', + reason, + verdict: { valid: false, score: 0 }, + } +} + +/** Read an outcome that Runtime attached to one of its own executor artifacts. */ +export function readSandboxOutcome(value: unknown): AgentRunOutcome | undefined { + if (!value || typeof value !== 'object') return undefined + const outcome = (value as SandboxOutcomeCarrier).outcome + if (!outcome || typeof outcome !== 'object') return undefined + return outcome +} diff --git a/src/runtime/sandbox-run.ts b/src/runtime/sandbox-run.ts index 9d67e1b6..7e9db85f 100644 --- a/src/runtime/sandbox-run.ts +++ b/src/runtime/sandbox-run.ts @@ -32,6 +32,7 @@ */ import type { PromptOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' +import { type AgentRunOutcome, createAgentRunOutcomeTracker } from '@tangle-network/sandbox/runtime' import type { RuntimeHooks, RuntimeHookTarget } from '../runtime-hooks' import { notifyRuntimeHookEvent } from '../runtime-hooks' import { boxReadErrorMessage, readBoxPathWithRetry } from './box-read-retry' @@ -64,6 +65,8 @@ export type Deliverable = export interface TurnResult { out: Out events: SandboxEvent[] + /** Outcome settled by the public Sandbox tracker after the stream drained. */ + outcome: AgentRunOutcome readError?: string } @@ -251,11 +254,13 @@ export async function openSandboxRun( turnKind: 'start' | 'resume', ): Promise> { const collected: SandboxEvent[] = [] + const outcomeTracker = createAgentRunOutcomeTracker() // The stream itself can throw an AbortError when the run is cancelled mid-drain; // re-throw it carrying the events drained so far so the partial trace is not lost. try { for await (const ev of events) { collected.push(ev) + outcomeTracker.observe(ev) notifySandboxEventObserver(ev, options.onSandboxEvent, { turnIndex, turnKind, @@ -266,8 +271,9 @@ export async function openSandboxRun( if (isAbortError(err)) throw new SandboxRunAbortError(collected) throw err } + const outcome = outcomeTracker.finish() if (deliverable.kind === 'events') { - return { out: deliverable.fromEvents(collected), events: collected } + return { out: deliverable.fromEvents(collected), events: collected, outcome } } if (options.signal.aborted) throw new SandboxRunAbortError(collected) // The data plane can transiently 404 a just-written artifact (write not yet @@ -288,6 +294,7 @@ export async function openSandboxRun( return { out: deliverable.fromArtifact(raw, collected), events: collected, + outcome, ...(readError !== undefined ? { readError } : {}), } } diff --git a/src/runtime/stream-agent-turn.test.ts b/src/runtime/stream-agent-turn.test.ts index 3f464a0f..f59d9c00 100644 --- a/src/runtime/stream-agent-turn.test.ts +++ b/src/runtime/stream-agent-turn.test.ts @@ -7,6 +7,7 @@ * paths. No network, no credentials. */ +import { type InteractionRequest, interactionRequestDigest } from '@tangle-network/agent-interface' import type { SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' @@ -28,6 +29,30 @@ function finalOf(events: RuntimeStreamEvent[]): RuntimeStreamEvent & { type: 'fi return final } +function doneEvent(data: Record = {}): SandboxEvent { + return { type: 'done', data: { outcome: { type: 'completed' }, ...data } } +} + +function questionRequest(): InteractionRequest { + const material = { + id: 'interaction-1', + kind: 'question', + title: 'Choose a target', + answerSpec: { + fields: [{ type: 'text' as const, name: 'target', label: 'Target' }], + }, + binding: { + runId: 'run-1', + provider: 'tangle-sandbox', + environmentId: 'sandbox-1', + sessionId: 'session-1', + executionId: 'execution-1', + interactionId: 'interaction-1', + }, + } + return { ...material, requestDigest: interactionRequestDigest(material) } +} + describe('streamAgentTurn: box backend', () => { async function makeBox(events: SandboxEvent[]) { const client = inProcessSandboxClient({ onPrompt: () => events }) @@ -40,6 +65,7 @@ describe('streamAgentTurn: box backend', () => { { type: 'message.part.updated', data: { part: { type: 'text' }, delta: 'world' } }, { type: 'llm_call', data: { model: 'kimi-k2', tokensIn: 100, tokensOut: 40, costUsd: 0.02 } }, { type: 'result', data: { finalText: 'Hello world' } }, + doneEvent(), ] as SandboxEvent[]) const seen: RuntimeStreamEvent[] = [] @@ -70,7 +96,7 @@ describe('streamAgentTurn: box backend', () => { it('collectAgentTurn round-trips the terminal summary', async () => { const box = await makeBox([ { type: 'message.part.updated', data: { part: { type: 'text' }, delta: '42' } }, - { type: 'done', data: { tokenUsage: { inputTokens: 7, outputTokens: 3 } } }, + doneEvent({ tokenUsage: { inputTokens: 7, outputTokens: 3 } }), ] as SandboxEvent[]) const turn = await collectAgentTurn( @@ -110,7 +136,7 @@ describe('streamAgentTurn: box backend', () => { const box = await inProcessSandboxClient({ onPrompt: (prompt): SandboxEvent[] => { prompts.push(prompt) - return [{ type: 'done', data: { finalText: 'answer' } }] + return [doneEvent({ finalText: 'answer' })] }, }).create() @@ -132,6 +158,120 @@ describe('streamAgentTurn: box backend', () => { }) }) +describe('streamAgentTurn: Sandbox outcome contract', () => { + const question = questionRequest() + const plan = { + id: 'plan-1', + revision: 1, + title: 'Run the checks', + body: 'Run the checks and report the result.', + submittedAt: '2026-08-19T00:00:00.000Z', + } + const approvalTool = { + toolName: 'send_email', + input: { to: 'user@example.com' }, + result: { + structuredContent: { + code: 'HUB_APPROVAL_REQUIRED', + message: 'Email approval required', + details: { approval: { id: 'approval-1', connectionId: 'connection-1' } }, + }, + }, + isError: true, + } + + const cases: Array<{ + name: string + events: SandboxEvent[] + finalStatus: 'blocked' | 'failed' + outcome: Record + }> = [ + { + name: 'question response request', + events: [{ type: 'interaction', data: { request: question } }, doneEvent()], + finalStatus: 'blocked', + outcome: { + success: false, + status: 'awaiting_question', + interaction: question, + question: { questionId: question.id, questions: [{ question: 'Target' }] }, + }, + }, + { + name: 'durable plan response request', + events: [ + { type: 'plan.submitted', data: { plan } }, + doneEvent({ outcome: { type: 'awaiting_plan_decision', plan } }), + ], + finalStatus: 'blocked', + outcome: { success: false, status: 'awaiting_plan_decision', plan }, + }, + { + name: 'approval response request', + events: [ + { type: 'result', data: { toolInvocations: [approvalTool] } }, + doneEvent({ toolInvocations: [approvalTool] }), + ], + finalStatus: 'blocked', + outcome: { + success: false, + status: 'blocked_on_approval', + toolInvocations: [approvalTool], + approval: { + message: 'Email approval required', + approvalId: 'approval-1', + connectionId: 'connection-1', + }, + }, + }, + { + name: 'run error after a response request', + events: [ + { type: 'interaction', data: { request: question } }, + { type: 'error', data: { message: 'late provider failure' } }, + ], + finalStatus: 'failed', + outcome: { success: false, status: 'failed', error: 'late provider failure' }, + }, + { + name: 'failed terminal after a response request', + events: [ + { type: 'interaction', data: { request: question } }, + doneEvent({ status: 'failed', error: 'terminal provider failure' }), + ], + finalStatus: 'failed', + outcome: { success: false, status: 'failed', error: 'terminal provider failure' }, + }, + { + name: 'result without a terminal', + events: [{ type: 'result', data: { finalText: 'not complete' } }], + finalStatus: 'failed', + outcome: { + success: false, + status: 'failed', + error: 'Agent stream ended without a terminal event', + }, + }, + ] + + it.each(cases)('settles $name through the public tracker', async (testCase) => { + const client = inProcessSandboxClient({ onPrompt: () => testCase.events }) + const box = await client.create() + const turn = await collectAgentTurn( + streamObservedAgentTurn({ kind: 'box', box }, { prompt: testCase.name }), + ) + const final = finalOf(turn.events) + + expect(turn.status).toBe(testCase.finalStatus) + expect(turn.sandboxOutcome).toMatchObject(testCase.outcome) + expect(final.status).toBe(testCase.finalStatus) + expect(final.metadata).toMatchObject({ + sandboxOutcome: testCase.outcome, + verdict: { valid: false, score: 0 }, + }) + }) +}) + describe('streamAgentTurn: current Sandbox prompt options', () => { it('forwards current prompt options and folds usage identically', async () => { const calls: { mode?: string; options?: Record }[] = [] @@ -141,14 +281,11 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { expect(ctx.signal).toBeInstanceOf(AbortSignal) return [ { type: 'message.part.updated', data: { part: { type: 'text' }, delta: 'task output' } }, - { - type: 'done', - data: { - tokenUsage: { inputTokens: 9, outputTokens: 4 }, - totalCostUsd: 0.01, - model: 'kimi-k2', - }, - }, + doneEvent({ + tokenUsage: { inputTokens: 9, outputTokens: 4 }, + totalCostUsd: 0.01, + model: 'kimi-k2', + }), ] as SandboxEvent[] }, }) @@ -288,10 +425,7 @@ describe('streamAgentTurn: canonical event precedence', () => { async (_label, source, type) => { const box = await makeBox([ source, - { - type: 'done', - data: { finalText: 'finished', tokenUsage: { inputTokens: 1, outputTokens: 1 } }, - }, + doneEvent({ finalText: 'finished', tokenUsage: { inputTokens: 1, outputTokens: 1 } }), ] as SandboxEvent[]) const turn = await collectAgentTurn( streamObservedAgentTurn({ kind: 'box', box }, { prompt: 'canonical' }), @@ -317,7 +451,7 @@ describe('streamAgentTurn: canonical event precedence', () => { }, }, }, - { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 1 } }), ] as SandboxEvent[]) const turn = await collectAgentTurn( streamObservedAgentTurn( @@ -334,7 +468,7 @@ describe('streamAgentTurn: canonical event precedence', () => { const observed: SandboxEvent[] = [] const box = await makeBox([ { type: 'provider.secret', data: { token: 'do-not-persist' } }, - { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 1 } }), ] as SandboxEvent[]) const turn = await collectAgentTurn( streamObservedAgentTurn( @@ -389,7 +523,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { }, }, { type: 'message.part.updated', data: { part: { type: 'text' }, delta: 'listed' } }, - { type: 'done', data: { tokenUsage: { inputTokens: 5, outputTokens: 2 } } }, + doneEvent({ tokenUsage: { inputTokens: 5, outputTokens: 2 } }), ] as SandboxEvent[] async function makeBox(events: SandboxEvent[]) { @@ -451,7 +585,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { }, }, }, - { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 1 } }), ] as SandboxEvent[]) const turn = await collectAgentTurn( streamObservedAgentTurn( @@ -473,7 +607,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { [ { type: 'tool.call', data: { id: 't-1', name: 'search', input: { q: 'tangle' } } }, { type: 'tool.result', data: { id: 't-1', name: 'search', output: 'hit' } }, - { type: 'done', data: { tokenUsage: { inputTokens: 3, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 3, outputTokens: 1 } }), ] as SandboxEvent[], }) const box = await client.create() @@ -505,7 +639,7 @@ describe('streamAgentTurn: raw-event tap (onRawEvent)', () => { // `step-start` has no chat-UX projection — the tap must still see it. { type: 'message.part.updated', data: { part: { type: 'step-start' } } }, { type: 'message.part.updated', data: { part: { type: 'text' }, delta: 'hi' } }, - { type: 'done', data: { tokenUsage: { inputTokens: 2, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 2, outputTokens: 1 } }), ] as SandboxEvent[], }) const box = await client.create() @@ -549,10 +683,7 @@ describe('streamAgentTurn: mid-stream lifecycle (pull-based, no extra API)', () data: { part: { type: 'text' }, delta: 'b' }, } as SandboxEvent log.push('produced:done') - yield { - type: 'done', - data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } }, - } as SandboxEvent + yield doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 1 } }) }, }) const box = await client.create() @@ -586,15 +717,13 @@ describe('streamAgentTurn: mid-stream lifecycle (pull-based, no extra API)', () const client = inProcessSandboxClient({ onPrompt: (_prompt, ctx) => ctx.round === 0 - ? ([ - { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 0 } } }, - ] as SandboxEvent[]) + ? ([doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 0 } })] as SandboxEvent[]) : ([ { type: 'message.part.updated', data: { part: { type: 'text' }, delta: 'real answer' }, }, - { type: 'done', data: { tokenUsage: { inputTokens: 2, outputTokens: 2 } } }, + doneEvent({ tokenUsage: { inputTokens: 2, outputTokens: 2 } }), ] as SandboxEvent[]), }) const box = await client.create() @@ -741,6 +870,57 @@ describe('streamAgentTurn: executor backend', () => { expect(toreDown).toBe(1) }) + it('projects one Sandbox executor result without a consumer wrapper', async () => { + const client = inProcessSandboxClient({ + onPrompt: () => + [ + { + type: 'message.part.updated', + data: { + part: { + type: 'tool', + callID: 'call-1', + tool: 'read', + state: { status: 'completed', input: { path: 'README.md' }, output: 'ok' }, + }, + }, + }, + { + type: 'message.part.updated', + data: { part: { id: 'answer-1', type: 'text', text: 'cloud answer' } }, + }, + { type: 'result', data: { finalText: 'tool noise\ncloud answer' } }, + doneEvent({ tokenUsage: { inputTokens: 9, outputTokens: 3 } }), + ] as SandboxEvent[], + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { + kind: 'executor', + factory: createExecutor({ backend: 'sandbox', sandboxClient: client }), + profile: { ...TEST_PROFILE, harness: 'opencode' }, + }, + { prompt: 'read the file' }, + { preserveToolParts: true }, + ), + ) + + expect(turn.status).toBe('completed') + expect(turn.finalText).toBe('cloud answer') + expect(turn.sandboxOutcome).toEqual({ success: true, status: 'success' }) + expect(turn.events).toContainEqual( + expect.objectContaining({ + type: 'tool_call', + toolCallId: 'call-1', + toolName: 'read', + args: '{"path":"README.md"}', + }), + ) + expect(finalOf(turn.events).metadata).toMatchObject({ + sandboxOutcome: { success: true, status: 'success' }, + }) + }) + it('uses one detached profile snapshot even when the caller mutates nested input mid-turn', async () => { const providerOptions = { mode: 'before' } const profile = { diff --git a/src/runtime/stream-agent-turn.ts b/src/runtime/stream-agent-turn.ts index d4abb380..54a8ae81 100644 --- a/src/runtime/stream-agent-turn.ts +++ b/src/runtime/stream-agent-turn.ts @@ -55,6 +55,7 @@ import { } from '@tangle-network/agent-interface' import type { AgentTurnInput } from '@tangle-network/agent-interface/environment-provider' import type { PromptOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' +import { type AgentRunOutcome, createAgentRunOutcomeTracker } from '@tangle-network/sandbox/runtime' import { normalizeBackendStreamEvent } from '../backends' import { BackendTransportError, ValidationError } from '../errors' import { newRuntimeSession, nowIso } from '../sessions' @@ -74,6 +75,7 @@ import { mapSandboxEvent, mapSandboxToolEvent, } from './sandbox-events' +import { projectSandboxOutcome, readSandboxOutcome } from './sandbox-outcome' import { executableAgentProfileSnapshot } from './supervise/executable-spec' import { authoredProfileDigest, @@ -406,6 +408,8 @@ export interface CollectedAgentTurn { events: RuntimeStreamEvent[] status: AgentTaskStatus error?: BackendErrorDetail + /** Public Sandbox outcome, when the turn ran through a Sandbox stream or executor. */ + sandboxOutcome?: AgentRunOutcome } /** Mutable per-turn accumulator threaded through the backend adapters. */ @@ -426,10 +430,13 @@ interface TurnAccumulator { tokensKnown: boolean usdKnown: boolean sawLlmCall: boolean + sawTokenUsage: boolean + sawCostUsage: boolean model?: string stopReason?: string transportAttempts?: number result?: ExecutorResult + sandboxOutcome?: AgentRunOutcome } /** @@ -524,6 +531,8 @@ async function* streamAgentTurnInternal( tokensKnown: false, usdKnown: false, sawLlmCall: false, + sawTokenUsage: false, + sawCostUsage: false, } const deadline = deriveTurnSignal(opts.signal, opts.timeoutMs ?? 0) const startedAt = Date.now() @@ -658,11 +667,14 @@ async function* streamAgentTurnInternal( assertExactExecutorEvidence(materialization, executionBinding) } + const terminalOutcome = acc.sandboxOutcome + ? projectSandboxOutcome(acc.sandboxOutcome) + : { status: 'completed' as const, reason: 'turn completed' } yield buildFinalEvent( task, session, acc, - { status: 'completed', reason: 'turn completed' }, + terminalOutcome, turnProvenance( startedAt, opts.timeoutMs, @@ -697,12 +709,18 @@ async function* streamAgentTurnInternal( materialization = terminal } const callerAborted = opts.signal?.aborted === true - const status: AgentTaskStatus = callerAborted ? 'aborted' : 'failed' - const message = err instanceof Error ? err.message : String(err) + const sandboxProjection = acc.sandboxOutcome + ? projectSandboxOutcome(acc.sandboxOutcome) + : undefined + const status: AgentTaskStatus = callerAborted + ? 'aborted' + : (sandboxProjection?.status ?? 'failed') + const message = sandboxProjection?.reason ?? (err instanceof Error ? err.message : String(err)) const error: BackendErrorDetail = - err instanceof BackendTransportError + sandboxProjection?.error ?? + (err instanceof BackendTransportError ? { kind: 'transport', message, status: err.status, body: err.body } - : { kind: 'backend', message } + : { kind: 'backend', message }) yield { type: 'backend_error', task, @@ -779,6 +797,10 @@ export async function collectAgentTurn( metadata.result && typeof metadata.result === 'object' ? (metadata.result as Record) : undefined + const sandboxOutcome = + metadata.sandboxOutcome && typeof metadata.sandboxOutcome === 'object' + ? (metadata.sandboxOutcome as AgentRunOutcome) + : undefined const tokenUsage = metadata.tokenUsage && typeof metadata.tokenUsage === 'object' ? (metadata.tokenUsage as Record) @@ -818,6 +840,7 @@ export async function collectAgentTurn( events, status: final.status, ...(final.error ? { error: final.error } : {}), + ...(sandboxOutcome ? { sandboxOutcome } : {}), } } @@ -876,7 +899,9 @@ async function* driveBoxTurn( signal, }) const toolParts = cfg.preserveToolParts ? createSandboxToolPartState() : undefined + const outcomeTracker = createAgentRunOutcomeTracker() for await (const event of abortableValues(stream, signal)) { + outcomeTracker.observe(event) if (cfg.onRawEvent) { await awaitAbortable( Promise.resolve().then(() => cfg.onRawEvent!(event)), @@ -908,6 +933,11 @@ async function* driveBoxTurn( // Unknown provider events stay available through onRawEvent. Do not copy // arbitrary provider payloads into the public stream. } + acc.sandboxOutcome = outcomeTracker.finish() + const projection = projectSandboxOutcome(acc.sandboxOutcome) + if (projection.status === 'failed') { + throw new Error(projection.reason) + } } /** One turn over an in-process backend: its own `stream()` surface, projected @@ -942,7 +972,8 @@ async function* driveExecutorTurn( const run = executor.execute(taskValue, signal) let result: ExecutorResult if (isAsyncIterable(run)) { - for await (const _usage of abortableValues(run, signal)) { + for await (const usage of abortableValues(run, signal)) { + foldUsageEvent(usage, acc) throwIfAborted(signal) } result = executor.resultArtifact() @@ -950,6 +981,7 @@ async function* driveExecutorTurn( result = await awaitAbortable(run, signal) } acc.result = result + acc.sandboxOutcome = readSandboxOutcome(result.out) acc.terminalText = executorResultText(result.out) acc.input = result.spent.tokens.input acc.output = result.spent.tokens.output @@ -1010,6 +1042,9 @@ async function* driveExecutorTurn( }, timestamp: nowIso(), } + if (acc.sandboxOutcome && projectSandboxOutcome(acc.sandboxOutcome).status === 'failed') { + throw new Error(projectSandboxOutcome(acc.sandboxOutcome).reason) + } } function executorTaskValue(input: AgentTurnInput): unknown { @@ -1138,6 +1173,27 @@ function executorResultToolCalls( }) } +/** Fold one normalized executor usage event into the turn accumulator. */ +function foldUsageEvent(event: UsageEvent, acc: TurnAccumulator): void { + if (event.kind === 'tokens') { + const known = event.tokensKnown !== false + acc.tokensKnown = acc.sawTokenUsage ? acc.tokensKnown && known : known + acc.sawTokenUsage = true + acc.input += event.input + acc.output += event.output + } else if (event.kind === 'cost') { + const known = event.usdKnown !== false + acc.usdKnown = acc.sawCostUsage ? acc.usdKnown && known : known + acc.sawCostUsage = true + acc.costUsd += event.usd + if (event.usdEstimated !== undefined) { + acc.estimatedCostUsd += event.usdEstimated + acc.sawEstimatedCost = true + } + } + if (event.kind === 'tokens' || event.kind === 'cost') acc.sawLlmCall = true +} + /** Fold one normalized event into the turn accumulator (text + usage). * `fallbackModelLabel` — a mapper-stamped run label to exclude from * `usage.model` (it is not a backend-reported model). */ @@ -1215,16 +1271,28 @@ function buildFinalEvent( ...(acc.model ? { model: acc.model } : {}), ...(acc.stopReason ? { stopReason: acc.stopReason } : {}), ...(acc.transportAttempts !== undefined ? { transportAttempts: acc.transportAttempts } : {}), + ...(acc.sandboxOutcome + ? { + sandboxOutcome: acc.sandboxOutcome, + verdict: projectSandboxOutcome(acc.sandboxOutcome).verdict, + } + : {}), ...(acc.result ? { result: { outRef: acc.result.outRef, output: acc.result.out, - ...(acc.result.verdict ? { verdict: acc.result.verdict } : {}), + ...(acc.sandboxOutcome + ? { verdict: projectSandboxOutcome(acc.sandboxOutcome).verdict } + : acc.result.verdict + ? { verdict: acc.result.verdict } + : {}), spent: acc.result.spent, }, } - : {}), + : acc.sandboxOutcome + ? { result: { verdict: projectSandboxOutcome(acc.sandboxOutcome).verdict } } + : {}), ...provenance, }, ...(outcome.error ? { error: outcome.error } : {}), diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 5b189ee0..240e0714 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -88,6 +88,7 @@ import { } from '../router-client' import type { RunAgentRoundsOptions } from '../run-loop' import { runAgentRounds } from '../run-loop' +import { type SandboxLeafOut, sandboxLeafOutputFromEvents } from '../sandbox-executor-output' import type { AgentRunSpec, Driver, @@ -244,6 +245,8 @@ export interface SandboxSeam { steering?: SandboxSteeringOptions } +export type { SandboxExecutorToolCall, SandboxLeafOut } from '../sandbox-executor-output' + /** * UNMETERED CLI subprocess seam. `bin` + `args` describe the process to spawn. * @@ -1402,11 +1405,11 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { ) } - // The leaf runs an opaque, self-parallelizing coding harness; the loop just - // refines once over it. Output is the raw event stream parsed to its tail text. + // The leaf runs an opaque, self-parallelizing coding harness. Runtime keeps + // its complete event archive and projects the visible answer and tool calls. const output: OutputAdapter = { parse(events: SandboxEvent[]): SandboxLeafOut { - return { events } + return sandboxLeafOutputFromEvents(events) }, } const driver = singleShotDriver(maxIterations) @@ -1451,12 +1454,6 @@ export const sandboxExecutor: ExecutorFactory = (spec, ctx) => { ) } -/** Parsed output of the sandbox leaf: the iteration's raw event stream. What a - * `SandboxSeam.validator` receives as its `output` argument. */ -export interface SandboxLeafOut { - events: SandboxEvent[] -} - interface StreamSandboxArgs { task: unknown signal: AbortSignal @@ -1524,12 +1521,24 @@ async function* streamSandboxLeaf(args: StreamSandboxArgs): AsyncIterable iteration.index === result.winner?.iterationIndex) + : result.iterations.at(-1) + const out = + winningIteration?.output ?? sandboxLeafOutputFromEvents(winningIteration?.events ?? []) + const sandboxOutcome = winningIteration?.sandboxOutcome + const outWithOutcome: SandboxLeafOut = { + ...out, + ...(sandboxOutcome ? { outcome: sandboxOutcome } : {}), + } + const verdict = + sandboxOutcome && !sandboxOutcome.success + ? { valid: false, score: 0 } + : (result.winner?.verdict ?? leafVerdict(result)) const tokensKnown = result.tokenUsage.tokensKnown !== false const usdKnown = result.costUsdKnown !== false const outWithUsage = { - ...out, + ...outWithOutcome, ...(result.estimatedCostUsd !== undefined ? { estimatedCostUsd: result.estimatedCostUsd } : {}), diff --git a/src/runtime/supervise/sandbox-session.ts b/src/runtime/supervise/sandbox-session.ts index e995a1ec..dbc2d05e 100644 --- a/src/runtime/supervise/sandbox-session.ts +++ b/src/runtime/supervise/sandbox-session.ts @@ -31,14 +31,12 @@ import type { AgentProfile } from '@tangle-network/agent-interface' import type { BackendType, PromptOptions, SandboxEvent } from '@tangle-network/sandbox' +import { type AgentRunOutcome, createAgentRunOutcomeTracker } from '@tangle-network/sandbox/runtime' import { ValidationError } from '../../errors' import { probeSandboxCapabilities } from '../sandbox-capabilities' -import { - assertSandboxEventSucceeded, - assertSandboxServedModel, - extractLlmCallEvent, -} from '../sandbox-events' +import { assertSandboxServedModel, extractLlmCallEvent } from '../sandbox-events' import { createSandboxLineage, type SandboxLineageHandle } from '../sandbox-lineage' +import { projectSandboxOutcome } from '../sandbox-outcome' import type { AgentRunSpec, ExecCtx, SandboxClient } from '../types' import { addTokenUsage, promptCacheTokenClasses, zeroTokenUsage } from '../util' import type { Inbox } from './inbox' @@ -51,7 +49,7 @@ import { type ExecutorProgress, } from './progress' import { createPushTraceSource, decodeToolPart, type TraceSource } from './trace-source' -import type { Spend, UsageEvent } from './types' +import type { DefaultVerdict, Spend, UsageEvent } from './types' /** Ceiling on continuation turns. Turn 0 is the task; every later turn is a folded steer, so * this bounds how many times a supervisor may redirect ONE worker before it must respawn. */ @@ -74,7 +72,7 @@ export interface SteerableSandboxSession { stream(task: unknown, signal: AbortSignal): AsyncIterable progress(): ExecutorProgress traceSource(): TraceSource - artifact(): { outRef: string; out: unknown; spent: Spend } | undefined + artifact(): { outRef: string; out: unknown; verdict?: DefaultVerdict; spent: Spend } | undefined teardown(): Promise } @@ -113,7 +111,10 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer turns: 0, lastText: '', seenToolCalls: new Set(), - artifact: undefined as { outRef: string; out: unknown; spent: Spend } | undefined, + artifact: undefined as + | { outRef: string; out: unknown; verdict?: DefaultVerdict; spent: Spend } + | undefined, + latestOutcome: undefined as AgentRunOutcome | undefined, teardown: undefined as (() => Promise) | undefined, note: 'starting', } @@ -224,6 +225,7 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer let events: AsyncIterable let sawLlmCall = false + const outcomeTracker = createAgentRunOutcomeTracker() try { if (!handle) { const opened = await lineage.start(spec, prompt, turnController.signal, promptOptions) @@ -233,41 +235,40 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer events = await lineage.continue(handle, prompt, turnController.signal, promptOptions) } for await (const event of events) { + outcomeTracker.observe(event) recordEvent(event) - // The same terminal truth boundary the single-shot leaf applies, after the event is - // recorded: an in-band SDK failure, or a box serving a model other than the one the - // exact profile asked for, ends the turn instead of settling it as an empty success. - assertSandboxEventSucceeded(event) - assertSandboxServedModel(event, requested) const call = extractLlmCallEvent(event, spec.name ?? String(args.harness)) - if (!call) continue - sawLlmCall = true - const callTokensKnown = - call.tokensKnown !== false && - typeof call.tokensIn === 'number' && - typeof call.tokensOut === 'number' - if (!callTokensKnown) tokensKnown = false - const input = call.tokensIn ?? 0 - const output = call.tokensOut ?? 0 - if (input || output || !callTokensKnown) { - const usage: Extract = { - kind: 'tokens', - input, - output, - ...(callTokensKnown ? {} : { tokensKnown: false }), - // Classify against THIS call's own prompt total, so the classes partition the - // number they belong to rather than a running sum from other calls. - ...promptCacheTokenClasses(call.tokensIn, call.promptCache), + if (call) { + sawLlmCall = true + const callTokensKnown = + call.tokensKnown !== false && + typeof call.tokensIn === 'number' && + typeof call.tokensOut === 'number' + if (!callTokensKnown) tokensKnown = false + const input = call.tokensIn ?? 0 + const output = call.tokensOut ?? 0 + if (input || output || !callTokensKnown) { + const usage: Extract = { + kind: 'tokens', + input, + output, + ...(callTokensKnown ? {} : { tokensKnown: false }), + // Classify against THIS call's own prompt total, so the classes partition the + // number they belong to rather than a running sum from other calls. + ...promptCacheTokenClasses(call.tokensIn, call.promptCache), + } + addTokenUsage(tokens, usage) + yield usage + } + if (typeof call.costUsd === 'number' && call.costUsd > 0) { + usd += call.costUsd + // Numeric sandbox cost has no billing-provenance/completeness receipt. + yield { kind: 'cost', usd: call.costUsd, usdKnown: false } } - addTokenUsage(tokens, usage) - yield usage - } - if (typeof call.costUsd === 'number' && call.costUsd > 0) { - usd += call.costUsd - // Numeric sandbox cost has no billing-provenance/completeness receipt. - yield { kind: 'cost', usd: call.costUsd, usdKnown: false } } + assertSandboxServedModel(event, requested) } + state.latestOutcome = outcomeTracker.finish() } catch (e) { cleanup() // Re-plan ONLY when a forceful steer (not external teardown) aborted the turn: the @@ -319,10 +320,15 @@ export function createSteerableSandboxSession(args: SteerableSandboxArgs): Steer .read() .filter((n) => n.kind === 'tool') .map((n) => n.label), + ...(state.latestOutcome ? { outcome: state.latestOutcome } : {}), } + const verdict = state.latestOutcome + ? projectSandboxOutcome(state.latestOutcome).verdict + : undefined state.artifact = { outRef: args.contentRef('sandbox-steerable', { harness: args.harness, out }), out, + ...(verdict ? { verdict } : {}), spent, } } diff --git a/src/runtime/types.ts b/src/runtime/types.ts index da63863f..91c34210 100644 --- a/src/runtime/types.ts +++ b/src/runtime/types.ts @@ -20,6 +20,7 @@ import type { SandboxEvent, SandboxInstance, } from '@tangle-network/sandbox' +import type { AgentRunOutcome } from '@tangle-network/sandbox/runtime' import type { RuntimeHooks } from '../runtime-hooks' import type { RuntimeRunHandle } from '../runtime-run' @@ -220,6 +221,8 @@ export interface Iteration { output?: Output verdict?: DefaultVerdict error?: Error + /** Public Sandbox outcome settled after the complete event stream. */ + sandboxOutcome?: AgentRunOutcome /** Raw sandbox event stream collected for this iteration. Present on a failed iteration too, * holding the events received before the failure — including the one that reported it. */ events: SandboxEvent[] diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 1003b30b..622ff9c7 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:b7013ba46d2e69b918f5b68fdb9598677be93996e65d59c7b5c8c55eb16e444a", + "digest": "sha256:48c661298166f66507443e0eec2f7eb29606502ecf140f0f53711eb4575bd831", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.142.3" + "runtimeVersion": "0.143.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:67fda76a6a9508fc1573c8f00d215051bd4be078ac85db538530e0d20411680b", - "runId": "agent-runtime-0.142.3-proposal-fixture", + "recordDigest": "sha256:37417d66cd978183738751bf0ba0bd7ccd05c69ac9e3ad9050e228681b488f9e", + "runId": "agent-runtime-0.143.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.142.3-proposal-fixture" + "runId": "agent-runtime-0.143.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 288d5753..ad9701fa 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:dd890bcb652aa108a8f0954fe0016853274cf96821f29dda07173738d1232d66", + "digest": "sha256:ac51cf5857d66e96b6cbdd2c9746ae1a44c95310bb89fb40cb947ebf0f8a0220", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.142.3" + "runtimeVersion": "0.143.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:b1c860be2dbc9e3a73bb296d76429f2554843a65d9c2e3c56a0072860e662de7", + "recordDigest": "sha256:45a87e9da971b040445440e2daf2a2cac7aff32ab3b4fc828dd93a1d5a9a4fbf", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/kernel/completion.test.ts b/tests/kernel/completion.test.ts index 5dd30299..4b54c02d 100644 --- a/tests/kernel/completion.test.ts +++ b/tests/kernel/completion.test.ts @@ -41,6 +41,7 @@ function echoClient() { return { async *streamPrompt(message: string) { yield { type: 'result', data: { answer: message } } satisfies SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } satisfies SandboxEvent }, } as unknown as SandboxInstance }, diff --git a/tests/kernel/loop-dispatch.test.ts b/tests/kernel/loop-dispatch.test.ts index c5920bbd..9aa7c0e4 100644 --- a/tests/kernel/loop-dispatch.test.ts +++ b/tests/kernel/loop-dispatch.test.ts @@ -51,7 +51,10 @@ function spec(): AgentRunSpec { const output: OutputAdapter = { parse: (events) => { - const data = events.at(-1)?.data as { attempt?: number } | undefined + const data = [...events] + .reverse() + .map((event) => event.data as { attempt?: number } | undefined) + .find((candidate) => typeof candidate?.attempt === 'number') return { attempt: typeof data?.attempt === 'number' ? data.attempt : -1 } }, } @@ -70,6 +73,7 @@ function stubClient(events: SandboxEvent[]): { return { async *streamPrompt() { for (const e of events) yield e + yield { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent }, } as unknown as SandboxInstance }, diff --git a/tests/kernel/sandbox-events.test.ts b/tests/kernel/sandbox-events.test.ts index 1d0d2ab8..4c96a4bb 100644 --- a/tests/kernel/sandbox-events.test.ts +++ b/tests/kernel/sandbox-events.test.ts @@ -4,7 +4,6 @@ import { assertSandboxServedModel, extractLlmCallEvent, mapSandboxEvent, - sandboxEventFailure, sandboxEventServedBackend, sumSandboxUsage, } from '../../src/runtime/sandbox-events' @@ -222,41 +221,6 @@ describe('the event decoders stay pure so a failed run can still be read', () => }) }) -describe('sandboxEventFailure — only the execution settles the execution', () => { - it('reads the SDK error event', () => { - expect( - sandboxEventFailure({ - type: 'error', - data: { error: { message: 'No API key found for anthropic' } }, - } as SandboxEvent), - ).toBe('No API key found for anthropic') - }) - - it('reads a terminal event that reports a failed status', () => { - expect(sandboxEventFailure({ type: 'done', data: { status: 'failed' } } as SandboxEvent)).toBe( - 'sandbox execution ended with status failed', - ) - }) - - it('reads a terminal event that reports success:false', () => { - expect( - sandboxEventFailure({ - type: 'result', - data: { success: false, message: 'provider rejected the request' }, - } as SandboxEvent), - ).toBe('provider rejected the request') - }) - - it('does not read a failing tool event as an execution failure', () => { - expect( - sandboxEventFailure({ - type: 'tool.result', - data: { status: 'error', success: false, name: 'bash' }, - } as SandboxEvent), - ).toBeUndefined() - }) -}) - describe('sandboxEventServedBackend — a request is not a receipt', () => { it('decodes the platform report', () => { expect( diff --git a/tests/kernel/sandbox-leaf-settle.test.ts b/tests/kernel/sandbox-leaf-settle.test.ts index 1181b561..e2a143ca 100644 --- a/tests/kernel/sandbox-leaf-settle.test.ts +++ b/tests/kernel/sandbox-leaf-settle.test.ts @@ -35,7 +35,23 @@ function sandboxClient(over: { createFails?: string } = {}) { return { id: 'box-0', async *streamPrompt(): AsyncGenerator { - yield { type: 'result', data: { ok: true, text: 'delivered' } } as SandboxEvent + yield { + type: 'message.part.updated', + data: { + part: { + type: 'tool', + callID: 'call-1', + tool: 'read', + state: { status: 'completed', input: { path: 'README.md' }, output: 'ok' }, + }, + }, + } as SandboxEvent + yield { + type: 'message.part.updated', + data: { part: { id: 'answer-1', type: 'text', text: 'delivered' } }, + } as SandboxEvent + yield { type: 'result', data: { finalText: 'tool noise\ndelivered' } } as SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent }, async delete() {}, } as unknown as SandboxInstance @@ -58,6 +74,10 @@ describe('sandbox leaf — the settle contract', () => { await drain(executor.execute('task', new AbortController().signal) as AsyncIterable) const artifact = executor.resultArtifact() expect(artifact.verdict?.valid).toBe(true) + expect(artifact.out).toMatchObject({ + content: 'delivered', + toolCalls: [{ id: 'call-1', name: 'read', arguments: { path: 'README.md' } }], + }) expect((artifact.out as { events: SandboxEvent[] }).events.length).toBeGreaterThan(0) }) diff --git a/tests/kernel/sandbox-lineage.test.ts b/tests/kernel/sandbox-lineage.test.ts index ace5fadc..5273d873 100644 --- a/tests/kernel/sandbox-lineage.test.ts +++ b/tests/kernel/sandbox-lineage.test.ts @@ -96,6 +96,7 @@ function createFakeClient(opts: FakeClientOpts) { }) opts.onStream?.(streamCalls.length) yield { type: 'result', data: { ok: true } } satisfies SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } satisfies SandboxEvent }, async checkpoint(_o?: { leaveRunning?: boolean }) { return { checkpointId: `cp-${checkpointSeq++}`, createdAt: new Date(), tags: [] } @@ -222,8 +223,10 @@ describe('runAgentRounds — streaming: poll (drop-resilient batch path)', () => }, } const pollOutput: OutputAdapter = { - parse: (events) => - String((events.at(-1)?.data as { finalText?: string } | undefined)?.finalText ?? ''), + parse: (events) => { + const result = [...events].reverse().find((event) => event.type === 'result') + return String((result?.data as { finalText?: string } | undefined)?.finalText ?? '') + }, } const moves: ScriptedMove[] = [{ kind: 'refine', task: { goal: 'g' } }, { kind: 'stop' }] let i = 0 diff --git a/tests/kernel/sandbox-terminal-truth.test.ts b/tests/kernel/sandbox-terminal-truth.test.ts index a123805e..5d5f7b05 100644 --- a/tests/kernel/sandbox-terminal-truth.test.ts +++ b/tests/kernel/sandbox-terminal-truth.test.ts @@ -1,7 +1,12 @@ import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import { createExecutor } from '../../src/runtime/supervise/runtime' -import type { AgentSpec, ExecutorContext, UsageEvent } from '../../src/runtime/supervise/types' +import type { + AgentSpec, + Executor, + ExecutorContext, + UsageEvent, +} from '../../src/runtime/supervise/types' import { testAgentProfile } from './test-agent-profile' const spec: AgentSpec = { @@ -12,6 +17,10 @@ const spec: AgentSpec = { harness: 'opencode', } +function doneEvent(data: Record = {}): SandboxEvent { + return { type: 'done', data: { ...data, outcome: { type: 'completed' } } } +} + function ctx(): ExecutorContext { return { signal: new AbortController().signal, seams: {} } } @@ -38,7 +47,7 @@ const failedEvents: readonly SandboxEvent[] = [ } as SandboxEvent, // The SDK may synthesize `done` after an error. It must not turn the // already-observed failure into an empty successful artifact. - { type: 'done', data: { status: 'failed' } } as SandboxEvent, + doneEvent({ status: 'failed' }), ] /** The measured substitution from agent-runtime#892: the box reports the model it really bound, @@ -55,7 +64,7 @@ const substitutedEvents: readonly SandboxEvent[] = [ }, }, } as SandboxEvent, - { type: 'done', data: { tokenUsage: { inputTokens: 10, outputTokens: 5 }, totalCostUsd: 0.1 } }, + doneEvent({ tokenUsage: { inputTokens: 10, outputTokens: 5 }, totalCostUsd: 0.1 }), ] as readonly SandboxEvent[] const honestEvents: readonly SandboxEvent[] = [ @@ -65,7 +74,7 @@ const honestEvents: readonly SandboxEvent[] = [ effectiveBackend: { provider: 'offline', model: 'offline-test-model', source: 'request' }, }, } as SandboxEvent, - { type: 'done', data: { tokenUsage: { inputTokens: 10, outputTokens: 5 }, totalCostUsd: 0.1 } }, + doneEvent({ tokenUsage: { inputTokens: 10, outputTokens: 5 }, totalCostUsd: 0.1 }), ] as readonly SandboxEvent[] async function drain(events: AsyncIterable): Promise { @@ -74,46 +83,54 @@ async function drain(events: AsyncIterable): Promise { } } -function run(events: readonly SandboxEvent[], steering?: { maxTurns: number }): Promise { +async function run( + events: readonly SandboxEvent[], + steering?: { maxTurns: number }, +): Promise> { const executor = createExecutor({ backend: 'sandbox', sandboxClient: sandboxClientEmitting(events), ...(steering ? { steering } : {}), })(spec, ctx()) - return drain(executor.execute('task', new AbortController().signal) as AsyncIterable) + await drain(executor.execute('task', new AbortController().signal) as AsyncIterable) + return executor } describe('sandbox terminal truth', () => { - it('fails the single-shot sandbox leaf on the SDK error event', async () => { - await expect(run(failedEvents)).rejects.toThrow(/No API key found for anthropic/) - }) - - it('fails the steerable sandbox session on the same SDK error event', async () => { - await expect(run(failedEvents, { maxTurns: 1 })).rejects.toThrow( - /No API key found for anthropic/, + it.each([ + { label: 'single-shot run error', steering: undefined }, + { label: 'steerable run error', steering: { maxTurns: 1 } }, + ])('$label settles failed with the public outcome', async ({ steering }) => { + const executor = await run(failedEvents, steering) + const artifact = executor.resultArtifact() + expect(artifact.verdict).toEqual({ valid: false, score: 0 }) + expect((artifact.out as { outcome: { status: string; error?: string } }).outcome).toMatchObject( + { + status: 'failed', + error: 'No API key found for anthropic', + }, ) }) - it('does not fail a run because one TOOL call failed', async () => { + it('does not fail a run because one tool call failed', async () => { // A tool that errors is a normal, reportable result. Only the execution's own error and // terminal events settle the execution — anything wider makes a working agent unrunnable. - await expect( - run([ - { - type: 'message.part.updated', - data: { - part: { - type: 'tool', - callID: 'call-1', - tool: 'bash', - state: { status: 'error', error: 'exit 1' }, - }, + const executor = await run([ + { + type: 'message.part.updated', + data: { + part: { + type: 'tool', + callID: 'call-1', + tool: 'bash', + state: { status: 'error', error: 'exit 1' }, }, - } as SandboxEvent, - { type: 'tool.result', data: { status: 'error', success: false, name: 'bash' } }, - { type: 'done', data: { tokenUsage: { inputTokens: 4, outputTokens: 2 } } }, - ] as readonly SandboxEvent[]), - ).resolves.toBeUndefined() + }, + } as SandboxEvent, + { type: 'tool.result', data: { status: 'error', success: false, name: 'bash' } }, + doneEvent({ tokenUsage: { inputTokens: 4, outputTokens: 2 } }), + ] as readonly SandboxEvent[]) + expect(executor.resultArtifact().verdict).toEqual({ valid: true, score: 1 }) }) }) @@ -137,7 +154,13 @@ describe('sandbox served-model truth', () => { }) it('accepts the run when the box reports serving the exact declared model', async () => { - await expect(run(honestEvents)).resolves.toBeUndefined() + const executor = await run(honestEvents) + expect(executor.resultArtifact().spent.tokens).toMatchObject({ input: 10, output: 5 }) + expect( + (executor.resultArtifact().out as { outcome: { status: string } }).outcome, + ).toMatchObject({ + status: 'success', + }) }) it('accepts a provider-prefixed spelling of the same model', async () => { @@ -153,9 +176,9 @@ describe('sandbox served-model truth', () => { }, }, } as SandboxEvent, - { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 1 } }), ] as readonly SandboxEvent[]), - ).resolves.toBeUndefined() + ).resolves.toBeDefined() }) it('never invents a verdict when the box reports no served model', async () => { @@ -164,8 +187,8 @@ describe('sandbox served-model truth', () => { await expect( run([ { type: 'execution.started', data: {} } as SandboxEvent, - { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, + doneEvent({ tokenUsage: { inputTokens: 1, outputTokens: 1 } }), ] as readonly SandboxEvent[]), - ).resolves.toBeUndefined() + ).resolves.toBeDefined() }) }) diff --git a/tests/kernel/supervise-live-box-validator.test.ts b/tests/kernel/supervise-live-box-validator.test.ts index 97894994..c0dbb3f3 100644 --- a/tests/kernel/supervise-live-box-validator.test.ts +++ b/tests/kernel/supervise-live-box-validator.test.ts @@ -75,6 +75,7 @@ function sandboxClient(answer: string): { client: { create: () => Promise { files.set('/work/answer.txt', answer) yield { type: 'result', data: { ok: true, text: 'wrote the answer' } } as SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent }, exec(command: string): Promise { // A destroyed container cannot answer. Throwing here is what makes "the validator saw diff --git a/tests/kernel/supervise-worker-trace.test.ts b/tests/kernel/supervise-worker-trace.test.ts index 9709ce43..8d9f82e1 100644 --- a/tests/kernel/supervise-worker-trace.test.ts +++ b/tests/kernel/supervise-worker-trace.test.ts @@ -408,6 +408,7 @@ function fakeSandboxClient() { id, async *streamPrompt(): AsyncGenerator { yield { type: 'result', data: { ok: true, text: 'done' } } as SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } as SandboxEvent }, async delete() {}, } as unknown as SandboxInstance diff --git a/tests/kernel/validator-tracing.test.ts b/tests/kernel/validator-tracing.test.ts index 6e7cea81..87660340 100644 --- a/tests/kernel/validator-tracing.test.ts +++ b/tests/kernel/validator-tracing.test.ts @@ -13,7 +13,8 @@ function makeSandboxClient() { return { id: 'box-1', async *streamPrompt() { - yield { type: 'text', content: 'done' } + yield { type: 'text', data: { content: 'done' } } + yield { type: 'done', data: { outcome: { type: 'completed' } } } }, async stop() {}, } as any diff --git a/tests/mcp/coder-delegate-selection.test.ts b/tests/mcp/coder-delegate-selection.test.ts index 3819611c..82c4cf75 100644 --- a/tests/mcp/coder-delegate-selection.test.ts +++ b/tests/mcp/coder-delegate-selection.test.ts @@ -47,6 +47,7 @@ function candidateClient() { return { async *streamPrompt() { yield { type: 'result', data: { result: out } } satisfies SandboxEvent + yield { type: 'done', data: { outcome: { type: 'completed' } } } satisfies SandboxEvent }, } as unknown as SandboxInstance }, @@ -133,6 +134,10 @@ describe('detachedSessionDelegate — reviewer gate + winner selection', () => { return { async *streamPrompt() { yield { type: 'result', data: { result: CANDIDATES[0] } } satisfies SandboxEvent + yield { + type: 'done', + data: { outcome: { type: 'completed' } }, + } satisfies SandboxEvent }, } as unknown as SandboxInstance }, diff --git a/tests/mcp/detached-turn.test.ts b/tests/mcp/detached-turn.test.ts index 32bce1a4..acedd95c 100644 --- a/tests/mcp/detached-turn.test.ts +++ b/tests/mcp/detached-turn.test.ts @@ -672,6 +672,7 @@ describe('detachedSessionDelegate detached path', () => { }, }, } + yield { type: 'done', data: { outcome: { type: 'completed' } } } }) const box = { ...fake.box, streamPrompt } const executor = createSiblingSandboxExecutor({ client: fakeClient(box) }) diff --git a/tests/runtime/sandbox-run.test.ts b/tests/runtime/sandbox-run.test.ts index 2fddeefd..5107bf15 100644 --- a/tests/runtime/sandbox-run.test.ts +++ b/tests/runtime/sandbox-run.test.ts @@ -30,6 +30,10 @@ interface StreamCall { signal?: AbortSignal } +function doneEvent(data: Record = {}): SandboxEvent { + return { type: 'done', data: { outcome: { type: 'completed' }, ...data } } +} + /** * A fake sandbox client whose box additionally exposes `fs.read` (the artifact * seam `openSandboxRun` adds over the pure events path) and a live `session` @@ -166,6 +170,37 @@ describe('openSandboxRun — events deliverable', () => { expect(run.sessionId).toBe(streamCalls[0]!.sessionId) expect(run.sessionId).toBeDefined() }) + + it.each([ + { + name: 'done is a live terminal', + events: [ + { type: 'result', data: { text: 'complete' } } as SandboxEvent, + doneEvent({ text: 'complete' }), + ], + outcome: { success: true, status: 'success' }, + }, + { + name: 'result alone is not a terminal', + events: [{ type: 'result', data: { text: 'partial' } } as SandboxEvent], + outcome: { + success: false, + status: 'failed', + error: 'Agent stream ended without a terminal event', + }, + }, + ])('$name', async ({ events, outcome }) => { + const { client } = createFakeClient({ events }) + const run = await openSandboxRun( + client, + { agentRun: spec(), signal: new AbortController().signal }, + eventsDeliverable, + ) + + const turn = await run.start('settle the outcome') + + expect(turn.outcome).toMatchObject(outcome) + }) }) describe('openSandboxRun — live sandbox event observer', () => {