diff --git a/CHANGELOG.md b/CHANGELOG.md index 502c7826..283eafe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 0.154.0 + +### A promotion may not call a candidate cheaper on dollars nobody measured + +`BenchmarkCell` carries `usd` and `usdKnown` as required siblings. `promotionGate` read `usd` from both arms and never read `usdKnown`, while non-inferiority mode promotes on a significant paired cost saving. A candidate whose dollars were a catalog estimate or an unreported floor could therefore be promoted for being cheaper than a receipted incumbent. + +`Spend.usdKnown` states the rule: a false value must not be treated as a measurement when enforcing a dollar-denominated comparison or limit. Promotion on cost savings is that comparison. + +`PromotionVerdict.reason` gains `'cost-unknown'`, and the verdict carries `costUnknownTasks` naming the tasks that caused the refusal. A caller that exhaustively switches on `reason` must handle the new member; a caller in `superiority` mode is unaffected, because only non-inferiority mode reads dollars. + ## 0.153.2 ### Eval 0.163.2 and Knowledge 10.7.0 reach this package diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index ecf6357d..39457d1f 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.153.2` and `@tangle-network/agent-eval@0.163.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.154.0` and `@tangle-network/agent-eval@0.163.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 9f2a4dcf..a8238610 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -5854,7 +5854,7 @@ Fixed by the substrate by default — the same report always yields the same ver ##### reason -> **reason**: `"identical-champion"` \| `"few-tasks"` \| `"no-margin"` \| `"significant"` \| `"non-inferior-and-cheaper"` \| `"non-inferiority-unproven"` \| `"not-cheaper"` +> **reason**: `"identical-champion"` \| `"few-tasks"` \| `"no-margin"` \| `"significant"` \| `"non-inferior-and-cheaper"` \| `"non-inferiority-unproven"` \| `"not-cheaper"` \| `"cost-unknown"` ##### mode @@ -5912,6 +5912,14 @@ non-inferiority mode: paired (incumbent − candidate) cost savings per task (us > **high**: `number` +##### costUnknownTasks? + +> `optional` **costUnknownTasks?**: `string`[] + +non-inferiority mode: the tasks whose dollars were not measured on at least one arm. + Present only with `reason: 'cost-unknown'`; naming them is what makes the refusal + actionable instead of a bare no. + ##### latency? > `optional` **latency?**: `object` diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 1997e1b0..1c43f754 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.153.2.** +> **Version 0.154.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.163.2 <0.164.0`. > `sandbox` must satisfy `>=0.31.0 <0.32.0`. diff --git a/package.json b/package.json index 12673f50..5bf05793 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.153.2", + "version": "0.154.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/runtime/promotion-gate.ts b/src/runtime/promotion-gate.ts index 4ce2a5fc..8175aaba 100644 --- a/src/runtime/promotion-gate.ts +++ b/src/runtime/promotion-gate.ts @@ -45,6 +45,7 @@ export interface PromotionVerdict { | 'non-inferior-and-cheaper' | 'non-inferiority-unproven' | 'not-cheaper' + | 'cost-unknown' mode: 'superiority' | 'non-inferiority' /** Paired tasks that carried both strategies' cells. */ n: number @@ -54,6 +55,10 @@ export interface PromotionVerdict { /** non-inferiority mode: paired (incumbent − candidate) cost savings per task (usd). * Positive means the candidate is cheaper; `low` and `high` carried the decision. */ costSavings?: { mean: number; median: number; low: number; high: number } + /** non-inferiority mode: the tasks whose dollars were not measured on at least one arm. + * Present only with `reason: 'cost-unknown'`; naming them is what makes the refusal + * actionable instead of a bare no. */ + costUnknownTasks?: string[] /** Paired (candidate − incumbent) wall-clock per task (ms) — negative = the candidate * is FASTER. Informational in every mode (never gates); the latency answer to "what * does this win actually cost the user?". */ @@ -79,6 +84,8 @@ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { const incMs: number[] = [] const candMs: number[] = [] const cellIds: string[] = [] + /** Tasks where either arm's dollars were never measured — see the refusal below. */ + const costUnknownTasks: string[] = [] for (const row of opts.report.perTask) { const inc = row.cells?.[opts.incumbent] const cand = row.cells?.[opts.candidate] @@ -87,6 +94,7 @@ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { after.push(cand.score) incUsd.push(inc.usd) candUsd.push(cand.usd) + if (inc.usdKnown === false || cand.usdKnown === false) costUnknownTasks.push(row.taskId) incMs.push(inc.ms) candMs.push(cand.ms) cellIds.push(row.taskId) @@ -147,6 +155,22 @@ export function promotionGate(opts: PromotionGateOptions): PromotionVerdict { ...(opts.resamples !== undefined ? { resamples: opts.resamples } : {}), }, ) + // `BenchmarkCell.usdKnown` false means the dollars are a floor, not a measurement, and + // `Spend.usdKnown` states the rule this gate has to obey: an unknown dollar amount "must not be + // treated as $0 when enforcing a dollar-denominated comparison or limit". Promotion on cost + // savings IS that comparison, so an unmeasured arm cannot be declared cheaper — it can only be + // declared unmeasured. Refuse rather than promote on a number no provider ever billed. + if (costUnknownTasks.length > 0) { + return { + promoted: false, + reason: 'cost-unknown', + mode, + n: before.length, + lift, + costUnknownTasks: [...costUnknownTasks], + latency, + } + } const costSig = heldoutSignificance( { before: candUsd, after: incUsd, cellIds }, { diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 0227b6e8..9c8a1f69 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:6af30ed1418652d00082d07f90766d0ee30681db2d13322885972ea6d732b3c0", + "digest": "sha256:f78c77e891758f0bb6d260e19c9eeb2a2171a3a8b000e51b6e934b4b79ffb0b8", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.153.2" + "runtimeVersion": "0.154.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:2cbc803968b26ba66887d9c096a5d5383d416b32990997d89c763a1e13503241", - "runId": "agent-runtime-0.153.2-proposal-fixture", + "recordDigest": "sha256:d60f130db2672f9ccb836ba15bded1d90481c38368810cbd9f64a9e4ef84a2fb", + "runId": "agent-runtime-0.154.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.153.2-proposal-fixture" + "runId": "agent-runtime-0.154.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 20f4dac1..5b62f57f 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:731dab22aaa07732a81639375387cf9ed15bfe3547601925ee51f2e629da5154", + "digest": "sha256:ca08433b17a8a4a7708f2f6da706dfd18d2dd9f8e549b17be0a8876e13303c88", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.153.2" + "runtimeVersion": "0.154.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:f50ab9d3e539e3a9c7a10964d69250a5ea719eeb9cebe4099086a9cdc7d4505f", + "recordDigest": "sha256:42a01d47c8b81447ff30d54dcc7ef7890eac946f9113e4e60045dc6ff64c7b08", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/kernel/strategy-suite.test.ts b/tests/kernel/strategy-suite.test.ts index bab8079a..2c403467 100644 --- a/tests/kernel/strategy-suite.test.ts +++ b/tests/kernel/strategy-suite.test.ts @@ -682,6 +682,7 @@ describe('promotionGate non-inferiority', () => { candScore: number incUsd: number candUsd: number + candUsdKnown?: boolean }>, ): BenchmarkReport => ({ n: rows.length, @@ -703,6 +704,7 @@ describe('promotionGate non-inferiority', () => { resolved: false, progression: [], usd: r.candUsd, + ...(r.candUsdKnown !== undefined ? { usdKnown: r.candUsdKnown } : {}), ms: 0, tokens: { input: 0, output: 0 }, }, @@ -737,6 +739,39 @@ describe('promotionGate non-inferiority', () => { expect(v.latency).toBeDefined() }) + it('refuses to call a candidate cheaper when its dollars were never measured', () => { + // The same inputs that PROMOTE above, with one arm's dollars marked unmeasured. `usdKnown: + // false` means `usd` is a floor, and `Spend.usdKnown` states the rule: an unknown amount must + // not be treated as a measurement "when enforcing a dollar-denominated comparison or limit". + // Promotion on cost savings is exactly that comparison. + const scoreDeltas = [-0.018, -0.012, -0.008, -0.004, 0, 0.006, 0.011, -0.006] + const costSavings = [0.011, 0.014, 0.016, 0.013, 0.018, 0.015, 0.012, 0.017] + const rows = Array.from({ length: 24 }, (_, i) => { + const incScore = 0.48 + (i % 7) * 0.055 + const incUsd = 0.024 + (i % 5) * 0.0014 + return { + id: `t${i}`, + incScore, + candScore: incScore + scoreDeltas[i % scoreDeltas.length]!, + incUsd, + candUsd: incUsd - costSavings[i % costSavings.length]!, + // Only the first task is unmeasured; one unmeasured pair is enough to void the comparison. + ...(i === 0 ? { candUsdKnown: false } : {}), + } + }) + const v = promotionGate({ + report: costReport(rows), + incumbent: 'incumbent', + candidate: 'candidate', + mode: 'non-inferiority', + }) + expect(v.promoted).toBe(false) + expect(v.reason).toBe('cost-unknown') + expect(v.costUnknownTasks).toEqual(['t0']) + // The refusal replaces the cost verdict rather than reporting one built on the same numbers. + expect(v.costSavings).toBeUndefined() + }) + it('cheaper but score-inferior beyond tolerance LOSES', () => { const rows = Array.from({ length: 12 }, (_, i) => ({ id: `t${i}`,