From ace153e1fe8b71ca065b3fc9a80957687af38644 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 20 Aug 2026 18:23:25 -0700 Subject: [PATCH] feat(history): record the candidate search into the canonical ledger --- CHANGELOG.md | 10 + clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- docs/search-history-receipts.md | 40 +- package.json | 2 +- src/analyst/benchmark-implementation.ts | 2 +- src/campaign/gepa-optimization-method.ts | 60 ++ src/campaign/index.ts | 11 + src/campaign/parent-selection.test.ts | 49 +- ...mpare-optimization-methods-history.test.ts | 152 ++- .../run-optimization-search-ledger.test.ts | 168 +++ src/campaign/presets/run-optimization.test.ts | 50 +- src/campaign/presets/run-optimization.ts | 59 ++ src/campaign/search-history-receipt.test.ts | 1 + src/campaign/search-ledger-recording.ts | 968 ++++++++++++++++++ src/campaign/search-ledger.test.ts | 146 ++- src/campaign/search-ledger.ts | 184 +++- src/contract/self-improve.ts | 11 + tests/contract-self-improve.test.ts | 64 -- 20 files changed, 1766 insertions(+), 217 deletions(-) create mode 100644 src/campaign/presets/run-optimization-search-ledger.test.ts create mode 100644 src/campaign/search-ledger-recording.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c795c14..5d57729a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.155.0] — 2026-08-21 + +### Added + +- `runOptimization({ searchLedger })` and `selfImprove({ searchLedger })` record the candidate search into the canonical `SearchLedger` and return a bounded `searchHistory` receipt (#633). The loop emits the plan (slots = generations x populationSize, one candidate-generation operation per generation, one selection operation, one task per designed scenario-replicate cell), one registration per candidate carrying the exact parent surface it mutated, one attempt per scored cell with the cell's own outcome and accounting, one decision per candidate, and the terminal event. `FileSearchLedger` now has a first-party caller in its own package. The terminal event is appended only when canonical replay accounts for the whole planned denominator, so an interrupted or partly unscored search reports the gap instead of claiming a closed search. +- `search-plan-extended`: a rolling search appends candidate slots and operations to an existing plan instead of opening a second ledger. Replay merges the first plan with every extension, the generation invariant continues across rounds, and the planless refusal is unchanged. The planned task denominator stays frozen. +- `gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's own candidate population — its parent graph and per-scenario selection scores — into the same ledger through `recordCandidatePopulationSearch()`. `compareOptimizationMethods({ searchHistoryPolicy: 'require-complete' })` now accepts a first-party method, which is what `docs/search-history-receipts.md` promised. + +--- + ## [0.154.0] — 2026-08-21 ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 8a180b68..c135b39f 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.154.0" +version = "0.155.0" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index bbc7bc5e..8a4a26c3 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.154.0" + __version__ = "0.155.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index a20f9dc4..b05000b9 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.154.0" +version = "0.155.0" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/docs/search-history-receipts.md b/docs/search-history-receipts.md index 6fe2aabc..84414b6f 100644 --- a/docs/search-history-receipts.md +++ b/docs/search-history-receipts.md @@ -64,7 +64,43 @@ const receipt = createSearchHistoryReceipt({ }) ``` -First-party optimizer adapters should do this automatically. Application code should not hand-author receipt JSON. +First-party optimizers do this for you. Application code should not hand-author receipt JSON. + +## Record a search from the loop + +`runOptimization()` and `selfImprove()` accept `searchLedger` and return the receipt on `searchHistory`: + +```ts +import { openSearchLedger, runOptimization } from '@tangle-network/agent-eval/campaign' + +const result = await runOptimization({ + // ...scenarios, dispatchWithSurface, judges, proposer, populationSize, maxGenerations, runDir + searchLedger: { + ledger: openSearchLedger({ path: `${runDir}/search-ledger.jsonl`, campaignId: runId }), + identity: { + agent: { uri: 'git+https://github.com/acme/agent.git', revision: agentCommit }, + proposer: { kind: 'deterministic', source: { uri: proposerUri, revision: proposerCommit } }, + search: { uri: searchUri, revision: searchCommit }, + model: { provider: 'openai', snapshot: 'gpt-5.4@2026-06-01' }, + }, + }, +}) +``` + +The loop emits the plan, one candidate-generation operation per generation, one registration per candidate with the exact parent it mutated, one task attempt per designed cell, one decision per candidate, and the terminal event. + +`identity` carries what the ledger requires and a campaign cannot infer: immutable revisions for the agent, proposer, and search implementations, plus the model the agent runs. A measured value wins wherever execution reported one; a cell that ran a moving model alias is refused rather than recorded as an immutable identity. + +`gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's own candidate population into the same ledger, so a comparison under `require-complete` accepts it. + +## Extend a plan for a rolling search + +A search whose length is not known when it starts appends `search-plan-extended` with the extra candidate slots and operations. +The first plan event stays first, the effective plan is the merge, and the generation invariant continues across rounds: a candidate whose parent is a round-one candidate is generation 2, not a restarted 0. + +The planned task denominator does not extend. Extending it would reopen candidates that already closed their tasks. + +A search still uses one ledger. A parent from an earlier ledger enters as a generation-0 `candidate-registered` whose surface artifact references the prior ledger, because a cross-file parent cannot be replayed and verified from these bytes. ## Complete means the planned denominator is closed @@ -78,6 +114,8 @@ A receipt is complete only when canonical replay reports: - no pending candidate decisions; - a terminal status of `selected` or `all-rejected`. +A first-party recorder appends the terminal event only when replay already accounts for the whole planned denominator. An interrupted run, or a candidate that left a designed cell unscored, stays `in-progress` and reports the exact gap. + Cost completeness remains a separate contract. Unknown spend stays unknown; it is never converted into zero merely because search history is complete. ## Compare methods without exposing final cases diff --git a/package.json b/package.json index de144870..b98984e8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.154.0", + "version": "0.155.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index ee470dff..f2fe8c6d 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - 'c306327eaf924a0f51ede71f5cd5e4efc17fc1dc4920d80e03a71a6c98a6e87d' + 'aca2e91e7764bdf5ee8ab439aa21147e4179df7aa7c885dea7ead18bbd7e6a13' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/campaign/gepa-optimization-method.ts b/src/campaign/gepa-optimization-method.ts index f0a1ac85..3db00155 100644 --- a/src/campaign/gepa-optimization-method.ts +++ b/src/campaign/gepa-optimization-method.ts @@ -61,6 +61,10 @@ import { type OptimizationMethod, optimizationTokenUsageFromSummary, } from './presets/compare-optimization-methods' +import type { SearchHistoryReceipt } from './search-history-receipt' +import type { SearchAttemptAccounting } from './search-ledger' +import { openSearchLedger } from './search-ledger' +import { recordCandidatePopulationSearch, type SearchRunIdentity } from './search-ledger-recording' import { fsCampaignStorage } from './storage' import type { Scenario } from './types' @@ -187,6 +191,16 @@ export interface GepaOptimizationMethodConfig/search-ledger.jsonl`. + */ + searchLedger?: { identity: SearchRunIdentity; path?: string } } /** @@ -520,9 +534,11 @@ export function gepaOptimizationMethod( `${name}: GEPA reported ${result.totalEvaluations} evaluations but the callback received ${callback.evaluations()}`, ) } + let searchHistory: SearchHistoryReceipt | undefined if (result.candidatePopulation) { const population = readGepaCandidatePopulationArtifact({ summary: result.candidatePopulation, + storage, }) const selected = population.candidates[population.bestIndex] const selectedHash = contentHash({ @@ -532,6 +548,22 @@ export function gepaOptimizationMethod( if (selected?.candidateHash !== selectedHash) { throw new Error(`${name}: GEPA candidate population identifies a different winner`) } + if (config.searchLedger) { + searchHistory = await recordCandidatePopulationSearch({ + ledger: openSearchLedger({ + path: config.searchLedger.path ?? `${runDir}/search-ledger.jsonl`, + campaignId: runId, + }), + storage, + runDir, + identity: config.searchLedger.identity, + population, + scenarios: input.selectionScenarios, + generationAccounting: optimizerAccounting(result.tokenUsage, result.proposerCostUsd), + producerId: name, + runId, + }) + } } const evaluationCost = costFromLedgerSummary( @@ -588,6 +620,7 @@ export function gepaOptimizationMethod( const externalTotalCostUsd = evaluationCost.totalCostUsd + reportedProposerCost return { winnerSurface: decodeExternalTextCandidate(result.bestCandidate), + ...(searchHistory ? { searchHistory } : {}), cost: modelProxy ? meteredCost! : { @@ -640,3 +673,30 @@ export function gepaOptimizationMethod( }, } } + +/** Spend the optimizer booked to its own candidate generation. Unknown stays + * unknown: the bridge reports proposer cost only when the engine measured it. */ +function optimizerAccounting( + tokenUsage: { inputTokens?: number; outputTokens?: number } | undefined, + proposerCostUsd: number | undefined, +): SearchAttemptAccounting { + return { + tokens: + tokenUsage?.inputTokens === undefined || tokenUsage.outputTokens === undefined + ? { status: 'unknown', reason: 'the optimizer bridge reported no token usage' } + : { + status: 'known', + inputTokens: tokenUsage.inputTokens, + outputTokens: tokenUsage.outputTokens, + cachedTokens: 0, + }, + cost: + proposerCostUsd === undefined + ? { + status: 'unknown', + knownLowerBoundUsd: 0, + reason: 'the optimizer bridge reported no proposer cost', + } + : { status: 'known', usd: proposerCostUsd, source: 'provider' }, + } +} diff --git a/src/campaign/index.ts b/src/campaign/index.ts index 65f05114..e8a68705 100644 --- a/src/campaign/index.ts +++ b/src/campaign/index.ts @@ -396,6 +396,7 @@ export { type SearchOperationKind, type SearchOperationRecordedEvent, type SearchPlan, + type SearchPlanExtendedEvent, type SearchPlannedEvent, type SearchPlannedOperation, type SearchPlannedTask, @@ -408,6 +409,16 @@ export { type SearchTokenAccounting, validateSearchLedgerEvent, } from './search-ledger' +export { + type MeasuredSearchCandidate, + type ProposedSearchCandidate, + recordCandidatePopulationSearch, + type SearchExecutionIdentity, + type SearchLedgerBinding, + SearchRecorder, + type SearchRecorderOptions, + type SearchRunIdentity, +} from './search-ledger-recording' export { acquireSingleRunLock, type SingleRunLock, diff --git a/src/campaign/parent-selection.test.ts b/src/campaign/parent-selection.test.ts index 37371d68..ca77e1ac 100644 --- a/src/campaign/parent-selection.test.ts +++ b/src/campaign/parent-selection.test.ts @@ -34,57 +34,14 @@ describe('crowdedFrontierParent', () => { const right = parent('right', { s1: 0, s2: 1 }) const middle = parent('middle', { s1: 0.5, s2: 0.5 }) - it('rejects a non-integer seed', () => { - expect(() => crowdedFrontierParent({ seed: 0.5 })).toThrow(/seed must be an integer/) - }) - - it('refuses an empty frontier', () => { - expect(() => crowdedFrontierParent({ seed: 1 })(context([], 0))).toThrow(/frontier is empty/) - }) - - it('returns the sole member of a one-parent frontier', () => { - expect(crowdedFrontierParent({ seed: 1 })(context([middle], 0))).toBe(middle) - }) - - it('is deterministic for the same seed, frontier, and generation', () => { - const frontier = [left, middle, right] - const a = crowdedFrontierParent({ seed: 11 }) - const b = crowdedFrontierParent({ seed: 11 }) - const drawsA = Array.from({ length: 20 }, (_, g) => a(context(frontier, g)).surfaceHash) - const drawsB = Array.from({ length: 20 }, (_, g) => b(context(frontier, g)).surfaceHash) - expect(drawsA).toEqual(drawsB) - }) - - it('prefers boundary parents in every tournament', () => { + it('prefers the isolated frontier parents the crowded tournament exists to keep', () => { const frontier = [middle, left, right] const select = crowdedFrontierParent({ seed: 7 }) const draws = new Set( Array.from({ length: 40 }, (_, g) => select(context(frontier, g)).surfaceHash), ) - // Every pair contains a boundary parent, so the interior one never wins. - expect(draws.has('middle')).toBe(false) + // Every pair contains a boundary parent, so the interior one never wins + // and the population cannot collapse onto the frontier's middle. expect(draws).toEqual(new Set(['left', 'right'])) }) - - it('breaks a distance tie by composite, then by surface hash', () => { - // Both parents are boundary points (infinite distance) with equal - // composite, so the smaller surface hash wins every tournament. - const a = parent('a', { s1: 1, s2: 0 }) - const b = parent('b', { s1: 0, s2: 1 }) - const select = crowdedFrontierParent({ seed: 3 }) - for (let g = 0; g < 10; g++) expect(select(context([b, a], g))).toBe(a) - // A higher composite beats the hash order. - const c = parent('c', { s1: 0.9, s2: 0.2 }) - for (let g = 0; g < 10; g++) expect(select(context([a, c], g))).toBe(c) - }) - - it('fails loud on a frontier member with a missing or non-finite objective', () => { - const select = crowdedFrontierParent({ seed: 1 }) - expect(() => select(context([left, parent('broken', { s1: Number.NaN, s2: 1 })], 0))).toThrow( - /has no finite objective "s1"/, - ) - expect(() => select(context([left, parent('partial', { s1: 0.2 })], 0))).toThrow( - /has no finite objective "s2"/, - ) - }) }) diff --git a/src/campaign/presets/compare-optimization-methods-history.test.ts b/src/campaign/presets/compare-optimization-methods-history.test.ts index 36434e9e..284a2e8c 100644 --- a/src/campaign/presets/compare-optimization-methods-history.test.ts +++ b/src/campaign/presets/compare-optimization-methods-history.test.ts @@ -1,6 +1,13 @@ -import { describe, expect, it } from 'vitest' +import { createHash } from 'node:crypto' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { readGepaCandidatePopulationArtifact } from '../gepa-candidate-population' import { type SearchHistoryReceipt, SearchHistoryRequiredError } from '../search-history-receipt' -import { inMemoryCampaignStorage } from '../storage' +import { openSearchLedger } from '../search-ledger' +import { recordCandidatePopulationSearch, type SearchRunIdentity } from '../search-ledger-recording' +import { fsCampaignStorage, inMemoryCampaignStorage } from '../storage' import type { DispatchContext, JudgeConfig, MutableSurface, Scenario } from '../types' import { type CompareOptimizationMethodsOptions, @@ -141,3 +148,144 @@ describe('compareOptimizationMethods search-history policy', () => { expect(dispatchCalls).toBe(0) }) }) + +describe('a first-party method that records its candidate population', () => { + let runDir: string + beforeEach(() => { + runDir = mkdtempSync(join(tmpdir(), 'population-history-')) + }) + afterEach(() => { + rmSync(runDir, { recursive: true, force: true }) + }) + + const identity: SearchRunIdentity = { + agent: { uri: 'git+https://github.com/example/agent.git', revision: 'a'.repeat(40) }, + proposer: { + kind: 'model', + model: { provider: 'example', snapshot: 'optimizer-model@2026-08-01' }, + source: { uri: 'git+https://github.com/gepa-ai/gepa.git', revision: 'b'.repeat(40) }, + }, + search: { + uri: 'git+https://github.com/tangle-network/agent-eval.git', + revision: 'c'.repeat(40), + }, + model: { provider: 'example', snapshot: 'agent-model@2026-08-01' }, + } + + /** The exact artifact shape a GEPA run writes: a candidate graph with + * parents and a score per selection scenario. */ + function populationArtifact(storage: ReturnType) { + const path = join(runDir, 'candidate-population.json') + const scenarioIds = selectionScenarios.map((scenario) => scenario.id) + const candidates = [ + { + index: 0, + candidate: 'baseline prompt', + parentIndices: [null], + aggregateScore: 0, + selectionScores: scenarioIds.map((scenarioId) => ({ scenarioId, score: 0 })), + discoveryEvaluationCount: 1, + }, + { + index: 1, + candidate: 'candidate prompt', + parentIndices: [0], + aggregateScore: 1, + selectionScores: scenarioIds.map((scenarioId) => ({ scenarioId, score: 1 })), + discoveryEvaluationCount: 1, + }, + ] + const contents = JSON.stringify({ + schemaVersion: 1, + scope: 'gepa-candidate-population', + runId: 'population-run', + bestIndex: 1, + candidates, + }) + storage.ensureDir(runDir) + storage.write(path, contents) + return readGepaCandidatePopulationArtifact({ + storage, + summary: { + scope: 'gepa-candidate-population', + path, + sha256: `sha256:${createHash('sha256').update(contents).digest('hex')}`, + bytes: new TextEncoder().encode(contents).byteLength, + runId: 'population-run', + candidates: candidates.length, + bestIndex: 1, + maxCandidates: 8, + maxCandidateChars: 4_000, + scenarioIds, + surfaceKind: 'text', + }, + }) + } + + it('passes require-complete with a receipt replayable from the ledger bytes', async () => { + const storage = fsCampaignStorage() + const population = populationArtifact(storage) + const ledgerPath = join(runDir, 'search-ledger.jsonl') + + const recorded: OptimizationMethod = { + name: 'method-with-population-history', + async optimize() { + const searchHistory = await recordCandidatePopulationSearch({ + ledger: openSearchLedger({ path: ledgerPath, campaignId: 'population-run' }), + storage, + runDir, + identity, + population, + scenarios: selectionScenarios, + generationAccounting: { + tokens: { status: 'known', inputTokens: 120, outputTokens: 40, cachedTokens: 0 }, + cost: { status: 'known', usd: 0.002, source: 'provider' }, + }, + producerId: 'method-with-population-history', + runId: 'population-run', + }) + return { + winnerSurface: 'candidate prompt', + cost: { + totalCostUsd: 0.002, + costProvenance: { kind: 'observed', usd: 0.002 }, + accountingComplete: true, + incompleteReasons: [], + }, + durationMs: 1, + searchHistory, + } + }, + } + + const comparison = await compareOptimizationMethods({ + ...options([recorded], async (surface, scenario) => { + const rendered = typeof surface === 'string' ? surface : JSON.stringify(surface) + return { text: `${rendered}:${scenario.id}` } + }), + searchHistoryPolicy: 'require-complete', + }) + + expect(comparison.searchHistory).toMatchObject({ + policy: 'require-complete', + allComplete: true, + producers: [{ producerId: 'method-with-population-history', status: 'complete' }], + }) + + // The receipt is a cover sheet over durable bytes: replaying the file + // reproduces the candidate graph the optimizer reported. + const replay = await openSearchLedger({ + path: ledgerPath, + campaignId: 'population-run', + }).replay() + expect(replay.candidates.map((event) => event.lineage.generation)).toEqual([0, 1]) + expect(replay.candidates[1]?.lineage.parentCandidateIds).toEqual([ + replay.candidates[0]?.candidateId, + ]) + expect(replay.audit).toMatchObject({ + status: 'selected', + attemptCount: 2 * selectionScenarios.length, + expected: { missingTaskOutcomes: [], missingCandidateSlots: [], missingOperations: [] }, + }) + }) +}) diff --git a/src/campaign/presets/run-optimization-search-ledger.test.ts b/src/campaign/presets/run-optimization-search-ledger.test.ts new file mode 100644 index 00000000..d7b01365 --- /dev/null +++ b/src/campaign/presets/run-optimization-search-ledger.test.ts @@ -0,0 +1,168 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { verifySearchHistoryReceipt } from '../search-history-receipt' +import { openSearchLedger, type SearchLedgerEvent } from '../search-ledger' +import type { SearchRunIdentity } from '../search-ledger-recording' +import { fsCampaignStorage } from '../storage' +import { surfaceHash } from '../surface-identity' +import type { JudgeConfig, Scenario, SurfaceProposer } from '../types' +import { runOptimization } from './run-optimization' + +interface LedgerScenario extends Scenario { + kind: 'ledger' +} + +interface LedgerArtifact { + surface: string +} + +const scenarios: LedgerScenario[] = [ + { id: 'alpha', kind: 'ledger' }, + { id: 'beta', kind: 'ledger' }, +] + +const judge: JudgeConfig = { + name: 'marker', + dimensions: [{ key: 'marker', description: 'candidate marker is present' }], + score: ({ artifact }) => { + const composite = artifact.surface === 'CANDIDATE' ? 1 : 0 + return { composite, dimensions: { marker: composite }, notes: '' } + }, +} + +const proposer: SurfaceProposer = { + kind: 'fixed-candidate', + async propose({ generation }) { + return generation === 0 ? ['CANDIDATE'] : [] + }, +} + +const identity: SearchRunIdentity = { + agent: { uri: 'git+https://github.com/example/agent.git', revision: 'a'.repeat(40) }, + proposer: { + kind: 'deterministic', + source: { uri: 'git+https://github.com/example/proposer.git', revision: 'b'.repeat(40) }, + }, + search: { uri: 'git+https://github.com/tangle-network/agent-eval.git', revision: 'c'.repeat(40) }, + model: { provider: 'example', snapshot: 'test-model@2026-08-01' }, +} + +let runDir: string +beforeEach(() => { + runDir = mkdtempSync(join(tmpdir(), 'search-ledger-loop-')) +}) +afterEach(() => { + rmSync(runDir, { recursive: true, force: true }) +}) + +describe('runOptimization search ledger', () => { + it('records the whole search and returns a complete, verifiable receipt', async () => { + const ledgerPath = join(runDir, 'search-ledger.jsonl') + const ledger = openSearchLedger({ path: ledgerPath, campaignId: 'ledger-run' }) + + const result = await runOptimization({ + baselineSurface: 'BASELINE', + scenarios, + dispatchWithSurface: async (surface) => ({ surface: String(surface) }), + dispatchRef: 'test:search-ledger', + judges: [judge], + proposer, + populationSize: 1, + // Generation 1 proposes nothing, so its planned slot must close rather + // than silently vanish from the denominator. + maxGenerations: 2, + seed: 7, + reps: 2, + resumable: false, + runDir, + storage: fsCampaignStorage(), + tracing: 'off', + expectUsage: 'off', + searchLedger: { ledger, identity }, + }) + + const receipt = result.searchHistory + if (!receipt) throw new Error('runOptimization returned no search history receipt') + expect(receipt.complete).toBe(true) + expect(receipt.incompleteReasons).toEqual([]) + // The receipt verifies as a canonical envelope, digest included. + expect(verifySearchHistoryReceipt(receipt).receiptDigest).toBe(receipt.receiptDigest) + + const replay = await openSearchLedger({ path: ledgerPath, campaignId: 'ledger-run' }).replay() + const kinds = new Set(replay.entries.map((entry) => entry.event.kind)) + expect(kinds).toEqual( + new Set([ + 'search-planned', + 'search-operation-recorded', + 'candidate-registered', + 'task-attempted', + 'candidate-slot-closed', + 'candidate-decided', + 'search-completed', + ]), + ) + + // The winner is the one registered candidate, its lineage roots at the + // baseline (measured before this ledger opened), and every designed cell + // is accounted for: 2 scenarios x 2 replicates. + const candidateId = surfaceHash('CANDIDATE') + expect(replay.candidates.map((event) => event.candidateId)).toEqual([candidateId]) + expect(replay.candidates[0]?.lineage.parentCandidateIds).toEqual([]) + expect(replay.attempts).toHaveLength(4) + expect(replay.audit).toMatchObject({ + status: 'selected', + selectedCandidateId: candidateId, + attemptCount: 4, + outcomes: { passed: 4, failed: 0, errored: 0 }, + expected: { + taskOutcomes: 4, + missingTaskOutcomes: [], + missingCandidateSlots: [], + missingOperations: [], + }, + }) + expect(result.winnerSurface).toBe('CANDIDATE') + }) + + it('reports the exact gap instead of a closed search when a cell fails', async () => { + const ledgerPath = join(runDir, 'search-ledger.jsonl') + const ledger = openSearchLedger({ path: ledgerPath, campaignId: 'ledger-gap' }) + + const result = await runOptimization({ + baselineSurface: 'BASELINE', + scenarios, + dispatchWithSurface: async (surface, scenario) => { + if (surface === 'CANDIDATE' && scenario.id === 'beta') { + throw new Error('dispatch refused the beta scenario') + } + return { surface: String(surface) } + }, + dispatchRef: 'test:search-ledger-gap', + judges: [judge], + proposer, + populationSize: 1, + maxGenerations: 1, + seed: 7, + reps: 1, + resumable: false, + runDir, + storage: fsCampaignStorage(), + tracing: 'off', + expectUsage: 'off', + searchLedger: { ledger, identity }, + }) + + const receipt = result.searchHistory + if (!receipt) throw new Error('runOptimization returned no search history receipt') + expect(receipt.complete).toBe(false) + expect(receipt.summary.missingTaskOutcomes).toBe(1) + expect(receipt.summary.hasCompletion).toBe(false) + expect(receipt.incompleteReasons.join(' ')).toMatch(/task outcome is unresolved/i) + + const replay = await openSearchLedger({ path: ledgerPath, campaignId: 'ledger-gap' }).replay() + expect(replay.audit.outcomes).toMatchObject({ passed: 1, errored: 1 }) + expect(replay.audit.status).toBe('in-progress') + }) +}) diff --git a/src/campaign/presets/run-optimization.test.ts b/src/campaign/presets/run-optimization.test.ts index a69d019c..fec80e03 100644 --- a/src/campaign/presets/run-optimization.test.ts +++ b/src/campaign/presets/run-optimization.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { crowdedFrontierParent, type ParentSelector } from '../parent-selection' +import type { ParentSelector } from '../parent-selection' import { runCampaign } from '../run-campaign' import { campaignMeanComposite, compareRankKeys } from '../score-utils' import { type CampaignStorage, inMemoryCampaignStorage } from '../storage' @@ -863,34 +863,6 @@ describe('runOptimization parent selection', () => { ) }) - it('a seeded crowded-frontier selector is deterministic across runs', async () => { - const plan = [['A', 'B'], ['C'], ['D', 'E']] - const first = await runWithParentPolicy({ - plan, - runDir: '/parent/seeded-1', - selectParent: crowdedFrontierParent({ seed: 5 }), - }) - const second = await runWithParentPolicy({ - plan, - runDir: '/parent/seeded-2', - selectParent: crowdedFrontierParent({ seed: 5 }), - }) - const lineage = (run: typeof first) => - run.result.generations.map((g) => g.record.candidates.map((c) => c.parentSurfaceHash)) - expect(lineage(first)).toEqual(lineage(second)) - expect(first.result.generations.map((g) => g.record)).toEqual( - second.result.generations.map((g) => g.record), - ) - // Generation 0 has a one-member frontier, so its parent is the baseline; - // later parents are frontier members. - expect(lineage(first)[0]).toEqual([baselineHash, baselineHash]) - // From generation 1 on, the crowded tournament prefers boundary frontier - // parents, so the interior baseline is never drawn again. - for (const hash of lineage(first).flat().slice(2)) { - expect(hash).not.toBe(baselineHash) - } - }) - it('refuses a parent the run never measured to completion', async () => { await expect( runWithParentPolicy({ @@ -904,24 +876,4 @@ describe('runOptimization parent selection', () => { }), ).rejects.toThrow(/has not measured to completion/) }) - - it('refuses a parent whose surface does not match its hash', async () => { - await expect( - runWithParentPolicy({ - plan: [['A', 'B'], ['C']], - runDir: '/parent/mismatch', - selectParent: ({ frontier }) => ({ ...frontier[0]!, surface: 'TAMPERED' }), - }), - ).rejects.toThrow(/surface does not match its surfaceHash/) - }) - - it('refuses a selector that does not return a ParetoParent', async () => { - await expect( - runWithParentPolicy({ - plan: [['A', 'B']], - runDir: '/parent/not-a-parent', - selectParent: (() => undefined) as unknown as ParentSelector, - }), - ).rejects.toThrow(/selectParent must return a ParetoParent/) - }) }) diff --git a/src/campaign/presets/run-optimization.ts b/src/campaign/presets/run-optimization.ts index 79dc6e2d..c038df70 100644 --- a/src/campaign/presets/run-optimization.ts +++ b/src/campaign/presets/run-optimization.ts @@ -33,6 +33,8 @@ import { campaignMeanCompositeOrNull, compareRankKeys, } from '../score-utils' +import type { SearchHistoryReceipt } from '../search-history-receipt' +import { type SearchLedgerBinding, SearchRecorder } from '../search-ledger-recording' import { createRunCostLedger, fsCampaignStorage } from '../storage' import { surfaceHash } from '../surface-identity' import { @@ -140,6 +142,18 @@ export interface RunOptimizationBaseOptions { baselineCampaign: CampaignResult /** Run-wide spend, including agents, proposers, analysts, and judges. */ cost: CostLedgerSummary + /** Bounded proof envelope over the canonical search ledger. Present only + * when `searchLedger` was supplied. `complete` is false when the search was + * interrupted or a candidate left a designed cell unscored. */ + searchHistory?: SearchHistoryReceipt /** The GEPA Pareto frontier across every scored surface (baseline + all * generations) by per-scenario objective vector — the non-dominated set. * Each generation's `propose()` received the frontier-so-far as @@ -238,6 +256,21 @@ export async function runOptimization( ) } + const recorder = opts.searchLedger + ? await SearchRecorder.open({ + binding: opts.searchLedger, + storage, + runDir: opts.runDir, + scenarios: opts.scenarios, + reps, + maxGenerations: opts.maxGenerations, + populationSize: opts.populationSize, + splitDigest: baselineCampaign.splitDigest, + proposerLabel: proposer.kind, + costLedger, + }) + : undefined + const generations: RunOptimizationResult['generations'] = [] const history: GenerationRecord[] = [] // Refreshed each generation by `analyzeGeneration`; seeded with the static @@ -390,6 +423,15 @@ export async function runOptimization( generationHashes.add(hash) } for (const hash of generationHashes) admittedCandidateHashes.add(hash) + await recorder?.recordGeneration({ + generation: gen, + parentSurfaceHash, + candidates: candidates.map(({ surface, label }) => ({ + surface, + surfaceHash: surfaceHash(surface), + ...(label ? { label } : {}), + })), + }) // Run each candidate as its own campaign. type SurfaceResult = { @@ -465,6 +507,16 @@ export async function runOptimization( } } + await recorder?.recordResults( + surfaceResults.map((result) => ({ + surface: result.surface, + surfaceHash: result.surfaceHash, + cells: result.campaign.cells, + runDir: result.campaign.runDir, + coverageComplete: result.coverage.complete, + })), + ) + // Rank only candidates with the complete designed denominator. Incomplete // rows follow the eligible rows for auditability but never promote. surfaceResults.sort((a, b) => { @@ -553,6 +605,12 @@ export async function runOptimization( } } + const searchHistory = await recorder?.finish({ + winnerSurfaceHash, + generationsRun: generations.length, + runId: opts.runDir, + }) + return { generations, baselineSurface, @@ -561,6 +619,7 @@ export async function runOptimization( winnerLabel, winnerRationale, baselineCampaign, + ...(searchHistory ? { searchHistory } : {}), paretoFrontier: computeParetoFrontier(scored), cost: costLedger.summary(), } diff --git a/src/campaign/search-history-receipt.test.ts b/src/campaign/search-history-receipt.test.ts index 3f6225be..479e0f0a 100644 --- a/src/campaign/search-history-receipt.test.ts +++ b/src/campaign/search-history-receipt.test.ts @@ -149,6 +149,7 @@ function completeReplay(): SearchLedgerReplay { return { entries, plan: entries[0]!.event as SearchLedgerReplay['plan'], + planExtensions: [], candidates: [entries[1]!.event as SearchLedgerReplay['candidates'][number]], closedCandidateSlots: [], attempts: [entries[3]!.event as SearchLedgerReplay['attempts'][number]], diff --git a/src/campaign/search-ledger-recording.ts b/src/campaign/search-ledger-recording.ts new file mode 100644 index 00000000..9a2d156e --- /dev/null +++ b/src/campaign/search-ledger-recording.ts @@ -0,0 +1,968 @@ +/** + * Record a candidate search into the canonical `SearchLedger`. + * + * `runOptimization` and the GEPA adapter produce the same three facts — a + * bounded plan, a candidate lineage, and a measured outcome per planned task — + * so both write them through this recorder into one ledger and one + * `SearchHistoryReceipt`. There is no second lineage record. + * + * The ledger contract requires identities a campaign cannot infer: an + * immutable revision for the agent and proposer implementations, and a model + * snapshot on every attempt. The caller declares them once in + * `SearchRunIdentity`. Measured values win wherever execution reported them; a + * declaration only fills what execution did not report. + */ + +import type { CostLedgerHandle, CostReceipt } from '../cost-ledger' +import { canonicalString, hashCanonical } from '../ledger-core/canonical' +import { modelHasSnapshot } from '../run-record' +import type { ExternalTextCandidate } from './external-optimizer-contracts' +import type { GepaCandidatePopulationArtifact } from './gepa-candidate-population' +import { createSearchHistoryReceipt, type SearchHistoryReceipt } from './search-history-receipt' +import type { + SearchArtifactRef, + SearchAttemptAccounting, + SearchCandidateSlot, + SearchCandidateSurface, + SearchFailureReason, + SearchLedger, + SearchLedgerEvent, + SearchModelIdentity, + SearchOperationRecordedEvent, + SearchPlannedOperation, + SearchPlannedTask, + SearchSourceRef, + SearchSurfaceEvidence, + SearchTaskOutcome, +} from './search-ledger' +import type { CampaignStorage } from './storage' +import type { CampaignCellResult, MutableSurface, Scenario } from './types' + +/** How a search operation executed. The shape the ledger event records. */ +export type SearchExecutionIdentity = SearchOperationRecordedEvent['execution'] + +/** Immutable identities the ledger requires and a campaign cannot infer. */ +export interface SearchRunIdentity { + /** The agent implementation under optimization. */ + agent: SearchSourceRef + /** The candidate generator: a model call or deterministic code. */ + proposer: SearchExecutionIdentity + /** The code that plans the search and selects its winner. */ + search: SearchSourceRef + /** Model the agent runs. Used only for a cell that reported none. */ + model: SearchModelIdentity +} + +export interface SearchLedgerBinding { + ledger: SearchLedger + identity: SearchRunIdentity +} + +/** One proposed candidate, before it is measured. */ +export interface ProposedSearchCandidate { + surface: MutableSurface + surfaceHash: string + label?: string +} + +/** One measured candidate, after its campaign scored. */ +export interface MeasuredSearchCandidate { + surface: MutableSurface + surfaceHash: string + cells: ReadonlyArray> + runDir: string + /** False when the candidate missed a designed cell. */ + coverageComplete: boolean +} + +export interface SearchRecorderOptions { + binding: SearchLedgerBinding + storage: CampaignStorage + runDir: string + scenarios: ReadonlyArray + reps: number + maxGenerations: number + populationSize: number + /** Identity of the exact campaign design; the task benchmark pin. */ + splitDigest: `sha256:${string}` + /** Proposer label recorded on every candidate lineage. */ + proposerLabel: string + costLedger: CostLedgerHandle +} + +interface RegisteredCandidate { + candidateId: string + slotId: string + generation: number +} + +const SEARCH_LEDGER_DIR = 'search-ledger' + +/** + * Recorder for one `runOptimization` run: `open()`, then `recordGeneration()` + * and `recordResults()` per generation, then `finish()`. + * + * Every event id is derived from the run, and an id already durable is not + * appended again, so a resumed run continues one ledger instead of conflicting + * with its own history. + */ +export class SearchRecorder { + private readonly opts: SearchRecorderOptions + private readonly tasks: SearchPlannedTask[] + private readonly registered = new Map() + private readonly order: RegisteredCandidate[] = [] + private readonly coverage = new Map() + private readonly openSlots = new Set() + private readonly durableEventIds = new Set() + private lastStampMs = 0 + private proposalReceiptCount = 0 + + private constructor(opts: SearchRecorderOptions) { + this.opts = opts + this.tasks = plannedTasks(opts.scenarios, opts.reps, opts.runDir, opts.splitDigest) + } + + /** Open the recorder and append the plan. An existing ledger for the same + * run is re-read first, so a resumed run keeps one plan and one lineage. */ + static async open( + opts: SearchRecorderOptions, + ): Promise> { + const recorder = new SearchRecorder(opts) + await recorder.hydrate() + await recorder.plan() + return recorder + } + + /** + * Record one generation's candidate-generation call and the candidates it + * produced. A proposal larger than the planned population extends the plan + * with the extra slots; a proposal that fills fewer closes the rest. + */ + async recordGeneration(input: { + generation: number + parentSurfaceHash: string + candidates: ReadonlyArray + }): Promise { + const { generation, candidates } = input + const planned = this.opts.populationSize + if (candidates.length > planned) { + const extra: SearchCandidateSlot[] = [] + for (let index = planned; index < candidates.length; index++) { + const slot = { + slotId: slotId(generation, index), + generationOperationId: generationOperationId(generation), + } + extra.push(slot) + this.openSlots.add(slot.slotId) + } + await this.append({ + kind: 'search-plan-extended', + eventId: `search:plan-extended:gen-${generation}`, + occurredAt: this.stamp(), + artifacts: [this.proposalArtifact(generation, candidates)], + extension: { candidateSlots: extra, operations: [] }, + }) + } + await this.recordGenerationOperation(generation, candidates, planned - candidates.length) + + const parent = this.registered.get(input.parentSurfaceHash) + for (const [index, candidate] of candidates.entries()) { + const candidateId = candidate.surfaceHash + const slot = slotId(generation, index) + const surfaceArtifact = this.writeArtifact( + 'candidate-surface', + `candidate-${candidateId}.json`, + { surfaceHash: candidateId, surface: candidate.surface, label: candidate.label ?? '' }, + ) + // A parent measured before this ledger opened — the run baseline, or a + // surface carried in from an earlier ledger — is not an in-file + // candidate, so a candidate mutating it is a lineage root. + const ledgerGeneration = parent ? parent.generation + 1 : 0 + await this.append({ + kind: 'candidate-registered', + eventId: `candidate:${candidateId}`, + occurredAt: this.stamp(), + artifacts: [surfaceArtifact], + slotId: slot, + generationOperationId: generationOperationId(generation), + candidateId, + lineage: { + lineageNodeId: candidateId, + parentCandidateIds: parent ? [parent.candidateId] : [], + generation: ledgerGeneration, + proposer: this.opts.proposerLabel, + proposerSource: this.opts.binding.identity.proposer.source, + }, + surfaces: candidateSurfaces(candidate.surface, surfaceArtifact), + }) + this.remember({ candidateId, slotId: slot, generation: ledgerGeneration }) + } + for (let index = candidates.length; index < planned; index++) { + await this.closeSlot(slotId(generation, index), generationOperationId(generation), { + code: 'proposal-short', + message: `generation ${generation} proposed ${candidates.length} of ${planned} planned candidates`, + }) + } + } + + /** Append one task attempt per designed cell of each candidate campaign. */ + async recordResults( + candidates: ReadonlyArray>, + ): Promise { + for (const candidate of candidates) { + const registered = this.registered.get(candidate.surfaceHash) + if (!registered) { + throw new Error( + `search ledger: candidate ${candidate.surfaceHash} produced results without a registration`, + ) + } + this.coverage.set(registered.candidateId, candidate.coverageComplete) + const evidence = this.writeArtifact( + 'candidate-cells', + `cells-${registered.candidateId}.json`, + { + surfaceHash: candidate.surfaceHash, + runDir: candidate.runDir, + cells: candidate.cells.map((cell) => ({ + cellId: cell.cellId, + scenarioId: cell.scenarioId, + rep: cell.rep, + judgeScores: cell.judgeScores, + costUsd: cell.costUsd, + costProvenance: cell.costProvenance, + tokenUsage: cell.tokenUsage, + ...(cell.error === undefined ? {} : { error: cell.error, errorStage: cell.errorStage }), + })), + }, + ) + const surfaceIds = candidateSurfaces(candidate.surface, evidence).map( + (surface) => surface.surfaceId, + ) + for (const cell of candidate.cells) { + const taskId = taskIdFor(cell.scenarioId, cell.rep) + const task = this.tasks.find((planned) => planned.taskId === taskId) + if (!task) { + throw new Error(`search ledger: cell ${cell.cellId} is outside the planned task set`) + } + await this.append({ + kind: 'task-attempted', + eventId: `attempt:${registered.candidateId}:${taskId}`, + occurredAt: this.stamp(), + artifacts: [evidence], + candidateId: registered.candidateId, + runId: `${registered.candidateId}:${cell.cellId}`, + attemptIndex: 0, + task: { taskId, source: task.source }, + identity: { + model: this.cellModel(cell), + agent: this.opts.binding.identity.agent, + benchmark: task.benchmark, + }, + outcome: cellOutcome(cell), + accounting: cellAccounting(cell), + surfaceEvidence: surfaceIds.map((surfaceId) => surfaceEvidenceFor(surfaceId, evidence)), + }) + } + } + } + + /** + * Close the search: unreached generations, the selection operation, one + * decision per candidate, then the terminal event. + * + * The terminal event is appended only when canonical replay accounts for the + * whole planned denominator. An interrupted or partly unscored search stays + * `in-progress` and its receipt reports the exact gap, instead of claiming a + * closed search. + */ + async finish(input: { + winnerSurfaceHash: string + generationsRun: number + runId: string + }): Promise { + const stopped: SearchFailureReason = { + code: 'generation-not-reached', + message: `the search stopped after ${input.generationsRun} generation(s)`, + } + for ( + let generation = input.generationsRun; + generation < this.opts.maxGenerations; + generation++ + ) { + await this.recordGenerationOperation(generation, [], this.opts.populationSize, stopped) + for (let index = 0; index < this.opts.populationSize; index++) { + await this.closeSlot(slotId(generation, index), generationOperationId(generation), stopped) + } + } + + const decisions = this.writeArtifact('search-decisions', 'decisions.json', { + winnerSurfaceHash: input.winnerSurfaceHash, + candidates: this.order.map((entry) => entry.candidateId), + }) + await this.append({ + kind: 'search-operation-recorded', + eventId: 'operation:selection', + occurredAt: this.stamp(), + artifacts: [decisions], + operationId: 'selection', + operationKind: 'selection', + execution: { kind: 'deterministic', source: this.opts.binding.identity.search }, + outcome: { status: 'completed' }, + accounting: { + tokens: { status: 'known', inputTokens: 0, outputTokens: 0, cachedTokens: 0 }, + cost: { status: 'known', usd: 0, source: 'free' }, + }, + }) + + const winner = this.registered.get(input.winnerSurfaceHash) + for (const entry of this.order) { + await this.append({ + kind: 'candidate-decided', + eventId: `decision:${entry.candidateId}`, + occurredAt: this.stamp(), + artifacts: [decisions], + candidateId: entry.candidateId, + decision: + winner?.candidateId === entry.candidateId + ? { status: 'selected' } + : { + status: 'rejected', + reason: + this.coverage.get(entry.candidateId) === false + ? { + code: 'coverage-incomplete', + message: 'the candidate missed a designed cell and could not be ranked', + } + : { + code: 'not-promoted', + message: 'the candidate did not beat the incumbent', + }, + }, + }) + } + + const replay = await this.opts.binding.ledger.replay() + const { missingCandidateSlots, missingTaskOutcomes, missingOperations } = replay.audit.expected + if ( + missingCandidateSlots.length === 0 && + missingTaskOutcomes.length === 0 && + missingOperations.length === 0 && + replay.audit.decisions.pending === 0 + ) { + await this.append({ + kind: 'search-completed', + eventId: 'search:completed', + occurredAt: this.stamp(), + artifacts: [decisions], + result: winner + ? { status: 'selected', candidateId: winner.candidateId } + : { + status: 'all-rejected', + reason: { + code: 'no-promoted-candidate', + message: 'no candidate beat the incumbent on the designed denominator', + }, + }, + }) + } + return this.receipt(input.runId) + } + + /** Bounded receipt over the exact ledger bytes this run produced. */ + async receipt(runId: string): Promise { + const { ledger } = this.opts.binding + const replay = await ledger.replay() + const bytes = replay.entries.map((entry) => `${canonicalString(entry)}\n`).join('') + return createSearchHistoryReceipt({ + producerId: this.opts.proposerLabel, + runId, + ledger: { + role: 'search-ledger', + uri: `file://${ledger.path}`, + sha256: hashCanonical(bytes), + byteLength: new TextEncoder().encode(bytes).byteLength, + }, + replay, + }) + } + + /** Read an existing ledger for this run so a resume continues it. */ + private async hydrate(): Promise { + const replay = await this.opts.binding.ledger.replay() + for (const entry of replay.entries) { + this.durableEventIds.add(entry.event.eventId) + const stamped = Date.parse(entry.event.occurredAt) + if (stamped > this.lastStampMs) this.lastStampMs = stamped + } + for (const slot of replay.plan?.plan.candidateSlots ?? []) this.openSlots.add(slot.slotId) + for (const extension of replay.planExtensions) { + for (const slot of extension.extension.candidateSlots) this.openSlots.add(slot.slotId) + } + for (const closed of replay.closedCandidateSlots) this.openSlots.delete(closed.slotId) + for (const candidate of replay.candidates) { + this.remember({ + candidateId: candidate.candidateId, + slotId: candidate.slotId, + generation: candidate.lineage.generation, + }) + this.openSlots.delete(candidate.slotId) + } + } + + private async plan(): Promise { + const candidateSlots: SearchCandidateSlot[] = [] + const operations: SearchPlannedOperation[] = [{ operationId: 'selection', kind: 'selection' }] + for (let generation = 0; generation < this.opts.maxGenerations; generation++) { + operations.push({ + operationId: generationOperationId(generation), + kind: 'candidate-generation', + }) + for (let index = 0; index < this.opts.populationSize; index++) { + const slot = { + slotId: slotId(generation, index), + generationOperationId: generationOperationId(generation), + } + candidateSlots.push(slot) + if (!this.registeredSlot(slot.slotId)) this.openSlots.add(slot.slotId) + } + } + await this.append({ + kind: 'search-planned', + eventId: 'search:plan', + occurredAt: this.stamp(), + artifacts: [ + this.writeArtifact('search-plan', 'plan.json', { + runDir: this.opts.runDir, + splitDigest: this.opts.splitDigest, + maxGenerations: this.opts.maxGenerations, + populationSize: this.opts.populationSize, + tasks: this.tasks, + }), + ], + plan: { candidateSlots, tasks: this.tasks, operations }, + }) + } + + private registeredSlot(slot: string): boolean { + return this.order.some((entry) => entry.slotId === slot) + } + + private remember(entry: RegisteredCandidate): void { + if (this.registered.has(entry.candidateId)) return + this.registered.set(entry.candidateId, entry) + this.order.push(entry) + } + + private async recordGenerationOperation( + generation: number, + candidates: ReadonlyArray, + unfilled: number, + failure?: SearchFailureReason, + ): Promise { + const shortfall: SearchFailureReason = { + code: 'proposal-short', + message: `generation ${generation} left ${unfilled} planned candidate slot(s) unfilled`, + } + const outcome: SearchOperationRecordedEvent['outcome'] = failure + ? { status: 'failed', failure } + : unfilled <= 0 + ? { status: 'completed' } + : candidates.length === 0 + ? { status: 'failed', failure: shortfall } + : { status: 'partial', failure: shortfall } + await this.append({ + kind: 'search-operation-recorded', + eventId: `operation:${generationOperationId(generation)}`, + occurredAt: this.stamp(), + artifacts: [this.proposalArtifact(generation, candidates)], + operationId: generationOperationId(generation), + operationKind: 'candidate-generation', + execution: this.opts.binding.identity.proposer, + outcome, + accounting: this.proposalAccounting(), + }) + } + + private async closeSlot( + slot: string, + generationOperation: string, + reason: SearchFailureReason, + ): Promise { + if (!this.openSlots.has(slot)) return + this.openSlots.delete(slot) + await this.append({ + kind: 'candidate-slot-closed', + eventId: `slot-closed:${slot}`, + occurredAt: this.stamp(), + artifacts: [this.writeArtifact('closed-slot', `slot-${slot}.json`, { slot, reason })], + slotId: slot, + generationOperationId: generationOperation, + reason, + }) + } + + /** Spend booked to candidate generation since the previous generation. */ + private proposalAccounting(): SearchAttemptAccounting { + const receipts = this.opts.costLedger.list({ phase: 'search.proposal' }) + const fresh = receipts.slice(this.proposalReceiptCount) + this.proposalReceiptCount = receipts.length + return receiptAccounting(fresh) + } + + private cellModel(cell: CampaignCellResult): SearchModelIdentity { + const resolved = cell.resolvedModel + if (resolved === undefined) return this.opts.binding.identity.model + if (!modelHasSnapshot(resolved)) { + throw new Error( + `search ledger: cell ${cell.cellId} ran model '${resolved}', which carries no immutable snapshot; the ledger cannot record a moving alias as execution identity`, + ) + } + return { provider: this.opts.binding.identity.model.provider, snapshot: resolved } + } + + private proposalArtifact( + generation: number, + candidates: ReadonlyArray, + ): SearchArtifactRef { + return this.writeArtifact('candidate-proposal', `proposal-gen-${generation}.json`, { + generation, + candidates: candidates.map((candidate) => ({ + surfaceHash: candidate.surfaceHash, + label: candidate.label ?? '', + })), + }) + } + + /** Write one canonical evidence document and return its content address. */ + private writeArtifact(role: string, name: string, body: unknown): SearchArtifactRef { + const directory = `${this.opts.runDir}/${SEARCH_LEDGER_DIR}` + const path = `${directory}/${name}` + const contents = canonicalString(body) + this.opts.storage.ensureDir(directory) + this.opts.storage.write(path, contents) + return { + role, + uri: `file://${path}`, + sha256: hashCanonical(contents), + byteLength: new TextEncoder().encode(contents).byteLength, + } + } + + private async append(event: SearchLedgerEvent): Promise { + if (this.durableEventIds.has(event.eventId)) return + await this.opts.binding.ledger.append(event) + this.durableEventIds.add(event.eventId) + } + + /** Non-decreasing ISO stamps; the ledger refuses an event that moves back. */ + private stamp(): string { + const now = Date.now() + this.lastStampMs = now > this.lastStampMs ? now : this.lastStampMs + 1 + return new Date(this.lastStampMs).toISOString() + } +} + +/** One task per designed (scenario, replicate) cell. */ +function plannedTasks( + scenarios: ReadonlyArray, + reps: number, + runDir: string, + splitDigest: `sha256:${string}`, +): SearchPlannedTask[] { + const tasks: SearchPlannedTask[] = [] + for (const scenario of scenarios) { + for (let rep = 0; rep < reps; rep++) { + tasks.push({ + taskId: taskIdFor(scenario.id, rep), + source: { uri: `scenario://${scenario.id}`, revision: hashCanonical(scenario) }, + benchmark: { uri: `campaign://${runDir}`, revision: splitDigest }, + maxAttempts: 1, + }) + } + } + return tasks +} + +function taskIdFor(scenarioId: string, rep: number): string { + return `${scenarioId}#rep-${rep}` +} + +function slotId(generation: number, index: number): string { + return `gen-${generation}-slot-${index}` +} + +function generationOperationId(generation: number): string { + return `candidate-generation:gen-${generation}` +} + +/** Declared surfaces of one candidate. A component surface declares one + * surface per named component, so per-component evidence stays addressable. */ +function candidateSurfaces( + surface: MutableSurface, + artifact: SearchArtifactRef, +): SearchCandidateSurface[] { + if (typeof surface === 'string') return [{ surfaceId: 'prompt', kind: 'prompt', artifact }] + if (surface.kind === 'code') return [{ surfaceId: 'code', kind: 'code', artifact }] + return Object.keys(surface.components) + .sort() + .map((name) => ({ surfaceId: `component:${name}`, kind: 'prompt' as const, artifact })) +} + +/** The campaign measures a candidate surface as a whole, so per-surface + * attribution stays unmeasured instead of inventing a per-component delta. */ +function surfaceEvidenceFor(surfaceId: string, evidence: SearchArtifactRef): SearchSurfaceEvidence { + return { + surfaceId, + fired: true, + firingCount: 1, + effect: { + status: 'not-measured', + reason: 'the campaign measures the candidate surface as a whole, not per surface', + }, + evidence: [evidence], + } +} + +function cellOutcome(cell: CampaignCellResult): SearchTaskOutcome { + const scores = Object.entries(cell.judgeScores).filter( + ([, score]) => score.failed !== true && Number.isFinite(score.composite), + ) + if (cell.error !== undefined || scores.length === 0) { + return { + status: 'errored', + metrics: {}, + error: { + code: cell.errorStage ?? 'unscored', + message: cell.error ?? 'the cell produced no complete judge score', + retryable: false, + }, + } + } + const composite = scores.reduce((sum, [, score]) => sum + score.composite, 0) / scores.length + const metrics: Record = { composite } + for (const [judge, score] of scores) metrics[`judge.${judge}`] = score.composite + return { status: 'passed', score: composite, metrics } +} + +function cellAccounting(cell: CampaignCellResult): SearchAttemptAccounting { + const usage = cell.tokenUsage + return { + tokens: + usage.tokensKnown === false + ? { status: 'unknown', reason: 'a paid call in this cell reported no token usage' } + : { + status: 'known', + inputTokens: usage.input, + outputTokens: usage.output, + cachedTokens: 0, + }, + cost: + cell.costProvenance.kind === 'uncaptured' + ? { + status: 'unknown', + knownLowerBoundUsd: cell.costUsd, + reason: 'the cell recorded spend without a provider receipt', + } + : { + status: 'known', + usd: cell.costUsd, + source: cell.costProvenance.kind === 'observed' ? 'provider' : 'pricing-table', + }, + } +} + +function receiptAccounting(receipts: ReadonlyArray): SearchAttemptAccounting { + let inputTokens = 0 + let outputTokens = 0 + let cachedTokens = 0 + let usd = 0 + let tokensKnown = true + let costKnown = true + for (const receipt of receipts) { + if (receipt.usageUnknown === true) tokensKnown = false + inputTokens += receipt.inputTokens + outputTokens += receipt.outputTokens + cachedTokens += receipt.cachedTokens ?? 0 + if (receipt.costUnknown) costKnown = false + else usd += receipt.costUsd + } + return { + tokens: tokensKnown + ? { status: 'known', inputTokens, outputTokens, cachedTokens } + : { status: 'unknown', reason: 'a candidate-generation call reported no token usage' }, + cost: costKnown + ? { status: 'known', usd, source: usd === 0 ? 'free' : 'provider' } + : { + status: 'unknown', + knownLowerBoundUsd: usd, + reason: 'a candidate-generation call recorded no provider cost', + }, + } +} + +/** + * Record an optimizer's own candidate graph into the same ledger. + * + * A complete optimization method searches inside its own process and reports + * one artifact when it finishes: the candidate population, with each + * candidate's parents and its score per selection scenario. This turns that + * artifact into the canonical event stream, so a first-party method returns + * the same `SearchHistoryReceipt` the in-process loop returns, and + * `compareOptimizationMethods({ searchHistoryPolicy: 'require-complete' })` + * accepts it. + * + * A candidate the optimizer left unscored on a planned scenario leaves the + * planned denominator open, so the receipt reports the gap instead of closing + * the search. + */ +export async function recordCandidatePopulationSearch(input: { + ledger: SearchLedger + storage: CampaignStorage + runDir: string + identity: SearchRunIdentity + population: GepaCandidatePopulationArtifact + /** Scenarios the optimizer selected on. Must cover the population's ids. */ + scenarios: ReadonlyArray + /** Spend the optimizer booked to its own candidate generation. */ + generationAccounting: SearchAttemptAccounting + producerId: string + runId: string +}): Promise { + const { population, ledger, identity } = input + const stamps = monotonicStamps() + const writeArtifact = artifactWriter(input.storage, input.runDir) + const populationArtifact: SearchArtifactRef = { + role: 'candidate-population', + uri: `file://${population.summary.path}`, + sha256: population.summary.sha256, + byteLength: population.summary.bytes, + } + const tasks = input.scenarios + .filter((scenario) => population.summary.scenarioIds.includes(scenario.id)) + .map((scenario) => ({ + taskId: taskIdFor(scenario.id, 0), + source: { uri: `scenario://${scenario.id}`, revision: hashCanonical(scenario) }, + benchmark: { uri: `optimizer://${input.producerId}`, revision: population.summary.sha256 }, + maxAttempts: 1, + })) + if (tasks.length !== population.summary.scenarioIds.length) { + throw new Error( + `search ledger: the candidate population scored ${population.summary.scenarioIds.length} scenarios, but ${tasks.length} were supplied`, + ) + } + + const generationOperation = 'candidate-generation:population' + await ledger.append({ + kind: 'search-planned', + eventId: 'search:plan', + occurredAt: stamps(), + artifacts: [populationArtifact], + plan: { + candidateSlots: population.candidates.map((candidate) => ({ + slotId: `candidate-${candidate.index}`, + generationOperationId: generationOperation, + })), + tasks, + operations: [ + { operationId: generationOperation, kind: 'candidate-generation' }, + { operationId: 'selection', kind: 'selection' }, + ], + }, + }) + await ledger.append({ + kind: 'search-operation-recorded', + eventId: `operation:${generationOperation}`, + occurredAt: stamps(), + artifacts: [populationArtifact], + operationId: generationOperation, + operationKind: 'candidate-generation', + execution: identity.proposer, + outcome: { status: 'completed' }, + accounting: input.generationAccounting, + }) + + const candidateIds = new Map() + const depths = new Map() + for (const candidate of population.candidates) { + const parents = candidate.parentIndices.filter((index): index is number => index !== null) + const candidateId = candidate.candidateHash + candidateIds.set(candidate.index, candidateId) + const depth = + parents.length === 0 ? 0 : Math.max(...parents.map((index) => depths.get(index) ?? 0)) + 1 + depths.set(candidate.index, depth) + const surfaceArtifact = writeArtifact( + 'candidate-surface', + `candidate-${candidate.index}.json`, + { + index: candidate.index, + candidateDigest: candidate.candidateDigest, + candidate: candidate.candidate, + }, + ) + await ledger.append({ + kind: 'candidate-registered', + eventId: `candidate:${candidate.index}`, + occurredAt: stamps(), + artifacts: [surfaceArtifact], + slotId: `candidate-${candidate.index}`, + generationOperationId: generationOperation, + candidateId, + lineage: { + lineageNodeId: candidateId.slice(0, 16), + parentCandidateIds: parents.map((index) => { + const parentId = candidateIds.get(index) + if (!parentId) { + throw new Error( + `search ledger: candidate ${candidate.index} names parent ${index}, which precedes no registered candidate`, + ) + } + return parentId + }), + generation: depth, + proposer: input.producerId, + proposerSource: identity.proposer.source, + }, + surfaces: candidateSurfaces(externalSurface(candidate.candidate), surfaceArtifact), + }) + + const surfaceIds = candidateSurfaces(externalSurface(candidate.candidate), surfaceArtifact).map( + (surface) => surface.surfaceId, + ) + for (const score of candidate.selectionScores) { + const task = tasks.find((planned) => planned.taskId === taskIdFor(score.scenarioId, 0)) + if (!task) { + throw new Error( + `search ledger: candidate ${candidate.index} scored unplanned scenario ${score.scenarioId}`, + ) + } + await ledger.append({ + kind: 'task-attempted', + eventId: `attempt:${candidate.index}:${task.taskId}`, + occurredAt: stamps(), + artifacts: [populationArtifact], + candidateId, + runId: `${input.runId}:${candidate.index}:${task.taskId}`, + attemptIndex: 0, + task: { taskId: task.taskId, source: task.source }, + identity: { + model: identity.model, + agent: identity.agent, + benchmark: task.benchmark, + }, + outcome: { status: 'passed', score: score.score, metrics: { composite: score.score } }, + accounting: { + tokens: { + status: 'unknown', + reason: 'the optimizer reports no token usage per candidate evaluation', + }, + cost: { + status: 'unknown', + knownLowerBoundUsd: 0, + reason: 'the optimizer reports no cost per candidate evaluation', + }, + }, + surfaceEvidence: surfaceIds.map((surfaceId) => + surfaceEvidenceFor(surfaceId, populationArtifact), + ), + }) + } + } + + await ledger.append({ + kind: 'search-operation-recorded', + eventId: 'operation:selection', + occurredAt: stamps(), + artifacts: [populationArtifact], + operationId: 'selection', + operationKind: 'selection', + execution: { kind: 'deterministic', source: identity.search }, + outcome: { status: 'completed' }, + accounting: { + tokens: { status: 'known', inputTokens: 0, outputTokens: 0, cachedTokens: 0 }, + cost: { status: 'known', usd: 0, source: 'free' }, + }, + }) + + const winnerId = candidateIds.get(population.bestIndex) + for (const candidate of population.candidates) { + await ledger.append({ + kind: 'candidate-decided', + eventId: `decision:${candidate.index}`, + occurredAt: stamps(), + artifacts: [populationArtifact], + candidateId: candidateIds.get(candidate.index)!, + decision: + candidate.index === population.bestIndex + ? { status: 'selected' } + : { + status: 'rejected', + reason: { + code: 'not-selected', + message: 'the optimizer selected another candidate', + }, + }, + }) + } + + const replay = await ledger.replay() + const { missingCandidateSlots, missingTaskOutcomes, missingOperations } = replay.audit.expected + if ( + winnerId && + missingCandidateSlots.length === 0 && + missingTaskOutcomes.length === 0 && + missingOperations.length === 0 && + replay.audit.decisions.pending === 0 + ) { + await ledger.append({ + kind: 'search-completed', + eventId: 'search:completed', + occurredAt: stamps(), + artifacts: [populationArtifact], + result: { status: 'selected', candidateId: winnerId }, + }) + } + + const final = await ledger.replay() + const bytes = final.entries.map((entry) => `${canonicalString(entry)}\n`).join('') + return createSearchHistoryReceipt({ + producerId: input.producerId, + runId: input.runId, + ledger: { + role: 'search-ledger', + uri: `file://${ledger.path}`, + sha256: hashCanonical(bytes), + byteLength: new TextEncoder().encode(bytes).byteLength, + }, + replay: final, + }) +} + +/** An optimizer candidate as the canonical mutable surface it represents. */ +function externalSurface(candidate: ExternalTextCandidate): MutableSurface { + return typeof candidate === 'string' ? candidate : { kind: 'components', components: candidate } +} + +function artifactWriter(storage: CampaignStorage, runDir: string) { + const directory = `${runDir}/${SEARCH_LEDGER_DIR}` + return (role: string, name: string, body: unknown): SearchArtifactRef => { + const contents = canonicalString(body) + storage.ensureDir(directory) + storage.write(`${directory}/${name}`, contents) + return { + role, + uri: `file://${directory}/${name}`, + sha256: hashCanonical(contents), + byteLength: new TextEncoder().encode(contents).byteLength, + } + } +} + +/** Non-decreasing ISO stamps; the ledger refuses an event that moves back. */ +function monotonicStamps(): () => string { + let last = 0 + return () => { + const now = Date.now() + last = now > last ? now : last + 1 + return new Date(last).toISOString() + } +} diff --git a/src/campaign/search-ledger.test.ts b/src/campaign/search-ledger.test.ts index ce241cca..55311d5d 100644 --- a/src/campaign/search-ledger.test.ts +++ b/src/campaign/search-ledger.test.ts @@ -14,6 +14,7 @@ import { SearchLedgerConflictError, SearchLedgerIntegrityError, type SearchOperationRecordedEvent, + type SearchPlanExtendedEvent, type SearchPlannedEvent, type SearchSurfaceEvidence, type SearchTaskAttemptedEvent, @@ -126,13 +127,13 @@ function candidate( function operation( cost: SearchCostAccounting = { status: 'known', usd: 0.02, source: 'provider' }, outcome: SearchOperationRecordedEvent['outcome'] = { status: 'completed' }, - options: { operationId?: string; eventId?: string } = {}, + options: { operationId?: string; eventId?: string; occurredAt?: string } = {}, ): SearchOperationRecordedEvent { const operationId = options.operationId ?? 'candidate-generation:a' return { kind: 'search-operation-recorded', eventId: options.eventId ?? `operation:${operationId}`, - occurredAt: '2026-07-11T11:59:30.000Z', + occurredAt: options.occurredAt ?? '2026-07-11T11:59:30.000Z', artifacts: [artifact('operation-receipt', HASHES.proposal)], operationId, operationKind: 'candidate-generation', @@ -213,12 +214,13 @@ function attempt( outcome?: SearchTaskOutcome cost?: SearchCostAccounting evidence?: SearchSurfaceEvidence[] + occurredAt?: string } = {}, ): SearchTaskAttemptedEvent { return { kind: 'task-attempted', eventId: options.eventId ?? `attempt:${candidateId}:task-1:${options.attemptIndex ?? 0}`, - occurredAt: '2026-07-11T12:01:00.000Z', + occurredAt: options.occurredAt ?? '2026-07-11T12:01:00.000Z', artifacts: [artifact('run-record', HASHES.run), artifact('trace', HASHES.trace)], candidateId, runId: options.runId ?? `run:${candidateId}:${options.attemptIndex ?? 0}`, @@ -286,6 +288,30 @@ function completion( } } +function planExtension( + options: { + eventId?: string + slotIds?: string[] + operations?: Array<{ operationId: string; kind: 'candidate-generation' | 'analysis' }> + generationOperationId?: string + occurredAt?: string + } = {}, +): SearchPlanExtendedEvent { + return { + kind: 'search-plan-extended', + eventId: options.eventId ?? 'search:plan-extended', + occurredAt: options.occurredAt ?? '2026-07-11T12:01:30.000Z', + artifacts: [artifact('search-round-manifest', HASHES.report)], + extension: { + candidateSlots: (options.slotIds ?? []).map((slotId) => ({ + slotId, + generationOperationId: options.generationOperationId ?? 'candidate-generation:b', + })), + operations: options.operations ?? [], + }, + } +} + async function ledgerPath(name: string): Promise { const dir = await mkdtemp(join(tmpdir(), `search-ledger-${name}-`)) return join(dir, 'nested', 'search.jsonl') @@ -1029,3 +1055,117 @@ describe('search ledger evidence completeness', () => { ]) }) }) + +describe('search ledger rolling extension', () => { + it('replays an appended round: merged plan, continued generations, complete audit', async () => { + const path = await ledgerPath('rolling') + const ledger = openSearchLedger({ path, campaignId: 'campaign-rolling' }) + await ledger.append(plan()) + await ledger.append(operation()) + await ledger.append(candidate('candidate-a')) + await ledger.append(attempt('candidate-a')) + + // Round two is authored only after round one produced a parent. + await ledger.append( + planExtension({ + slotIds: ['slot-b'], + operations: [{ operationId: 'candidate-generation:b', kind: 'candidate-generation' }], + }), + ) + await ledger.append( + operation(undefined, undefined, { + operationId: 'candidate-generation:b', + occurredAt: '2026-07-11T12:01:40.000Z', + }), + ) + await ledger.append( + candidate('candidate-b', { + slotId: 'slot-b', + generationOperationId: 'candidate-generation:b', + lineageNodeId: 'f'.repeat(16), + // The generation invariant continues across the extension: the child of + // a round-one candidate is generation 1, not a restarted 0. + parents: ['candidate-a'], + generation: 1, + occurredAt: '2026-07-11T12:01:45.000Z', + }), + ) + await ledger.append( + attempt('candidate-b', { + eventId: 'attempt:candidate-b:task-1:0', + runId: 'run:candidate-b:0', + occurredAt: '2026-07-11T12:01:50.000Z', + }), + ) + await ledger.append(decision('candidate-a', 'rejected')) + await ledger.append(decision('candidate-b', 'selected')) + const terminal = await ledger.append(completion('selected', 'candidate-b')) + + expect(terminal.replay.planExtensions).toHaveLength(1) + expect(terminal.replay.audit).toMatchObject({ + status: 'selected', + selectedCandidateId: 'candidate-b', + candidateCount: 2, + expected: { + candidateSlots: 2, + operations: 2, + taskOutcomes: 2, + missingCandidateSlots: [], + missingOperations: [], + missingTaskOutcomes: [], + }, + }) + + // The durable file replays to the same merged plan in a new process. + const reopened = await openSearchLedger({ path, campaignId: 'campaign-rolling' }).replay() + expect(reopened.audit.expected.candidateSlots).toBe(2) + expect(reopened.candidates.map((event) => event.lineage.generation)).toEqual([0, 1]) + }) + + it('still refuses a planless ledger, including one that opens with an extension', async () => { + const first = openSearchLedger({ + path: await ledgerPath('rolling-planless'), + campaignId: 'campaign-planless', + }) + await expect( + first.append( + planExtension({ + operations: [{ operationId: 'candidate-generation:b', kind: 'candidate-generation' }], + }), + ), + ).rejects.toThrow(/appears before the required search plan/) + + const second = openSearchLedger({ + path: await ledgerPath('rolling-planless-candidate'), + campaignId: 'campaign-planless-candidate', + }) + await expect(second.append(candidate())).rejects.toThrow( + /appears before the required search plan/, + ) + }) + + it('counts an unfilled extended slot as missing until it is bound or closed', async () => { + const ledger = openSearchLedger({ + path: await ledgerPath('rolling-open-slot'), + campaignId: 'campaign-rolling-open', + }) + await ledger.append(plan()) + await ledger.append(operation()) + await ledger.append(candidate('candidate-a')) + await ledger.append(attempt('candidate-a')) + await ledger.append( + planExtension({ + slotIds: ['slot-b'], + operations: [{ operationId: 'candidate-generation:b', kind: 'candidate-generation' }], + }), + ) + await ledger.append(decision('candidate-a', 'selected')) + + await expect(ledger.append(completion('selected'))).rejects.toThrow( + /missing candidate slots: slot-b/, + ) + const replay = await ledger.replay() + expect(replay.audit.expected.missingCandidateSlots).toEqual(['slot-b']) + expect(replay.audit.expected.missingOperations).toEqual(['candidate-generation:b']) + }) +}) diff --git a/src/campaign/search-ledger.ts b/src/campaign/search-ledger.ts index 90fee3ee..4e6e5cfe 100644 --- a/src/campaign/search-ledger.ts +++ b/src/campaign/search-ledger.ts @@ -213,6 +213,18 @@ export interface SearchPlannedEvent extends SearchLedgerEventBase { plan: SearchPlan } +/** Additional candidate slots and operations for a search whose length is not + * known when it starts. The plan stays the first event and the planned task + * denominator stays frozen: extending tasks would retroactively reopen + * candidates that already closed theirs. */ +export interface SearchPlanExtendedEvent extends SearchLedgerEventBase { + kind: 'search-plan-extended' + extension: { + candidateSlots: SearchCandidateSlot[] + operations: SearchPlannedOperation[] + } +} + export interface SearchCandidateRegisteredEvent extends SearchLedgerEventBase { kind: 'candidate-registered' slotId: string @@ -295,6 +307,7 @@ export interface SearchCompletedEvent extends SearchLedgerEventBase { export type SearchLedgerEvent = | SearchPlannedEvent + | SearchPlanExtendedEvent | SearchCandidateRegisteredEvent | SearchCandidateSlotClosedEvent | SearchTaskAttemptedEvent @@ -356,6 +369,9 @@ export interface SearchLedgerAudit { export interface SearchLedgerReplay { entries: SearchLedgerEntry[] plan: SearchPlannedEvent | null + /** Appended plan extensions, in ledger order. The effective plan is the + * first plan event merged with these; `audit.expected` counts the merge. */ + planExtensions: SearchPlanExtendedEvent[] candidates: SearchCandidateRegisteredEvent[] closedCandidateSlots: SearchCandidateSlotClosedEvent[] attempts: SearchTaskAttemptedEvent[] @@ -425,22 +441,45 @@ const OperationKindSchema = z.enum([ 'other', ]) +const CandidateSlotSchema = z + .object({ + slotId: NON_EMPTY, + generationOperationId: NON_EMPTY, + }) + .strict() + +const PlannedOperationSchema = z + .object({ + operationId: NON_EMPTY, + kind: OperationKindSchema, + }) + .strict() + +const SearchPlanExtendedSchema = z + .object({ + ...EventBaseShape, + kind: z.literal('search-plan-extended'), + extension: z + .object({ + candidateSlots: z.array(CandidateSlotSchema), + operations: z.array(PlannedOperationSchema), + }) + .strict() + .superRefine((extension, ctx) => { + if (extension.candidateSlots.length === 0 && extension.operations.length === 0) { + ctx.addIssue({ code: 'custom', message: 'a plan extension must add slots or operations' }) + } + }), + }) + .strict() + const SearchPlannedSchema = z .object({ ...EventBaseShape, kind: z.literal('search-planned'), plan: z .object({ - candidateSlots: z - .array( - z - .object({ - slotId: NON_EMPTY, - generationOperationId: NON_EMPTY, - }) - .strict(), - ) - .min(1), + candidateSlots: z.array(CandidateSlotSchema).min(1), tasks: z .array( z @@ -453,16 +492,7 @@ const SearchPlannedSchema = z .strict(), ) .min(1), - operations: z - .array( - z - .object({ - operationId: NON_EMPTY, - kind: OperationKindSchema, - }) - .strict(), - ) - .min(1), + operations: z.array(PlannedOperationSchema).min(1), }) .strict(), }) @@ -763,6 +793,7 @@ const SearchCompletedSchema = z const EventSchema = z.discriminatedUnion('kind', [ SearchPlannedSchema, + SearchPlanExtendedSchema, CandidateRegisteredSchema, CandidateSlotClosedSchema, TaskAttemptedSchema, @@ -950,6 +981,12 @@ function createSearchLedgerProjector( campaignId: string, ): LedgerProjector { const candidates = new Map() + // The effective plan: the first plan event merged with every later + // extension. Every slot and operation lookup reads these, so a rolling + // search that appends slots keeps one plan, one denominator, one audit. + const plannedSlots = new Map() + const plannedOperations = new Map() + const planExtensions: SearchPlanExtendedEvent[] = [] const candidateBySlot = new Map() const closedSlots = new Map() const lineageNodes = new Map() @@ -1000,15 +1037,12 @@ function createSearchLedgerProjector( 'planned operationId', event.eventId, ) + for (const operation of event.plan.operations) { + plannedOperations.set(operation.operationId, operation) + } for (const slot of event.plan.candidateSlots) { - const generationOperation = event.plan.operations.find( - (operation) => operation.operationId === slot.generationOperationId, - ) - if (generationOperation?.kind !== 'candidate-generation') { - throw new SearchLedgerIntegrityError( - `candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`, - ) - } + assertSlotGenerationOperation(slot, plannedOperations) + plannedSlots.set(slot.slotId, slot) } planEvent = event return @@ -1020,11 +1054,43 @@ function createSearchLedgerProjector( ) } + if (event.kind === 'search-plan-extended') { + assertUnique( + event.extension.candidateSlots.map((slot) => slot.slotId), + 'candidate slot', + event.eventId, + ) + assertUnique( + event.extension.operations.map((operation) => operation.operationId), + 'planned operationId', + event.eventId, + ) + for (const operation of event.extension.operations) { + if (plannedOperations.has(operation.operationId)) { + throw new SearchLedgerIntegrityError( + `plan extension ${event.eventId} re-plans operation ${operation.operationId}`, + ) + } + plannedOperations.set(operation.operationId, operation) + } + for (const slot of event.extension.candidateSlots) { + if (plannedSlots.has(slot.slotId)) { + throw new SearchLedgerIntegrityError( + `plan extension ${event.eventId} re-plans candidate slot ${slot.slotId}`, + ) + } + assertSlotGenerationOperation(slot, plannedOperations) + plannedSlots.set(slot.slotId, slot) + } + planExtensions.push(event) + return + } + if (event.kind === 'candidate-registered') { if (candidates.has(event.candidateId)) { throw new SearchLedgerIntegrityError(`candidate ${event.candidateId} was registered twice`) } - const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId) + const plannedSlot = plannedSlots.get(event.slotId) if (!plannedSlot) { throw new SearchLedgerIntegrityError( `candidate ${event.candidateId} binds unknown slot ${event.slotId}`, @@ -1177,9 +1243,7 @@ function createSearchLedgerProjector( } if (event.kind === 'search-operation-recorded') { - const plannedOperation = planEvent.plan.operations.find( - (operation) => operation.operationId === event.operationId, - ) + const plannedOperation = plannedOperations.get(event.operationId) if (!plannedOperation) { throw new SearchLedgerIntegrityError( `operation ${event.operationId} was not declared in the search plan`, @@ -1199,7 +1263,7 @@ function createSearchLedgerProjector( } if (event.kind === 'candidate-slot-closed') { - const plannedSlot = planEvent.plan.candidateSlots.find((slot) => slot.slotId === event.slotId) + const plannedSlot = plannedSlots.get(event.slotId) if (!plannedSlot) { throw new SearchLedgerIntegrityError( `candidate slot closure ${event.eventId} references unknown slot ${event.slotId}`, @@ -1259,7 +1323,7 @@ function createSearchLedgerProjector( return } - const missingCandidateSlots = planEvent.plan.candidateSlots + const missingCandidateSlots = [...plannedSlots.values()] .filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)) .map((slot) => slot.slotId) if (missingCandidateSlots.length > 0) { @@ -1273,7 +1337,7 @@ function createSearchLedgerProjector( `search completed with missing task outcomes: ${missingTaskOutcomes.join(', ')}`, ) } - const missingOperations = planEvent.plan.operations + const missingOperations = [...plannedOperations.values()] .filter((operation) => !operationsById.has(operation.operationId)) .map((operation) => operation.operationId) if (missingOperations.length > 0) { @@ -1281,10 +1345,10 @@ function createSearchLedgerProjector( `search completed with missing search operations: ${missingOperations.join(', ')}`, ) } - for (const operation of planEvent.plan.operations) { + for (const operation of plannedOperations.values()) { if (operation.kind !== 'candidate-generation') continue const generationOutcome = operationsById.get(operation.operationId)!.outcome.status - const slots = planEvent.plan.candidateSlots.filter( + const slots = [...plannedSlots.values()].filter( (slot) => slot.generationOperationId === operation.operationId, ) if (slots.length === 0) continue @@ -1388,18 +1452,17 @@ function createSearchLedgerProjector( : completion?.result.status === 'all-rejected' ? 'all-rejected' : 'in-progress' - const missingCandidateSlots = - planEvent?.plan.candidateSlots - .filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)) - .map((slot) => slot.slotId) ?? [] + const missingCandidateSlots = [...plannedSlots.values()] + .filter((slot) => !candidateBySlot.has(slot.slotId) && !closedSlots.has(slot.slotId)) + .map((slot) => slot.slotId) const missingTaskOutcomes = planEvent ? plannedTaskOutcomeKeys(planEvent, candidates) : [] - const missingOperations = - planEvent?.plan.operations - .filter((operation) => !operationsById.has(operation.operationId)) - .map((operation) => operation.operationId) ?? [] + const missingOperations = [...plannedOperations.values()] + .filter((operation) => !operationsById.has(operation.operationId)) + .map((operation) => operation.operationId) return { entries: [...entries], plan: planEvent, + planExtensions, candidates: candidateEvents, closedCandidateSlots: closedSlotEvents, attempts, @@ -1421,9 +1484,9 @@ function createSearchLedgerProjector( pending: candidates.size - decisions.length, }, expected: { - candidateSlots: planEvent?.plan.candidateSlots.length ?? 0, + candidateSlots: plannedSlots.size, taskOutcomes: candidates.size * (planEvent?.plan.tasks.length ?? 0), - operations: planEvent?.plan.operations.length ?? 0, + operations: plannedOperations.size, missingCandidateSlots, missingTaskOutcomes, missingOperations, @@ -1439,6 +1502,19 @@ function createSearchLedgerProjector( return { apply, finish } } +/** Every candidate slot must name a planned candidate-generation operation, + * whether it arrives with the plan or with a later extension. */ +function assertSlotGenerationOperation( + slot: SearchCandidateSlot, + plannedOperations: ReadonlyMap, +): void { + if (plannedOperations.get(slot.generationOperationId)?.kind !== 'candidate-generation') { + throw new SearchLedgerIntegrityError( + `candidate slot ${slot.slotId} references unplanned candidate-generation operation ${slot.generationOperationId}`, + ) + } +} + function plannedTaskOutcomeKeys( planEvent: SearchPlannedEvent, candidates: Map, @@ -1476,6 +1552,20 @@ function normalizeEvent(event: SearchLedgerEvent): SearchLedgerEvent { }, } } + if (event.kind === 'search-plan-extended') { + return { + ...event, + artifacts, + extension: { + candidateSlots: [...event.extension.candidateSlots].sort((a, b) => + compareStrings(a.slotId, b.slotId), + ), + operations: [...event.extension.operations].sort((a, b) => + compareStrings(a.operationId, b.operationId), + ), + }, + } + } if (event.kind === 'candidate-registered') { return { ...event, diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 87f6f46b..6d25723e 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -37,6 +37,7 @@ import { campaignCellTaskScore, campaignCellToRunRecord, } from '../campaign/run-record' +import type { SearchHistoryReceipt } from '../campaign/search-history-receipt' import { type CampaignStorage, createRunCostLedger, @@ -306,6 +307,11 @@ export interface SelfImproveOptions { * from the Pareto frontier; promotion still compares against the incumbent. * Proposer mode only. See `RunOptimizationOptions.selectParent`. */ selectParent?: RunOptimizationOptions['selectParent'] + + /** Record this run's candidate search into a durable `SearchLedger` and + * return the bounded receipt on `searchHistory`. See + * `RunOptimizationOptions.searchLedger`. */ + searchLedger?: RunOptimizationOptions['searchLedger'] } export interface SelfImproveResult { @@ -357,6 +363,9 @@ export interface SelfImproveResult { /** Run-wide receipts across proposal, search, holdout, judging, analysis, * and promotion work, with phase and actor attribution. */ receipts: CostReceipt[] + /** Bounded proof envelope over this run's canonical search ledger. Present + * only when `searchLedger` was supplied. */ + searchHistory?: SearchHistoryReceipt /** Exact external method and source identity, when `method` was used. */ optimization?: { name: string @@ -742,6 +751,7 @@ async function runSelfImprove( findings: opts.findings, selectionRankKey: opts.selectionRankKey, selectParent: opts.selectParent, + searchLedger: opts.searchLedger, }) // Deferred holdout ran zero holdout cells, so the summary stats come from @@ -902,6 +912,7 @@ async function runSelfImprove( }, } : {}), + ...(result.searchHistory ? { searchHistory: result.searchHistory } : {}), insight, ...(power ? { power } : {}), raw: result, diff --git a/tests/contract-self-improve.test.ts b/tests/contract-self-improve.test.ts index 059ed1e8..bb179e39 100644 --- a/tests/contract-self-improve.test.ts +++ b/tests/contract-self-improve.test.ts @@ -245,70 +245,6 @@ describe('selfImprove — forwarded loop knobs', () => { expect(observedAbort).toBe(true) }) - it('forwards selectParent — the parent policy fires once per generation', async () => { - const generationsSeen: number[] = [] - const parentsSeen: string[] = [] - await selfImprove({ - ...base, - budget: { generations: 2, populationSize: 1 }, - expectUsage: 'off', - selectParent: ({ frontier, generation }) => { - generationsSeen.push(generation) - const parent = frontier[0]! - parentsSeen.push(String(parent.surface)) - return parent - }, - proposer: { - kind: 'fake:marker', - async propose({ currentSurface }) { - // The loop hands the selected parent to the proposer as currentSurface. - expect(String(currentSurface)).toBe(parentsSeen.at(-1)) - return [ - { surface: `${String(currentSurface)} ${MARKER}`, label: LABEL, rationale: RATIONALE }, - ] - }, - }, - }) - expect(generationsSeen).toEqual([0, 1]) - }) - - it('forwards cellRetry — a transient holdout hiccup is retried instead of failing the loop', async () => { - const train = SCENARIOS[0]! - const holdout = SCENARIOS[1]! - const flakyOnce = () => { - let failed = false - return async (surface: MutableSurface, scenario: S): Promise => { - if (scenario.id === holdout.id && !failed) { - failed = true - throw new Error('router returned HTTP 503 Service Unavailable') - } - return { text: String(surface) } - } - } - - // Without the policy the transient failure leaves the holdout incomplete. - await expect( - selfImprove({ - ...base, - agent: flakyOnce(), - scenarios: [train, holdout], - budget: { generations: 0, holdoutScenarios: [holdout] }, - expectUsage: 'off', - }), - ).rejects.toThrow(/holdout is incomplete/) - - // With it, the same slot is re-dispatched and the loop completes. - const ok = await selfImprove({ - ...base, - agent: flakyOnce(), - scenarios: [train, holdout], - budget: { generations: 0, holdoutScenarios: [holdout] }, - cellRetry: { attempts: 2, retryable: transientDispatchFailure() }, - expectUsage: 'off', - }) - expect(ok.gateDecision).toBe('hold') - }) - it('forwards analyzeGeneration — the per-generation findings producer fires', async () => { let calls = 0 await selfImprove({