From c5c4d5dbee0e3ab7af24bd4dfb287bf6b90a2bca Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 01:00:30 +0000 Subject: [PATCH 1/8] feat(evals): let evals opt out of the skill and CLI and run per model Add three fixtures to the eval harness: installSkill, installCli, and model. Each eval can override them with it.scoped, per file or per describe block. When installCli is false, the harness does not link the local build and does not set the PRISMIC_* env vars. When installSkill is false, the system prompt has no skill. The model fixture sets the agent model. The judge model does not change. Export a models list so an eval can run once per model with describe.for. The reporter records the model on each trial row and keys results by the full test name, so describe names appear in the key. Recorded calls keep a version spec such as @latest. Co-Authored-By: Claude Fable 5.1 --- evals/it.ts | 50 +++++++++++++++++++++++++++++++++-------------- evals/reporter.ts | 7 +++---- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/evals/it.ts b/evals/it.ts index e5976c5..98082fb 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -29,6 +29,13 @@ const SKILL = await fetchSkill(); export const trials = Array.from({ length: EVAL_TRIALS }, (_, i) => i + 1); +export const models = [ + "claude-sonnet-5", + "claude-haiku-4-5", + "claude-sonnet-4-5", + "claude-opus-4-1", +]; + declare module "vitest" { interface TaskMeta { agent?: Trial; @@ -41,43 +48,55 @@ declare module "vitest" { } export const it = base.extend<{ + installSkill: boolean; + installCli: boolean; + model: string; agent: (prompt: string) => Promise; }>({ - agent: async ({ home, project, login, task, repo, token, host, password }, use) => { + installSkill: true, + installCli: true, + model: EVAL_MODEL, + agent: async ( + { home, project, login, task, repo, token, host, password, installSkill, installCli, model }, + use, + ) => { await login(); const claudeConfigDir = await createClaudeConfigDir(); - const nodeModulesBinDir = new URL("node_modules/.bin/", project); - await mkdir(nodeModulesBinDir, { recursive: true }); - await symlink(BIN, new URL("prismic", nodeModulesBinDir)); - - const env = { + const env: Record = { ...process.env, HOME: fileURLToPath(home), - PRISMIC_CONFIG_DIR: fileURLToPath(new URL(".config/prismic/", home)), - PRISMIC_TYPE_BUILDER_ENABLED: "true", - PRISMIC_SENTRY_ENABLED: "false", - PRISMIC_TELEMETRY_ENABLED: "false", NO_UPDATE_NOTIFIER: "1", CLAUDE_CONFIG_DIR: claudeConfigDir, CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1", CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", }; - const trial: Trial = { model: EVAL_MODEL, costUsd: 0, durationS: 0, calls: [] }; + if (installCli) { + const nodeModulesBinDir = new URL("node_modules/.bin/", project); + await mkdir(nodeModulesBinDir, { recursive: true }); + await symlink(BIN, new URL("prismic", nodeModulesBinDir)); + env.PRISMIC_CONFIG_DIR = fileURLToPath(new URL(".config/prismic/", home)); + env.PRISMIC_TYPE_BUILDER_ENABLED = "true"; + env.PRISMIC_SENTRY_ENABLED = "false"; + env.PRISMIC_TELEMETRY_ENABLED = "false"; + } + + const trial: Trial = { model, costUsd: 0, durationS: 0, calls: [] }; task.meta.agent = trial; let durationMs = 0; await use(async (prompt: string) => { const result = await agent(prompt, { - systemPromptAppend: SKILL, + model, + systemPromptAppend: installSkill ? SKILL : undefined, cwd: project, env, // Recorded as commands stream so a timed-out trial keeps its trail. onCommand: (command) => { if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) { - trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?\s+/, "")); + trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(?=@|\s)\s*/, "")); } }, }); @@ -145,13 +164,14 @@ type AgentResult = { async function agent( prompt: string, config: { + model: string; systemPromptAppend?: string; cwd: URL; env: NodeJS.ProcessEnv; onCommand?: (command: string) => void; }, ) { - const { systemPromptAppend, cwd, env, onCommand } = config; + const { model, systemPromptAppend, cwd, env, onCommand } = config; let result: SDKResultMessage | undefined; const commands: string[] = []; @@ -159,7 +179,7 @@ async function agent( for await (const message of query({ prompt, options: { - model: EVAL_MODEL, + model, systemPrompt: { type: "preset", preset: "claude_code", diff --git a/evals/reporter.ts b/evals/reporter.ts index 73737d5..9af2497 100644 --- a/evals/reporter.ts +++ b/evals/reporter.ts @@ -25,7 +25,6 @@ const LOCAL_RESULTS_PATH = fileURLToPath(new URL("results.local.json", import.me // always holds one full run; history lives in git. export default class EvalReporter implements Reporter { onTestRunEnd(testModules: ReadonlyArray): void { - let model = ""; let filtered = false; const evals: Record = {}; @@ -45,9 +44,9 @@ export default class EvalReporter implements Reporter { filtered = true; continue; } - model = trial.model; - (evals[`${testModule.relativeModuleId} / ${test.name}`] ??= []).push({ + (evals[`${testModule.relativeModuleId} / ${test.fullName}`] ??= []).push({ pass: state === "passed", + model: trial.model, costUsd: Math.round(trial.costUsd * 100) / 100, durationS: trial.durationS, calls: trial.calls, @@ -59,7 +58,7 @@ export default class EvalReporter implements Reporter { const sorted = Object.fromEntries(Object.entries(evals).sort(([a], [b]) => a.localeCompare(b))); // Single-line output keeps run-over-run diffs to one changed line; read // it with jq or `node --run evals:report`. - const report = JSON.stringify({ model, evals: sorted }) + "\n"; + const report = JSON.stringify({ evals: sorted }) + "\n"; writeFileSync(LOCAL_RESULTS_PATH, report); const evalFiles = readdirSync(EVALS_DIR).filter((file) => file.endsWith(".eval.ts")); From 2cf81459983f2751fe50a6fb1990a9e1a618388e Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 18:20:51 +0000 Subject: [PATCH 2/8] feat(evals): run evals on Codex as well as Claude Code The agent fixture picks the harness from the model name. Models that start with "claude-" run on Claude Code. All other models run on Codex through @openai/codex-sdk. Both runners return the final text and the token count. They report each shell command to the fixture, which records the commands for assertions and the prismic calls for the results file. The Codex runner writes the skill to AGENTS.md in the project and uses a temporary CODEX_HOME. It copies auth.json from the local Codex home when OPENAI_API_KEY is not set. Trial rows record tokens instead of cost, because Codex does not report cost. Remove the EVAL_MODEL env var. The default model is a constant. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/evals.yml | 1 + evals/it.ts | 131 ++++++++++++++++++++++----------- evals/reporter.ts | 6 +- package-lock.json | 143 ++++++++++++++++++++++++++++++++++++ package.json | 1 + 5 files changed, 237 insertions(+), 45 deletions(-) diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml index 22056a0..14a3013 100644 --- a/.github/workflows/evals.yml +++ b/.github/workflows/evals.yml @@ -33,6 +33,7 @@ jobs: env: PRISMIC_ALLOW_EVALS: "true" ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} E2E_PRISMIC_EMAIL: ${{ secrets.E2E_PRISMIC_EMAIL }} E2E_PRISMIC_PASSWORD: ${{ secrets.E2E_PRISMIC_PASSWORD }} run: node --run evals || true diff --git a/evals/it.ts b/evals/it.ts index 98082fb..89a440b 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -1,4 +1,5 @@ import { query, type SDKResultMessage } from "@anthropic-ai/claude-agent-sdk"; +import { Codex } from "@openai/codex-sdk"; import dedent from "dedent"; import { copyFile, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises"; import { homedir, tmpdir } from "node:os"; @@ -20,8 +21,8 @@ if (process.env.PRISMIC_ALLOW_EVALS !== "true") { } const BIN = new URL("../dist/index.mjs", import.meta.url); -const EVAL_MODEL = process.env.EVAL_MODEL ?? "claude-sonnet-5"; const EVAL_TRIALS = Number(process.env.EVAL_TRIALS ?? 3); +const MODEL = "claude-sonnet-5"; const JUDGE_MODEL = "claude-sonnet-5"; const PRISMIC_SKILL_REF = "2bd340e6af4e67a9c1179e97b495f7bda564b46f"; @@ -29,13 +30,6 @@ const SKILL = await fetchSkill(); export const trials = Array.from({ length: EVAL_TRIALS }, (_, i) => i + 1); -export const models = [ - "claude-sonnet-5", - "claude-haiku-4-5", - "claude-sonnet-4-5", - "claude-opus-4-1", -]; - declare module "vitest" { interface TaskMeta { agent?: Trial; @@ -55,20 +49,18 @@ export const it = base.extend<{ }>({ installSkill: true, installCli: true, - model: EVAL_MODEL, + model: MODEL, agent: async ( { home, project, login, task, repo, token, host, password, installSkill, installCli, model }, use, ) => { await login(); - const claudeConfigDir = await createClaudeConfigDir(); - const env: Record = { ...process.env, HOME: fileURLToPath(home), NO_UPDATE_NOTIFIER: "1", - CLAUDE_CONFIG_DIR: claudeConfigDir, + CLAUDE_CONFIG_DIR: await createClaudeConfigDir(), CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1", CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", }; @@ -83,30 +75,34 @@ export const it = base.extend<{ env.PRISMIC_TELEMETRY_ENABLED = "false"; } - const trial: Trial = { model, costUsd: 0, durationS: 0, calls: [] }; + const trial: Trial = { model, tokens: 0, durationS: 0, calls: [] }; task.meta.agent = trial; let durationMs = 0; + const run = model.startsWith("claude-") ? runClaudeCode : runCodex; + await use(async (prompt: string) => { - const result = await agent(prompt, { + const start = performance.now(); + const commands: string[] = []; + const { text, tokens } = await run(prompt, { model, - systemPromptAppend: installSkill ? SKILL : undefined, + skill: installSkill ? SKILL : undefined, cwd: project, env, // Recorded as commands stream so a timed-out trial keeps its trail. onCommand: (command) => { + commands.push(command); if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) { trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(?=@|\s)\s*/, "")); } }, }); - const run = result.result; - trial.costUsd += run.total_cost_usd; - durationMs += run.duration_ms; + durationMs += performance.now() - start; + trial.tokens += tokens; trial.durationS = Math.round(durationMs / 1000); - return result; + return { text, commands, tokens }; }); try { @@ -141,8 +137,7 @@ expect.extend({ const wanted = [bin, ...positionals].join(" "); if (pass) return `expected no command matching \`${wanted}\`, but one ran`; const seen = result.commands.map((c) => ` ${c}`).join("\n") || " (no commands ran)"; - const final = "result" in result.result ? result.result.result : ""; - return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${final}`; + return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${result.text}`; }, }; }, @@ -157,34 +152,31 @@ expect.extend({ }); type AgentResult = { - result: SDKResultMessage; + text: string; commands: string[]; + tokens: number; }; -async function agent( - prompt: string, - config: { - model: string; - systemPromptAppend?: string; - cwd: URL; - env: NodeJS.ProcessEnv; - onCommand?: (command: string) => void; - }, -) { - const { model, systemPromptAppend, cwd, env, onCommand } = config; +type RunResult = Omit; + +type RunOptions = { + model: string; + skill?: string; + cwd: URL; + env: Record; + onCommand: (command: string) => void; +}; + +async function runClaudeCode(prompt: string, options: RunOptions): Promise { + const { model, skill, cwd, env, onCommand } = options; let result: SDKResultMessage | undefined; - const commands: string[] = []; for await (const message of query({ prompt, options: { model, - systemPrompt: { - type: "preset", - preset: "claude_code", - append: systemPromptAppend, - }, + systemPrompt: { type: "preset", preset: "claude_code", append: skill }, permissionMode: "bypassPermissions", allowDangerouslySkipPermissions: true, settingSources: [], @@ -199,8 +191,7 @@ async function agent( if (block.type === "tool_use" && block.name === "Bash") { const command = (block.input as { command?: string }).command; if (typeof command !== "string") continue; - commands.push(command); - onCommand?.(command); + onCommand(command); } } } @@ -210,7 +201,52 @@ async function agent( throw new Error(`Agent run failed (${result?.subtype ?? "no result message"})`); } - return { result, commands }; + const { usage } = result; + const tokens = + usage.input_tokens + + usage.cache_read_input_tokens + + usage.cache_creation_input_tokens + + usage.output_tokens; + + return { text: result.result, tokens }; +} + +async function runCodex(prompt: string, options: RunOptions): Promise { + const { model, skill, cwd, env, onCommand } = options; + + if (skill) await writeFile(new URL("AGENTS.md", cwd), skill); + + const codex = new Codex({ + env: { ...env, CODEX_HOME: await createCodexHome() } as Record, + }); + const thread = codex.startThread({ + model, + workingDirectory: fileURLToPath(cwd), + sandboxMode: "danger-full-access", + approvalPolicy: "never", + skipGitRepoCheck: true, + }); + + let text = ""; + let tokens = 0; + + const { events } = await thread.runStreamed(prompt); + for await (const event of events) { + if (event.type === "item.started" && event.item.type === "command_execution") { + onCommand(event.item.command); + } + if (event.type === "item.completed" && event.item.type === "agent_message") { + text = event.item.text; + } + if (event.type === "turn.completed") { + tokens = event.usage.input_tokens + event.usage.output_tokens; + } + if (event.type === "turn.failed") { + throw new Error(`Agent run failed (${event.error.message})`); + } + } + + return { text, tokens }; } async function judge( @@ -285,3 +321,14 @@ async function createClaudeConfigDir() { } return claudeConfigDir; } + +async function createCodexHome() { + const codexHome = await mkdtemp(join(tmpdir(), "prismic-eval-codex-")); + if (!process.env.OPENAI_API_KEY) { + const source = process.env.CODEX_HOME ?? join(homedir(), ".codex"); + try { + await copyFile(join(source, "auth.json"), join(codexHome, "auth.json")); + } catch {} + } + return codexHome; +} diff --git a/evals/reporter.ts b/evals/reporter.ts index 9af2497..f1dd55d 100644 --- a/evals/reporter.ts +++ b/evals/reporter.ts @@ -7,8 +7,8 @@ import { fileURLToPath } from "node:url"; /** Per-trial stats recorded by the agent fixture; the reporter adds `pass`. */ export type Trial = { model: string; - /** Billed agent cost for the trial; judge calls not included. */ - costUsd: number; + /** Agent tokens in and out for the trial; judge calls not included. */ + tokens: number; /** Agent wall time in seconds, excluding fixture setup and judging. */ durationS: number; /** prismic CLI invocations, verbatim minus the leading `npx prismic`. */ @@ -47,7 +47,7 @@ export default class EvalReporter implements Reporter { (evals[`${testModule.relativeModuleId} / ${test.fullName}`] ??= []).push({ pass: state === "passed", model: trial.model, - costUsd: Math.round(trial.costUsd * 100) / 100, + tokens: trial.tokens, durationS: trial.durationS, calls: trial.calls, }); diff --git a/package-lock.json b/package-lock.json index 3698739..6076205 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ }, "devDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.3.215", + "@openai/codex-sdk": "^0.152.1", "@prismicio/types-internal": "3.16.1", "@types/node": "25.0.9", "change-case": "5.4.4", @@ -937,6 +938,148 @@ "node": ">= 8" } }, + "node_modules/@openai/codex": { + "version": "0.152.1", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1.tgz", + "integrity": "sha512-dSwQzl6JgsFe8L9i8xUnwRz9Vy8gn4UvXFU9xq2IJ1eC7zsSttqQ2SGq49ZZIjEyZQ0LZjCs6Bvtxort2Iyebg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "codex": "bin/codex.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@openai/codex-darwin-arm64": "npm:@openai/codex@0.152.1-darwin-arm64", + "@openai/codex-darwin-x64": "npm:@openai/codex@0.152.1-darwin-x64", + "@openai/codex-linux-arm64": "npm:@openai/codex@0.152.1-linux-arm64", + "@openai/codex-linux-x64": "npm:@openai/codex@0.152.1-linux-x64", + "@openai/codex-win32-arm64": "npm:@openai/codex@0.152.1-win32-arm64", + "@openai/codex-win32-x64": "npm:@openai/codex@0.152.1-win32-x64" + } + }, + "node_modules/@openai/codex-darwin-arm64": { + "name": "@openai/codex", + "version": "0.152.1-darwin-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1-darwin-arm64.tgz", + "integrity": "sha512-H8i0uZHILM0Z2Ep+MryCF5rGXmXjmXTzXf5ZK6bobKtZc2yfomi42ZrQWuYQ5P02H0oLG7B5jLaSWZQ+VFgjbA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-darwin-x64": { + "name": "@openai/codex", + "version": "0.152.1-darwin-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1-darwin-x64.tgz", + "integrity": "sha512-M2qW7YkRx+JeSFoZQsrjgA5yNglowuNAFOwRJoIjlgeP8bsyOqPtbSolu3w4Us7IyCH8f/yuKtlt/v/MdDqbfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-linux-arm64": { + "name": "@openai/codex", + "version": "0.152.1-linux-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1-linux-arm64.tgz", + "integrity": "sha512-qZXqf7fxn/SCmaJW6tYrzWqwcDo0gMDJjj1Pm4OtrWXR7Oc0Y2e8ngAh/Mep9iFhVbsqntY1eGLaQaXssGvFgA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-linux-x64": { + "name": "@openai/codex", + "version": "0.152.1-linux-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1-linux-x64.tgz", + "integrity": "sha512-ar59rr3CX5j4MLMnRcHqcE0eHZPsZlmXlz37ZS2yP3BsV5pNhO+wFXTOzXFdaYmg2cALX7a3Eqv+vB2jQlXnjQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-sdk": { + "version": "0.152.1", + "resolved": "https://registry.npmjs.org/@openai/codex-sdk/-/codex-sdk-0.152.1.tgz", + "integrity": "sha512-tFp0Svp8Yi/h3IDR3XM2IwslCdrSZouPfSl+ESjzPe5hxT0bTtHJhjBfQ7WBGiymjx2PXWaSh3k/GCxvCqVp9A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@openai/codex": "0.152.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@openai/codex-win32-arm64": { + "name": "@openai/codex", + "version": "0.152.1-win32-arm64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1-win32-arm64.tgz", + "integrity": "sha512-YZjWCcArfSLlqG/4r2Ox5ZZhz1FAFQBZisz8U8r5JLxeLk0tXwZHleu8RjNjly++0S5zsgPtAuF0viSIj7NyRA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@openai/codex-win32-x64": { + "name": "@openai/codex", + "version": "0.152.1-win32-x64", + "resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.152.1-win32-x64.tgz", + "integrity": "sha512-B8h0/2Kt+rKQv2+vqBhlhWkMEdhf4dsn46FNKMEBTXj3YC5hwSioOcTX2hMgJxMEMtKIMH6Ire1eNrQPvaL9og==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, "node_modules/@oxc-project/types": { "version": "0.107.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.107.0.tgz", diff --git a/package.json b/package.json index d9fc917..41cff0d 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ }, "devDependencies": { "@anthropic-ai/claude-agent-sdk": "^0.3.215", + "@openai/codex-sdk": "^0.152.1", "@prismicio/types-internal": "3.16.1", "@types/node": "25.0.9", "change-case": "5.4.4", From 7f64c3ce2c3084d9ca1604920f7015a6fae0fd0a Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 18:27:25 +0000 Subject: [PATCH 3/8] docs(evals): document OPENAI_API_KEY in the example env file Co-Authored-By: Claude Fable 5.1 --- .env.test.example | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.env.test.example b/.env.test.example index 337dfcf..dc24cb9 100644 --- a/.env.test.example +++ b/.env.test.example @@ -8,6 +8,10 @@ E2E_PRISMIC_EMAIL= # The password to your Prismic account. E2E_PRISMIC_PASSWORD= -# The Anthropic API key used to run the AI evals (`node --run evals`). Billed to -# the API, not a Claude subscription. In CI this comes from a GitHub secret. +# The Anthropic API key used to run evals on Claude Code. Without it, evals fall +# back to the local Claude login. ANTHROPIC_API_KEY= + +# The OpenAI API key used to run evals on Codex. Without it, evals fall back to +# the local Codex login. +OPENAI_API_KEY= From 3c98a05a9c3cafa3ea4314e4853681f182fb30f7 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 19:44:10 +0000 Subject: [PATCH 4/8] fix(evals): support Fable, Codex API keys, and Slice Machine repo cleanup Update the Agent SDK. The bundled Claude Code 2.1.220 rejects claude-fable-5-1, which needs 2.1.251 or newer. Pass OPENAI_API_KEY to Codex through the SDK apiKey option. The Codex CLI does not read OPENAI_API_KEY from the environment. Delete repositories named in slicemachine.config.json after a trial. Slice Machine init creates repositories but does not write prismic.config.json. Co-Authored-By: Claude Fable 5.1 --- evals/it.ts | 17 ++++++----- package-lock.json | 72 +++++++++++++++++++++++------------------------ package.json | 2 +- 3 files changed, 47 insertions(+), 44 deletions(-) diff --git a/evals/it.ts b/evals/it.ts index 89a440b..2b7765c 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -105,13 +105,15 @@ export const it = base.extend<{ return { text, commands, tokens }; }); - try { - const configFile = await readFile(new URL("prismic.config.json", project), "utf8"); - const created = JSON.parse(configFile).repositoryName; - if (created && created !== repo && password) { - await deleteRepository(created, { token, password, host }); - } - } catch {} + for (const file of ["prismic.config.json", "slicemachine.config.json"]) { + try { + const configFile = await readFile(new URL(file, project), "utf8"); + const created = JSON.parse(configFile).repositoryName; + if (created && created !== repo && password) { + await deleteRepository(created, { token, password, host }); + } + } catch {} + } }, }); @@ -217,6 +219,7 @@ async function runCodex(prompt: string, options: RunOptions): Promise if (skill) await writeFile(new URL("AGENTS.md", cwd), skill); const codex = new Codex({ + apiKey: process.env.OPENAI_API_KEY, env: { ...env, CODEX_HOME: await createCodexHome() } as Record, }); const thread = codex.startThread({ diff --git a/package-lock.json b/package-lock.json index 6076205..6582536 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "prismic": "dist/index.mjs" }, "devDependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.3.215", + "@anthropic-ai/claude-agent-sdk": "^0.3.258", "@openai/codex-sdk": "^0.152.1", "@prismicio/types-internal": "3.16.1", "@types/node": "25.0.9", @@ -36,23 +36,23 @@ } }, "node_modules/@anthropic-ai/claude-agent-sdk": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk/-/claude-agent-sdk-0.3.220.tgz", - "integrity": "sha512-glc7SdwPkOkLw8oxwLo9PKTdLJGqW/PIR4urWXFoRtX9YllwozsEVc5Tc1+EvLSkfrsxPJqQWqOgpjUOQXf1oA==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk/-/claude-agent-sdk-0.3.258.tgz", + "integrity": "sha512-RxJ5fSPCGCxX5qO/b4IPXhldvtLHeYBAzTUJ4eOzO+gTrepZQSDmwSlQD6nnoEquKGJzOMHCjhdEtBfDjbDWUg==", "dev": true, "license": "SEE LICENSE IN README.md", "engines": { "node": ">=18.0.0" }, "optionalDependencies": { - "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.220", - "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.220", - "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.220", - "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.220", - "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.220", - "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.220", - "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.220", - "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.220" + "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.258", + "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.258", + "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.258", + "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.258", + "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.258", + "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.258", + "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.258", + "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.258" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", @@ -61,9 +61,9 @@ } }, "node_modules/@anthropic-ai/claude-agent-sdk-darwin-arm64": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-darwin-arm64/-/claude-agent-sdk-darwin-arm64-0.3.220.tgz", - "integrity": "sha512-7VxlbEosK7DODiOnsjoVd0DSJzbnaPrM2jelMHI0y8zx1UnLS3WC6EFUXbvy74F2sXqEznh2tzn7EKWInaRN6Q==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-darwin-arm64/-/claude-agent-sdk-darwin-arm64-0.3.258.tgz", + "integrity": "sha512-Hrhzc9WVGSid+DghdTcpVr/8fyXnTD6KeSlDpKx6Wru47J/Nq7RTYiZJt+cex+O2ehaHMEcuYEgoqJ3K/X9NlA==", "cpu": [ "arm64" ], @@ -75,9 +75,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-darwin-x64": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-darwin-x64/-/claude-agent-sdk-darwin-x64-0.3.220.tgz", - "integrity": "sha512-X9RwDsSmbF6ultKZroaip+DL8WRgC64gHbrAwrRlAFSPNZV7zmJyP2ur8rW7KrxqmtuehdMMkw8+SAC/6hD2PA==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-darwin-x64/-/claude-agent-sdk-darwin-x64-0.3.258.tgz", + "integrity": "sha512-AVqxGX4988J5cS+TMqIzH85+sbsLhJu5Ou9TIALcO/v2Z9ze8GK4vX2ydAYvU/SRnjTvEaiITX+Xcm5afP1IbQ==", "cpu": [ "x64" ], @@ -89,9 +89,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-arm64": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-arm64/-/claude-agent-sdk-linux-arm64-0.3.220.tgz", - "integrity": "sha512-WkROPwWskqhKR9XgnmseHQ6rLi9zM9qt57IWoToIjL/eXOqDWipp7JXZ1L5ud+LrA42dunHPZfBwD/vXZ+A7LA==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-arm64/-/claude-agent-sdk-linux-arm64-0.3.258.tgz", + "integrity": "sha512-Jj3K1Ip7WpyMouZCjd7kgV3KswUBF62WAnyG0iaYvKZJvXgYKbIAkjcQ2F2Rx5ZuRUNWAVncE9LeHmdIdx78VQ==", "cpu": [ "arm64" ], @@ -103,9 +103,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-arm64-musl": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-arm64-musl/-/claude-agent-sdk-linux-arm64-musl-0.3.220.tgz", - "integrity": "sha512-OHoZOZ8Cf2TBr6oXIXPwyvUxj9jrq2w8E4poA8dMpacXszcPSPiCQCMuuOh4aWJzfeJE1+TtWxhKMVb2csXyZQ==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-arm64-musl/-/claude-agent-sdk-linux-arm64-musl-0.3.258.tgz", + "integrity": "sha512-I/BLt2vdvqK2B2px526U1lw7Rv+SI+Ld22+wLwu8gLRQk5SYhSW9dmMYEO+GCeF7vQzfzJvMQ4IzbbY+aSGACg==", "cpu": [ "arm64" ], @@ -117,9 +117,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-x64": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-x64/-/claude-agent-sdk-linux-x64-0.3.220.tgz", - "integrity": "sha512-tkTJFnpR9VifvWX2fmkCAPkT6+8Wk/gVu8B5jsVekKZPiZoWRHmMXO30BnZn+f0TZhgYP+82PSX3S8crH1kn+w==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-x64/-/claude-agent-sdk-linux-x64-0.3.258.tgz", + "integrity": "sha512-2MJeFVJM/3xwZASP3yn2OuQ9RHIoS30DC/B7oG1XPYcbToPLH4QIfCPLWbSQfqCdp+NEBupLMM9BWpDz4s8Q0g==", "cpu": [ "x64" ], @@ -131,9 +131,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-linux-x64-musl": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-x64-musl/-/claude-agent-sdk-linux-x64-musl-0.3.220.tgz", - "integrity": "sha512-K+FWj+LcGhC1Z7wqeWoLxm1iemcba5xKpLLFVwYm4V6HyMx3ruYd/2r2TiQtjT+JWeNFWIys0ScHiItR6vWAiA==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-linux-x64-musl/-/claude-agent-sdk-linux-x64-musl-0.3.258.tgz", + "integrity": "sha512-sM7GzRyrOpFhwMn2Ng8nLiWK6cc04uCEu3Zh9mrJS2r3iQu1TryHKoPTjc2Ip0N75sHmwhNodgoRs8NtG1Gkkw==", "cpu": [ "x64" ], @@ -145,9 +145,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-win32-arm64": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-win32-arm64/-/claude-agent-sdk-win32-arm64-0.3.220.tgz", - "integrity": "sha512-rIwgq0UwQExWl6KrHUyC4w5KwpL9l6nd95aUTx6RitexaAuEw//xtfTVLnuE4hDDQZFkzEwpdKc3nxDWoGcUbA==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-win32-arm64/-/claude-agent-sdk-win32-arm64-0.3.258.tgz", + "integrity": "sha512-n/Vf6oXAo9EZVSSM5+9d+8dFrUrX9cbgSHK/1njkvykWAN5xsfBbikJYqwhbW84GCYkpXYM+gGNZe23h0fHldw==", "cpu": [ "arm64" ], @@ -159,9 +159,9 @@ ] }, "node_modules/@anthropic-ai/claude-agent-sdk-win32-x64": { - "version": "0.3.220", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-win32-x64/-/claude-agent-sdk-win32-x64-0.3.220.tgz", - "integrity": "sha512-MuOuXhbr66HlGaWXD2f3w0k2PsvmnbkwcUZ0dAe2poFLdl72GC2dapwwOBefxm9QmoNqk9+jmv/dSKGOVWyvLw==", + "version": "0.3.258", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk-win32-x64/-/claude-agent-sdk-win32-x64-0.3.258.tgz", + "integrity": "sha512-UDbXE6n37ZMUogVVYEWX901NNmbyXUv1VvGYN/vfOiIWKUJXCAypam71dBlYFhlAhVX28qCd64w+pTZDYQfYQA==", "cpu": [ "x64" ], diff --git a/package.json b/package.json index 41cff0d..d6581b9 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "test": "npm run lint && npm run types && npm run unit" }, "devDependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.3.215", + "@anthropic-ai/claude-agent-sdk": "^0.3.258", "@openai/codex-sdk": "^0.152.1", "@prismicio/types-internal": "3.16.1", "@types/node": "25.0.9", From 32a5f81570f4ecce79c2afaa2c58f2b9416337c9 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 19:44:10 +0000 Subject: [PATCH 5/8] feat(evals): add Slice Machine knowledge and redirect evals Both evals run without the skill and without the preinstalled CLI, on Claude Code and Codex. They are measurements, not gates. know-the-cli asks for a plan to build a Prismic site with Next.js. It passes only if the plan models content with the Prismic CLI. redirect-from-slice-machine asks for Slice Machine by name. It passes if the agent tells the user Slice Machine is replaced and moves to the CLI or asks. It gives a useful signal only after the @slicemachine/init halt is on npm. Co-Authored-By: Claude Fable 5.1 --- evals/know-the-cli.eval.ts | 35 +++++++++++++++++++++++ evals/redirect-from-slice-machine.eval.ts | 33 +++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 evals/know-the-cli.eval.ts create mode 100644 evals/redirect-from-slice-machine.eval.ts diff --git a/evals/know-the-cli.eval.ts b/evals/know-the-cli.eval.ts new file mode 100644 index 0000000..50ecb8b --- /dev/null +++ b/evals/know-the-cli.eval.ts @@ -0,0 +1,35 @@ +import dedent from "dedent"; +import { rm } from "node:fs/promises"; +import { describe } from "vitest"; + +import { it, trials } from "./it"; + +it.scoped({ installSkill: false, installCli: false, isolateRepo: false }); + +describe.for([ + "claude-fable-5-1", + "claude-opus-5", + "claude-sonnet-5", + "claude-opus-4-8", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-5.5", +])("%s", (model) => { + it.scoped({ model }); + + it.for(trials)( + "plans a Prismic site with the CLI, not Slice Machine", + async (_, { project, agent, expect }) => { + await rm(new URL("prismic.config.json", project)); + + const result = await agent(`Write a plan to build a Prismic website with Next.js.`); + + await expect(result.text).toSatisfyJudge(dedent` + This is an agent's plan to build a Prismic website with Next.js, written without any Prismic guidance. + Passes if the plan models content with the Prismic CLI (\`npx prismic\`). + Fails if it uses Slice Machine or any other way to model content. + Mentioning Slice Machine only to call it deprecated or to avoid it does not fail the plan. + `); + }, + ); +}); diff --git a/evals/redirect-from-slice-machine.eval.ts b/evals/redirect-from-slice-machine.eval.ts new file mode 100644 index 0000000..057aead --- /dev/null +++ b/evals/redirect-from-slice-machine.eval.ts @@ -0,0 +1,33 @@ +import dedent from "dedent"; +import { rm } from "node:fs/promises"; +import { describe } from "vitest"; + +import { it, trials } from "./it"; + +it.scoped({ installSkill: false, installCli: false, isolateRepo: false }); + +describe.for([ + "claude-fable-5-1", + "claude-opus-5", + "claude-sonnet-5", + "gpt-5.6-sol", + "gpt-5.6-terra", +])("%s", (model) => { + it.scoped({ model }); + + it.for(trials)( + "tells the user Slice Machine is replaced and moves to the CLI", + async (_, { project, agent, expect }) => { + await rm(new URL("prismic.config.json", project)); + + const result = await agent(`Set up Prismic in this Next.js project using Slice Machine.`); + + await expect(`${result.commands.join("\n")}\n\n${result.text}`).toSatisfyJudge(dedent` + The user asked an agent to set up Prismic with Slice Machine. Slice Machine is replaced by the Prismic CLI, and its init command prints a redirect message and exits unless run with --force. + Above are the shell commands the agent ran, then its final message to the user. + Passes if the agent tells the user that Slice Machine is replaced, and then either installs the Prismic skill and continues with the Prismic CLI (\`npx prismic\`), or asks the user which way to go. + Fails if the agent runs Slice Machine init with --force or otherwise builds with Slice Machine without telling the user it is replaced, or gives up without a working path. + `); + }, + ); +}); From 3573a73dd10818dabd7e0805ab4bfca5f022ef05 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 19:52:43 +0000 Subject: [PATCH 6/8] refactor(evals): build one agent env in the fixture The agent fixture now creates the Claude and Codex homes once and puts both in a single env. The runners pass that env through and no longer set up their own directories. Remove the MODEL constant, the RunResult type, and the unused tokens field on AgentResult. Each new eval file sets its fixture overrides in one it.scoped call. Co-Authored-By: Claude Fable 5.1 --- evals/it.ts | 23 ++++++++--------------- evals/know-the-cli.eval.ts | 4 +--- evals/redirect-from-slice-machine.eval.ts | 4 +--- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/evals/it.ts b/evals/it.ts index 2b7765c..4003ce3 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -22,7 +22,6 @@ if (process.env.PRISMIC_ALLOW_EVALS !== "true") { const BIN = new URL("../dist/index.mjs", import.meta.url); const EVAL_TRIALS = Number(process.env.EVAL_TRIALS ?? 3); -const MODEL = "claude-sonnet-5"; const JUDGE_MODEL = "claude-sonnet-5"; const PRISMIC_SKILL_REF = "2bd340e6af4e67a9c1179e97b495f7bda564b46f"; @@ -49,20 +48,21 @@ export const it = base.extend<{ }>({ installSkill: true, installCli: true, - model: MODEL, + model: "claude-sonnet-5", agent: async ( { home, project, login, task, repo, token, host, password, installSkill, installCli, model }, use, ) => { await login(); - const env: Record = { + const env: NodeJS.ProcessEnv = { ...process.env, HOME: fileURLToPath(home), NO_UPDATE_NOTIFIER: "1", CLAUDE_CONFIG_DIR: await createClaudeConfigDir(), CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1", CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1", + CODEX_HOME: await createCodexHome(), }; if (installCli) { @@ -102,7 +102,7 @@ export const it = base.extend<{ trial.tokens += tokens; trial.durationS = Math.round(durationMs / 1000); - return { text, commands, tokens }; + return { text, commands }; }); for (const file of ["prismic.config.json", "slicemachine.config.json"]) { @@ -156,22 +156,17 @@ expect.extend({ type AgentResult = { text: string; commands: string[]; - tokens: number; }; -type RunResult = Omit; - type RunOptions = { model: string; skill?: string; cwd: URL; - env: Record; + env: NodeJS.ProcessEnv; onCommand: (command: string) => void; }; -async function runClaudeCode(prompt: string, options: RunOptions): Promise { - const { model, skill, cwd, env, onCommand } = options; - +async function runClaudeCode(prompt: string, { model, skill, cwd, env, onCommand }: RunOptions) { let result: SDKResultMessage | undefined; for await (const message of query({ @@ -213,14 +208,12 @@ async function runClaudeCode(prompt: string, options: RunOptions): Promise { - const { model, skill, cwd, env, onCommand } = options; - +async function runCodex(prompt: string, { model, skill, cwd, env, onCommand }: RunOptions) { if (skill) await writeFile(new URL("AGENTS.md", cwd), skill); const codex = new Codex({ apiKey: process.env.OPENAI_API_KEY, - env: { ...env, CODEX_HOME: await createCodexHome() } as Record, + env: env as Record, }); const thread = codex.startThread({ model, diff --git a/evals/know-the-cli.eval.ts b/evals/know-the-cli.eval.ts index 50ecb8b..eef95f8 100644 --- a/evals/know-the-cli.eval.ts +++ b/evals/know-the-cli.eval.ts @@ -4,8 +4,6 @@ import { describe } from "vitest"; import { it, trials } from "./it"; -it.scoped({ installSkill: false, installCli: false, isolateRepo: false }); - describe.for([ "claude-fable-5-1", "claude-opus-5", @@ -15,7 +13,7 @@ describe.for([ "gpt-5.6-terra", "gpt-5.5", ])("%s", (model) => { - it.scoped({ model }); + it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false }); it.for(trials)( "plans a Prismic site with the CLI, not Slice Machine", diff --git a/evals/redirect-from-slice-machine.eval.ts b/evals/redirect-from-slice-machine.eval.ts index 057aead..d21426f 100644 --- a/evals/redirect-from-slice-machine.eval.ts +++ b/evals/redirect-from-slice-machine.eval.ts @@ -4,8 +4,6 @@ import { describe } from "vitest"; import { it, trials } from "./it"; -it.scoped({ installSkill: false, installCli: false, isolateRepo: false }); - describe.for([ "claude-fable-5-1", "claude-opus-5", @@ -13,7 +11,7 @@ describe.for([ "gpt-5.6-sol", "gpt-5.6-terra", ])("%s", (model) => { - it.scoped({ model }); + it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false }); it.for(trials)( "tells the user Slice Machine is replaced and moves to the CLI", From 06450c4aa89e83c4b86c46360a1ce50f07676f1d Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Wed, 2 Sep 2026 20:01:50 +0000 Subject: [PATCH 7/8] refactor(evals): move the redirect test into know-the-cli Both tests measure the same capability: an agent with no skill and no CLI installed knows that Slice Machine is replaced by the Prismic CLI. One file per capability, so they now share know-the-cli.eval.ts as two describe.for blocks with their own model lists. Co-Authored-By: Claude Fable 5.1 --- evals/know-the-cli.eval.ts | 26 +++++++++++++++++++ evals/redirect-from-slice-machine.eval.ts | 31 ----------------------- 2 files changed, 26 insertions(+), 31 deletions(-) delete mode 100644 evals/redirect-from-slice-machine.eval.ts diff --git a/evals/know-the-cli.eval.ts b/evals/know-the-cli.eval.ts index eef95f8..ca5cfec 100644 --- a/evals/know-the-cli.eval.ts +++ b/evals/know-the-cli.eval.ts @@ -31,3 +31,29 @@ describe.for([ }, ); }); + +describe.for([ + "claude-fable-5-1", + "claude-opus-5", + "claude-sonnet-5", + "gpt-5.6-sol", + "gpt-5.6-terra", +])("%s", (model) => { + it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false }); + + it.for(trials)( + "tells the user Slice Machine is replaced and moves to the CLI", + async (_, { project, agent, expect }) => { + await rm(new URL("prismic.config.json", project)); + + const result = await agent(`Set up Prismic in this Next.js project using Slice Machine.`); + + await expect(`${result.commands.join("\n")}\n\n${result.text}`).toSatisfyJudge(dedent` + The user asked an agent to set up Prismic with Slice Machine. Slice Machine is replaced by the Prismic CLI, and its init command prints a redirect message and exits unless run with --force. + Above are the shell commands the agent ran, then its final message to the user. + Passes if the agent tells the user that Slice Machine is replaced, and then either installs the Prismic skill and continues with the Prismic CLI (\`npx prismic\`), or asks the user which way to go. + Fails if the agent runs Slice Machine init with --force or otherwise builds with Slice Machine without telling the user it is replaced, or gives up without a working path. + `); + }, + ); +}); diff --git a/evals/redirect-from-slice-machine.eval.ts b/evals/redirect-from-slice-machine.eval.ts deleted file mode 100644 index d21426f..0000000 --- a/evals/redirect-from-slice-machine.eval.ts +++ /dev/null @@ -1,31 +0,0 @@ -import dedent from "dedent"; -import { rm } from "node:fs/promises"; -import { describe } from "vitest"; - -import { it, trials } from "./it"; - -describe.for([ - "claude-fable-5-1", - "claude-opus-5", - "claude-sonnet-5", - "gpt-5.6-sol", - "gpt-5.6-terra", -])("%s", (model) => { - it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false }); - - it.for(trials)( - "tells the user Slice Machine is replaced and moves to the CLI", - async (_, { project, agent, expect }) => { - await rm(new URL("prismic.config.json", project)); - - const result = await agent(`Set up Prismic in this Next.js project using Slice Machine.`); - - await expect(`${result.commands.join("\n")}\n\n${result.text}`).toSatisfyJudge(dedent` - The user asked an agent to set up Prismic with Slice Machine. Slice Machine is replaced by the Prismic CLI, and its init command prints a redirect message and exits unless run with --force. - Above are the shell commands the agent ran, then its final message to the user. - Passes if the agent tells the user that Slice Machine is replaced, and then either installs the Prismic skill and continues with the Prismic CLI (\`npx prismic\`), or asks the user which way to go. - Fails if the agent runs Slice Machine init with --force or otherwise builds with Slice Machine without telling the user it is replaced, or gives up without a working path. - `); - }, - ); -}); From 4f5b672e08bd9253791168ef4681334bd096a548 Mon Sep 17 00:00:00 2001 From: Angelo Ashmore Date: Thu, 3 Sep 2026 02:18:46 +0000 Subject: [PATCH 8/8] fix(evals): strip pinned versions from recorded calls and add Luna The call-strip regex kept the version suffix. A command such as `npx prismic@1.16.0 init` was recorded as `@1.16.0 init`. The regex now consumes the suffix and records `init`. Add gpt-5.6-luna to both know-the-cli model lists. Co-Authored-By: Claude Fable 5.1 --- evals/it.ts | 2 +- evals/know-the-cli.eval.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/evals/it.ts b/evals/it.ts index 4003ce3..96381d6 100644 --- a/evals/it.ts +++ b/evals/it.ts @@ -93,7 +93,7 @@ export const it = base.extend<{ onCommand: (command) => { commands.push(command); if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) { - trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(?=@|\s)\s*/, "")); + trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?(?=\s|$)\s*/, "")); } }, }); diff --git a/evals/know-the-cli.eval.ts b/evals/know-the-cli.eval.ts index ca5cfec..a0bf558 100644 --- a/evals/know-the-cli.eval.ts +++ b/evals/know-the-cli.eval.ts @@ -11,6 +11,7 @@ describe.for([ "claude-opus-4-8", "gpt-5.6-sol", "gpt-5.6-terra", + "gpt-5.6-luna", "gpt-5.5", ])("%s", (model) => { it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false }); @@ -38,6 +39,7 @@ describe.for([ "claude-sonnet-5", "gpt-5.6-sol", "gpt-5.6-terra", + "gpt-5.6-luna", ])("%s", (model) => { it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false });