test(evals): measure agent knowledge of the CLI without Prismic guidance

.env.test.example

@@ -8,6 +8,10 @@ E2E_PRISMIC_EMAIL=
 # The password to your Prismic account.
 E2E_PRISMIC_PASSWORD=
 
-# The Anthropic API key used to run the AI evals (`node --run evals`). Billed to
-# the API, not a Claude subscription. In CI this comes from a GitHub secret.
+# The Anthropic API key used to run evals on Claude Code. Without it, evals fall
+# back to the local Claude login.
 ANTHROPIC_API_KEY=
+
+# The OpenAI API key used to run evals on Codex. Without it, evals fall back to
+# the local Codex login.
+OPENAI_API_KEY=

.github/workflows/evals.yml

@@ -33,6 +33,7 @@ jobs:
         env:
           PRISMIC_ALLOW_EVALS: "true"
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           E2E_PRISMIC_EMAIL: ${{ secrets.E2E_PRISMIC_EMAIL }}
           E2E_PRISMIC_PASSWORD: ${{ secrets.E2E_PRISMIC_PASSWORD }}
         run: node --run evals || true

evals/it.ts

@@ -1,4 +1,5 @@
 import { query, type SDKResultMessage } from "@anthropic-ai/claude-agent-sdk";
+import { Codex } from "@openai/codex-sdk";
 import dedent from "dedent";
 import { copyFile, mkdir, mkdtemp, readFile, symlink, writeFile } from "node:fs/promises";
 import { homedir, tmpdir } from "node:os";
@@ -20,7 +21,6 @@ if (process.env.PRISMIC_ALLOW_EVALS !== "true") {
 }
 
 const BIN = new URL("../dist/index.mjs", import.meta.url);
-const EVAL_MODEL = process.env.EVAL_MODEL ?? "claude-sonnet-5";
 const EVAL_TRIALS = Number(process.env.EVAL_TRIALS ?? 3);
 const JUDGE_MODEL = "claude-sonnet-5";
 const PRISMIC_SKILL_REF = "2bd340e6af4e67a9c1179e97b495f7bda564b46f";
@@ -41,62 +41,79 @@ declare module "vitest" {
 }
 
 export const it = base.extend<{
+	installSkill: boolean;
+	installCli: boolean;
+	model: string;
 	agent: (prompt: string) => Promise<AgentResult>;
 }>({
-	agent: async ({ home, project, login, task, repo, token, host, password }, use) => {
+	installSkill: true,
+	installCli: true,
+	model: "claude-sonnet-5",
+	agent: async (
+		{ home, project, login, task, repo, token, host, password, installSkill, installCli, model },
+		use,
+	) => {
 		await login();
 
-		const claudeConfigDir = await createClaudeConfigDir();
-
-		const nodeModulesBinDir = new URL("node_modules/.bin/", project);
-		await mkdir(nodeModulesBinDir, { recursive: true });
-		await symlink(BIN, new URL("prismic", nodeModulesBinDir));
-
-		const env = {
+		const env: NodeJS.ProcessEnv = {
 			...process.env,
 			HOME: fileURLToPath(home),
-			PRISMIC_CONFIG_DIR: fileURLToPath(new URL(".config/prismic/", home)),
-			PRISMIC_TYPE_BUILDER_ENABLED: "true",
-			PRISMIC_SENTRY_ENABLED: "false",
-			PRISMIC_TELEMETRY_ENABLED: "false",
 			NO_UPDATE_NOTIFIER: "1",
-			CLAUDE_CONFIG_DIR: claudeConfigDir,
+			CLAUDE_CONFIG_DIR: await createClaudeConfigDir(),
 			CLAUDE_CODE_DISABLE_AUTO_MEMORY: "1",
 			CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1",
+			CODEX_HOME: await createCodexHome(),
 		};
 
-		const trial: Trial = { model: EVAL_MODEL, costUsd: 0, durationS: 0, calls: [] };
+		if (installCli) {
+			const nodeModulesBinDir = new URL("node_modules/.bin/", project);
+			await mkdir(nodeModulesBinDir, { recursive: true });
+			await symlink(BIN, new URL("prismic", nodeModulesBinDir));
+			env.PRISMIC_CONFIG_DIR = fileURLToPath(new URL(".config/prismic/", home));
+			env.PRISMIC_TYPE_BUILDER_ENABLED = "true";
+			env.PRISMIC_SENTRY_ENABLED = "false";
+			env.PRISMIC_TELEMETRY_ENABLED = "false";
+		}
+
+		const trial: Trial = { model, tokens: 0, durationS: 0, calls: [] };
 		task.meta.agent = trial;
 		let durationMs = 0;
 
+		const run = model.startsWith("claude-") ? runClaudeCode : runCodex;
+
 		await use(async (prompt: string) => {
-			const result = await agent(prompt, {
-				systemPromptAppend: SKILL,
+			const start = performance.now();
+			const commands: string[] = [];
+			const { text, tokens } = await run(prompt, {
+				model,
+				skill: installSkill ? SKILL : undefined,
 				cwd: project,
 				env,
 				// Recorded as commands stream so a timed-out trial keeps its trail.
 				onCommand: (command) => {
+					commands.push(command);
 					if (/(^|\s)(npx\s+)?prismic(@|\s|$)/.test(command)) {
-						trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?\s+/, ""));
+						trial.calls.push(command.replace(/^.*?(^|\s)(npx\s+)?prismic(@\S+)?(?=\s|$)\s*/, ""));
 					}
 				},
 			});
 
-			const run = result.result;
-			trial.costUsd += run.total_cost_usd;
-			durationMs += run.duration_ms;
+			durationMs += performance.now() - start;
+			trial.tokens += tokens;
 			trial.durationS = Math.round(durationMs / 1000);
 
-			return result;
+			return { text, commands };
 		});
 
-		try {
-			const configFile = await readFile(new URL("prismic.config.json", project), "utf8");
-			const created = JSON.parse(configFile).repositoryName;
-			if (created && created !== repo && password) {
-				await deleteRepository(created, { token, password, host });
-			}
-		} catch {}
+		for (const file of ["prismic.config.json", "slicemachine.config.json"]) {
+			try {
+				const configFile = await readFile(new URL(file, project), "utf8");
+				const created = JSON.parse(configFile).repositoryName;
+				if (created && created !== repo && password) {
+					await deleteRepository(created, { token, password, host });
+				}
+			} catch {}
+		}
 	},
 });
 
@@ -122,8 +139,7 @@ expect.extend({
 				const wanted = [bin, ...positionals].join(" ");
 				if (pass) return `expected no command matching \`${wanted}\`, but one ran`;
 				const seen = result.commands.map((c) => `  ${c}`).join("\n") || "  (no commands ran)";
-				const final = "result" in result.result ? result.result.result : "";
-				return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${final}`;
+				return `expected a command matching \`${wanted}\`, but saw:\n${seen}\n\nagent's final message:\n${result.text}`;
 			},
 		};
 	},
@@ -138,33 +154,26 @@ expect.extend({
 });
 
 type AgentResult = {
-	result: SDKResultMessage;
+	text: string;
 	commands: string[];
 };
 
-async function agent(
-	prompt: string,
-	config: {
-		systemPromptAppend?: string;
-		cwd: URL;
-		env: NodeJS.ProcessEnv;
-		onCommand?: (command: string) => void;
-	},
-) {
-	const { systemPromptAppend, cwd, env, onCommand } = config;
+type RunOptions = {
+	model: string;
+	skill?: string;
+	cwd: URL;
+	env: NodeJS.ProcessEnv;
+	onCommand: (command: string) => void;
+};
 
+async function runClaudeCode(prompt: string, { model, skill, cwd, env, onCommand }: RunOptions) {
 	let result: SDKResultMessage | undefined;
-	const commands: string[] = [];
 
 	for await (const message of query({
 		prompt,
 		options: {
-			model: EVAL_MODEL,
-			systemPrompt: {
-				type: "preset",
-				preset: "claude_code",
-				append: systemPromptAppend,
-			},
+			model,
+			systemPrompt: { type: "preset", preset: "claude_code", append: skill },
 			permissionMode: "bypassPermissions",
 			allowDangerouslySkipPermissions: true,
 			settingSources: [],
@@ -179,8 +188,7 @@ async function agent(
 				if (block.type === "tool_use" && block.name === "Bash") {
 					const command = (block.input as { command?: string }).command;
 					if (typeof command !== "string") continue;
-					commands.push(command);
-					onCommand?.(command);
+					onCommand(command);
 				}
 			}
 		}
@@ -190,7 +198,51 @@ async function agent(
 		throw new Error(`Agent run failed (${result?.subtype ?? "no result message"})`);
 	}
 
-	return { result, commands };
+	const { usage } = result;
+	const tokens =
+		usage.input_tokens +
+		usage.cache_read_input_tokens +
+		usage.cache_creation_input_tokens +
+		usage.output_tokens;
+
+	return { text: result.result, tokens };
+}
+
+async function runCodex(prompt: string, { model, skill, cwd, env, onCommand }: RunOptions) {
+	if (skill) await writeFile(new URL("AGENTS.md", cwd), skill);
+
+	const codex = new Codex({
+		apiKey: process.env.OPENAI_API_KEY,
+		env: env as Record<string, string>,
+	});
+	const thread = codex.startThread({
+		model,
+		workingDirectory: fileURLToPath(cwd),
+		sandboxMode: "danger-full-access",
+		approvalPolicy: "never",
+		skipGitRepoCheck: true,
+	});
+
+	let text = "";
+	let tokens = 0;
+
+	const { events } = await thread.runStreamed(prompt);
+	for await (const event of events) {
+		if (event.type === "item.started" && event.item.type === "command_execution") {
+			onCommand(event.item.command);
+		}
+		if (event.type === "item.completed" && event.item.type === "agent_message") {
+			text = event.item.text;
+		}
+		if (event.type === "turn.completed") {
+			tokens = event.usage.input_tokens + event.usage.output_tokens;
+		}
+		if (event.type === "turn.failed") {
+			throw new Error(`Agent run failed (${event.error.message})`);
+		}
+	}
+
+	return { text, tokens };
 }
 
 async function judge(
@@ -265,3 +317,14 @@ async function createClaudeConfigDir() {
 	}
 	return claudeConfigDir;
 }
+
+async function createCodexHome() {
+	const codexHome = await mkdtemp(join(tmpdir(), "prismic-eval-codex-"));
+	if (!process.env.OPENAI_API_KEY) {
+		const source = process.env.CODEX_HOME ?? join(homedir(), ".codex");
+		try {
+			await copyFile(join(source, "auth.json"), join(codexHome, "auth.json"));
+		} catch {}
+	}
+	return codexHome;
+}

evals/know-the-cli.eval.ts

@@ -0,0 +1,61 @@
+import dedent from "dedent";
+import { rm } from "node:fs/promises";
+import { describe } from "vitest";
+
+import { it, trials } from "./it";
+
+describe.for([
+	"claude-fable-5-1",
+	"claude-opus-5",
+	"claude-sonnet-5",
+	"claude-opus-4-8",
+	"gpt-5.6-sol",
+	"gpt-5.6-terra",
+	"gpt-5.6-luna",
+	"gpt-5.5",
+])("%s", (model) => {
+	it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false });
+
+	it.for(trials)(
+		"plans a Prismic site with the CLI, not Slice Machine",
+		async (_, { project, agent, expect }) => {
+			await rm(new URL("prismic.config.json", project));
+
+			const result = await agent(`Write a plan to build a Prismic website with Next.js.`);
+
+			await expect(result.text).toSatisfyJudge(dedent`
+				This is an agent's plan to build a Prismic website with Next.js, written without any Prismic guidance.
+				Passes if the plan models content with the Prismic CLI (\`npx prismic\`).
+				Fails if it uses Slice Machine or any other way to model content.
+				Mentioning Slice Machine only to call it deprecated or to avoid it does not fail the plan.
+			`);
+		},
+	);
+});
+
+describe.for([
+	"claude-fable-5-1",
+	"claude-opus-5",
+	"claude-sonnet-5",
+	"gpt-5.6-sol",
+	"gpt-5.6-terra",
+	"gpt-5.6-luna",
+])("%s", (model) => {
+	it.scoped({ model, installSkill: false, installCli: false, isolateRepo: false });
+
+	it.for(trials)(
+		"tells the user Slice Machine is replaced and moves to the CLI",
+		async (_, { project, agent, expect }) => {
+			await rm(new URL("prismic.config.json", project));
+
+			const result = await agent(`Set up Prismic in this Next.js project using Slice Machine.`);
+
+			await expect(`${result.commands.join("\n")}\n\n${result.text}`).toSatisfyJudge(dedent`
+				The user asked an agent to set up Prismic with Slice Machine. Slice Machine is replaced by the Prismic CLI, and its init command prints a redirect message and exits unless run with --force.
+				Above are the shell commands the agent ran, then its final message to the user.
+				Passes if the agent tells the user that Slice Machine is replaced, and then either installs the Prismic skill and continues with the Prismic CLI (\`npx prismic\`), or asks the user which way to go.
+				Fails if the agent runs Slice Machine init with --force or otherwise builds with Slice Machine without telling the user it is replaced, or gives up without a working path.
+			`);
+		},
+	);
+});

evals/reporter.ts

@@ -7,8 +7,8 @@ import { fileURLToPath } from "node:url";
 /** Per-trial stats recorded by the agent fixture; the reporter adds `pass`. */
 export type Trial = {
 	model: string;
-	/** Billed agent cost for the trial; judge calls not included. */
-	costUsd: number;
+	/** Agent tokens in and out for the trial; judge calls not included. */
+	tokens: number;
 	/** Agent wall time in seconds, excluding fixture setup and judging. */
 	durationS: number;
 	/** prismic CLI invocations, verbatim minus the leading `npx prismic`. */
@@ -25,7 +25,6 @@ const LOCAL_RESULTS_PATH = fileURLToPath(new URL("results.local.json", import.me
 // always holds one full run; history lives in git.
 export default class EvalReporter implements Reporter {
 	onTestRunEnd(testModules: ReadonlyArray<TestModule>): void {
-		let model = "";
 		let filtered = false;
 		const evals: Record<string, object[]> = {};
 
@@ -45,10 +44,10 @@ export default class EvalReporter implements Reporter {
 					filtered = true;
 					continue;
 				}
-				model = trial.model;
-				(evals[`${testModule.relativeModuleId} / ${test.name}`] ??= []).push({
+				(evals[`${testModule.relativeModuleId} / ${test.fullName}`] ??= []).push({
 					pass: state === "passed",
-					costUsd: Math.round(trial.costUsd * 100) / 100,
+					model: trial.model,
+					tokens: trial.tokens,
 					durationS: trial.durationS,
 					calls: trial.calls,
 				});
@@ -59,7 +58,7 @@ export default class EvalReporter implements Reporter {
 		const sorted = Object.fromEntries(Object.entries(evals).sort(([a], [b]) => a.localeCompare(b)));
 		// Single-line output keeps run-over-run diffs to one changed line; read
 		// it with jq or `node --run evals:report`.
-		const report = JSON.stringify({ model, evals: sorted }) + "\n";
+		const report = JSON.stringify({ evals: sorted }) + "\n";
 		writeFileSync(LOCAL_RESULTS_PATH, report);
 
 		const evalFiles = readdirSync(EVALS_DIR).filter((file) => file.endsWith(".eval.ts"));