feat(appkit): eval suite features — concurrency, retries, reporters (stack 4/5)

packages/appkit/src/evals/dataset.ts

@@ -21,6 +21,30 @@ export interface ReadEvalDatasetOptions {
   limit?: number;
 }
 
+/**
+ * Extract every user-message content, in order, from an MLflow
+ * `{"messages":[{"role":"user","content":"..."}]}` input. A dataset row can
+ * carry a full multi-turn conversation; replaying these against one thread (one
+ * `t.send` per returned string) lets the agent see the accumulating history.
+ *
+ * Only `role === "user"` turns are returned — any interleaved `assistant`/
+ * `system` messages in the row are ignored, since the agent generates its own
+ * responses; you never inject the dataset's assistant turns. A single-user-turn
+ * row yields a one-element array (backward compatible); a row with no `messages`
+ * yields `[]`.
+ */
+export function userTurns(input: Record<string, unknown>): string[] {
+  const messages = Array.isArray(input.messages) ? input.messages : [];
+  // Dataset rows are external data: skip null/non-object entries; coerce non-string content to "".
+  return messages
+    .filter(
+      (m): m is { role?: unknown; content?: unknown } =>
+        typeof m === "object" && m !== null,
+    )
+    .filter((m) => m.role === "user")
+    .map((m) => (typeof m.content === "string" ? m.content : ""));
+}
+
 /** A managed eval dataset is a UC table; only 3-level names are valid. */
 const UC_TABLE = /^[A-Za-z0-9_]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+$/;
 

packages/appkit/src/evals/define-eval.ts

@@ -1,4 +1,4 @@
-import type { EvalDefinition } from "./types";
+import type { EvalConfig, EvalDefinition } from "./types";
 
 /**
  * Define an agent eval. Default-export the result from a
@@ -25,3 +25,8 @@ export function defineEval(def: EvalDefinition): EvalDefinition {
   }
   return def;
 }
+
+/** Define per-directory eval config. Default-export from `evals.config.ts`. */
+export function defineEvalConfig(config: EvalConfig): EvalConfig {
+  return config;
+}

packages/appkit/src/evals/discover.ts

@@ -1,4 +1,4 @@
-import { type Dirent, readdirSync } from "node:fs";
+import { type Dirent, existsSync, readdirSync } from "node:fs";
 import path from "node:path";
 
 import { agentDirNames } from "../core/agent/agent-dirs";
@@ -14,6 +14,14 @@ export interface DiscoveredEval {
   agent: string;
 }
 
+/** A per-agent `evals.config.ts` found under `server/agents/<agent>/evals/`. */
+export interface DiscoveredEvalConfig {
+  /** Absolute path to the `evals.config.ts` file. */
+  file: string;
+  /** The agent id whose evals this config applies to. */
+  agent: string;
+}
+
 /** Recursively collect `*.eval.ts` files under `dir`. Empty when `dir` is absent. */
 function evalFilesIn(dir: string): string[] {
   try {
@@ -58,3 +66,28 @@ export function discoverEvalFiles(rootDir: string): DiscoveredEval[] {
     (a, b) => a.agent.localeCompare(b.agent) || a.id.localeCompare(b.id),
   );
 }
+
+/**
+ * Discover the per-agent `evals.config.ts` (from {@link defineEvalConfig}) at
+ * `<rootDir>/server/agents/<agent>/evals/evals.config.ts`. Config is per-agent:
+ * each agent's config applies only to that agent's evals. Agents without a
+ * config file are omitted. Returns a stable, sorted list.
+ */
+export function discoverEvalConfigs(rootDir: string): DiscoveredEvalConfig[] {
+  const agentsDir = path.join(rootDir, CODE_AGENTS_SOURCE_DIR);
+  const out: DiscoveredEvalConfig[] = [];
+
+  let entries: Dirent[];
+  try {
+    entries = readdirSync(agentsDir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+
+  for (const agent of agentDirNames(entries)) {
+    const file = path.join(agentsDir, agent, "evals", "evals.config.ts");
+    if (existsSync(file)) out.push({ file, agent });
+  }
+
+  return out.sort((a, b) => a.agent.localeCompare(b.agent));
+}

packages/appkit/src/evals/http-driver.ts

@@ -26,22 +26,34 @@ export interface HttpDriverOptions {
 /** Mutable running totals accumulated while draining one turn's SSE stream. */
 interface DriveState {
   reply: string;
-  toolCalls: string[];
-  seen: Set<string>;
+  /** Captured tool calls, keyed by `call_id ?? name` (dedupe). */
+  toolCalls: Map<string, ToolCall>;
   ok: boolean;
   traceId?: string;
 }
 
-/** Record a `function_call` output item once per call id (deduped). */
+/**
+ * Record a `function_call` output item, keyed by `call_id ?? name`, capturing
+ * its parsed arguments. A later `done` event's fuller args win over the initial
+ * `added` event's (often empty) args.
+ */
 function recordToolCall(
-  item: { type?: string; name?: string; call_id?: string } | undefined,
+  item:
+    | { type?: string; name?: string; call_id?: string; arguments?: string }
+    | undefined,
   state: DriveState,
 ): void {
   if (item?.type !== "function_call" || !item.name) return;
   const key = item.call_id ?? item.name;
-  if (state.seen.has(key)) return;
-  state.seen.add(key);
-  state.toolCalls.push(item.name);
+  const args = parseArgs(item.arguments);
+  const existing = state.toolCalls.get(key);
+  if (!existing) {
+    state.toolCalls.set(key, { name: item.name, args });
+  } else if (Object.keys(args).length > 0) {
+    // The initial `added` event may carry empty args while the later `done`
+    // carries the full arguments — keep the fuller set.
+    existing.args = args;
+  }
 }
 
 /** Apply an `appkit.metadata` event's thread/trace ids. */
@@ -54,6 +66,24 @@ function applyMetadata(
   if (data?.mlflowTraceId) state.traceId = data.mlflowTraceId;
 }
 
+/** A single captured tool call, deduped by `call_id ?? name`. */
+type ToolCall = { name: string; args: Record<string, unknown> };
+
+/** Parse a function-call `arguments` JSON string; `{}` on missing/invalid. */
+function parseArgs(raw: unknown): Record<string, unknown> {
+  if (typeof raw !== "string" || raw.trim() === "") return {};
+  try {
+    const parsed = JSON.parse(raw);
+    return parsed !== null &&
+      typeof parsed === "object" &&
+      !Array.isArray(parsed)
+      ? (parsed as Record<string, unknown>)
+      : {};
+  } catch {
+    return {};
+  }
+}
+
 /** Parse a single Responses-API SSE `data:` payload into the running totals. */
 function applyEvent(
   event: Record<string, unknown>,
@@ -73,6 +103,7 @@ function applyEvent(
       type?: string;
       name?: string;
       call_id?: string;
+      arguments?: string;
       content?: Array<{ text?: string }>;
     };
     recordToolCall(item, state);
@@ -129,10 +160,17 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
     reset(): void {
       threadId = undefined;
     },
-    async send(message: string): Promise<DriveResult> {
+    async send(
+      message: string,
+      opts?: { signal?: AbortSignal },
+    ): Promise<DriveResult> {
       // Bounds connect + the entire read below. Passed to both the fetch and
       // the SSE reader: on expiry the reader is cancelled and the turn fails.
-      const signal = AbortSignal.timeout(timeoutMs);
+      // Composed with the caller's signal so a per-eval timeout also aborts this turn.
+      const timeout = AbortSignal.timeout(timeoutMs);
+      const signal = opts?.signal
+        ? AbortSignal.any([timeout, opts.signal])
+        : timeout;
       let res: Response;
       try {
         res = await fetch(`${options.baseUrl}${chatPath}`, {
@@ -147,22 +185,27 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
           signal,
         });
       } catch {
-        return { reply: "", toolCalls: [], succeeded: false };
+        return {
+          reply: "",
+          toolCalls: [],
+          toolCallDetails: [],
+          succeeded: false,
+        };
       }
 
       if (!res.ok || !res.body) {
         return {
           reply: "",
           toolCalls: [],
+          toolCallDetails: [],
           succeeded: false,
           sessionId: threadId,
         };
       }
 
       const state: DriveState = {
         reply: "",
-        toolCalls: [],
-        seen: new Set<string>(),
+        toolCalls: new Map<string, ToolCall>(),
         ok: true,
       };
       const setThread = (id: string) => {
@@ -193,9 +236,11 @@ export function createHttpDriver(options: HttpDriverOptions): EvalDriver {
       // throwing, so mark an aborted turn failed explicitly.
       if (signal.aborted) state.ok = false;
 
+      const toolCallDetails = [...state.toolCalls.values()];
       return {
         reply: state.reply,
-        toolCalls: state.toolCalls,
+        toolCalls: toolCallDetails.map((c) => c.name),
+        toolCallDetails,
         succeeded: state.ok,
         sessionId: threadId,
         traceId: state.traceId,

packages/appkit/src/evals/index.ts

@@ -11,9 +11,15 @@ export {
   type DatasetRow,
   type ReadEvalDatasetOptions,
   readEvalDataset,
+  userTurns,
 } from "./dataset";
-export { defineEval } from "./define-eval";
-export { type DiscoveredEval, discoverEvalFiles } from "./discover";
+export { defineEval, defineEvalConfig } from "./define-eval";
+export {
+  type DiscoveredEval,
+  type DiscoveredEvalConfig,
+  discoverEvalConfigs,
+  discoverEvalFiles,
+} from "./discover";
 export { createHttpDriver, type HttpDriverOptions } from "./http-driver";
 export {
   configureJudge,
@@ -34,6 +40,8 @@ export {
   formatEvalDetail,
   formatEvalHeadline,
   formatEvalResults,
+  formatResultsJson,
+  formatResultsJUnit,
   formatSummaryLine,
   summarize,
 } from "./report";
@@ -43,6 +51,7 @@ export {
   type EvalRunSummary,
   type RunEvalsOptions,
   runEvalsInDir,
+  runWithRetries,
 } from "./run-evals";
 export type {
   AssertionHandle,