diff --git a/apps/vis/web/src/components/analysis/LoopEvalSection.tsx b/apps/vis/web/src/components/analysis/LoopEvalSection.tsx new file mode 100644 index 0000000000..30bf3bff7c --- /dev/null +++ b/apps/vis/web/src/components/analysis/LoopEvalSection.tsx @@ -0,0 +1,294 @@ +import type { + LoopEvaluation, + PromptPhaseEvaluation, + SteerComparison, +} from '../../lib/loop-eval'; +import { CopyButton } from '../shared/CopyButton'; +import { Pill } from '../shared/Pill'; + +interface LoopEvalSectionProps { + evaluation: LoopEvaluation; +} + +export function LoopEvalSection({ evaluation }: LoopEvalSectionProps) { + const phases = evaluation.phases.filter( + (phase) => phase.toolCallCount > 0 || phase.markers.length > 0, + ); + if (phases.length === 0) return null; + + return ( +
+
+ repetition eval + +
+ + + +
+ ); +} + +function EvalSummary({ evaluation }: { evaluation: LoopEvaluation }) { + const summary = evaluation.summary; + const longest = summary.longestExactRun; + const peak = summary.peakRepetitionWindow; + const steerOverlap = summary.meanCompleteSteerHistogramOverlap; + + return ( +
+ + + + +
+ ); +} + +function Metric({ + label, + value, + detail, + title, +}: { + label: string; + value: string; + detail: string; + title?: string; +}) { + return ( +
+
+ {label} +
+
+ {value} +
+
{detail}
+
+ ); +} + +function PhaseTable({ + phases, + repetitionWindowCalls, +}: { + phases: readonly PromptPhaseEvaluation[]; + repetitionWindowCalls: number; +}) { + return ( +
+ + + + + + + + + + + + + + + + {phases.map((phase) => ( + + ))} + +
phasecallsdistinctrepeatsmax run{repetitionWindowCalls}-call peaksteerscancelscmp markers
+
+ ); +} + +function PhaseRow({ phase }: { phase: PromptPhaseEvaluation }) { + const steerCount = markerCount(phase, 'steer'); + const cancelCount = markerCount(phase, 'cancel'); + const compactionCount = markerCount(phase, 'compaction'); + const run = phase.longestExactRun; + const peak = phase.peakRepetitionWindow; + const phaseTitle = + phase.promptLineNo === null + ? `before first prompt at line ${String(phase.nextPromptLineNo ?? '-')}` + : `prompt line ${String(phase.promptLineNo)}, next prompt ${String(phase.nextPromptLineNo ?? '-')}`; + + return ( + + + {phase.index === 0 ? '0 (preamble)' : phase.index} + + {phase.toolCallCount} + {phase.distinctCallCount} + + {percent(phase.repeatedCallRate)} + + + {run === null ? '-' : `${String(run.length)}x`} + + + {peak === null ? '-' : percent(peak.repeatedCallRate)} + + {steerCount} + {cancelCount} + {compactionCount} + + ); +} + +function SteerTable({ + comparisons, + comparisonCalls, +}: { + comparisons: readonly SteerComparison[]; + comparisonCalls: number; +}) { + if (comparisons.length === 0) return null; + return ( +
+
+ steer response, {comparisonCalls} calls per side +
+
+ + + + + + + + + + + + + {comparisons.map((comparison) => ( + + + + + + + + + ))} + +
steerphasecalls before / afterdistinct before / afterhistogram overlapwindow
+ line {comparison.steerLineNo} + {comparison.phaseIndex}{comparison.beforeCallCount} / {comparison.afterCallCount}{comparison.beforeDistinctCallCount} / {comparison.afterDistinctCallCount} + {comparison.histogramOverlap === null + ? '-' + : percent(comparison.histogramOverlap)} + + + {comparison.complete ? 'complete' : 'partial'} + +
+
+
+ ); +} + +function markerCount( + phase: PromptPhaseEvaluation, + kind: PromptPhaseEvaluation['markers'][number]['kind'], +): number { + return phase.markers.filter((marker) => marker.kind === kind).length; +} + +function percent(value: number): string { + return `${(value * 100).toFixed(1)}%`; +} + +function SectionTitle({ children }: { children: import('react').ReactNode }) { + return ( +

+ {children} +

+ ); +} + +function Th({ + children, + align = 'right', +}: { + children: import('react').ReactNode; + align?: 'left' | 'right'; +}) { + return ( + + {children} + + ); +} + +function Td({ + children, + title, +}: { + children: import('react').ReactNode; + title?: string; +}) { + return ( + + {children} + + ); +} diff --git a/apps/vis/web/src/components/analysis/TimelineTab.tsx b/apps/vis/web/src/components/analysis/TimelineTab.tsx index 1b082142ec..dc3ed419af 100644 --- a/apps/vis/web/src/components/analysis/TimelineTab.tsx +++ b/apps/vis/web/src/components/analysis/TimelineTab.tsx @@ -13,6 +13,7 @@ import type { WireEntry } from '../../types'; import { formatBytes } from '../shared/SizePreview'; import { formatDuration, formatTokens } from '../../util/time'; import { Pill } from '../shared/Pill'; +import { LoopEvalSection } from './LoopEvalSection'; interface TimelineTabProps { sessionId: string; @@ -72,6 +73,7 @@ export function TimelineTab({ sessionId }: TimelineTabProps) { +
diff --git a/apps/vis/web/src/components/shared/CopyButton.tsx b/apps/vis/web/src/components/shared/CopyButton.tsx index d287dd0443..4f59b32ff6 100644 --- a/apps/vis/web/src/components/shared/CopyButton.tsx +++ b/apps/vis/web/src/components/shared/CopyButton.tsx @@ -4,9 +4,10 @@ interface CopyButtonProps { value: string; label?: string; className?: string; + title?: string; } -export function CopyButton({ value, label = 'copy', className = '' }: CopyButtonProps) { +export function CopyButton({ value, label = 'copy', className = '', title }: CopyButtonProps) { const [state, setState] = useState<'idle' | 'ok' | 'err'>('idle'); return ( @@ -20,7 +21,7 @@ export function CopyButton({ value, label = 'copy', className = '' }: CopyButton .finally(() => setTimeout(() =>{ setState('idle'); }, 1200)); }} className={`font-mono text-[10px] text-fg-3 transition-colors hover:text-fg-1 ${className}`} - title={`Copy ${value}`} + title={title ?? `Copy ${value}`} > {state === 'idle' ? label : state === 'ok' ? '✓ copied' : '✗ err'} diff --git a/apps/vis/web/src/lib/analysis.ts b/apps/vis/web/src/lib/analysis.ts index 24a83e7ae3..1c6ef7359b 100644 --- a/apps/vis/web/src/lib/analysis.ts +++ b/apps/vis/web/src/lib/analysis.ts @@ -9,11 +9,13 @@ // - tool-result truncation / size / error flags // - tool usage stats (count, error rate, latency) // - idle gaps (large wall-clock gaps between records → waiting) +// - prompt-phase repetition and steer-response evaluation metrics // // Pure: consumes the same `WireEntry[]` the Wire tab already fetches, so the // Timeline view needs no extra server round-trip. import type { TokenUsage, WireEntry } from '../types'; +import { evaluateLoopTrace, type LoopEvaluation } from './loop-eval'; export interface ContentSummary { textChars: number; @@ -157,6 +159,7 @@ export interface Analysis { toolStats: ToolStat[]; idleGaps: IdleGap[]; configChanges: ConfigChange[]; + loopEvaluation: LoopEvaluation; } const ZERO_USAGE: TokenUsage = { @@ -432,6 +435,7 @@ export function analyzeWire(entries: readonly WireEntry[]): Analysis { toolStats, idleGaps: sortedGaps, configChanges, + loopEvaluation: evaluateLoopTrace(entries), }; } diff --git a/apps/vis/web/src/lib/loop-eval.ts b/apps/vis/web/src/lib/loop-eval.ts new file mode 100644 index 0000000000..dd412aa9a9 --- /dev/null +++ b/apps/vis/web/src/lib/loop-eval.ts @@ -0,0 +1,568 @@ +import type { WireEntry } from '../types'; + +export const LOOP_EVAL_REPORT_VERSION = 1; +export const DEFAULT_REPETITION_WINDOW_CALLS = 20; +export const DEFAULT_STEER_COMPARISON_CALLS = 10; + +export interface LoopEvalOptions { + /** Exact-size rolling window used for local repetition measurements. */ + repetitionWindowCalls?: number; + /** Calls sampled on each side of a turn.steer marker. */ + steerComparisonCalls?: number; +} + +export interface LoopEvalSettings { + repetitionWindowCalls: number; + steerComparisonCalls: number; +} + +export type LoopEvalMarker = + | { kind: 'steer'; lineNo: number; recordType: 'turn.steer' } + | { kind: 'cancel'; lineNo: number; recordType: 'turn.cancel' } + | { + kind: 'compaction'; + lineNo: number; + recordType: + | 'full_compaction.begin' + | 'full_compaction.complete' + | 'full_compaction.cancel' + | 'context.apply_compaction'; + }; + +export interface ExactRun { + length: number; + toolName: string; + startLineNo: number; + endLineNo: number; +} + +export interface RepetitionWindow { + callCount: number; + repeatedCallCount: number; + repeatedCallRate: number; + startLineNo: number; + endLineNo: number; +} + +export interface PromptPhaseEvaluation { + /** Phase 0 is the preamble before the first accepted turn.prompt. */ + index: number; + /** Accepted prompt that starts this phase; null for phase 0. Prompt text is excluded. */ + promptLineNo: number | null; + /** Accepted prompt that ends this phase; null for the final phase. */ + nextPromptLineNo: number | null; + toolCallCount: number; + distinctCallCount: number; + /** Calls whose exact (tool, canonical args) pair appeared earlier in this phase. */ + repeatedCallCount: number; + repeatedCallRate: number; + longestExactRun: ExactRun | null; + /** Null until a phase contains one complete configured-size window. */ + peakRepetitionWindow: RepetitionWindow | null; + /** Marker metadata only. Prompt/steer content is intentionally excluded. */ + markers: LoopEvalMarker[]; +} + +export interface SteerComparison { + phaseIndex: number; + steerLineNo: number; + beforeCallCount: number; + afterCallCount: number; + beforeDistinctCallCount: number; + afterDistinctCallCount: number; + /** True when both sides contain the configured number of calls. */ + complete: boolean; + /** + * Histogram intersection for exact (tool, canonical args) pairs. + * 0 means disjoint distributions and 1 means identical distributions. + * Null means at least one side has no calls. + */ + histogramOverlap: number | null; + beforeStartLineNo: number | null; + beforeEndLineNo: number | null; + afterStartLineNo: number | null; + afterEndLineNo: number | null; +} + +export interface LoopEvalSummary { + phaseCount: number; + toolCallCount: number; + repeatedCallCount: number; + repeatedCallRate: number; + steerCount: number; + cancelCount: number; + compactionApplyCount: number; + longestExactRun: (ExactRun & { phaseIndex: number }) | null; + peakRepetitionWindow: (RepetitionWindow & { phaseIndex: number }) | null; + completeSteerComparisonCount: number; + /** Mean overlap across complete steer windows only. */ + meanCompleteSteerHistogramOverlap: number | null; +} + +export interface LoopEvaluation { + version: typeof LOOP_EVAL_REPORT_VERSION; + settings: LoopEvalSettings; + summary: LoopEvalSummary; + phases: PromptPhaseEvaluation[]; + steerComparisons: SteerComparison[]; +} + +type CompactionRecordType = Extract< + LoopEvalMarker, + { kind: 'compaction' } +>['recordType']; + +interface FingerprintedCall { + key: string; + lineNo: number; + toolName: string; +} + +interface PendingSteer { + lineNo: number; + callIndex: number; +} + +interface MutablePhase { + index: number; + promptLineNo: number | null; + nextPromptLineNo: number | null; + calls: FingerprintedCall[]; + markers: LoopEvalMarker[]; + steers: PendingSteer[]; +} + +const COMPACTION_RECORD_TYPES = new Set([ + 'full_compaction.begin', + 'full_compaction.complete', + 'full_compaction.cancel', + 'context.apply_compaction', +]); + +/** + * Evaluate repetition and steer-response signals from a persisted wire. + * + * The report intentionally contains no prompt text, tool arguments, tool + * results, or fingerprints. Those values exist only while this pure function + * is running and are omitted from the aggregate report. + */ +export function evaluateLoopTrace( + entries: readonly WireEntry[], + options: LoopEvalOptions = {}, +): LoopEvaluation { + const settings: LoopEvalSettings = { + repetitionWindowCalls: positiveInteger( + options.repetitionWindowCalls, + DEFAULT_REPETITION_WINDOW_CALLS, + ), + steerComparisonCalls: positiveInteger( + options.steerComparisonCalls, + DEFAULT_STEER_COMPARISON_CALLS, + ), + }; + + if (entries.length === 0) return emptyEvaluation(settings); + + const acceptedPromptLineNos = findAcceptedPromptLineNos(entries); + const mutablePhases: MutablePhase[] = []; + let current = createPhase(0, null); + + for (const entry of entries) { + const record = entry.data; + + if (record.type === 'turn.prompt') { + if (!acceptedPromptLineNos.has(entry.lineNo)) continue; + current.nextPromptLineNo = entry.lineNo; + mutablePhases.push(current); + current = createPhase(mutablePhases.length, entry.lineNo); + continue; + } + + if (record.type === 'turn.steer') { + current.markers.push({ + kind: 'steer', + lineNo: entry.lineNo, + recordType: 'turn.steer', + }); + current.steers.push({ lineNo: entry.lineNo, callIndex: current.calls.length }); + continue; + } + + if (record.type === 'turn.cancel') { + current.markers.push({ + kind: 'cancel', + lineNo: entry.lineNo, + recordType: 'turn.cancel', + }); + continue; + } + + if (isCompactionRecordType(record.type)) { + current.markers.push({ + kind: 'compaction', + lineNo: entry.lineNo, + recordType: record.type, + }); + continue; + } + + if ( + record.type === 'context.append_loop_event' && + record.event.type === 'tool.call' + ) { + current.calls.push({ + key: toolCallKey(record.event.name, record.event.args), + lineNo: entry.lineNo, + toolName: record.event.name, + }); + } + } + mutablePhases.push(current); + + const phases = mutablePhases.map((phase) => + evaluatePhase(phase, settings.repetitionWindowCalls), + ); + const steerComparisons = mutablePhases.flatMap((phase) => + compareSteers(phase, settings.steerComparisonCalls), + ); + + return { + version: LOOP_EVAL_REPORT_VERSION, + settings, + summary: summarize(phases, steerComparisons), + phases, + steerComparisons, + }; +} + +function findAcceptedPromptLineNos(entries: readonly WireEntry[]): ReadonlySet { + const accepted = new Set(); + let currentTurnId: string | undefined; + let pendingPromptLineNo: number | undefined; + + for (const entry of entries) { + const record = entry.data; + if (record.type === 'turn.prompt') { + pendingPromptLineNo = entry.lineNo; + continue; + } + if ( + record.type !== 'context.append_loop_event' || + !('turnId' in record.event) + ) { + continue; + } + + const nextTurnId = record.event.turnId; + if (pendingPromptLineNo !== undefined && nextTurnId !== currentTurnId) { + accepted.add(pendingPromptLineNo); + } + pendingPromptLineNo = undefined; + currentTurnId = nextTurnId; + } + + return accepted; +} + +function createPhase(index: number, promptLineNo: number | null): MutablePhase { + return { + index, + promptLineNo, + nextPromptLineNo: null, + calls: [], + markers: [], + steers: [], + }; +} + +function evaluatePhase( + phase: MutablePhase, + repetitionWindowCalls: number, +): PromptPhaseEvaluation { + const seen = new Set(); + let repeatedCallCount = 0; + for (const call of phase.calls) { + if (seen.has(call.key)) repeatedCallCount += 1; + else seen.add(call.key); + } + + return { + index: phase.index, + promptLineNo: phase.promptLineNo, + nextPromptLineNo: phase.nextPromptLineNo, + toolCallCount: phase.calls.length, + distinctCallCount: seen.size, + repeatedCallCount, + repeatedCallRate: rate(repeatedCallCount, phase.calls.length), + longestExactRun: longestExactRun(phase.calls), + peakRepetitionWindow: peakRepetitionWindow( + phase.calls, + repetitionWindowCalls, + ), + markers: [...phase.markers], + }; +} + +function longestExactRun(calls: readonly FingerprintedCall[]): ExactRun | null { + const first = calls[0]; + if (first === undefined) return null; + + let best: ExactRun = { + length: 1, + toolName: first.toolName, + startLineNo: first.lineNo, + endLineNo: first.lineNo, + }; + let currentKey = first.key; + let currentToolName = first.toolName; + let currentStartLineNo = first.lineNo; + let currentLength = 1; + + for (let index = 1; index < calls.length; index += 1) { + const call = calls[index]!; + if (call.key === currentKey) { + currentLength += 1; + } else { + currentKey = call.key; + currentToolName = call.toolName; + currentStartLineNo = call.lineNo; + currentLength = 1; + } + if (currentLength > best.length) { + best = { + length: currentLength, + toolName: currentToolName, + startLineNo: currentStartLineNo, + endLineNo: call.lineNo, + }; + } + } + + return best; +} + +function peakRepetitionWindow( + calls: readonly FingerprintedCall[], + windowCalls: number, +): RepetitionWindow | null { + if (calls.length < windowCalls) return null; + + const counts = new Map(); + for (let index = 0; index < windowCalls; index += 1) { + increment(counts, calls[index]!.key); + } + + let bestStart = 0; + let bestRepeated = windowCalls - counts.size; + for (let start = 1; start + windowCalls <= calls.length; start += 1) { + decrement(counts, calls[start - 1]!.key); + increment(counts, calls[start + windowCalls - 1]!.key); + const repeated = windowCalls - counts.size; + if (repeated > bestRepeated) { + bestStart = start; + bestRepeated = repeated; + } + } + + return { + callCount: windowCalls, + repeatedCallCount: bestRepeated, + repeatedCallRate: bestRepeated / windowCalls, + startLineNo: calls[bestStart]!.lineNo, + endLineNo: calls[bestStart + windowCalls - 1]!.lineNo, + }; +} + +function compareSteers( + phase: MutablePhase, + comparisonCalls: number, +): SteerComparison[] { + return phase.steers.map((steer) => { + const before = phase.calls.slice( + Math.max(0, steer.callIndex - comparisonCalls), + steer.callIndex, + ); + const after = phase.calls.slice( + steer.callIndex, + steer.callIndex + comparisonCalls, + ); + return { + phaseIndex: phase.index, + steerLineNo: steer.lineNo, + beforeCallCount: before.length, + afterCallCount: after.length, + beforeDistinctCallCount: distinctCount(before), + afterDistinctCallCount: distinctCount(after), + complete: + before.length === comparisonCalls && after.length === comparisonCalls, + histogramOverlap: histogramOverlap(before, after), + beforeStartLineNo: before[0]?.lineNo ?? null, + beforeEndLineNo: before.at(-1)?.lineNo ?? null, + afterStartLineNo: after[0]?.lineNo ?? null, + afterEndLineNo: after.at(-1)?.lineNo ?? null, + }; + }); +} + +function summarize( + phases: readonly PromptPhaseEvaluation[], + steerComparisons: readonly SteerComparison[], +): LoopEvalSummary { + let toolCallCount = 0; + let repeatedCallCount = 0; + let steerCount = 0; + let cancelCount = 0; + let compactionApplyCount = 0; + let longest: LoopEvalSummary['longestExactRun'] = null; + let peak: LoopEvalSummary['peakRepetitionWindow'] = null; + + for (const phase of phases) { + toolCallCount += phase.toolCallCount; + repeatedCallCount += phase.repeatedCallCount; + for (const marker of phase.markers) { + if (marker.kind === 'steer') steerCount += 1; + else if (marker.kind === 'cancel') cancelCount += 1; + else if (marker.recordType === 'context.apply_compaction') { + compactionApplyCount += 1; + } + } + + if ( + phase.longestExactRun !== null && + (longest === null || phase.longestExactRun.length > longest.length) + ) { + longest = { ...phase.longestExactRun, phaseIndex: phase.index }; + } + if ( + phase.peakRepetitionWindow !== null && + (peak === null || + phase.peakRepetitionWindow.repeatedCallRate > peak.repeatedCallRate) + ) { + peak = { ...phase.peakRepetitionWindow, phaseIndex: phase.index }; + } + } + + const completeOverlaps = steerComparisons.flatMap((comparison) => + comparison.complete && comparison.histogramOverlap !== null + ? [comparison.histogramOverlap] + : [], + ); + + return { + phaseCount: phases.length, + toolCallCount, + repeatedCallCount, + repeatedCallRate: rate(repeatedCallCount, toolCallCount), + steerCount, + cancelCount, + compactionApplyCount, + longestExactRun: longest, + peakRepetitionWindow: peak, + completeSteerComparisonCount: completeOverlaps.length, + meanCompleteSteerHistogramOverlap: + completeOverlaps.length === 0 + ? null + : completeOverlaps.reduce((sum, value) => sum + value, 0) / + completeOverlaps.length, + }; +} + +function histogramOverlap( + before: readonly FingerprintedCall[], + after: readonly FingerprintedCall[], +): number | null { + if (before.length === 0 || after.length === 0) return null; + const beforeCounts = histogram(before); + const afterCounts = histogram(after); + let overlap = 0; + for (const [key, beforeCount] of beforeCounts) { + const afterCount = afterCounts.get(key) ?? 0; + overlap += Math.min( + beforeCount / before.length, + afterCount / after.length, + ); + } + return overlap; +} + +function histogram(calls: readonly FingerprintedCall[]): Map { + const counts = new Map(); + for (const call of calls) increment(counts, call.key); + return counts; +} + +function distinctCount(calls: readonly FingerprintedCall[]): number { + return new Set(calls.map((call) => call.key)).size; +} + +function increment(counts: Map, key: string): void { + counts.set(key, (counts.get(key) ?? 0) + 1); +} + +function decrement(counts: Map, key: string): void { + const count = counts.get(key); + if (count === undefined || count <= 1) counts.delete(key); + else counts.set(key, count - 1); +} + +function rate(numerator: number, denominator: number): number { + return denominator === 0 ? 0 : numerator / denominator; +} + +function positiveInteger(value: number | undefined, fallback: number): number { + return value !== undefined && Number.isFinite(value) && value >= 1 + ? Math.floor(value) + : fallback; +} + +function isCompactionRecordType( + type: string, +): type is CompactionRecordType { + return COMPACTION_RECORD_TYPES.has(type as CompactionRecordType); +} + +function toolCallKey(toolName: string, args: unknown): string { + const canonicalArgs = JSON.stringify(sortJsonValue(args)) ?? String(args); + return JSON.stringify([toolName, canonicalArgs]); +} + +// Mirrors agent-core's recursive object-key ordering. Kept local because vis +// is also shipped as a standalone browser bundle and the runtime helper is not +// part of agent-core's public API. +function sortJsonValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortJsonValue); + if (!isPlainRecord(value)) return value; + const sorted: Record = {}; + for (const key of Object.keys(value).toSorted()) { + sorted[key] = sortJsonValue(value[key]); + } + return sorted; +} + +function isPlainRecord(value: unknown): value is Record { + if (value === null || typeof value !== 'object') return false; + const prototype: unknown = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function emptyEvaluation(settings: LoopEvalSettings): LoopEvaluation { + return { + version: LOOP_EVAL_REPORT_VERSION, + settings, + summary: { + phaseCount: 0, + toolCallCount: 0, + repeatedCallCount: 0, + repeatedCallRate: 0, + steerCount: 0, + cancelCount: 0, + compactionApplyCount: 0, + longestExactRun: null, + peakRepetitionWindow: null, + completeSteerComparisonCount: 0, + meanCompleteSteerHistogramOverlap: null, + }, + phases: [], + steerComparisons: [], + }; +} diff --git a/apps/vis/web/test/analysis.test.ts b/apps/vis/web/test/analysis.test.ts index f8986674d5..c9e941cfd1 100644 --- a/apps/vis/web/test/analysis.test.ts +++ b/apps/vis/web/test/analysis.test.ts @@ -72,6 +72,11 @@ describe('analyzeWire', () => { expect(a.summary.toolErrorCount).toBe(1); expect(a.summary.truncatedToolCount).toBe(1); + // Offline loop-eval report is derived from the same wire without raw args. + expect(a.loopEvaluation.summary.toolCallCount).toBe(2); + expect(a.loopEvaluation.summary.repeatedCallCount).toBe(0); + expect(a.loopEvaluation.phases).toHaveLength(3); + // Tool stats const read = a.toolStats.find((s) => s.name === 'Read')!; expect(read.count).toBe(2); diff --git a/apps/vis/web/test/loop-eval.test.ts b/apps/vis/web/test/loop-eval.test.ts new file mode 100644 index 0000000000..a26629cb2c --- /dev/null +++ b/apps/vis/web/test/loop-eval.test.ts @@ -0,0 +1,330 @@ +import { describe, expect, it } from 'vitest'; + +import { + DEFAULT_REPETITION_WINDOW_CALLS, + DEFAULT_STEER_COMPARISON_CALLS, + evaluateLoopTrace, +} from '../src/lib/loop-eval'; +import type { WireEntry } from '../src/types'; + +let line = 0; + +function record(data: Record): WireEntry { + line += 1; + return { lineNo: line, data, raw: data } as unknown as WireEntry; +} + +function prompt(text = 'prompt'): WireEntry { + return record({ + type: 'turn.prompt', + input: [{ type: 'text', text }], + origin: { kind: 'user' }, + }); +} + +function steer(text = 'steer'): WireEntry { + return record({ + type: 'turn.steer', + input: [{ type: 'text', text }], + origin: { kind: 'user' }, + }); +} + +function call(name: string, args: unknown, turnId = '0'): WireEntry { + return record({ + type: 'context.append_loop_event', + event: { + type: 'tool.call', + uuid: `uuid-${String(line + 1)}`, + turnId, + step: 1, + stepUuid: 'step', + toolCallId: `call-${String(line + 1)}`, + name, + args, + }, + }); +} + +describe('evaluateLoopTrace', () => { + it('segments prompt phases and canonicalizes argument object keys', () => { + line = 0; + const entries = [ + prompt('first'), + call('Read', { path: '/tmp/a', offset: 1 }), + call('Read', { offset: 1, path: '/tmp/a' }), + call('Read', { path: '/tmp/a', offset: 1 }), + call('Read', { offset: 1, path: '/tmp/a' }), + call('Read', { path: '/tmp/a', offset: 1 }), + prompt('second'), + call('Read', { offset: 1, path: '/tmp/a' }, '1'), + ]; + + const evaluation = evaluateLoopTrace(entries, { + repetitionWindowCalls: 4, + }); + + expect(evaluation.phases).toHaveLength(3); + expect(evaluation.phases[0]).toMatchObject({ + index: 0, + promptLineNo: null, + nextPromptLineNo: 1, + toolCallCount: 0, + }); + expect(evaluation.phases[1]).toMatchObject({ + index: 1, + promptLineNo: 1, + nextPromptLineNo: 7, + toolCallCount: 5, + distinctCallCount: 1, + repeatedCallCount: 4, + repeatedCallRate: 0.8, + longestExactRun: { + length: 5, + toolName: 'Read', + startLineNo: 2, + endLineNo: 6, + }, + peakRepetitionWindow: { + callCount: 4, + repeatedCallCount: 3, + repeatedCallRate: 0.75, + startLineNo: 2, + endLineNo: 5, + }, + }); + expect(evaluation.phases[2]).toMatchObject({ + toolCallCount: 1, + distinctCallCount: 1, + repeatedCallCount: 0, + }); + expect(evaluation.summary).toMatchObject({ + phaseCount: 3, + toolCallCount: 6, + repeatedCallCount: 4, + longestExactRun: { phaseIndex: 1, length: 5 }, + peakRepetitionWindow: { phaseIndex: 1, repeatedCallRate: 0.75 }, + }); + }); + + it('does not split a phase when an active turn rejects a prompt', () => { + line = 0; + const entries = [ + prompt('start the active turn'), + call('Bash', { cmd: 'poll' }, '0'), + prompt('rejected while busy'), + call('Bash', { cmd: 'poll' }, '0'), + call('Bash', { cmd: 'poll' }, '0'), + ]; + + const evaluation = evaluateLoopTrace(entries, { + repetitionWindowCalls: 3, + }); + + expect(evaluation.phases).toHaveLength(2); + expect(evaluation.phases[1]).toMatchObject({ + promptLineNo: 1, + nextPromptLineNo: null, + toolCallCount: 3, + distinctCallCount: 1, + repeatedCallCount: 2, + longestExactRun: { + length: 3, + startLineNo: 2, + endLineNo: 5, + }, + peakRepetitionWindow: { + callCount: 3, + repeatedCallCount: 2, + repeatedCallRate: 2 / 3, + }, + }); + }); + + it('detects a rotating small alphabet without relying on consecutive runs', () => { + line = 0; + const pattern = ['true', 'true', 'echo standby', 'echo .']; + const entries: WireEntry[] = [prompt()]; + for (let index = 0; index < 20; index += 1) { + entries.push(call('Bash', { cmd: pattern[index % pattern.length] })); + } + + const evaluation = evaluateLoopTrace(entries); + const phase = evaluation.phases[1]; + + expect(phase.toolCallCount).toBe(20); + expect(phase.distinctCallCount).toBe(3); + expect(phase.repeatedCallCount).toBe(17); + expect(phase.repeatedCallRate).toBe(0.85); + expect(phase.longestExactRun?.length).toBe(2); + expect(phase.peakRepetitionWindow).toMatchObject({ + callCount: 20, + repeatedCallCount: 17, + repeatedCallRate: 0.85, + }); + }); + + it('keeps unique argument drift at zero repetition', () => { + line = 0; + const entries: WireEntry[] = [prompt()]; + for (let index = 0; index < 20; index += 1) { + entries.push(call('Bash', { cmd: `echo ${String(index)}` })); + } + + const phase = evaluateLoopTrace(entries).phases[1]; + + expect(phase.distinctCallCount).toBe(20); + expect(phase.repeatedCallCount).toBe(0); + expect(phase.repeatedCallRate).toBe(0); + expect(phase.longestExactRun?.length).toBe(1); + expect(phase.peakRepetitionWindow?.repeatedCallRate).toBe(0); + }); + + it('compares exact-call histograms before and after steers', () => { + line = 0; + const entries = [ + prompt('same distribution'), + call('Bash', { cmd: 'a' }), + call('Bash', { cmd: 'b' }), + call('Bash', { cmd: 'a' }), + call('Bash', { cmd: 'b' }), + steer('change course'), + call('Bash', { cmd: 'b' }), + call('Bash', { cmd: 'a' }), + call('Bash', { cmd: 'b' }), + call('Bash', { cmd: 'a' }), + prompt('disjoint distribution'), + call('Read', { path: 'a' }, '1'), + call('Read', { path: 'b' }, '1'), + call('Read', { path: 'c' }, '1'), + call('Read', { path: 'd' }, '1'), + steer('look elsewhere'), + call('Bash', { cmd: 'e' }, '1'), + call('Bash', { cmd: 'f' }, '1'), + call('Bash', { cmd: 'g' }, '1'), + call('Bash', { cmd: 'h' }, '1'), + ]; + + const evaluation = evaluateLoopTrace(entries, { steerComparisonCalls: 4 }); + + expect(evaluation.steerComparisons).toHaveLength(2); + expect(evaluation.steerComparisons[0]).toMatchObject({ + phaseIndex: 1, + steerLineNo: 6, + beforeCallCount: 4, + afterCallCount: 4, + beforeDistinctCallCount: 2, + afterDistinctCallCount: 2, + complete: true, + histogramOverlap: 1, + beforeStartLineNo: 2, + beforeEndLineNo: 5, + afterStartLineNo: 7, + afterEndLineNo: 10, + }); + expect(evaluation.steerComparisons[1]).toMatchObject({ + phaseIndex: 2, + steerLineNo: 16, + complete: true, + histogramOverlap: 0, + }); + expect(evaluation.summary.completeSteerComparisonCount).toBe(2); + expect(evaluation.summary.meanCompleteSteerHistogramOverlap).toBe(0.5); + }); + + it('marks incomplete steer windows without inventing an overlap', () => { + line = 0; + const evaluation = evaluateLoopTrace([ + prompt(), + call('Read', { path: 'a' }), + steer(), + ], { steerComparisonCalls: 4 }); + + expect(evaluation.steerComparisons[0]).toMatchObject({ + beforeCallCount: 1, + afterCallCount: 0, + complete: false, + histogramOverlap: null, + }); + expect(evaluation.summary.completeSteerComparisonCount).toBe(0); + expect(evaluation.summary.meanCompleteSteerHistogramOverlap).toBeNull(); + }); + + it('returns marker metadata without prompts, arguments, outputs, or fingerprints', () => { + line = 0; + const secret = 'do-not-export-this-value'; + const entries = [ + prompt(secret), + call('Bash', { cmd: secret }), + record({ + type: 'context.append_loop_event', + event: { + type: 'tool.result', + toolCallId: 'call-2', + parentUuid: 'uuid-2', + result: { output: secret }, + }, + }), + steer(secret), + record({ type: 'turn.cancel', turnId: 1 }), + record({ type: 'full_compaction.begin', source: 'auto' }), + record({ + type: 'context.apply_compaction', + summary: secret, + compactedCount: 2, + tokensBefore: 100, + tokensAfter: 20, + }), + record({ type: 'full_compaction.complete' }), + ]; + + const evaluation = evaluateLoopTrace(entries); + const serialized = JSON.stringify(evaluation); + + expect(serialized).not.toContain(secret); + expect(evaluation.phases[1]?.markers).toEqual([ + { kind: 'steer', lineNo: 4, recordType: 'turn.steer' }, + { kind: 'cancel', lineNo: 5, recordType: 'turn.cancel' }, + { + kind: 'compaction', + lineNo: 6, + recordType: 'full_compaction.begin', + }, + { + kind: 'compaction', + lineNo: 7, + recordType: 'context.apply_compaction', + }, + { + kind: 'compaction', + lineNo: 8, + recordType: 'full_compaction.complete', + }, + ]); + expect(evaluation.summary).toMatchObject({ + steerCount: 1, + cancelCount: 1, + compactionApplyCount: 1, + }); + }); + + it('uses stable defaults and returns an empty report for an empty wire', () => { + const evaluation = evaluateLoopTrace([], { + repetitionWindowCalls: 0, + steerComparisonCalls: Number.NaN, + }); + + expect(evaluation.settings).toEqual({ + repetitionWindowCalls: DEFAULT_REPETITION_WINDOW_CALLS, + steerComparisonCalls: DEFAULT_STEER_COMPARISON_CALLS, + }); + expect(evaluation.phases).toEqual([]); + expect(evaluation.steerComparisons).toEqual([]); + expect(evaluation.summary).toMatchObject({ + phaseCount: 0, + toolCallCount: 0, + repeatedCallCount: 0, + repeatedCallRate: 0, + }); + }); +});