diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6284ff67..761b2165 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -36,6 +36,23 @@ jobs: main-branch-name: main - name: Run Checks run: pnpm run test:pr + delivery: + name: Delivery (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup Tools + uses: TanStack/config/.github/setup@b313637fa7d314532b98638f6b57b7b9c169d390 # main + - name: Run Managed Link Tests + run: pnpm --filter @tanstack/intent run test:delivery preview: name: Preview runs-on: ubuntu-latest diff --git a/benchmarks/intent/catalog.bench.ts b/benchmarks/intent/catalog.bench.ts new file mode 100644 index 00000000..f947f9ff --- /dev/null +++ b/benchmarks/intent/catalog.bench.ts @@ -0,0 +1,107 @@ +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import { afterAll, beforeAll, bench, describe } from 'vitest' +import { scanForIntents } from '../../packages/intent/src/discovery/scanner.js' +import { buildCurrentLockfileSources } from '../../packages/intent/src/core/lockfile/lockfile-state.js' +import { writeIntentLockfile } from '../../packages/intent/src/core/lockfile/lockfile.js' +import { + createCliRunner, + createConsoleSilencer, + createTempDir, + writeFile, + writeJson, + writePackage, +} from './helpers.js' + +const consoleSilencer = createConsoleSilencer() +const root = createTempDir('catalog') +const runner = createCliRunner({ cwd: root }) + +const PACKAGES = [ + { + name: '@bench/query', + skills: [ + 'queries', + 'mutations', + 'invalidation', + 'prefetching', + 'suspense', + 'pagination', + 'optimistic-updates', + 'ssr-hydration', + ], + }, + { + name: '@bench/router', + skills: [ + 'routing', + 'loaders', + 'search-params', + 'navigation', + 'code-splitting', + 'route-masking', + 'not-found', + ], + }, + { + name: '@bench/table', + skills: ['columns', 'sorting', 'filtering', 'grouping', 'virtualization'], + }, +] + +let getIntentCatalogContext: (options: { + cwd: string + refresh?: boolean +}) => Promise + +beforeAll(async () => { + consoleSilencer.silence() + writeJson(join(root, 'package.json'), { + name: 'intent-catalog-benchmark', + private: true, + intent: { skills: ['@bench/*'] }, + dependencies: Object.fromEntries( + PACKAGES.map((pkg) => [pkg.name, '1.0.0']), + ), + }) + writeFile(join(root, 'pnpm-lock.yaml'), 'lockfileVersion: "9.0"\n') + for (const pkg of PACKAGES) { + writePackage(join(root, 'node_modules'), pkg.name, '1.0.0', { + skills: pkg.skills, + }) + } + + // Without a lockfile the catalogue skips verification entirely, so the warm + // path would measure an empty loop instead of the per-skill hashing it runs + // on every session start. + writeIntentLockfile(join(root, 'intent.lock'), { + lockfileVersion: 1, + sources: buildCurrentLockfileSources( + scanForIntents(root, { scope: 'local' }).packages, + ), + }) + + await runner.setup() + const catalog = await import('../../packages/intent/dist/catalog.mjs') + getIntentCatalogContext = catalog.getIntentCatalogContext +}) + +afterAll(() => { + runner.teardown() + rmSync(root, { recursive: true, force: true }) + consoleSilencer.restore() +}) + +describe('intent catalog', () => { + bench('cold catalogue generation through API', async () => { + await getIntentCatalogContext({ cwd: root, refresh: true }) + }) + + bench('warm cached catalogue retrieval through API', async () => { + await getIntentCatalogContext({ cwd: root }) + }) + + bench('warm cached catalogue retrieval through CLI', async () => { + await runner.run(['catalog', '--json']) + }) +}) diff --git a/benchmarks/intent/helpers.ts b/benchmarks/intent/helpers.ts index 011fd52b..c3918955 100644 --- a/benchmarks/intent/helpers.ts +++ b/benchmarks/intent/helpers.ts @@ -1,6 +1,7 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' +import type { IntentPackage } from '../../packages/intent/src/shared/types.js' let builtCliMainPromise: Promise< (argv?: Array) => Promise @@ -20,6 +21,7 @@ type ConsoleSnapshot = { export type SkillOptions = { description: string + name?: string bodyLines?: number type?: 'core' | 'framework' requires?: Array @@ -36,6 +38,22 @@ export type PackageOptions = { brokenIntent?: boolean } +export function createRepresentativeIntentPackages(): Array { + return Array.from({ length: 20 }, (_, packageIndex) => ({ + name: `@bench/package-${String(packageIndex).padStart(2, '0')}`, + version: '1.0.0', + kind: 'npm', + source: 'local', + packageRoot: `node_modules/@bench/package-${packageIndex}`, + intent: { version: 1, repo: 'bench/packages', docs: 'docs/' }, + skills: Array.from({ length: 5 }, (_, skillIndex) => ({ + name: `skill-${skillIndex}`, + path: `skills/skill-${skillIndex}/SKILL.md`, + description: `Skill ${skillIndex}`, + })), + })) +} + const noop = () => undefined export function createBenchOptions( @@ -142,7 +160,7 @@ async function loadBuiltCliMain(): Promise< ) } - return module.main as (argv?: Array) => Promise + return module.main }, ) @@ -168,7 +186,7 @@ export function writeSkill( options: SkillOptions, ): void { const frontmatter = [ - `name: ${JSON.stringify(skillName)}`, + `name: ${JSON.stringify(options.name ?? skillName)}`, `description: ${JSON.stringify(options.description)}`, ] diff --git a/benchmarks/intent/install-plan.bench.ts b/benchmarks/intent/install-plan.bench.ts new file mode 100644 index 00000000..234df780 --- /dev/null +++ b/benchmarks/intent/install-plan.bench.ts @@ -0,0 +1,32 @@ +import { bench, describe } from 'vitest' +import { updateIntentConsumerConfigText } from '../../packages/intent/src/commands/install/config.js' +import { buildSkillSelectionPlan } from '../../packages/intent/src/commands/install/plan.js' +import { createRepresentativeIntentPackages } from './helpers.js' + +const packages = createRepresentativeIntentPackages() + +const packageJson = `${JSON.stringify( + { + name: 'install-plan-benchmark', + private: true, + intent: { skills: [], exclude: [] }, + }, + null, + 2, +)}\n` + +const selection = buildSkillSelectionPlan(packages, { mode: 'all-found' }) + +describe('installer planning', () => { + bench('plans 100 discovered skills', () => { + buildSkillSelectionPlan(packages, { mode: 'all-found' }) + }) + + bench('updates consumer JSONC configuration', () => { + updateIntentConsumerConfigText(packageJson, { + skills: selection.skills, + exclude: selection.exclude, + install: { method: 'symlink', targets: ['agents'] }, + }) + }) +}) diff --git a/benchmarks/intent/lockfile-hash.bench.ts b/benchmarks/intent/lockfile-hash.bench.ts new file mode 100644 index 00000000..58d79949 --- /dev/null +++ b/benchmarks/intent/lockfile-hash.bench.ts @@ -0,0 +1,25 @@ +import { rmSync } from 'node:fs' +import { join } from 'node:path' +import { afterAll, beforeAll, bench, describe } from 'vitest' +import { computeSkillContentHash } from '../../packages/intent/src/core/lockfile/hash.js' +import { createTempDir, writeFile } from './helpers.js' + +const root = createTempDir('lockfile-hash') +const skillDir = join(root, 'skills', 'representative') + +beforeAll(() => { + writeFile(join(skillDir, 'SKILL.md'), '# Guidance\n'.repeat(200)) + writeFile(join(skillDir, 'references', 'api.md'), '# API\n'.repeat(200)) + writeFile(join(skillDir, 'assets', 'example.json'), '{"enabled":true}\n') + writeFile(join(skillDir, 'scripts', 'check.mjs'), 'process.exit(0)\n') +}) + +afterAll(() => { + rmSync(root, { recursive: true, force: true }) +}) + +describe('per-skill lock hashing', () => { + bench('hashes a representative skill folder', () => { + computeSkillContentHash({ packageRoot: root, skillDir }) + }) +}) diff --git a/benchmarks/intent/sync.bench.ts b/benchmarks/intent/sync.bench.ts new file mode 100644 index 00000000..e7e25b4c --- /dev/null +++ b/benchmarks/intent/sync.bench.ts @@ -0,0 +1,62 @@ +import { bench, describe } from 'vitest' +import { buildInstallDeltaInventory } from '../../packages/intent/src/commands/install/plan.js' +import { createSyncAliases } from '../../packages/intent/src/commands/sync/targets.js' +import { createRepresentativeIntentPackages } from './helpers.js' +import type { IntentLockfileSource } from '../../packages/intent/src/core/lockfile/lockfile.js' +import type { IntentConsumerConfig } from '../../packages/intent/src/commands/install/config.js' + +const packages = createRepresentativeIntentPackages() + +const sources: Array = packages.map((pkg) => ({ + kind: pkg.kind, + id: pkg.name, + skills: pkg.skills.map((skill) => ({ + path: `skills/${skill.name}`, + contentHash: `${pkg.name}-${skill.name}`, + })), +})) + +const config: IntentConsumerConfig = { + skills: ['@bench/*'], + exclude: [], + install: { method: 'symlink', targets: ['agents'] }, +} + +describe('sync planning', () => { + bench('plans unchanged representative sources and aliases', () => { + buildInstallDeltaInventory( + packages, + sources, + { status: 'found', lockfile: { lockfileVersion: 1, sources } }, + config, + ) + createSyncAliases( + packages.flatMap((pkg) => + pkg.skills.map((skill) => ({ + kind: pkg.kind, + id: pkg.name, + skill: skill.name, + })), + ), + ) + }) + + bench('plans changed and pending sources', () => { + const changed = sources.map((source, index) => + index === 0 + ? { + ...source, + skills: source.skills.map((skill, skillIndex) => + skillIndex === 0 ? { ...skill, contentHash: 'changed' } : skill, + ), + } + : source, + ) + buildInstallDeltaInventory( + packages, + changed, + { status: 'found', lockfile: { lockfileVersion: 1, sources } }, + { ...config, skills: ['@bench/package-00'] }, + ) + }) +}) diff --git a/benchmarks/intent/tsconfig.json b/benchmarks/intent/tsconfig.json index fa8caa6e..8ce9be06 100644 --- a/benchmarks/intent/tsconfig.json +++ b/benchmarks/intent/tsconfig.json @@ -1,7 +1,6 @@ { "extends": "../../tsconfig.json", "compilerOptions": { - "rootDir": ".", "noEmit": true }, "include": ["*.ts"] diff --git a/benchmarks/intent/validate.bench.ts b/benchmarks/intent/validate.bench.ts index b05e998e..6a0cdef0 100644 --- a/benchmarks/intent/validate.bench.ts +++ b/benchmarks/intent/validate.bench.ts @@ -44,7 +44,6 @@ function createFixture(): ValidateFixture { writeSkill(root, domain, { description: `${domain} overview and guardrails`, bodyLines: 20, - type: 'core', }) for (let index = 1; index <= 4; index++) { @@ -53,8 +52,8 @@ function createFixture(): ValidateFixture { writeSkill(root, skillName, { description: `${domain} workflow ${index}`, + name: `workflow-${index}`, bodyLines: 18, - type: isFrameworkSkill ? 'framework' : 'core', requires: isFrameworkSkill ? [domain] : undefined, }) } @@ -124,10 +123,8 @@ describe('intent validate', () => { 'checks a shipped skills tree', async () => { const state = getFixture() - for (let index = 0; index < 3; index++) { - await state.runner.run(['validate']) - } + await state.runner.run(['validate']) }, - createBenchOptions(setup, teardown), + { ...createBenchOptions(setup, teardown), warmupIterations: 1 }, ) }) diff --git a/docs/cli/intent-catalog.md b/docs/cli/intent-catalog.md new file mode 100644 index 00000000..f044a882 --- /dev/null +++ b/docs/cli/intent-catalog.md @@ -0,0 +1,53 @@ +--- +title: intent catalog +id: intent-catalog +--- + +`intent catalog` prints bounded, lock-verified skill context for coding agents. Humans can inspect trusted packages with [`intent list`](./intent-list). + + +@tanstack/intent@latest catalog [package] [--json] [--refresh] + + +## Options + +- `package`: include only one exact package name. +- `--json`: print structured skills, counts, warnings, cache status, and rendered context. +- `--refresh`: ignore a valid cache entry and rebuild the catalog. + +## Output + +The global catalog distributes its skill budget across packages so a package with many skills cannot hide every later package. Output is capped at 50 skills, 180 characters per description, and 8 KB. When the global catalog omits skills, run `intent catalog ` for the relevant package. Package catalogs use the same limits; a known omitted skill can still be loaded directly with `intent load `. + +Only skills accepted by `intent.lock` appear. New, changed, or unverifiable skills are withheld and reported by count. If trust or lock state is missing, catalog tells the agent to pause and ask the user to run `intent install` interactively. + +The text footer tells the agent to load a matching skill with `intent load ` and continue normally when none match. + +## JSON output + +`--json` prints the same skills included in the byte-bounded `context`: + +```json +{ + "cacheStatus": "hit", + "context": "Available Intent skills: ...", + "omittedSkillCount": 2, + "skills": [ + { + "id": "@tanstack/query#fetching", + "description": "Query data fetching patterns" + } + ], + "totalSkillCount": 3, + "warnings": [] +} +``` + +`cacheStatus` is `miss` for a new entry, `hit` for verified cached content, and `refresh` when an existing entry is rebuilt. Global and package catalogs have separate cache entries. The cache fingerprint includes dependency manifests, lockfiles, workspace configuration, trust policy, and accepted skill content. + +## Related + +- [`intent install`](./intent-install) +- [`intent list`](./intent-list) +- [`intent load`](./intent-load) +- [Trust model](../concepts/trust-model) diff --git a/docs/cli/intent-exclude.md b/docs/cli/intent-exclude.md index 1c1d4db2..fa692269 100644 --- a/docs/cli/intent-exclude.md +++ b/docs/cli/intent-exclude.md @@ -1,43 +1,39 @@ ---- -title: intent exclude -id: intent-exclude ---- - -`intent exclude` manages `package.json#intent.exclude` entries. - -```bash -npx @tanstack/intent@latest exclude [list|add|remove] [pattern] [--json] -``` - -## Options - -- `--json`: print the configured exclude patterns as JSON - -## Actions - -1. `list` (default): print current excludes -2. `add `: append one exclude pattern -3. `remove `: remove one exclude pattern - -## Examples - -```bash -npx @tanstack/intent@latest exclude -npx @tanstack/intent@latest exclude list --json -npx @tanstack/intent@latest exclude add @tanstack/router#experimental-* -npx @tanstack/intent@latest exclude remove @tanstack/router#experimental-* -``` - -## Behavior - -- Reads and writes the current working directory `package.json` -- Creates `intent.exclude` when missing -- Keeps existing excludes and appends new patterns in order -- Validates pattern syntax before writing -- Refuses invalid `package.json` structures for `intent` and `intent.exclude` - -## Related - -- [Configuration](../concepts/configuration) -- [intent list](./intent-list) -- [intent load](./intent-load) +--- +title: intent exclude +id: intent-exclude +--- + +`intent exclude` manages the `intent.exclude` list in your `package.json`. Excludes remove packages or individual skills after the `intent.skills` allowlist resolves, so an excluded skill never reaches your agent even when its package is trusted. + + +@tanstack/intent@latest exclude [list|add|remove] [pattern] [--json] + + +## Actions + +- `list` (default): print the configured excludes. Add `--json` for machine-readable output. Humans and agents may list policy. +- `add `: append one exclude pattern. This is a user-owned policy change; agents pause and ask the user to run it. +- `remove `: remove one exclude pattern. This is a user-owned policy change; agents pause and ask the user to run it. + +```bash +npx @tanstack/intent@latest exclude +npx @tanstack/intent@latest exclude list --json +npx @tanstack/intent@latest exclude add @tanstack/router#experimental-* +npx @tanstack/intent@latest exclude remove @tanstack/router#experimental-* +``` + +For the pattern grammar - whole packages, single skills, and globs - see [Configuration](../concepts/configuration). + +## Behavior + +`add` and `remove` edit the project policy `package.json`, using the workspace root when one owns the current package. They create `intent.exclude` if it is missing and keep existing entries in order. Intent validates a pattern before writing and refuses an invalid `intent` or `intent.exclude` structure. `list` prints `Configured excludes:` with one entry per line, or `No excludes configured.` when the list is empty. `--json` is available only with `list`. + +After a mutation, Intent immediately reconciles configured symlink delivery without prompting, so a newly excluded skill is no longer exposed through an existing managed link. Hook delivery reads policy dynamically and needs no reconciliation. `intent.lock` remains unchanged; removing an exclusion can restore content that the user previously accepted. + +An excluded package does not trigger the unlisted-source warning, because excluding it is an explicit decision. + +## Related + +- [Configuration](../concepts/configuration) - the exclude pattern grammar. +- [`intent list`](./intent-list) - see what remains after excludes. +- [`intent load`](./intent-load) - excluded skills refuse to load. diff --git a/docs/cli/intent-hooks.md b/docs/cli/intent-hooks.md index 44e1c9f0..8767535c 100644 --- a/docs/cli/intent-hooks.md +++ b/docs/cli/intent-hooks.md @@ -3,49 +3,40 @@ title: intent hooks id: intent-hooks --- -`intent hooks install` installs lifecycle hooks that surface available Intent skills and enforce loading matching guidance before edits in supported agents. +`intent hooks run` runs the agent lifecycle hook that shows your coding agent which skills are available when a new context starts. You do not usually run it yourself: `intent install` wires it into the agent's supported session and subagent hooks when you choose hook delivery. -```bash -npx @tanstack/intent@latest hooks install [--scope project|user] [--agents copilot,claude,codex|all] -``` + +@tanstack/intent@latest hooks run --agent copilot|claude|codex + ## Options -- `--scope `: hook install scope, either `project` or `user`; defaults to `project` -- `--agents `: comma-separated hook agents to configure (`copilot`, `claude`, `codex`) or `all`; defaults to `all` +- `--agent `: the agent whose hook format to emit, one of `copilot`, `claude`, or `codex`. Required. -## Behavior +## What it does -- Installs hook behavior without writing an `intent-skills` guidance block. -- Adds a session-start skill catalog for supported agents so the agent sees available `skill-id: description` entries before it starts work. -- Keeps edit enforcement in place: supported edit tools are blocked until the agent runs `intent load ` for matching guidance. -- `--scope project` writes project-local hook config for agents that support it. -- `--scope user` writes user-level agent config and stores runner scripts under `~/.tanstack/intent/hooks`. -- `--agents all` is the default. In project scope, Copilot is skipped because the supported Copilot CLI hook location is user-scoped. -- Run `intent install` separately when you also want to write project guidance. -- Use `package.json#intent.skills` and `package.json#intent.exclude` to control which skills are surfaced in the session catalog. +At a supported context boundary, the hook prints a short catalogue of the skills your project trusts, each with the command to load it, so the agent knows what is available before it starts work. Claude Code and Codex replay their persisted catalogue on ordinary resumes without reinjecting it. GitHub Copilot CLI reinjects the catalogue when a resumed process starts because its hook context is process-local. -## Hook support +Claude Code and Codex receive the catalogue for new, cleared, and compacted contexts and when a subagent starts. Claude Code also receives it for forked sessions. GitHub Copilot CLI receives it for new and resumed process starts and subagents; its hook API does not provide a post-compaction context event. -| Agent | Project scope | User scope | Hooks installed | -| --- | --- | --- | --- | -| Claude Code | `.claude/settings.json` | `~/.claude/settings.json` | `SessionStart` skill catalog plus `PreToolUse` edit gate | -| Codex | `.codex/hooks.json` | `~/.codex/hooks.json` | `SessionStart` skill catalog plus `PreToolUse` edit gate; Codex hook interception is not a complete security boundary | -| GitHub Copilot CLI | Guidance via `.github/copilot-instructions.md`; blocking hooks are not project-scoped | `$COPILOT_HOME/hooks/hooks.json` or `~/.copilot/hooks/hooks.json` | `SessionStart` skill catalog plus `PreToolUse` edit gate in user scope | -| Cursor | Guidance only | Guidance only | Use `AGENTS.md` or Cursor rules; no blocking hook is installed | -| Generic `AGENTS.md` agents | Guidance only | Guidance only | Use the `intent-skills` guidance block; no blocking hook is installed | +The catalogue lists only skills accepted in `intent.lock`, and it is capped to keep sessions small: at most 50 skills and about 8 KB, with long descriptions trimmed. If it cannot build the catalogue it fails open, so the session continues and the hook prints a note to run `intent catalog` outside the session to see why. -`.github/copilot-instructions.md` is a supported project guidance target for `intent install`. GitHub Copilot CLI hook enforcement uses the user-scoped Copilot hooks directory because that is the supported hook location. +Hooks surface skills; they do not block edits. They are a convenience for getting skills in front of your agent, not a security boundary - the trust guarantees come from the source policy and the lockfile. See the [trust model](../concepts/trust-model). -Codex requires users to review and trust non-managed hooks before they run. If Codex reports hooks awaiting review, open its hook browser and trust the generated Intent hook. +## Installing hooks -## Status messages +Choose hook delivery when you run [`intent install`](./intent-install). Because the supported hook locations live in your home directory, these hooks are user-scoped and apply across your repositories, so `install` asks before writing them. It configures the supported session and subagent hooks for the agents you target and removes any earlier Intent edit-gate hooks. -- Hook installed: `Installed Intent hooks for claude (project) in .claude/settings.json.` -- Hook skipped: `Skipped Intent hooks for copilot: project scope is not supported; use --scope user` +| Agent | Hook config | +| --- | --- | +| Claude Code | `~/.claude/settings.json` | +| Codex | `~/.codex/hooks.json` | +| GitHub Copilot CLI | `~/.copilot/hooks/hooks.json` (or `$COPILOT_HOME/hooks/hooks.json`) | + +Codex may hold new hooks for review; open its hook browser and trust the Intent hook if prompted. ## Related -- [intent install](./intent-install) -- [intent list](./intent-list) -- [intent load](./intent-load) +- [`intent install`](./intent-install) - set up hook delivery. +- [`intent list`](./intent-list) - see which skills are available. +- [`intent load`](./intent-load) - print a skill's guidance. diff --git a/docs/cli/intent-install.md b/docs/cli/intent-install.md index 85b70e92..9394890c 100644 --- a/docs/cli/intent-install.md +++ b/docs/cli/intent-install.md @@ -3,92 +3,69 @@ title: intent install id: intent-install --- -`intent install` creates or updates an `intent-skills` guidance block in a project guidance file. +`intent install` sets up trusted skill delivery for your project. It records which packages you trust, locks the skill content you accept, and delivers those skills to your coding agents. Run it once to set up a project, and again after dependencies change to review and accept updates. For a step-by-step walkthrough, see the [consumer quick start](../getting-started/quick-start-consumers). -```bash -npx @tanstack/intent@latest install [--map] [--dry-run] [--print-prompt] [--global] [--global-only] [--no-notices] -``` +The default is an interactive setup where you choose how skills are delivered: symlinks, lifecycle hooks, or a static guidance block. `--map` writes the static guidance block directly, without the interactive delivery prompts and without a terminal. + + +@tanstack/intent@latest install [--map] [--dry-run] [--debug] [--no-notices] + ## Options -### Guidance output +- `--map`: write catalog loading guidance directly, without managed delivery. +- `--dry-run`: report what install would write, and change nothing. +- `--debug`: include package paths in diagnostic output. +- `--no-notices`: suppress non-critical notices on stderr. -- `--map`: write explicit task-to-skill mappings instead of lightweight loading guidance -- `--dry-run`: print the generated block without writing files -- `--print-prompt`: print the agent setup prompt instead of writing files +## Interactive setup -### Mapping scan scope +The default `install` runs an interactive setup, so it needs a terminal. For CI or a non-interactive shell, use `--map`. -- `--global`: include global packages after project packages when `--map` is passed -- `--global-only`: install mappings from global packages only when `--map` is passed -- `--no-notices`: suppress non-critical notices on stderr +Intent asks how to deliver skills, where to deliver them, and which skills to trust, then confirms before writing. There are three delivery choices: -## Behavior +- **Symlinks** link the accepted skill folders into your agent directories. +- **Lifecycle hooks** surface accepted skills at the start of an agent session. +- **Static guidance block** writes an `intent-skills` block into a file such as `AGENTS.md`. -- Writes lightweight skill loading guidance by default. -- Creates `AGENTS.md` when no managed block exists. -- Updates an existing managed block in a supported config file. -- Preserves all content outside the managed block. -- Scans packages and writes compact `id`, `run`, and `for` mappings only when `--map` is passed. -- Surfaces packages permitted by `package.json#intent.skills` in `--map` mode. See [Configuration](../concepts/configuration). -- Skips reference, meta, maintainer, and maintainer-only skills in `--map` mode. -- Writes compact skill identities and runnable guidance commands instead of local file paths in `--map` mode. -- Verifies the managed block before reporting success. -- Prints `No intent-enabled skills found.` and does not create a config file when `--map` finds no actionable skills. +Every choice records your trusted sources in `package.json` as explicit `intent.skills` and `intent.exclude` arrays, and the content you accepted in `intent.lock`. -Supported config files: `AGENTS.md`, `CLAUDE.md`, `.cursorrules`, `.github/copilot-instructions.md`. +Symlinks and hooks are managed delivery: Intent also writes `.intent/delivery.json`, adds `.intent/` to the project `.gitignore`, and keeps the skills in place. With symlinks it runs [`intent sync`](./intent-sync) once, adds generated link paths to the checkout's `.git/info/exclude`, and adds a `prepare: intent sync` script when Intent is a dev dependency, then prints a line such as `Installed 5 skills using symlink.` Static guidance is committed agent instructions without managed delivery; Intent prints a line such as `Installed 5 skills to AGENTS.md as a static guidance block.` -## Default output +Use `--dry-run` to preview any of this without writing files. See the [trust model](../concepts/trust-model) for how trusted sources and accepted content combine. -The default block tells agents to discover skills and load matching guidance on demand: +## Portable guidance with --map -```markdown - -## Skill Loading - -Before editing files for a substantial task: -- Run `npx @tanstack/intent@latest list` from the workspace root to see available local skills. -- If a listed skill matches the task, run `npx @tanstack/intent@latest load #` before changing files. -- Use the loaded `SKILL.md` guidance while making the change. -- Monorepos: when working across packages, run the skill check from the workspace root and prefer the local skill for the package being changed. -- Multiple matches: prefer the most specific local skill for the package or concern you are changing; load additional skills only when the task spans multiple packages or concerns. - -``` +`install --map` writes a compact static block without managed delivery. The block tells an agent to run `intent catalog` once when the session does not already contain an Intent catalog, then run `intent load ` only when a catalog entry matches the task. It does not embed every skill or description in the agent file. -## Mapping output +Supported files are `AGENTS.md`, `CLAUDE.md`, `.cursorrules`, and `.github/copilot-instructions.md`. In a terminal, Intent asks which file to use or lets you name another project file. On a project with no policy yet, a terminal run also helps you pick which skills to trust and writes `intent.skills`, `intent.exclude`, and `intent.lock`. An agent may regenerate guidance only when committed trust and lock state already exist; otherwise it stops and asks the user to run `intent install` interactively. -`--map` writes compact skill identities and commands: +The block stores portable identities and commands, never local file paths: ```yaml -# TanStack Intent - before editing files, run the matching guidance command. -tanstackIntent: - - id: "@tanstack/query#fetching" - run: "npx @tanstack/intent@latest load @tanstack/query#fetching" - for: "Query data fetching patterns" +## Intent Skills + +If an Intent catalog is not already present in this session context, run `npx @tanstack/intent@latest catalog` once. +If the catalog omits relevant skills, run `npx @tanstack/intent@latest catalog ` for the relevant package. +If a catalog entry matches the task, run `npx @tanstack/intent@latest load #` before editing. +Do not rerun the catalog for every task. If no skill matches, continue normally. ``` -- `id`: portable skill identity in `#` format -- `run`: package-manager-aware command agents should run before editing -- `for`: task-routing phrase for agents -- The block does not store `load` paths, absolute paths, or package-manager-internal paths +When `@tanstack/intent` is a project dev dependency, generated guidance and hooks use `npx @tanstack/intent`, which resolves the installed package without relying on the ambiguous `intent` binary name. Otherwise they use a pinned one-off runner such as `npx @tanstack/intent@0.4`. -## Status messages +Intent verifies the block after writing and reports whether it created, updated, or left the target unchanged. If it finds no usable skills it prints `No intent-enabled skills found.` and writes nothing. `--dry-run` prints the target and proposed trust/delivery changes without writing the block, trust config, lockfile, or local delivery state. Warnings remain visible during review; use `--debug` when package paths are needed. -- Created: `Created AGENTS.md with 1 mapping.` -- Updated: `Updated AGENTS.md with 2 mappings.` -- Unchanged: `No changes to AGENTS.md; 2 mappings already current.` -- Guidance created: `Created AGENTS.md with skill loading guidance.` -- Guidance unchanged: `No changes to AGENTS.md; skill loading guidance already current.` -- Placement tip: `Tip: Keep the intent-skills block near the top of AGENTS.md so agents read it before task-specific instructions.` -- No actionable skills in `--map` mode: `No intent-enabled skills found.` +## When install stops -To suppress trust and migration notices in automation, pass `--no-notices`. +- **No terminal.** The interactive setup needs a TTY. Without one, install stops and points you to `--map`. +- **Symlinks not possible.** Archive-backed and Yarn Plug'n'Play sources cannot be symlinked. Install stops and tells you to choose hook delivery or the static guidance block instead. +- **Target conflict.** If a delivery target already contains a conflicting file, install stops and lists the paths so you can move them. ## Related -- [intent list](./intent-list) -- [intent load](./intent-load) -- [intent hooks](./intent-hooks) -- [Quick Start for Consumers](../getting-started/quick-start-consumers) +- [Consumer quick start](../getting-started/quick-start-consumers) +- [Trust model](../concepts/trust-model) +- [`intent list`](./intent-list) +- [`intent hooks`](./intent-hooks) diff --git a/docs/cli/intent-list.md b/docs/cli/intent-list.md index c1fe1e9a..6ffcdf8c 100644 --- a/docs/cli/intent-list.md +++ b/docs/cli/intent-list.md @@ -3,42 +3,43 @@ title: intent list id: intent-list --- -`intent list` discovers skill-enabled packages and prints available skills. +`intent list` shows the packages your project trusts. Pass a package name or `--verbose` to inspect skills. -```bash -npx @tanstack/intent@latest list [--json] [--debug] [--global] [--global-only] [--show-hidden] [--no-notices] -``` + +@tanstack/intent@latest list [package] [--verbose] [--json] [--debug] [--global] [--global-only] [--show-hidden] [--why] [--no-notices] + ## Options -- `--json`: print JSON instead of text output -- `--debug`: print discovery debug details to stderr -- `--global`: include global packages after project packages -- `--global-only`: list global packages only -- `--show-hidden`: show unlisted hidden skill sources when run outside an agent session -- `--no-notices`: suppress non-critical notices on stderr - -## What you get - -- Scans project and workspace dependencies for intent-enabled packages and skills -- Surfaces packages permitted by `package.json#intent.skills` (see [Allowlist](#allowlist)) -- Includes global packages only when `--global` or `--global-only` is passed -- Includes warnings from discovery -- Excludes packages and skills matched by package.json `intent.exclude` -- Prints debug details to stderr when `--debug` is passed -- If no packages are discovered, prints `No intent-enabled packages found.` -- Summary line with package count and skill count -- Package table columns: `PACKAGE`, `SOURCE`, `VERSION`, `SKILLS` -- Skill tree grouped by package -- Optional warnings section (`⚠ ...` per warning) -- Optional notices section on stderr (`ℹ ...` per notice), suppressed by `--no-notices` - -`SOURCE` is a lightweight indicator showing whether the selected package came from local discovery or explicit global scanning. -When both local and global packages are scanned, local packages take precedence. +- `package`: show skills for one exact package name. +- `--verbose`: show every visible skill with its description and load command. +- `--json`: print the machine-readable list instead of the text tables. +- `--global`: include global packages after the project packages. +- `--global-only`: list global packages only. +- `--show-hidden`: also list sources that `intent.skills` does not permit, so you can decide what to enable. Has no effect in an agent session. +- `--why`: explain why each skill is shown or hidden. Has no effect in an agent session. +- `--debug`: print discovery details to stderr. +- `--no-notices`: suppress non-critical notices on stderr for this run. + +## What it shows + +For a human, plain `list` prints a summary line and package table with `PACKAGE`, `SOURCE`, `VERSION`, and `SKILLS` columns. `list ` shows that package's skill tree; `--verbose` shows every package's tree. `SOURCE` shows whether a package came from local discovery or global scanning; when the same package is found both locally and globally, the local one is used. If nothing is found, `list` prints `No intent-enabled packages found.` If `--show-hidden` finds only unpermitted sources, it prints `No permitted intent-enabled packages found.` before listing them. + +For an agent, plain `list` prints a compact package inventory and directs task matching to `intent catalog`. A package argument prints only portable skill IDs for that package. Hidden source names and local paths remain redacted. + +Warnings print under `Warnings:` (each prefixed `⚠`), and notices print under `Notices:` on stderr (each prefixed `ℹ`). Suppress notices for one run with `--no-notices`, or set `INTENT_NO_NOTICES=1` for CI and wrapper scripts. + +`list` scans the project's `node_modules`, or Yarn's Plug'n'Play API when there is no usable `node_modules`. It reads package files only and never runs package code; see the [trust model](../concepts/trust-model). + +## Which packages appear + +`list` shows only packages permitted by `package.json#intent.skills`, then removes anything matched by `intent.exclude`. A missing or empty `intent.skills` permits nothing, so `list` shows no packages until you add a source; the exact `"*"` entry shows every discovered package. See [Configuration](../concepts/configuration) for the entry grammar and special forms. + +A package that ships skills but is not permitted is hidden. Outside an agent session, `list` names hidden sources; `--show-hidden` lists them and `--why` explains each decision. In an agent session, hidden sources are reported by count only, so run `intent list --show-hidden` outside the session to review candidates. An entry that matches no discovered package is reported too. ## JSON output -`--json` prints an adapter-friendly skill list: +`--json` prints a stable shape for tools and agents: ```json { @@ -50,9 +51,7 @@ When both local and global packages are scanned, local packages take precedence. "packageVersion": "5.0.0", "packageSource": "local", "skillName": "fetching", - "description": "Query data fetching patterns", - "type": "skill (optional)", - "framework": "react (optional)" + "description": "Query data fetching patterns" } ], "packages": [ @@ -72,74 +71,26 @@ When both local and global packages are scanned, local packages take precedence. } ], "warnings": ["string"], + "notices": ["string"], "conflicts": [ { "packageName": "string", - "chosen": { - "version": "string", - "packageRoot": "string" - }, - "variants": [ - { - "version": "string", - "packageRoot": "string" - } - ] + "chosen": { "version": "string", "packageRoot": "string" }, + "variants": [{ "version": "string", "packageRoot": "string" }] } ] } ``` -When the same package exists both locally and globally and global scanning is enabled, `intent list` prefers the local package. -When project `node_modules` exists, `intent list` scans it. In Yarn PnP projects without usable `node_modules`, `intent list` uses Yarn's PnP API. - -## Allowlist - -`package.json#intent.skills` is the allowlist that decides which discovered packages are surfaced. Only listed packages contribute skills. - -```json -{ - "intent": { - "skills": ["@tanstack/query", "workspace:@scope/internal"] - } -} -``` - -Each entry is one source: - -- `@scope/pkg` or `pkg`: an npm package reachable through the dependency tree. -- `workspace:@scope/pkg`: a package in the current workspace. -- `@scope/*` or `workspace:@scope/*`: every discovered package of that kind whose name matches the pattern. -- `git:/#`: reserved, and not yet supported. - -The list as a whole has three special forms: - -- **Absent** (no `intent.skills` key): every discovered package is surfaced, with a deprecation notice printed to stderr on each run until you set `intent.skills`. This is the upgrade path for existing projects. A future version will require an explicit allowlist. -- **Empty** (`"skills": []`): no package is surfaced, with an info notice printed to stderr. -- **Wildcard** (`"skills": ["*"]`): every discovered package is surfaced, with an acknowledged-risk notice printed to stderr. This exact trust-all entry is distinct from a scoped package pattern such as `@tanstack/*`. - -A package that ships skills but is not listed or matched by a pattern is dropped. When packages are dropped this way, Intent prints one summary line naming them so you can opt in. In agent sessions, hidden sources are reported by count only; run `intent list --show-hidden` outside the agent session to review candidates. An exact entry or pattern that matches no discovered package is reported as well. Package patterns support `*` wildcards. Matching is currently by package name. See [Configuration](../concepts/configuration) and [Trust model](../concepts/trust-model). - -## Excludes - -Package excludes are hard filters for packages that should not be used in a repo, applied after the allowlist. -Intent reads `intent.exclude` arrays from package.json files while walking from the workspace or project root to the current working directory. -Manage persistent excludes with `intent exclude add|remove|list`. +Each skill also carries `type` and `framework` when the skill sets them. In an agent session, package paths are blanked, hidden source details are empty, and `conflicts` is empty; `hiddenSourceCount` still reports how many sources were withheld. -```json -{ - "intent": { - "exclude": ["@tanstack/*devtools*", "@tanstack/router#experimental-*"] - } -} -``` - -A pattern without `#` excludes a whole package. A pattern with `#` excludes a single skill (`@scope/pkg#search-params`), and the skill segment may itself be a glob (`@scope/pkg#experimental-*`). A pattern may cross package boundaries at skill granularity (`*#experimental-*`). The `#*` shortcut (`@scope/pkg#*`) excludes the whole package. Only exact names and `*` wildcards are supported on each segment. Bare package-name patterns keep working unchanged. +## Common errors -An excluded package never triggers the unlisted-source warning, because an exclude is an explicit decision rather than an oversight. +- Scanner failures print as errors. +- Unsupported environments, such as Deno projects without `node_modules`. `list` needs a resolvable `node_modules` or a Yarn Plug'n'Play setup. -## Common errors +## Related -- Scanner failures are printed as errors -- Unsupported environments: - - Deno projects without `node_modules` +- [Configuration](../concepts/configuration) - the `intent.skills` and `intent.exclude` grammar. +- [Trust model](../concepts/trust-model) - why discovery does not grant trust. +- [`intent load`](./intent-load) - print a matching skill's `SKILL.md`. diff --git a/docs/cli/intent-load.md b/docs/cli/intent-load.md index 28043d29..6ca22696 100644 --- a/docs/cli/intent-load.md +++ b/docs/cli/intent-load.md @@ -3,43 +3,34 @@ title: intent load id: intent-load --- -`intent load` loads a compact skill identity from the current install and prints the matching `SKILL.md` content. +`intent load` prints the `SKILL.md` for a skill in one of your trusted packages, matched to the version installed in your project. Your coding agent runs it to pull a skill's guidance into context, and you can run it yourself to read one. -```bash -npx @tanstack/intent@latest load # [--path] [--json] [--debug] [--global] [--global-only] -``` + +@tanstack/intent@latest load # [--path] [--json] [--debug] + + +The package may be scoped or unscoped, and the skill may include slash-separated sub-skill names. An unambiguous short skill name works when only one package-prefixed skill matches. + + +@tanstack/intent@latest load @tanstack/query#fetching +@tanstack/intent@latest load @tanstack/query#core/fetching +@tanstack/intent@latest load some-lib#core --path + ## Options -- `--path`: print the resolved skill path instead of the file content -- `--json`: print structured JSON with metadata and content -- `--debug`: print resolution debug details to stderr -- `--global`: load from project packages first, then global packages -- `--global-only`: load from global packages only - -## What you get - -- Validates `#` before scanning -- Scans project-local packages by default -- Includes global packages only when `--global` or `--global-only` is passed -- Refuses before scanning when the target package is not permitted by `package.json#intent.skills` -- Refuses before scanning when the target package or skill matches `intent.exclude` -- Prefers local packages when `--global` is used and the same package exists locally and globally -- Accepts an unambiguous short skill name when a package-prefixed skill exists -- Prints raw `SKILL.md` content by default -- Prints the scanner-reported path when `--path` is passed -- Prints debug details to stderr when `--debug` is passed - -The package can be scoped or unscoped. The skill can include slash-separated sub-skill names. - -Examples: - -```bash -npx @tanstack/intent@latest load @tanstack/query#fetching -npx @tanstack/intent@latest load @tanstack/query#core/fetching -npx @tanstack/intent@latest load @tanstack/router-core#auth-and-guards -npx @tanstack/intent@latest load some-lib#core --path -``` +- `--path`: print the resolved file path instead of the content. Cannot be combined with `--json`. +- `--json`: print the content plus metadata as JSON. Cannot be combined with `--path`. +- `--debug`: print resolution details to stderr. + +## What it checks + +Before printing anything, `load` confirms the skill is one you are allowed to use: + +- The package must be permitted by `package.json#intent.skills`, and must not be removed by `intent.exclude`. +- `intent.lock` must exist, the skill must be recorded in it, and its content must still match the accepted hash. `load` refuses missing, unaccepted, or changed content. Run `intent install` interactively to review and accept a baseline. + +`load` reads project packages accepted in `intent.lock`. ## JSON output @@ -58,21 +49,22 @@ npx @tanstack/intent@latest load some-lib#core --path } ``` +For explicit agent output (`INTENT_AUDIENCE=agent`), `path` and `packageRoot` are blank unless `--debug` is also present. `--path` always prints the requested path. + ## Common errors -- Missing separator: `Invalid skill use "@tanstack/query": expected #.` -- Empty package: `Invalid skill use "#core": package is required.` -- Empty skill: `Invalid skill use "@tanstack/query#": skill is required.` -- Missing package: `Cannot resolve skill use "...": package "..." was not found.` -- Missing skill: `Cannot resolve skill use "...": skill "..." was not found in package "...".` -- Skill suggestion: `Did you mean @tanstack/router-core#router-core/auth-and-guards?` -- Unlisted package: `Cannot load skill use "...": package "..." is not listed in intent.skills.` -- Excluded package: `Cannot load skill use "...": package "..." is excluded by Intent configuration.` -- Excluded skill: `Cannot load skill use "...": skill "..." is excluded by Intent configuration.` +- **Malformed use.** `Invalid skill use "@tanstack/query": expected #.`, or a similar message for an empty package or skill. +- **Missing lock.** `Cannot load skill use "...": intent.lock is missing.` A human is told to run interactive install; an agent is told to pause and ask the user. +- **Invalid lock.** `Cannot load skill use "...": intent.lock is invalid: ...` followed by the same human or agent review instruction. +- **Not found.** Missing packages direct humans to `intent list`; missing skills provide up to three portable suggestions or direct to `intent list `. Hidden package and skill names are not enumerated. +- **Not trusted.** `Cannot load skill use "...": package "..." is not listed in intent.skills.` +- **Excluded.** `Cannot load skill use "...": package "..." is excluded by Intent configuration.`, or the same for a skill. +- **Not accepted.** `Cannot load skill use "...": skill is not accepted in intent.lock.` +- **Content changed.** `Cannot load skill use "...": installed content does not match intent.lock.` Not-accepted and changed-content errors tell a human to run interactive install and tell an agent to pause and ask the user. ## Related -- [intent list](./intent-list) -- [intent install](./intent-install) -- [Trust model](../concepts/trust-model) -- [Configuration](../concepts/configuration) +- [`intent list`](./intent-list) - find loadable skills. +- [`intent install`](./intent-install) - accept skills into the lockfile. +- [Trust model](../concepts/trust-model) - how the policy and lockfile gate loading. +- [Configuration](../concepts/configuration) - the `intent.skills` and `intent.exclude` grammar. diff --git a/docs/cli/intent-sync.md b/docs/cli/intent-sync.md new file mode 100644 index 00000000..93d5c0ac --- /dev/null +++ b/docs/cli/intent-sync.md @@ -0,0 +1,72 @@ +--- +title: intent sync +id: intent-sync +--- + +`intent sync` updates the managed symlinks for symlink delivery so your agent sees the skills you have accepted. Run it after installing dependencies or pulling changes. With symlink delivery, `install` also adds a `prepare` script so `sync` runs after each `npm install`. + + +@tanstack/intent@latest sync [--dry-run] [--json] + + +## Options + +- `--dry-run`: report what sync would change without touching any links, then print `No files changed.` +- `--json`: print compact structured link and review results. + +## Prerequisites + +`sync` works only with symlink delivery, and it needs the state that `intent install` writes: + +- `.intent/delivery.json` set to symlink delivery. Without it, sync exits with an error and tells you to run `intent install` interactively. +- `package.json` policy and `intent.lock`. Without them, sync exits with an error and tells you to run `intent install` interactively. + +For hook delivery, run `intent install` instead; `sync` does not manage hooks. + +## What it does + +`sync` reconciles the symlinks in your agent directories against `intent.lock`: it adds links for accepted skills and removes links that no longer belong. It reports anything that needs your attention rather than accepting it silently: + +- **New dependencies** that ship skills you have not trusted. +- **New skills** in a package you already trust. +- **Changed skill content** that no longer matches `intent.lock`. + +New and changed skills are held for review. In a human terminal, sync offers to enable or exclude new dependencies and names packages that need review. Explicit agent runs and the generated `prepare` script never prompt or reveal untrusted package names; they report counts and tell the agent to pause and ask the user to run interactive install. To accept changed content as a new baseline, run `intent install`. + +When no work is needed, a human run prints one summary line. Agent and `prepare` runs stay silent. Human link changes include project-relative paths; agent and `prepare` output reports counts only. + +## JSON output + +```json +{ + "dryRun": true, + "links": { + "created": [], + "repaired": [], + "removed": [], + "conflicts": [], + "unchangedCount": 41 + }, + "review": { + "newDependencies": [], + "newSkills": [], + "changed": [] + } +} +``` + +Each review entry contains `name` and `skillCount`. Explicit agent JSON blanks names for untrusted new dependencies. + +## When sync stops + +- **No delivery configured.** sync exits with an error and points to interactive install. +- **Missing policy or lockfile.** sync stops and points you to `intent install`. +- **Hook delivery.** sync manages symlinks only; use `intent install` to repair hooks. +- **Symlinks not possible.** Archive-backed and Yarn Plug'n'Play sources cannot be symlinked; sync stops and tells you to use hook delivery. +- **Link conflict.** If a managed target already holds an unmanaged file, sync stops and lists the paths. +- **Malformed install state.** sync stops and explains how to restore or reset the Intent-managed links and state. + +## Related + +- [`intent install`](./intent-install) - configure delivery and accept a new baseline. +- [Trust model](../concepts/trust-model) - how the lockfile gates content. diff --git a/docs/cli/intent-validate.md b/docs/cli/intent-validate.md index b54b4d2a..c66a7f58 100644 --- a/docs/cli/intent-validate.md +++ b/docs/cli/intent-validate.md @@ -1,93 +1,113 @@ ---- -title: intent validate -id: intent-validate ---- - -`intent validate` checks `SKILL.md` files and artifacts for structural problems. - -```bash -npx @tanstack/intent@latest validate [] [--github-summary] [--fix] [--check] -``` - -## Arguments - -- ``: directory containing skills; default is `skills` -- Relative paths are resolved from the current working directory - -## Options - -- `--github-summary`: write a GitHub Actions step summary when `GITHUB_STEP_SUMMARY` is set -- `--check`: fail if any `SKILL.md` has fixable frontmatter migrations pending, without writing files -- `--fix`: rewrite fixable `SKILL.md` frontmatter migrations, then validate the result - -## Frontmatter migration fixes - -Use `--check` in CI to detect mechanical frontmatter migrations that have not been applied: - -```bash -npx @tanstack/intent@latest validate --check -``` - -Use `--fix` locally to apply the mechanical frontmatter migrations: - -```bash -npx @tanstack/intent@latest validate --fix -``` - -`--fix` only rewrites unambiguous frontmatter migrations: - -- `name` values are rewritten to the parent directory leaf when the parent directory is already a legal skill name -- Top-level string fields `type`, `library`, `library_version`, and `framework` are moved under `metadata` - -`--fix` does not rewrite authoring-judgment validation errors: - -- Missing or invalid `description` -- Length-limit failures -- Invalid `metadata` shape or non-string `metadata` values -- Missing `requires` for framework skills -- Artifact validation failures - -## Validation checks - -For each discovered `SKILL.md`: - -- Frontmatter delimiter and structure are valid -- YAML frontmatter parses successfully -- Required fields exist: `name`, `description` -- `name` is a single leaf segment matching the skill's parent directory (no slashes); the namespace is carried by the directory path -- `name` uses only lowercase letters, numbers, and hyphens -- `name` is at most 64 characters -- Only spec top-level keys are allowed (`name`, `description`, `license`, `compatibility`, `metadata`, `allowed-tools`); Intent-specific scalars (`type`, `library`, `library_version`, `framework`) must live under `metadata` -- `metadata`, when present, is a mapping of string values -- `description` length is at most 1024 characters -- `type: framework` requires `requires` to be an array -- Total file length is at most 500 lines - -If `/_artifacts` exists, it also validates artifacts: - -- Required files: `domain_map.yaml`, `skill_spec.md`, `skill_tree.yaml` -- Required files must be non-empty -- `.yaml` artifacts must parse successfully - -## Packaging warnings - -Packaging warnings are always computed from `package.json` in the current working directory: - -- `@tanstack/intent` missing from `devDependencies` -- Missing `tanstack-intent` in keywords array -- Missing `files` entries when `files` array exists: - - `skills` - - `!skills/_artifacts` - -Warnings are informational; they are printed on both pass and fail paths. - -## Common errors - -- Missing target directory: `Skills directory not found: ` -- No skill files discovered: `No SKILL.md files found` -- Validation failures: aggregated file-specific errors and count - -## Related - -- [intent scaffold](./intent-scaffold) -- [setup commands](./intent-setup) +--- +title: intent validate +id: intent-validate +--- + +`intent validate` checks `SKILL.md` files, artifacts, and the rendered session catalogue. + +```bash +npx @tanstack/intent@latest validate [] [--github-summary] [--fix] [--check] +``` + +## Arguments + +- ``: directory containing skills; default is `skills` +- Relative paths are resolved from the current working directory + +## Options + +- `--github-summary`: write a GitHub Actions step summary when `GITHUB_STEP_SUMMARY` is set +- `--check`: fail on catalogue warnings or fixable frontmatter migrations, without writing files +- `--fix`: rewrite fixable `SKILL.md` frontmatter migrations, then validate the result + +## Frontmatter migration fixes + +Use `--check` in CI to detect catalogue warnings and mechanical frontmatter migrations that have not been applied: + +```bash +npx @tanstack/intent@latest validate --check +``` + +Use `--fix` locally to apply the mechanical frontmatter migrations: + +```bash +npx @tanstack/intent@latest validate --fix +``` + +`--fix` only rewrites unambiguous frontmatter migrations: + +- `name` values are rewritten to the parent directory leaf when the parent directory is already a legal skill name +- Top-level string fields `type`, `library`, `library_version`, and `framework` are moved under `metadata` + +`--fix` does not rewrite authoring-judgment validation errors: + +- Missing or invalid `description` +- Length-limit failures +- Invalid `metadata` shape or non-string `metadata` values +- Missing `requires` for framework skills +- Artifact validation failures + +## Validation checks + +For each discovered `SKILL.md`, validation checks the [Agent Skills specification](https://agentskills.io/specification) fields and Intent-specific constraints: + +- Frontmatter delimiter and structure are valid +- YAML frontmatter parses as a mapping +- Required fields `name` and `description` are non-empty strings +- `name` is a single leaf segment matching the skill's parent directory (no slashes); the namespace is carried by the directory path +- `name` uses only lowercase letters, numbers, and hyphens +- `name` is at most 64 characters +- Only spec top-level keys are allowed (`name`, `description`, `license`, `compatibility`, `metadata`, `allowed-tools`); Intent-specific scalars (`type`, `library`, `library_version`, `framework`) must live under `metadata` +- `metadata`, when present, is a mapping of string values +- `description` length is at most 1024 characters +- `license`, when present, is a non-empty string +- `compatibility`, when present, is a non-empty string of at most 500 characters +- `allowed-tools`, when present, is a non-empty space-separated string +- `type: framework` requires `requires` to be an array +- Total file length is at most 500 lines + +If `/_artifacts` exists, it also validates artifacts: + +- Required files: `domain_map.yaml`, `skill_spec.md`, `skill_tree.yaml` +- Required files must be non-empty +- `.yaml` artifacts must parse successfully + +## Catalogue warnings + +Validation builds minimal skill summaries from the parsed frontmatter and passes them through the same catalogue builder and formatter used for agent sessions. It reports these warnings per skill: + +- Description truncation beyond 180 characters, including the number of characters lost +- Description blanking when it contains a local filesystem path +- Unknown `metadata.type` values and whether the skill remains in the catalogue +- Known types excluded from the catalogue, because agents will not see those skills +- Duplicate descriptions after catalogue normalization and truncation +- Malformed `#` uses + +For each package, validation also reports the full rendered byte count against the 8000-byte budget and lists skills omitted by the 50-skill or byte limit. + +Catalogue findings are warnings by default. `--check` promotes them to validation errors. + +Agent Skills field warnings and packaging warnings remain informational in `--check` mode. + +## Packaging warnings + +Packaging warnings are always computed from `package.json` in the current working directory: + +- `@tanstack/intent` missing from `devDependencies` +- Missing `tanstack-intent` in keywords array +- Missing `files` entries when `files` array exists: + - `skills` + - `!skills/_artifacts` + +Warnings are informational; they are printed on both pass and fail paths. + +## Common errors + +- Missing target directory: `Skills directory not found: ` +- No skill files discovered: `No SKILL.md files found` +- Validation failures: aggregated file-specific errors and count + +## Related + +- [intent scaffold](./intent-scaffold) +- [setup commands](./intent-setup) diff --git a/docs/concepts/configuration.md b/docs/concepts/configuration.md index 8b9d27ad..8efa8d1f 100644 --- a/docs/concepts/configuration.md +++ b/docs/concepts/configuration.md @@ -18,7 +18,7 @@ Intent merges these keys from every `package.json` between the current working d ## `intent.skills` -`intent.skills` is the allowlist. Only packages it permits contribute skills to `list`, `load`, `install`, and `stale`. See [Trust model](./trust-model) for the reasoning. +`intent.skills` is the allowlist. Only packages it permits contribute skills to commands like `list` and `load`. Interactive `install` is where you set this list. See [Trust model](./trust-model) for the reasoning. ### Source entries @@ -31,21 +31,21 @@ Each array entry names one source: | `@scope/*` or `workspace:@scope/*` | npm or workspace | Every discovered package of that kind whose name matches the pattern. | | `git:/#` | git | Reserved. Not yet supported, and rejected until a future version adds it. | -A malformed entry fails the whole command, and every bad entry is reported at once. Package patterns support `*` wildcards, including scoped patterns such as `@tanstack/*`. Intent matches allowlist entries against discovered package names. This matching will tighten in a future version. +A malformed entry fails the whole command, and every bad entry is reported at once. Package patterns support `*` wildcards, including scoped patterns such as `@tanstack/*`. Intent matches allowlist entries against discovered package names. ### Special forms The list as a whole has three special forms: -- **Absent.** No `intent.skills` key. Every discovered package is surfaced, and Intent prints a deprecation notice to stderr on each run until you set `intent.skills`. This is the upgrade path for existing projects. A future version will require an explicit allowlist. -- **Empty.** `"skills": []`. No package is surfaced. Intent prints an info notice to stderr. +- **Absent or empty.** No `intent.skills` key, or `"skills": []`. Intent permits no sources, so nothing is surfaced until you list at least one entry. It prints a notice to stderr saying no sources are permitted. +- **Explicit entries.** Intent surfaces only the packages that match a listed entry. - **Wildcard.** `"skills": ["*"]`. Every discovered package is surfaced. Unlike a package pattern such as `@tanstack/*`, this exact entry crosses package scopes and source kinds. Intent prints an acknowledged-risk notice to stderr, since unvetted skills may reach your agent. A package that ships skills but is not listed is dropped. When packages are dropped this way, Intent prints one summary line naming them so you can opt in. A listed package that was not discovered is reported as well. ### Existing projects -A project that has not set `intent.skills` keeps working. Intent surfaces every discovered package and prints the deprecation notice described under the absent form. Nothing breaks. Add an allowlist when you are ready, before a future version requires one. Run `intent list` to confirm which packages are surfaced. +A project that has not set `intent.skills` surfaces no skills until you list at least one source. Configuration written for earlier versions still parses, but Intent no longer treats a missing `intent.skills` as permission to surface every package, so you have to opt in. Run `intent install` to choose which packages to trust, or add entries to `intent.skills` by hand, then run `intent list` to confirm what is surfaced. ### Suppressing notices temporarily diff --git a/docs/concepts/trust-model.md b/docs/concepts/trust-model.md index 65ea0b4b..f26fbb4c 100644 --- a/docs/concepts/trust-model.md +++ b/docs/concepts/trust-model.md @@ -3,26 +3,45 @@ title: Trust model id: trust-model --- -Intent surfaces skills from your dependencies into your coding agent's guidance. A skill is instructions an agent follows, so the set of packages allowed to contribute skills is a trust decision. Intent makes that decision explicit through the `intent.skills` allowlist. +A skill is instructions your coding agent follows, so letting a dependency contribute one carries the same weight as running its code. Intent keeps that decision explicit and in your hands: a skill reaches your agent only when you have trusted its package and accepted its content. -## Explicit sources +## Two gates: sources and content + +Trust has two parts, and a skill has to pass both: + +- **Sources**: which packages may contribute skills at all, set by `intent.skills` in `package.json`. +- **Content**: which exact skill files you have accepted, recorded in `intent.lock`. + +The source policy decides who is allowed in. The lockfile pins what you actually agreed to, so a later change cannot slip through unnoticed. + +## Which packages you trust A package ships skills in a `skills/` directory. Discovery finds every installed package that has one, including transitive dependencies. Discovery does not grant trust. -`package.json#intent.skills` is the gate. A discovered package contributes skills only when an exact entry or `*` pattern in the allowlist matches it. An unlisted package is dropped, and Intent reports it so you can opt in or ignore it. +`intent.skills` is the gate, and it is required. With no `intent.skills` key, or an empty list, Intent permits no sources and surfaces nothing until you list at least one. An explicit entry matches packages by name; the exact `"*"` entry allows every discovered package, which is why Intent warns when you use it. See the [special forms](./configuration#special-forms) in Configuration for each case. + +Trust does not propagate. A package you trust may depend on another package that ships skills, but that dependency stays untrusted unless a separate entry matches it. A package that ships skills but is not listed is dropped, and Intent names it so you can opt in or leave it out. -The gate is opt-in today. A project with no `intent.skills` key still surfaces every discovered package, and Intent prints a deprecation notice to stderr on each run until you set `intent.skills`. A future version will require an explicit allowlist. See the [special forms](./configuration#special-forms) in Configuration. +## Which content you accepted -Trust does not propagate. A listed package may depend on another package that ships skills, but that dependency stays unlisted unless another entry matches it. Exact entries allow one source; patterns such as `@tanstack/*` explicitly allow every matching source. +`intent.lock` records every accepted skill as a path and a content hash. When a lockfile is present, `load` enforces it: it refuses a skill that is not in the lock, and refuses one whose installed content no longer matches the recorded hash. -## Static discovery +This is what stops a dependency update from quietly changing what your agent reads. When an update adds a skill or changes an accepted one, `sync` reports it for review instead of accepting it, and you take the new content by running `install` again. Without a lockfile, Intent falls back to the source policy alone and does not check content. + +## Discovery never runs package code Intent reads package data as files. It never imports, requires, or executes the code of a discovered package to find or load a skill. Adding a package to your dependency tree cannot run that package's code through Intent. -One exception is sanctioned: in Yarn Plug'n'Play projects, Intent loads Yarn's PnP runtime (`.pnp.cjs`) to map package identities to readable locations. It loads no package entry points, bins, lifecycle scripts, or other package-provided JavaScript. An ESLint rule enforces this invariant in the discovery code. +One exception is sanctioned: in Yarn Plug'n'Play projects, Intent loads Yarn's PnP runtime (`.pnp.cjs`) to map package identities to readable locations. It loads no package entry points, bins, lifecycle scripts, or other package-provided JavaScript, and an ESLint rule enforces that invariant in the discovery code. + +## Delivery affects the guarantee + +The lockfile check runs when Intent runs. How skills reach your agent between those runs depends on the delivery method you chose at install. + +Symlink delivery links the live package folders into your agent's directories. A package update can change linked content before Intent re-checks `intent.lock`; Intent detects the drift the next time it runs, but it cannot stop an agent from reading changed content in the meantime. Hook delivery surfaces only skills already accepted in the lockfile, so changes are held for review before they can reach the agent. Choose hooks when you want every change reviewed first. -## What the allowlist does not cover yet +## Current limits -Matching is currently by package name. A `workspace:foo` entry and a bare `foo` entry both authorize a discovered package named `foo`, because the scanner does not yet distinguish a workspace member from a published package of the same name. This errs toward permitting a same-named package, never toward denying one you listed. A future version tightens matching once the scanner carries that signal. +Matching is currently by package name. A `workspace:foo` entry and a bare `foo` entry both authorize a discovered package named `foo`, because the scanner does not yet distinguish a workspace member from a published package of the same name. This errs toward permitting a same-named package, never toward denying one you listed. -The `git:` source kind is reserved. Intent parses and validates the shape, then rejects it until a future version can pin the resolved ref and content hash. A git entry never loads silently. +The `git:` source kind is reserved. Intent validates the shape but rejects it for now, so a git entry never loads silently. diff --git a/docs/config.json b/docs/config.json index 11e09090..1f8b96e0 100644 --- a/docs/config.json +++ b/docs/config.json @@ -17,6 +17,10 @@ "label": "Quick Start (Consumers)", "to": "getting-started/quick-start-consumers" }, + { + "label": "Troubleshooting", + "to": "getting-started/troubleshooting" + }, { "label": "Quick Start (Maintainers)", "to": "getting-started/quick-start-maintainers" @@ -47,6 +51,10 @@ "label": "intent install", "to": "cli/intent-install" }, + { + "label": "intent sync", + "to": "cli/intent-sync" + }, { "label": "intent hooks", "to": "cli/intent-hooks" @@ -59,6 +67,10 @@ "label": "intent list", "to": "cli/intent-list" }, + { + "label": "intent catalog", + "to": "cli/intent-catalog" + }, { "label": "intent load", "to": "cli/intent-load" diff --git a/docs/getting-started/quick-start-consumers.md b/docs/getting-started/quick-start-consumers.md index 374742a8..c8753b88 100644 --- a/docs/getting-started/quick-start-consumers.md +++ b/docs/getting-started/quick-start-consumers.md @@ -3,123 +3,103 @@ title: Quick Start for Consumers id: quick-start-consumers --- -Get started using Intent to help your agent discover and load package skills. +When a library you depend on ships Agent Skills, Intent puts that guidance in front of your coding agent. A skill tells your agent what to do, so you choose which dependencies to trust and how their skills reach your agent. -## 1. Run install +## Before you start -The install command guides your agent through the setup process: +You need a project with a `package.json` and at least one installed dependency that ships skills. To check what dependencies in your project offer skills before you set anything up, Intent can scan your `node_modules` and report the candidates: -```bash -npx @tanstack/intent@latest install -``` + -Examples use `npx` for npm projects. In pnpm, Yarn, or Bun projects, use the matching runner: `pnpm dlx`, `yarn dlx`, or `bunx`. +@tanstack/intent@latest list --show-hidden -This creates or updates an `intent-skills` guidance block. It: + -1. Checks for existing `intent-skills` guidance in your config files (`AGENTS.md`, `CLAUDE.md`, `.cursorrules`, etc.) -2. Writes lightweight instructions for skill discovery and loading -3. Preserves content outside the managed block -4. Verifies the managed block before reporting success +Until you trust a package, its skills stay hidden, so `--show-hidden` is what reveals the candidates. Without it, a fresh project reports no packages even when a dependency ships skills. -If an `intent-skills` block already exists, Intent updates that file in place. -If no block exists, `AGENTS.md` is the default target. +## How to run Intent -Intent creates guidance like: +Every command in this guide works with `npx @tanstack/intent@latest` and no install. That is fine for a quick start or a one-off, but `@latest` fetches whatever version is current, so a new release can change how a command behaves. -```markdown - -## Skill Loading +For the most stable experience, add Intent as a dev dependency: -Before editing files for a substantial task: -- Run `pnpm dlx @tanstack/intent@latest list` from the workspace root to see available local skills. -- If a listed skill matches the task, run `pnpm dlx @tanstack/intent@latest load #` before changing files. -- Use the loaded `SKILL.md` guidance while making the change. -- Monorepos: when working across packages, run the skill check from the workspace root and prefer the local skill for the package being changed. -- Multiple matches: prefer the most specific local skill for the package or concern you are changing; load additional skills only when the task spans multiple packages or concerns. - -``` + -Intent detects the package manager when generating this block, so the runner may be `npx`, `pnpm dlx`, `yarn dlx`, or `bunx`. +@tanstack/intent -To enforce loading guidance before edits in supported agents, opt in to hooks: + -```bash -npx @tanstack/intent@latest hooks install -``` +Your lockfile then records the exact version, so everyone on your team runs the same Intent and upgrades happen when you choose. With Intent installed and symlink delivery, `install` also adds a `prepare` script that runs `intent sync` after each `npm install`, so your managed links stay current without anyone remembering to run it. -Project-scoped hooks are installed for Claude Code and Codex. `intent install` can write project guidance to `.github/copilot-instructions.md`, but GitHub Copilot CLI hook enforcement is user-scoped, so configure it explicitly: +## Install skills -```bash -npx @tanstack/intent@latest hooks install --scope user --agents copilot -``` +`install` runs an interactive setup, so run it in a terminal. For committed agent loading instructions without managed delivery, use [portable guidance](#portable-guidance) instead. -Cursor and generic `AGENTS.md` agents use the guidance block only. + -Hooks add the available Intent skill catalog to supported agent sessions and keep the edit gate active until the agent loads matching full guidance. To tailor what appears in the session catalog, configure `intent.skills` and `intent.exclude` in `package.json`. +@tanstack/intent@latest install -## 2. Choose which packages' skills to use + -`package.json#intent.skills` is an allowlist of the packages whose skills you want surfaced. +Intent asks: -```json -{ - "intent": { - "skills": ["@tanstack/*"] - } -} -``` +- **How to deliver skills.** You can symlink the skill folders into your agent's directories, install lifecycle hooks that list available skills at the start of a session, or write static catalog loading guidance into an agent file such as `AGENTS.md`. +- **Where to put them.** Intent pre-selects the agent tools it can detect, such as GitHub Copilot, Cursor, Claude Code, Codex, VS Code, or a shared `.agents` directory. Symlinks support all of these; hooks at this time only support GitHub Copilot, Claude Code, and Codex (if you'd like to add hooks for other agents or platforms, we welcome contributions). You can also choose a custom folder for symlinks or hooks. +- **Which skills to trust.** Enable every skill it found, everything under a certain package name (eg. `@tanstack/*`), or pick individual skills. Only the packages you enable here can provide skills to your agent. +- **A final confirmation** before it writes anything. -List the packages or `*` package patterns you trust. Intent then surfaces skills from matching packages and leaves the rest out. See the [source entries](../concepts/configuration#source-entries) in Configuration for the forms an entry can take, and [Trust model](../concepts/trust-model) for why the allowlist exists. +> [!WARNING] +> Using symlinks can expose live package content before Intent can re-checks it. This means a skill can be updated in a dependency without Intent reviewing it first. If you want to review new or changed skills before they reach your agent, choose hook delivery instead. -## 3. Use skills in your workflow +Once finished, Intent prints a line describing how many skills were installed, e.g., `Installed 5 skills using symlink.` -When your agent works on a task that matches an available skill, it loads the matching `SKILL.md` into context. +## What install writes -Load a skill manually: +Intent records your choices in three files: -```bash -npx @tanstack/intent@latest load @tanstack/react-query#core -``` +- `package.json` holds explicit `intent.skills` and `intent.exclude` arrays for the sources you trust and the skills or packages you do not want. +- `intent.lock` holds contains the accepted skill contents, so teams can share the same baseline. It also records the package versions that shipped those skills, so Intent can detect when a dependency update changes its skills, or the contents of a skill you already accepted have changed. +- `.intent/delivery.json` holds your local delivery method and targets. -This prints the skill content for the installed package version. +> [!NOTE] +> Intent adds `.intent/` to the project `.gitignore`. If you chose symlinks, it also adds the generated link paths to the checkout's `.git/info/exclude` so they do not get committed. -If you want explicit task-to-skill mappings in your agent config, opt in: +Commit `package.json` and `intent.lock` if you're looking for the project to share the same trusted sources and accepted skills. `.intent/` stays local to your checkout. -```bash -npx @tanstack/intent@latest install --map -``` +## Check that it worked -## 4. Keep skills up-to-date +List the skills your project now trusts: -Skills version with library releases. When you update a library: + -```bash -npm update @tanstack/react-query -``` +@tanstack/intent@latest list -The new version brings updated skills automatically. The skills are shipped with the library, so you get the version that matches your installed code. If a package is installed both locally and globally and global scanning is enabled, Intent prefers the local version. + -If you need to see what skills have changed, run: +Intent prints a summary such as `5 intent-enabled packages, 12 skills`, then the packages you trusted and their skills. Load one to read its guidance: -```bash -npx @tanstack/intent@latest list -``` + -Use `--json` for machine-readable output: +@tanstack/intent@latest load @tanstack/query#fetching -```bash -npx @tanstack/intent@latest list --json -``` + -Global package scanning is opt-in: +Replace `@tanstack/query#fetching` with a package and skill from your own list. `load` prints the `SKILL.md` shipped with the version installed in your project, and your agent can run the same command when it needs that guidance. -```bash -npx @tanstack/intent@latest list --global -``` +If a command does not behave as described, see [Troubleshooting](./troubleshooting). -You can also check if any skills reference outdated source documentation: +## Keep skills current -```bash -npx @tanstack/intent@latest stale -``` +Updating a dependency can add, remove, or change its skills. With symlink delivery, run [`intent sync`](../cli/intent-sync) to update the links; it flags new or changed skills for review before they reach your agent. Run `install` again when you are ready to accept a new baseline. See the [trust model](../concepts/trust-model) for how that review works. + +## Portable guidance + +`install --map` writes compact catalog loading instructions into an agent file such as `AGENTS.md` instead of setting up managed delivery: + + + +@tanstack/intent@latest install --map + + + +The instructions tell the agent to run `intent catalog` once if a catalog is not already present in session context, then load only a matching skill with `intent load `. When Intent is a dev dependency, generated commands use the local binary; otherwise they use a pinned one-off runner. Hooks inject the same catalog automatically, while symlinks expose accepted skill folders directly. diff --git a/docs/getting-started/troubleshooting.md b/docs/getting-started/troubleshooting.md new file mode 100644 index 00000000..5e3fc941 --- /dev/null +++ b/docs/getting-started/troubleshooting.md @@ -0,0 +1,30 @@ +--- +title: Troubleshooting +id: troubleshooting +--- + +Fixes for common problems when you consume skills with Intent. New to Intent? Start with the [consumer quick start](./quick-start-consumers). + +## `list` reports no packages + +Before you install, nothing is trusted, so `intent list` reports no packages even when a dependency ships skills. Run `intent list --show-hidden` to see the candidates, then enable the ones you want during `install`. + +## A skill you expected is missing + +Add `--why` to see how Intent classified each source: + + + +@tanstack/intent@latest list --show-hidden --why + + + +A skill can be missing because you did not enable its package during install, or because an `intent.exclude` pattern removed it. + +## Install exits without prompting + +Interactive install needs a terminal. For committed instructions without managed delivery, use [portable guidance](./quick-start-consumers#portable-guidance) instead. + +## Symlinks are not available + +Some setups, such as Yarn Plug'n'Play, cannot expose package skills as real folders. Choose another method of delivery instead when `install` asks how to deliver skills. diff --git a/docs/overview.md b/docs/overview.md index 237fffeb..9886ee8c 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -1,88 +1,22 @@ ---- -title: Overview -id: overview ---- - -`@tanstack/intent` is a CLI for shipping and consuming Agent Skills as package artifacts. - -Skills are markdown documents that teach AI coding agents how to use your library correctly. Intent versions them with your releases and ships them inside npm packages. It discovers skills from your project and workspace dependencies, then helps agents load them when working on matching tasks. - -## What Intent does - -Intent provides tooling for two workflows: - -**For consumers:** - -- Discover skills from your project and workspace dependencies -- Control which packages' skills are surfaced with an allowlist -- Add lightweight skill loading guidance to your agent config -- Add hook enforcement for agents that support blocking lifecycle hooks -- Keep skills synchronized with library versions - -**For maintainers (library teams):** - -- Scaffold skills through AI-assisted domain discovery -- Validate SKILL.md format and packaging -- Ship skills in the same release pipeline as code -- Track staleness when source docs change - -## How it works - -### Discovery and installation - -Examples use `npx` for npm projects. In pnpm, Yarn, or Bun projects, use the matching runner: - -| Tool | Pattern | -| ---- | -------------------------------------------- | -| npm | `npx @tanstack/intent@latest ` | -| pnpm | `pnpm dlx @tanstack/intent@latest ` | -| Yarn | `yarn dlx @tanstack/intent@latest ` | -| Bun | `bunx @tanstack/intent@latest ` | - -```bash -npx @tanstack/intent@latest list -``` - -Scans the current project's installed dependencies for intent-enabled packages, including `node_modules`, workspace dependencies, and Yarn PnP projects without `node_modules`. You can narrow which packages are surfaced with `package.json#intent.skills`. See the [Trust model](./concepts/trust-model) and [Configuration](./concepts/configuration) for how the allowlist works. -Global package scanning is explicit; pass `--global` to include global packages or `--global-only` to ignore local packages. -When both local and global packages are scanned, local packages take precedence. - -```bash -npx @tanstack/intent@latest install -``` - -Creates or updates lightweight `intent-skills` guidance in your config files (`AGENTS.md`, `CLAUDE.md`, `.cursorrules`, etc.). Existing guidance is updated in place; otherwise `AGENTS.md` is the default target. Pass `--map` to opt in to explicit task-to-skill mappings. - -```bash -npx @tanstack/intent@latest hooks install -``` - -Installs hook enforcement for supported agents. Project-scoped hooks are available for Claude Code and Codex. GitHub Copilot CLI project guidance can live in `.github/copilot-instructions.md`, while blocking hooks are user-scoped. Cursor and generic `AGENTS.md` agents use guidance only. - -```bash -npx @tanstack/intent@latest load @tanstack/query#fetching -``` - -Loads the matching `SKILL.md` content for the installed package version. Pass `--path` when you need the resolved skill file path for debugging. - -### Scaffolding and validation - -```bash -npx @tanstack/intent@latest scaffold -``` - -Guides your agent through domain discovery, tree generation, and skill authoring with interactive maintainer interviews. - -```bash -npx @tanstack/intent@latest validate -``` - -Enforces SKILL.md format rules and packaging requirements before publish. - -### Staleness tracking - -```bash -npx @tanstack/intent@latest stale -``` - -Detects when skills reference outdated source documentation or library versions. +--- +title: Overview +id: overview +--- + +An Agent Skill is a set of instructions that helps a coding agent work with a library or handle a particular task. `@tanstack/intent` lets libraries include these skills in their packages, so each package version can carry matching guidance. + +Skills come from your dependencies, and a skill tells your agent what to do, so Intent only uses skills from the packages you choose to trust. You decide which packages may provide skills and how your agents receive them. + +## Use skills from a dependency + +If a dependency already includes skills, start with the [consumer quick start](./getting-started/quick-start-consumers). During installation, you approve the packages you trust to provide skills and choose how to deliver those skills to your agents. + +Intent records the sources you approved in `package.json`, the accepted skill contents in `intent.lock`, and your local delivery choice in `.intent/delivery.json`. A package that ships skills contributes nothing until you list it among the sources you trust. + +Symlink installs use `sync` to keep agent links current, and it flags new or changed skills for review before they reach your agent. Hooks are another delivery option, and a static guidance block can write the skills into a file such as `AGENTS.md`. Read the [trust model](./concepts/trust-model) for how packages and skill changes are approved, or [configuration](./concepts/configuration) for the available settings. + +## Publish skills with a library + +If you maintain a library, keep its skills in the package alongside the code they describe, so each release ships the guidance written for it. Start with the [maintainer quick start](./getting-started/quick-start-maintainers). + +Intent scaffolds skills with your agent, validates their format and packaging before you publish, and reports when a skill looks stale as the library changes. The [registry](./registry) explains how to make a published package discoverable. diff --git a/evals/intent-discovery/README.md b/evals/intent-discovery/README.md index 76f36476..0f5c3c62 100644 --- a/evals/intent-discovery/README.md +++ b/evals/intent-discovery/README.md @@ -4,7 +4,7 @@ Opt-in eval suite for measuring whether Copilot discovers and invokes Intent sur ## Commands -- `pnpm eval:intent-discovery` runs the saved-transcript eval suite. +- `pnpm eval:intent-discovery` runs grader and harness regression fixtures. It does not measure live product efficacy. - `pnpm eval:intent-discovery:json` writes `evals/intent-discovery/runs/latest/vitest-results.json`. - `pnpm eval:intent-discovery:live` runs the eval suite with the local Copilot CLI adapter enabled. - `pnpm eval:intent-discovery:live:json` writes a JSON report that includes live Copilot condition cases. @@ -20,33 +20,34 @@ pnpm eval:intent-discovery:summary pnpm eval:intent-discovery:report ``` -Set `INTENT_DISCOVERY_RUN_COUNT=3` with the live commands to run each live condition three times and include `pass@k` / `pass^k` in the generated summary. - -## Live eval speed +## Live matrix Only the live `copilot -p` subprocess runs are slow; the saved-transcript suite (`pnpm eval:intent-discovery`) is unaffected. -- `INTENT_DISCOVERY_LIVE_CONCURRENCY` bounds how many live runs execute at once (default `1`, clamped to an integer `>= 1`). Values above `1` measured slower here: concurrent `copilot -p` calls on one account contend upstream (a run with its own isolated `COPILOT_HOME` still slowed ~2x), so raise it only with separate accounts or dedicated infrastructure. -- `COPILOT_MODEL` selects the Copilot model end-to-end. The adapter passes the process environment through to `copilot -p`, and the CLI honors `COPILOT_MODEL`. `INTENT_DISCOVERY_COPILOT_MODEL` only sets the model label recorded in report metadata; it does not change the model the CLI runs. -- `INTENT_DISCOVERY_RUN_COUNT` stays `1` by default for iteration. Set it to `3` only when measuring `pass@k` / `pass^k`. +- The default matrix contains 20 isolated sessions: five model/reasoning profiles paired across unaided, symlink, map, and hook delivery. +- Every session preserves one Copilot session and workspace across six turns: three related tasks, two clearly unrelated tasks, and one table-named distractor that must not load the TanStack Table skill. +- Profiles are `claude-haiku-4.5/default`, `claude-sonnet-4.6/medium`, `claude-opus-4.8/high`, `gpt-5.4-mini/low`, and `gpt-5.6-sol/high`. `default` means the model rejects configurable reasoning effort; no silent fallback is allowed. +- Sessions run serially by default. `INTENT_DISCOVERY_LIVE_CONCURRENCY` can raise concurrency, but concurrent `copilot -p` calls on one account previously measured slower. -The optional LLM judge is secondary. It can annotate whether final answers appear to apply loaded guidance, but it never changes deterministic scores such as `StrictIntentInvocation`, `CorrectSkillLoaded`, or `AutonomousDiscoverySuccess`. +The optional LLM judge is secondary. It never changes deterministic session, catalog, discovery, abstention, or task-completion scores. ## Current scope This executable slice grades synthetic saved transcripts with Vitest plus `vitest-evals` harness normalization helpers. It attaches `vitest-evals`-compatible metadata to the Vitest JSON artifact for the local report UI because this repo's current Vitest runtime does not expose the APIs used by `vitest-evals/reporter` and `describeEval()`. -The controlled fixture corpus is limited to current skill-backed surfaces. For this slice, that means TanStack Router, TanStack Start, and TanStack Table v9. +The controlled fixture corpus is limited to TanStack Router, TanStack Start, and TanStack Table v9. It generates synthetic benchmark skills with capability-oriented descriptions and task-relevant guidance because this repository does not contain the published skills for those packages. The live matrix measures autonomous discovery and exact loading. It does not establish that published skill guidance improves task outcomes. + +Live sessions compare four delivery conditions: -Live Router runs compare four setup conditions: +- `no-intent`: no Intent package policy, catalog guidance, hooks, or native skill links. This is the unaided task-completion control; discovery metrics are reported as `n/a`. +- `symlink-intent`: package skills are symlinked into `.github/skills` for native GitHub Copilot discovery. +- `mapped-intent`: production `install --map` guidance asks the agent to catalog once, load a match for related turns, and continue normally for unrelated turns. +- `hooked-intent`: the production Copilot lifecycle hook injects the trusted catalog when each new or resumed CLI process and subagent starts; the agent loads matches for related turns and continues normally for unrelated turns. Copilot does not persist hook context across process resumes or expose a post-compaction context injection event. -- `no-intent`: no Intent guidance or allowlist is added. -- `current-intent`: `package.json#intent.skills` plus the current install-style `AGENTS.md` skill-loading guidance. -- `mapped-intent`: `package.json#intent.skills` plus `AGENTS.md` task-to-skill mappings like `install --map`. -- `explicit-intent-control`: current install-style setup plus a prompt that explicitly asks the agent to run Intent. This condition is diagnostic and excluded from autonomous scoring. +The live Copilot harness can run an opt-in command backend through `INTENT_DISCOVERY_COPILOT_COMMAND`. When unset, it returns a normalized `unsupported` run. Each live session uses a fresh composite fixture, valid trust lock, isolated `COPILOT_HOME`, explicit model and reasoning effort, and one UUID resumed across six separate `copilot -p` processes. Hook delivery requires exactly one catalog injection per process. -The live Copilot harness can run an opt-in command backend through `INTENT_DISCOVERY_COPILOT_COMMAND`. When that environment variable is unset, it returns a normalized `unsupported` run with no tool calls and an explicit `LiveCopilotRunnerUnavailableError`. The command runs inside a prepared fixture workspace with task metadata in `INTENT_DISCOVERY_TASK_ID`, `INTENT_DISCOVERY_FIXTURE`, `INTENT_DISCOVERY_PROMPT`, `INTENT_DISCOVERY_RUN_ID`, and `INTENT_DISCOVERY_WORKSPACE`. +`pnpm eval:intent-discovery:live` sets the repo-local Copilot CLI adapter. Structured events in the isolated home provide turn-local native and shell evidence; malformed or incomplete event evidence fails the session instead of becoming a silent discovery miss. Share transcripts remain diagnostic artifacts. Do not put API keys or tokens in commands or prompts. -`pnpm eval:intent-discovery:live` sets `INTENT_DISCOVERY_RUN_LIVE=1` and `INTENT_DISCOVERY_COPILOT_COMMAND` to the repo-local Copilot CLI adapter. The adapter calls `copilot -p` in the prepared fixture workspace, writes a Copilot share transcript under the generated run directory, and prints the transcript for command capture. Live runs attach the same strict efficacy scores as saved transcripts, so a passing harness run can still report `AutonomousDiscoverySuccess: 0` when Copilot did not invoke Intent or loaded the wrong skill. Do not put API keys or tokens in the command or prompt; provide credentials through the normal Copilot CLI login or secret environment configuration. +The primary discovery metric is exact related-turn loading with wrong-load and distractor penalties. Strict session success remains the reliability bar: a delivery session passes only when catalog behavior is correct, all three related turns use the exact expected guidance, all three unrelated turns abstain, all six tasks complete, every runner turn completes, and no wrong guidance loads occur. Task completion is reported against `no-intent`, but guidance-value claims require a separate benchmark using published skills and held-out acceptance criteria. -Harness integrity failures fail the eval. Product findings such as reference-only behavior, no discovery attempt, or wrong skill selection are recorded as diagnostic failures, not passing scores. The headline success signal is strict Intent invocation plus the expected skill loaded for autonomous cases. +This suite executes GitHub Copilot CLI only. Model names selected inside Copilot do not test Claude Code or Codex lifecycle behavior. Cross-agent claims require agent-native runners and evidence capture. diff --git a/evals/intent-discovery/bin/copilot-cli-adapter.mjs b/evals/intent-discovery/bin/copilot-cli-adapter.mjs index 80a7c860..59348c7b 100644 --- a/evals/intent-discovery/bin/copilot-cli-adapter.mjs +++ b/evals/intent-discovery/bin/copilot-cli-adapter.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { existsSync, mkdirSync, readFileSync } from 'node:fs' +import { mkdirSync } from 'node:fs' import { dirname, join } from 'node:path' import { spawnSync } from 'node:child_process' @@ -9,10 +9,14 @@ const taskId = requiredEnv('INTENT_DISCOVERY_TASK_ID') const fixture = requiredEnv('INTENT_DISCOVERY_FIXTURE') const prompt = requiredEnv('INTENT_DISCOVERY_PROMPT') const runId = requiredEnv('INTENT_DISCOVERY_RUN_ID') +const sessionId = requiredEnv('INTENT_DISCOVERY_SESSION_ID') +const turnId = requiredEnv('INTENT_DISCOVERY_TURN_ID') +const model = requiredEnv('INTENT_DISCOVERY_COPILOT_MODEL') +const effort = requiredEnv('INTENT_DISCOVERY_REASONING_EFFORT') const sharePath = join( workspace, '.intent-eval', - `${sanitizeFileName(runId)}.md`, + `${sanitizeFileName(runId)}-${sanitizeFileName(turnId)}.md`, ) mkdirSync(dirname(sharePath), { recursive: true }) @@ -37,6 +41,12 @@ const args = [ '--no-ask-user', '--no-color', '--plain-diff', + '--silent', + '--session-id', + sessionId, + '--model', + model, + ...(effort === 'default' ? [] : ['--effort', effort]), '--share', sharePath, ] @@ -60,10 +70,7 @@ if (result.stdout.trim()) { console.log(result.stdout.trim()) } -if (existsSync(sharePath)) { - console.log(`\nTRANSCRIPT_PATH: ${sharePath}`) - console.log(readFileSync(sharePath, 'utf8')) -} +console.log(`TRANSCRIPT_PATH: ${sharePath}`) if (result.stderr.trim()) { console.error(result.stderr.trim()) diff --git a/evals/intent-discovery/bin/llm-judge.mjs b/evals/intent-discovery/bin/llm-judge.mjs index 0e3bd57d..e19006c7 100644 --- a/evals/intent-discovery/bin/llm-judge.mjs +++ b/evals/intent-discovery/bin/llm-judge.mjs @@ -44,22 +44,22 @@ function reportCases(report) { .map((test) => { const run = test.meta.harness?.run ?? {} const artifacts = run.artifacts ?? {} + const scoreEntries = test.meta.eval.scores ?? [] const scores = Object.fromEntries( - (test.meta.eval.scores ?? []).map((score) => [ - score.name, - score.score ?? 0, - ]), + scoreEntries.map((score) => [score.name, score.score ?? 0]), + ) + const loaded = scoreEntries.find( + (score) => score.name === 'CorrectSkillLoaded', ) return { - artifacts: pick(artifacts, [ - 'condition', - 'expectedSkillAreas', - 'intentCommandsInvoked', - 'loadedSkills', - 'runnerStatus', - 'taskId', - ]), + artifacts: { + condition: artifacts.condition, + expectedSkillAreas: artifacts.expectedSkillAreas, + loadedSkills: loaded?.metadata?.loadedSkills ?? [], + runnerStatus: artifacts.runnerStatus, + taskId: artifacts.taskId, + }, finalAnswer: test.meta.eval.output?.finalAnswer ?? '', scores, title: test.title, @@ -139,11 +139,3 @@ async function judgeCase({ apiKey, item, model }) { title: item.title, } } - -function pick(value, keys) { - return Object.fromEntries( - keys - .filter((key) => Object.prototype.hasOwnProperty.call(value, key)) - .map((key) => [key, value[key]]), - ) -} diff --git a/evals/intent-discovery/bin/summarize-results.mjs b/evals/intent-discovery/bin/summarize-results.mjs index 7032e14c..8d18095d 100644 --- a/evals/intent-discovery/bin/summarize-results.mjs +++ b/evals/intent-discovery/bin/summarize-results.mjs @@ -17,128 +17,203 @@ writeFileSync( writeFileSync(join(outDir, 'summary.md'), `${formatSummaryMarkdown(summary)}\n`) console.log(formatSummaryMarkdown(summary)) -export function summarizeReport(report) { - const cases = reportCases(report) - const byCondition = groupBy(cases, (item) => item.condition ?? 'unknown') - const conditionSummaries = Object.fromEntries( - [...byCondition.entries()].map(([condition, items]) => [ - condition, - summarizeCases(items), - ]), +function summarizeReport(value) { + const cases = reportCases(value) + const liveSessions = cases.filter( + (item) => + item.runKind === 'live-copilot' && + item.runnerStatus === 'completed' && + item.sessionScore, + ) + const byCondition = Object.fromEntries( + [...groupBy(liveSessions, (item) => item.condition).entries()].map( + ([condition, items]) => [condition, summarizeSessions(items)], + ), + ) + const byProfile = liveSessions.map((item) => ({ + ...summarizeSessions([item]), + condition: item.condition, + effort: item.effort, + model: item.model, + profileId: item.profileId, + })) + const turnOutcomes = Object.fromEntries( + [ + ...groupBy( + liveSessions.flatMap((session) => + session.turns.map((turn) => ({ session, turn })), + ), + ({ session, turn }) => `${session.condition}/${turn.id}`, + ).entries(), + ].map(([key, items]) => [key, summarizeTurns(items)]), ) return { generatedAt: new Date().toISOString(), totals: { + liveSessions: liveSessions.length, reportCases: cases.length, - testFailures: report.numFailedTests ?? 0, - testPasses: report.numPassedTests ?? 0, - testSuites: report.numTotalTestSuites ?? 0, + testFailures: value.numFailedTests ?? 0, + testPasses: value.numPassedTests ?? 0, + testSuites: value.numTotalTestSuites ?? 0, }, - byCondition: conditionSummaries, - failureClasses: countBy( - cases.map((item) => item.failureClass ?? 'unknown'), - ), - repeatedRuns: repeatedRunSummary(cases), + byCondition, + byProfile, + turnOutcomes, } } -function reportCases(report) { - return (report.testResults ?? []).flatMap((suite) => +function reportCases(value) { + return (value.testResults ?? []).flatMap((suite) => (suite.assertionResults ?? []) .filter((test) => test.meta?.eval) .map((test) => { const artifacts = test.meta.harness?.run?.artifacts ?? {} + const profile = artifacts.profile ?? {} const scores = Object.fromEntries( - (test.meta.eval.scores ?? []).map((score) => [ - score.name, - score.score ?? 0, + (test.meta.eval.scores ?? []).map((entry) => [ + entry.name, + entry.score ?? 0, ]), ) - const firstScore = test.meta.eval.scores?.[0] return { - condition: artifacts.condition, - failureClass: firstScore?.metadata?.failureClass, - fixture: artifacts.fixture, - loadedSkills: artifacts.loadedSkills ?? [], + condition: artifacts.condition ?? 'unknown', + effort: profile.effort ?? artifacts.effort ?? 'unknown', + model: profile.model ?? artifacts.model ?? 'unknown', + profileId: profile.id ?? artifacts.profileId ?? 'unknown', + runKind: artifacts.runKind, + runnerStatus: artifacts.runnerStatus, scores, - taskId: artifacts.taskId ?? test.title, - title: test.title, + sessionScore: artifacts.sessionScore, + turns: artifacts.turns ?? [], } }), ) } -function summarizeCases(cases) { +function summarizeSessions(cases) { + const scores = cases.map((item) => item.sessionScore) + const discoveryExpected = cases.every( + (item) => item.condition !== 'no-intent', + ) + const observedTurns = cases.reduce( + (total, item) => total + item.turns.length, + 0, + ) return { - autonomousSuccessRate: rate(cases, 'AutonomousDiscoverySuccess'), - correctSkillLoadedRate: rate(cases, 'CorrectSkillLoaded'), - count: cases.length, - referenceOnlyFalsePositiveRate: rate(cases, 'NoReferenceOnlyFalsePositive'), - strictInvocationRate: rate(cases, 'StrictIntentInvocation'), + agentCatalogCommands: sum(scores, 'agentCatalogCount'), + catalogBehaviorRate: discoveryExpected + ? rate(cases, 'CatalogBehavior') + : null, + discoveryExpected, + hookCatalogInjections: sum(scores, 'hookCatalogInjections'), + relatedDiscoveryRate: discoveryExpected + ? ratio(scores, 'relatedCorrect', 'relatedTotal') + : null, + runnerCompletionRate: + observedTurns === 0 + ? 0 + : sum(scores, 'runnerCompletionCount') / observedTurns, + sessionSuccessRate: discoveryExpected + ? rate(cases, 'SessionSuccess') + : null, + sessions: cases.length, + strictSuccesses: discoveryExpected + ? cases.filter((item) => item.scores.SessionSuccess === 1).length + : null, + taskCompletionRate: + observedTurns === 0 + ? 0 + : sum(scores, 'taskCompletionCount') / observedTurns, + unrelatedAbstentionRate: discoveryExpected + ? ratio(scores, 'unrelatedCorrect', 'unrelatedTotal') + : null, + wrongSkillLoads: discoveryExpected ? sum(scores, 'wrongSkillLoads') : null, } } -function repeatedRunSummary(cases) { - const liveCases = cases.filter((item) => item.title.includes('/run-')) - const grouped = groupBy(liveCases, (item) => - item.title.replace(/\/run-\d+$/, ''), +function summarizeTurns(items) { + const discoveryExpected = items.every( + ({ session }) => session.condition !== 'no-intent', ) + let discoveryCorrect = 0 + let taskCompleted = 0 - return Object.fromEntries( - [...grouped.entries()].map(([key, items]) => { - const successes = items.map( - (item) => item.scores.AutonomousDiscoverySuccess === 1, - ) - - return [ - key, - { - passAtK: successes.some(Boolean), - passHatK: successes.every(Boolean), - runs: items.length, - successes: successes.filter(Boolean).length, - }, - ] - }), - ) + for (const { session, turn } of items) { + const result = session.sessionScore.turnResults?.find( + (candidate) => candidate.id === turn.id, + ) + if (result?.discoveryCorrect) discoveryCorrect++ + if (turn.taskPassed) taskCompleted++ + } + + return { + agentCatalogCommands: items.reduce( + (total, { turn }) => total + turn.catalogCommands.length, + 0, + ), + discoveryExpected, + discoveryRate: + !discoveryExpected || items.length === 0 + ? null + : discoveryCorrect / items.length, + hookCatalogInjections: items.reduce( + (total, { turn }) => total + turn.hookCatalogInjections, + 0, + ), + sessions: items.length, + taskCompletionRate: items.length === 0 ? 0 : taskCompleted / items.length, + } } function formatSummaryMarkdown(summary) { const lines = [ - '# Intent discovery eval summary', + '# Intent discovery live session summary', '', - `Report cases: ${summary.totals.reportCases}`, + `Live sessions: ${summary.totals.liveSessions}`, `Tests: ${summary.totals.testPasses} passed, ${summary.totals.testFailures} failed`, '', - '## By condition', + '## Strict session success', '', - '| Condition | Cases | Strict invocation | Correct skill | Autonomous success | No reference-only false positive |', - '| --- | ---: | ---: | ---: | ---: | ---: |', + '| Mode | Sessions | Strict success | Catalog | Related discovery | Unrelated abstention | Tasks completed | Wrong loads | Agent catalogs | Hook catalogs |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ] for (const [condition, item] of Object.entries(summary.byCondition)) { + const strictSuccess = item.discoveryExpected + ? `${item.strictSuccesses}/${item.sessions} (${metric(item.sessionSuccessRate)})` + : 'n/a' + const wrongLoads = item.discoveryExpected ? item.wrongSkillLoads : 'n/a' lines.push( - `| ${condition} | ${item.count} | ${percent(item.strictInvocationRate)} | ${percent(item.correctSkillLoadedRate)} | ${percent(item.autonomousSuccessRate)} | ${percent(item.referenceOnlyFalsePositiveRate)} |`, + `| ${condition} | ${item.sessions} | ${strictSuccess} | ${metric(item.catalogBehaviorRate)} | ${metric(item.relatedDiscoveryRate)} | ${metric(item.unrelatedAbstentionRate)} | ${metric(item.taskCompletionRate)} | ${wrongLoads} | ${item.agentCatalogCommands} | ${item.hookCatalogInjections} |`, ) } - lines.push('', '## Failure classes', '') - for (const [failureClass, count] of Object.entries(summary.failureClasses)) { - lines.push(`- ${failureClass}: ${count}`) + lines.push( + '', + '## By model profile', + '', + '| Profile | Model | Effort | Mode | Pass | Catalog | Related | Unrelated | Tasks |', + '| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: |', + ) + for (const item of summary.byProfile) { + lines.push( + `| ${item.profileId} | ${item.model} | ${item.effort} | ${item.condition} | ${item.discoveryExpected ? `${item.strictSuccesses}/${item.sessions}` : 'n/a'} | ${metric(item.catalogBehaviorRate)} | ${metric(item.relatedDiscoveryRate)} | ${metric(item.unrelatedAbstentionRate)} | ${metric(item.taskCompletionRate)} |`, + ) } - lines.push('', '## Repeated runs', '') - const repeated = Object.entries(summary.repeatedRuns) - if (repeated.length === 0) { - lines.push('No repeated live runs found.') - } else { - for (const [key, item] of repeated) { - lines.push( - `- ${key}: pass@k=${item.passAtK}, pass^k=${item.passHatK}, successes=${item.successes}/${item.runs}`, - ) - } + lines.push( + '', + '## Per-turn outcomes', + '', + '| Mode / turn | Sessions | Discovery | Task completion | Agent catalogs | Hook catalogs |', + '| --- | ---: | ---: | ---: | ---: | ---: |', + ) + for (const [key, item] of Object.entries(summary.turnOutcomes)) { + lines.push( + `| ${key} | ${item.sessions} | ${metric(item.discoveryRate)} | ${metric(item.taskCompletionRate)} | ${item.agentCatalogCommands} | ${item.hookCatalogInjections} |`, + ) } return lines.join('\n') @@ -153,15 +228,6 @@ function groupBy(items, keyFn) { return grouped } -function countBy(items) { - return Object.fromEntries( - [...groupBy(items, (item) => item).entries()].map(([key, values]) => [ - key, - values.length, - ]), - ) -} - function rate(cases, scoreName) { if (cases.length === 0) return 0 return ( @@ -169,6 +235,23 @@ function rate(cases, scoreName) { ) } +function ratio(items, numerator, denominator) { + const total = items.reduce( + (value, item) => value + Number(item[denominator] ?? 0), + 0, + ) + if (total === 0) return 0 + return sum(items, numerator) / total +} + +function sum(items, key) { + return items.reduce((total, item) => total + Number(item[key] ?? 0), 0) +} + function percent(value) { return `${Math.round(value * 100)}%` } + +function metric(value) { + return value === null ? 'n/a' : percent(value) +} diff --git a/evals/intent-discovery/condition-setup.eval.ts b/evals/intent-discovery/condition-setup.eval.ts index 0fec098a..4ba8aee3 100644 --- a/evals/intent-discovery/condition-setup.eval.ts +++ b/evals/intent-discovery/condition-setup.eval.ts @@ -1,4 +1,11 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { + existsSync, + lstatSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' @@ -16,7 +23,7 @@ describe('Intent discovery condition setup', () => { workspacePath: prepared.workspacePath, }) - expect(result.filesWritten).toEqual([]) + expect(result).toEqual([]) expect(existsSync(join(prepared.workspacePath, 'AGENTS.md'))).toBe(false) expect( readFileSync(join(prepared.workspacePath, 'package.json'), 'utf8'), @@ -26,48 +33,47 @@ describe('Intent discovery condition setup', () => { } }) - it('writes current Intent guidance without mappings', () => { + it('symlinks package skills for native GitHub Copilot discovery', () => { const prepared = prepareInTemp() try { const result = applyIntentCondition({ - condition: 'current-intent', + condition: 'symlink-intent', expectedSkillAreas: ['router'], workspacePath: prepared.workspacePath, }) - const agents = readFileSync( - join(prepared.workspacePath, 'AGENTS.md'), - 'utf8', + const linkPath = join( + prepared.workspacePath, + '.github', + 'skills', + 'npm-tanstack-router-routing', ) - const packageJson = readFileSync( - join(prepared.workspacePath, 'package.json'), - 'utf8', + const skillPath = join( + prepared.workspacePath, + 'node_modules', + '@tanstack', + 'router', + 'skills', + 'routing', ) - expect(result.filesWritten).toHaveLength(4) - expect(agents).toContain('Skill Loading') - expect(agents).toContain('npx @tanstack/intent@latest list') - expect(agents).not.toContain('\ntanstackIntent:\n') - expect(packageJson).toContain('"@tanstack/router"') - expect( - existsSync( - join( - prepared.workspacePath, - 'node_modules', - '@tanstack', - 'router', - 'skills', - 'routing', - 'SKILL.md', - ), - ), - ).toBe(true) + expect(result).toContain(linkPath) + expect(existsSync(join(prepared.workspacePath, 'AGENTS.md'))).toBe(false) + expect(existsSync(join(prepared.workspacePath, 'intent.lock'))).toBe(true) + expect(lstatSync(linkPath).isSymbolicLink()).toBe(true) + expect(realpathSync(linkPath)).toBe(realpathSync(skillPath)) + const skill = readFileSync(join(skillPath, 'SKILL.md'), 'utf8') + expect(skill).toContain('TanStack Router route loaders') + expect(skill).toContain( + 'Read loader data through `Route.useLoaderData()`', + ) + expect(skill).not.toMatch(/\beval\b/i) } finally { prepared.cleanup() } }) - it('writes mapped Intent guidance with use values', () => { + it('writes catalog-once guidance for mapped delivery', () => { const prepared = prepareInTemp() try { @@ -81,11 +87,28 @@ describe('Intent discovery condition setup', () => { 'utf8', ) - expect(agents).toContain('tanstackIntent:') - expect(agents).toContain('id: "@tanstack/router#routing"') - expect(agents).toContain( - 'run: "npx @tanstack/intent@latest load @tanstack/router#routing"', - ) + expect(agents).toContain('## Intent Skills') + expect(agents).toContain('npx @tanstack/intent catalog') + expect(agents).toContain('npx @tanstack/intent load #') + expect(agents).not.toContain('tanstackIntent:') + expect(existsSync(join(prepared.workspacePath, 'intent.lock'))).toBe(true) + } finally { + prepared.cleanup() + } + }) + + it('prepares trusted skills for hook delivery without agent guidance', () => { + const prepared = prepareInTemp() + + try { + applyIntentCondition({ + condition: 'hooked-intent', + expectedSkillAreas: ['router'], + workspacePath: prepared.workspacePath, + }) + + expect(existsSync(join(prepared.workspacePath, 'AGENTS.md'))).toBe(false) + expect(existsSync(join(prepared.workspacePath, 'intent.lock'))).toBe(true) } finally { prepared.cleanup() } diff --git a/evals/intent-discovery/corpus/conditions.ts b/evals/intent-discovery/corpus/conditions.ts index 0190f373..6307b598 100644 --- a/evals/intent-discovery/corpus/conditions.ts +++ b/evals/intent-discovery/corpus/conditions.ts @@ -1,50 +1,6 @@ -const intentDiscoveryConditions = [ - { - id: 'no-intent', - countsTowardAutonomousScore: true, - }, - { - id: 'plain-docs', - countsTowardAutonomousScore: true, - }, - { - id: 'current-intent', - countsTowardAutonomousScore: true, - }, - { - id: 'mapped-intent', - countsTowardAutonomousScore: true, - }, - { - id: 'hooked-intent', - countsTowardAutonomousScore: true, - }, - { - id: 'explicit-intent-control', - countsTowardAutonomousScore: false, - }, -] as const - export type IntentDiscoveryCondition = - (typeof intentDiscoveryConditions)[number]['id'] - -const promptExplicitnessLevels = [0, 1, 2, 3, 4] as const - -export type PromptExplicitnessLevel = (typeof promptExplicitnessLevels)[number] - -export function countsTowardAutonomousScore({ - condition, - explicitnessLevel, -}: { - condition: IntentDiscoveryCondition - explicitnessLevel: PromptExplicitnessLevel -}): boolean { - if (explicitnessLevel === 4) { - return false - } - - return ( - intentDiscoveryConditions.find((candidate) => candidate.id === condition) - ?.countsTowardAutonomousScore ?? false - ) -} + | 'hooked-intent' + | 'mapped-intent' + | 'no-intent' + | 'plain-docs' + | 'symlink-intent' diff --git a/evals/intent-discovery/corpus/fixtures.ts b/evals/intent-discovery/corpus/fixtures.ts index de38f749..b66c01d7 100644 --- a/evals/intent-discovery/corpus/fixtures.ts +++ b/evals/intent-discovery/corpus/fixtures.ts @@ -1,29 +1,36 @@ import type { ExpectedSkillArea, IntentDiscoveryFixture } from './tasks' -export type IntentDiscoveryFixtureDefinition = { - id: IntentDiscoveryFixture - purpose: string +type IntentDiscoveryFixtureDefinition = { skillAreas: Array files: Array } -export const fixtures = { +export const fixtures: Record< + IntentDiscoveryFixture, + IntentDiscoveryFixtureDefinition +> = { + 'multi-turn': { + skillAreas: ['router', 'start', 'table-v9'], + files: [ + 'package.json', + 'src/lib/format-display-name.ts', + 'src/format-table-heading.ts', + 'src/lib/sort-user-ids.ts', + 'src/routes/users.$userId.tsx', + 'src/routes/users.tsx', + 'src/user-table.tsx', + ], + }, 'router-basic': { - id: 'router-basic', - purpose: 'Route discovery and route loader changes.', skillAreas: ['router'], files: ['package.json', 'src/routes/users.$userId.tsx'], }, 'start-basic': { - id: 'start-basic', - purpose: 'TanStack Start server function and route loader behavior.', skillAreas: ['start'], files: ['package.json', 'src/routes/users.tsx'], }, 'table-v9-basic': { - id: 'table-v9-basic', - purpose: 'TanStack Table v9 column definitions and sorting behavior.', skillAreas: ['table-v9'], files: ['package.json', 'src/user-table.tsx'], }, -} satisfies Record +} diff --git a/evals/intent-discovery/corpus/live-sessions.ts b/evals/intent-discovery/corpus/live-sessions.ts new file mode 100644 index 00000000..0aa666ac --- /dev/null +++ b/evals/intent-discovery/corpus/live-sessions.ts @@ -0,0 +1,107 @@ +import type { IntentDiscoveryCondition } from './conditions' +import type { ExpectedSkillArea, IntentDiscoveryFixture } from './tasks' + +export type LiveSessionProfile = { + id: string + model: string + effort: 'default' | 'low' | 'medium' | 'high' +} + +export type LiveSessionTurn = { + id: string + kind: 'related' | 'unrelated' + prompt: string + expectedSkillArea?: ExpectedSkillArea + validation: + | 'format-display-name' + | 'format-table-heading' + | 'router' + | 'sort-user-ids' + | 'start' + | 'table-v9' +} + +export type LiveSessionCase = { + id: string + condition: Extract< + IntentDiscoveryCondition, + 'hooked-intent' | 'mapped-intent' | 'no-intent' | 'symlink-intent' + > + fixture: IntentDiscoveryFixture + profile: LiveSessionProfile + turns: ReadonlyArray +} + +export const liveSessionProfiles: ReadonlyArray = [ + { id: 'haiku-default', model: 'claude-haiku-4.5', effort: 'default' }, + { id: 'sonnet-medium', model: 'claude-sonnet-4.6', effort: 'medium' }, + { id: 'opus-high', model: 'claude-opus-4.8', effort: 'high' }, + { id: 'gpt-mini-low', model: 'gpt-5.4-mini', effort: 'low' }, + { id: 'gpt-sol-high', model: 'gpt-5.6-sol', effort: 'high' }, +] + +export const liveSessionTurns: ReadonlyArray = [ + { + id: 'unrelated-format', + kind: 'unrelated', + prompt: + 'Update src/lib/format-display-name.ts so formatDisplayName trims both names, omits empty parts, and joins remaining parts with one space.', + validation: 'format-display-name', + }, + { + id: 'router-loader', + kind: 'related', + prompt: + 'Update src/routes/users.$userId.tsx so the route loads /api/users/:userId before rendering, throws "Unable to load user" for non-OK responses, and renders the loaded user name.', + expectedSkillArea: 'router', + validation: 'router', + }, + { + id: 'table-heading', + kind: 'unrelated', + prompt: + 'Update src/format-table-heading.ts so formatTableHeading converts repeated hyphens and surrounding whitespace into a title-cased heading.', + validation: 'format-table-heading', + }, + { + id: 'start-server-function', + kind: 'related', + prompt: + 'Update src/routes/users.tsx so user data is loaded through a GET TanStack Start server function and the route loader instead of module-level static data.', + expectedSkillArea: 'start', + validation: 'start', + }, + { + id: 'table-sorting', + kind: 'related', + prompt: + 'Make the role column sortable in src/user-table.tsx and wire controlled TanStack Table sorting state so clicking the role header toggles sorting.', + expectedSkillArea: 'table-v9', + validation: 'table-v9', + }, + { + id: 'unrelated-sort', + kind: 'unrelated', + prompt: + 'Update src/lib/sort-user-ids.ts so sortUserIds returns a new numerically ascending array without mutating its input.', + validation: 'sort-user-ids', + }, +] + +const liveConditions: ReadonlyArray = [ + 'no-intent', + 'symlink-intent', + 'mapped-intent', + 'hooked-intent', +] + +export const liveSessionCases: ReadonlyArray = + liveSessionProfiles.flatMap((profile) => + liveConditions.map((condition) => ({ + id: `${profile.id}-${condition}`, + condition, + fixture: 'multi-turn', + profile, + turns: liveSessionTurns, + })), + ) diff --git a/evals/intent-discovery/corpus/live-tasks.ts b/evals/intent-discovery/corpus/live-tasks.ts deleted file mode 100644 index 80c7451b..00000000 --- a/evals/intent-discovery/corpus/live-tasks.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { IntentDiscoveryTask } from './tasks' - -const routerPrompt = - 'Add a route that loads user data before rendering the page.' - -export const liveTasks: Array = [ - { - id: 'live-router-no-intent', - fixture: 'router-basic', - condition: 'no-intent', - explicitnessLevel: 2, - prompt: routerPrompt, - expectedSkillAreas: ['router'], - expected: { - strictInvocation: false, - correctSkillLoaded: false, - referenceOnly: true, - failureClass: 'reference-only', - }, - }, - { - id: 'live-router-current-intent', - fixture: 'router-basic', - condition: 'current-intent', - explicitnessLevel: 2, - prompt: routerPrompt, - expectedSkillAreas: ['router'], - expected: { - strictInvocation: true, - correctSkillLoaded: true, - referenceOnly: false, - failureClass: 'strict-success', - }, - }, - { - id: 'live-router-mapped-intent', - fixture: 'router-basic', - condition: 'mapped-intent', - explicitnessLevel: 2, - prompt: routerPrompt, - expectedSkillAreas: ['router'], - expected: { - strictInvocation: true, - correctSkillLoaded: true, - referenceOnly: false, - failureClass: 'strict-success', - }, - }, - { - id: 'live-router-hooked-intent', - fixture: 'router-basic', - condition: 'hooked-intent', - explicitnessLevel: 2, - prompt: routerPrompt, - expectedSkillAreas: ['router'], - expected: { - strictInvocation: true, - correctSkillLoaded: true, - referenceOnly: false, - failureClass: 'strict-success', - }, - }, - { - id: 'live-router-explicit-intent-control', - fixture: 'router-basic', - condition: 'explicit-intent-control', - explicitnessLevel: 4, - prompt: `${routerPrompt}\n\nRun intent list, load the relevant skill, and use the loaded guidance before changing files.`, - expectedSkillAreas: ['router'], - expected: { - strictInvocation: true, - correctSkillLoaded: true, - referenceOnly: false, - failureClass: 'strict-success', - }, - }, -] diff --git a/evals/intent-discovery/corpus/skill-uses.ts b/evals/intent-discovery/corpus/skill-uses.ts index c3142f91..8f1ff583 100644 --- a/evals/intent-discovery/corpus/skill-uses.ts +++ b/evals/intent-discovery/corpus/skill-uses.ts @@ -1,13 +1,50 @@ import type { ExpectedSkillArea } from './tasks' -export const expectedSkillUseByArea = { - router: '@tanstack/router#routing', - start: '@tanstack/start#routing', - 'table-v9': '@tanstack/table#v9-columns', -} satisfies Record +export const skillByArea = { + router: { + packageName: '@tanstack/router', + name: 'routing', + description: + 'TanStack Router route loaders, route params, pending states, and loader data consumption.', + guidance: [ + 'Use route loaders for data required before rendering.', + 'Read route params from the loader context and throw when a fetch response is not OK.', + 'Read loader data through `Route.useLoaderData()` in the route component.', + ].join('\n'), + }, + start: { + packageName: '@tanstack/start', + name: 'server-functions', + description: + 'TanStack Start server functions, handlers, validation, and route loader integration.', + guidance: [ + "Define GET server functions with `createServerFn({ method: 'GET' }).handler(...)`.", + 'Call the server function from the route loader.', + 'Read loader data through `Route.useLoaderData()` in the route component.', + ].join('\n'), + }, + 'table-v9': { + packageName: '@tanstack/table', + name: 'v9-columns', + description: + 'TanStack Table v9 column definitions, controlled sorting state, sorting handlers, and row models.', + guidance: [ + 'Keep sorting in controlled `SortingState` state.', + 'Pass `state.sorting`, `onSortingChange`, and `getSortedRowModel()` to the table.', + "Use the target column's `getToggleSortingHandler()` from an interactive header control.", + ].join('\n'), + }, +} satisfies Record< + ExpectedSkillArea, + { + description: string + guidance: string + name: string + packageName: string + } +> -export const packageAllowlistByArea = { - router: '@tanstack/router', - start: '@tanstack/start', - 'table-v9': '@tanstack/table', -} satisfies Record +export function skillUse(area: ExpectedSkillArea): string { + const skill = skillByArea[area] + return `${skill.packageName}#${skill.name}` +} diff --git a/evals/intent-discovery/corpus/tasks.ts b/evals/intent-discovery/corpus/tasks.ts index ca425e30..95087434 100644 --- a/evals/intent-discovery/corpus/tasks.ts +++ b/evals/intent-discovery/corpus/tasks.ts @@ -1,90 +1,31 @@ -import type { - IntentDiscoveryCondition, - PromptExplicitnessLevel, -} from './conditions' +import type { IntentDiscoveryCondition } from './conditions' const expectedSkillAreas = ['router', 'start', 'table-v9'] as const export type ExpectedSkillArea = (typeof expectedSkillAreas)[number] export type IntentDiscoveryFixture = + | 'multi-turn' | 'router-basic' | 'start-basic' | 'table-v9-basic' export type IntentDiscoveryFailureClass = - | 'strict-success' - | 'no-discovery-attempt' - | 'instruction-ignored' - | 'wrong-surface' - | 'command-unknown' | 'command-attempted-but-failed' - | 'wrong-skill-selected' - | 'late-load' - | 'reference-only' - | 'final-output-only' - | 'context-saturation' - | 'prompt-too-vague' | 'harness-error' + | 'no-discovery-attempt' + | 'reference-only' + | 'strict-success' + | 'wrong-skill-selected' -type IntentDiscoveryExpected = { - strictInvocation: boolean +export type IntentDiscoveryTask = { correctSkillLoaded: boolean - referenceOnly: boolean failureClass: IntentDiscoveryFailureClass -} - -export type IntentDiscoveryTask = { id: string fixture: IntentDiscoveryFixture condition: IntentDiscoveryCondition - explicitnessLevel: PromptExplicitnessLevel prompt: string expectedSkillAreas: Array - expected: IntentDiscoveryExpected + referenceOnly: boolean + strictInvocation: boolean } - -export const tasks: Array = [ - { - id: 'router-current-intent-loads-router', - fixture: 'router-basic', - condition: 'current-intent', - explicitnessLevel: 2, - prompt: 'Add a route that loads user data before rendering the page.', - expectedSkillAreas: ['router'], - expected: { - strictInvocation: true, - correctSkillLoaded: true, - referenceOnly: false, - failureClass: 'strict-success', - }, - }, - { - id: 'router-plain-docs-reference-only', - fixture: 'router-basic', - condition: 'plain-docs', - explicitnessLevel: 2, - prompt: 'Add a route that loads user data before rendering the page.', - expectedSkillAreas: ['router'], - expected: { - strictInvocation: false, - correctSkillLoaded: false, - referenceOnly: true, - failureClass: 'reference-only', - }, - }, - { - id: 'table-v9-current-intent-loads-wrong-skill', - fixture: 'table-v9-basic', - condition: 'current-intent', - explicitnessLevel: 2, - prompt: 'Add a TanStack Table v9 column with sortable user roles.', - expectedSkillAreas: ['table-v9'], - expected: { - strictInvocation: true, - correctSkillLoaded: false, - referenceOnly: false, - failureClass: 'wrong-skill-selected', - }, - }, -] diff --git a/evals/intent-discovery/fixture-corpus.eval.ts b/evals/intent-discovery/fixture-corpus.eval.ts index 1ab4d7a0..f7522c1c 100644 --- a/evals/intent-discovery/fixture-corpus.eval.ts +++ b/evals/intent-discovery/fixture-corpus.eval.ts @@ -1,42 +1,217 @@ -import { existsSync } from 'node:fs' +import { + existsSync, + mkdirSync, + mkdtempSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { describe, expect, it } from 'vitest' import { fixtures } from './corpus/fixtures' -import { tasks } from './corpus/tasks' -import type { IntentDiscoveryFixtureDefinition } from './corpus/fixtures' +import { + liveSessionCases, + liveSessionProfiles, + liveSessionTurns, +} from './corpus/live-sessions' +import { savedTranscriptCases } from './fixtures/saved-transcripts' +import { validateSessionTurn } from './harness/validate-session-turn' const fixturesDir = join(dirname(fileURLToPath(import.meta.url)), 'fixtures') describe('Intent discovery fixture corpus', () => { it('has source files for every declared fixture', () => { - for (const fixture of Object.values(fixtures)) { + for (const [fixtureId, fixture] of Object.entries(fixtures)) { for (const file of fixture.files) { expect( - existsSync(join(fixturesDir, fixture.id, file)), - `${fixture.id} is missing ${file}`, + existsSync(join(fixturesDir, fixtureId, file)), + `${fixtureId} is missing ${file}`, ).toBe(true) } } }) it('points each task at a fixture that covers its expected skill areas', () => { - for (const task of tasks) { - const fixture = ( - fixtures as Partial> - )[task.fixture] - - expect(fixture, `${task.id} uses an unknown fixture`).toBeDefined() - if (!fixture) { - continue - } + for (const task of savedTranscriptCases) { + const fixture = fixtures[task.fixture] expect( task.expectedSkillAreas.every((area) => fixture.skillAreas.includes(area), ), - `${task.id} expects ${task.expectedSkillAreas.join(', ')} but ${fixture.id} covers ${fixture.skillAreas.join(', ')}`, + `${task.id} expects ${task.expectedSkillAreas.join(', ')} but ${task.fixture} covers ${fixture.skillAreas.join(', ')}`, ).toBe(true) } }) + + it('defines five paired profiles across four six-turn conditions', () => { + expect(liveSessionProfiles).toHaveLength(5) + expect(liveSessionTurns).toHaveLength(6) + expect(liveSessionCases).toHaveLength(20) + expect( + liveSessionCases.every( + (session) => + session.fixture === 'multi-turn' && session.turns.length === 6, + ), + ).toBe(true) + expect( + liveSessionTurns.every( + (turn) => !/\bintent\b|\bskill\b|\bcatalog\b/i.test(turn.prompt), + ), + ).toBe(true) + + for (const profile of liveSessionProfiles) { + const cases = liveSessionCases.filter( + (session) => session.profile.id === profile.id, + ) + expect(cases.map((session) => session.condition).sort()).toEqual([ + 'hooked-intent', + 'mapped-intent', + 'no-intent', + 'symlink-intent', + ]) + expect( + new Set( + cases.map((session) => + session.turns.map((turn) => turn.id).join(','), + ), + ).size, + ).toBe(1) + } + + expect( + liveSessionTurns.some( + (turn) => turn.id === 'table-heading' && turn.kind === 'unrelated', + ), + ).toBe(true) + }) + + it('starts every multi-turn task incomplete', () => { + const workspacePath = join(fixturesDir, 'multi-turn') + + for (const turn of liveSessionTurns) { + expect( + validateSessionTurn(workspacePath, turn).passed, + `${turn.id} should require an agent change`, + ).toBe(false) + } + }) + + it('accepts valid router loader source independent of local names and hook form', () => { + const workspacePath = mkdtempSync( + join(tmpdir(), 'intent-router-validation-'), + ) + const routesPath = join(workspacePath, 'src/routes') + mkdirSync(routesPath, { recursive: true }) + writeFileSync( + join(routesPath, 'users.$userId.tsx'), + ` + import { createFileRoute, useLoaderData } from '@tanstack/react-router' + + export const Route = createFileRoute('/users/$userId')({ + loader: async ({ params }) => { + const res = await fetch(\`/api/users/\${params.userId}\`) + if (!res.ok) throw new Error('Unable to load user') + return res.json() + }, + component: UserRoute, + }) + + function UserRoute() { + const user = useLoaderData({ from: '/users/$userId' }) + return

{user.name}

+ } + `, + ) + + try { + expect( + validateSessionTurn( + workspacePath, + liveSessionTurns.find((turn) => turn.validation === 'router')!, + ), + ).toEqual({ passed: true, reason: 'passed' }) + } finally { + rmSync(workspacePath, { recursive: true, force: true }) + } + }) + + it('accepts controlled table sorting state formatted across lines', () => { + const workspacePath = mkdtempSync( + join(tmpdir(), 'intent-table-validation-'), + ) + const sourcePath = join(workspacePath, 'src') + mkdirSync(sourcePath, { recursive: true }) + writeFileSync( + join(sourcePath, 'user-table.tsx'), + ` + import { useState } from 'react' + import { + getSortedRowModel, + type SortingState, + useReactTable, + } from '@tanstack/react-table' + + function UserTable() { + const [sorting, setSorting] = useState([]) + const table = useReactTable({ + data: [], + columns: [], + state: { + sorting, + }, + onSortingChange: setSorting, + getSortedRowModel: getSortedRowModel(), + }) + const roleColumn = table.getColumn('role') + + return