feat: typecheck skill code blocks against the library's own source

.changeset/skill-code-block-checks.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/intent': minor
+---
+
+`validate` checks every fenced TypeScript and JavaScript block in a `SKILL.md` against the owning package's own types when TypeScript is available in the repository. A renamed export, a removed option, or a changed shape fails validation with the skill file and line. Imports of exports marked `@deprecated` produce warnings, and relative Markdown links must resolve. Names, modules, and globals a partial snippet leaves out are not reported. Repositories without TypeScript skip the code checks with a notice. Pending review items for skills report whether their examples still compile.

benchmarks/intent/startup.bench.ts

@@ -1,6 +1,9 @@
 import { spawnSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { bench, describe } from 'vitest'
+import { afterAll, beforeAll, bench, describe } from 'vitest'
 
 const cliPath = fileURLToPath(
   new URL('../../packages/intent/dist/cli.mjs', import.meta.url),
@@ -11,8 +14,9 @@ const coldStartBenchOptions = {
   time: 3_000,
 }
 
-function runNode(args: Array<string>): void {
+function runNode(args: Array<string>, cwd?: string): void {
   const result = spawnSync(process.execPath, args, {
+    cwd,
     stdio: 'ignore',
     timeout: 10_000,
   })
@@ -24,6 +28,27 @@ function runNode(args: Array<string>): void {
 }
 
 describe('cold start', () => {
+  let root: string | undefined
+  function setup() {
+    if (root) return
+    root = mkdtempSync(join(tmpdir(), 'intent-prose-startup-'))
+    writeFileSync(
+      join(root, 'package.json'),
+      '{"name":"prose-library","version":"1.0.0"}\n',
+    )
+    mkdirSync(join(root, 'skills', 'guide'), { recursive: true })
+    writeFileSync(
+      join(root, 'skills', 'guide', 'SKILL.md'),
+      '---\nname: guide\ndescription: Use when reading the guide.\n---\nProse-only guidance.\n',
+    )
+  }
+  function teardown() {
+    if (root) rmSync(root, { recursive: true, force: true })
+    root = undefined
+  }
+  beforeAll(setup)
+  afterAll(teardown)
+
   bench(
     'empty node process (baseline)',
     () => {
@@ -39,4 +64,13 @@ describe('cold start', () => {
     },
     coldStartBenchOptions,
   )
+
+  bench(
+    'intent validate prose-only skills',
+    () => {
+      setup()
+      runNode([cliPath, 'validate'], root)
+    },
+    { ...coldStartBenchOptions, setup, teardown },
+  )
 })

packages/intent/src/commands/maintainer.ts

@@ -19,6 +19,7 @@ import {
 } from '../maintainer/distribution.js'
 import { runSetupGithubActions } from '../setup/index.js'
 import { detectIntentCommandPackageManager } from '../shared/command-runner.js'
+import { describeSkillExamples } from '../validate/blocks.js'
 import { getMetaDir } from './support.js'
 import {
   buildMaintainerGuidanceBlock,
@@ -447,6 +448,12 @@ export async function runMaintainerCommand(
     for (const problem of status.problems) console.log(`  ${problem}`)
     for (const path of status.staleFiles)
       console.log(`  Run intent maintainer sync: ${path}`)
+    const examples = describeSkillExamples(
+      project.root,
+      review.items
+        .filter((item) => item.kind === 'skill' && !item.problems.length)
+        .map((item) => item.path),
+    )
     for (const item of review.items) {
       const label =
         item.kind === 'skill'
@@ -459,7 +466,10 @@ export async function runMaintainerCommand(
         : item.changedFiles.length
           ? `changed ${item.changedFiles.join(', ')}`
           : 'no recorded review'
-      console.log(`  ${label} ${item.path}: ${detail}`)
+      const example = examples.get(item.path)
+      console.log(
+        `  ${label} ${item.path}: ${detail}${example ? `; ${example}` : ''}`,
+      )
     }
   }
   if (action === 'check') {

packages/intent/src/commands/validate.ts

@@ -9,6 +9,7 @@ import { fail, isCliFailure } from '../shared/cli-error.js'
 import { resolveProjectContext } from '../core/project-context.js'
 import { findWorkspacePackages } from '../setup/workspace-patterns.js'
 import { createIntentFsCache } from '../discovery/fs-cache.js'
+import { checkSkillBlocks } from '../validate/blocks.js'
 import { printWarnings } from './support.js'
 import type { ProjectContext } from '../core/project-context.js'
 
@@ -462,6 +463,7 @@ async function runValidateCommandInternal(
 
   const errors: Array<ValidationError> = []
   const warnings: Array<string> = []
+  const skippedBlockChecks = new Set<string>()
   const fixPlans: Array<FrontmatterFixPlan> = []
   const setVersionPlans: Array<SetVersionPlan> = []
   let validatedCount = 0
@@ -482,6 +484,11 @@ async function runValidateCommandInternal(
       targetPath: skillsDir,
     })
 
+    const checkedSkills: Array<{
+      file: string
+      content: string
+      library: string | undefined
+    }> = []
     for (const filePath of skillFiles) {
       const rel = relative(process.cwd(), filePath)
       const content = readFileSync(filePath, 'utf8')
@@ -611,6 +618,12 @@ async function runValidateCommandInternal(
         ...collectAgentSkillSpecWarnings({ fm, rel }).map(formatWarning),
       )
 
+      checkedSkills.push({
+        file: rel,
+        content,
+        library: readScalarField(fm, 'library'),
+      })
+
       const lineCount = content.split(/\r?\n/).length
       if (lineCount > 500) {
         errors.push({
@@ -620,6 +633,43 @@ async function runValidateCommandInternal(
       }
     }
 
+    // Code blocks and links are checked against the owning package's own
+    // source, so a renamed export or option fails here with a skill line.
+    if (validateContext.packageRoot && checkedSkills.length) {
+      let packageName: string | undefined
+      try {
+        packageName = JSON.parse(
+          readFileSync(validateContext.targetPackageJsonPath!, 'utf8'),
+        ).name
+      } catch {
+        packageName = undefined
+      }
+      const byLibrary: Record<string, typeof checkedSkills> =
+        Object.create(null)
+      for (const skill of checkedSkills) {
+        const library = skill.library ?? packageName
+        if (library) (byLibrary[library] ??= []).push(skill)
+      }
+      for (const [library, skills] of Object.entries(byLibrary)) {
+        const result = checkSkillBlocks({
+          root: process.cwd(),
+          packageDir: validateContext.packageRoot,
+          library,
+          skills,
+        })
+        if (result.skipped) skippedBlockChecks.add(result.skipped)
+        for (const finding of result.findings) {
+          if (finding.severity === 'error')
+            errors.push({
+              file: `${finding.file}:${finding.line}`,
+              message: finding.message,
+            })
+          else
+            warnings.push(`${finding.file}:${finding.line}: ${finding.message}`)
+        }
+      }
+    }
+
     // In monorepos, _artifacts lives at the workspace root, not under each package's skills/ dir.
     const artifactsDir = join(skillsDir, '_artifacts')
     if (!validateContext.isMonorepo && existsSync(artifactsDir)) {
@@ -668,6 +718,9 @@ async function runValidateCommandInternal(
     )
   }
 
+  for (const reason of skippedBlockChecks)
+    warnings.push(`Skill code blocks were not typechecked: ${reason}`)
+
   if (options.check) {
     for (const plan of fixPlans) {
       errors.push({