diff --git a/.changeset/skill-code-block-checks.md b/.changeset/skill-code-block-checks.md new file mode 100644 index 0000000..c0baa20 --- /dev/null +++ b/.changeset/skill-code-block-checks.md @@ -0,0 +1,5 @@ +--- +'@tanstack/intent': minor +--- + +`validate` checks every fenced TypeScript and JavaScript block in a `SKILL.md` against the owning package's own types when TypeScript is available in the repository. A renamed export, a removed option, or a changed shape fails validation with the skill file and line. Imports of exports marked `@deprecated` produce warnings, and relative Markdown links must resolve. Names, modules, and globals a partial snippet leaves out are not reported. Repositories without TypeScript skip the code checks with a notice. Pending review items for skills report whether their examples still compile. diff --git a/benchmarks/intent/startup.bench.ts b/benchmarks/intent/startup.bench.ts index cf0159d..241f5bf 100644 --- a/benchmarks/intent/startup.bench.ts +++ b/benchmarks/intent/startup.bench.ts @@ -1,6 +1,9 @@ import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { fileURLToPath } from 'node:url' -import { bench, describe } from 'vitest' +import { afterAll, beforeAll, bench, describe } from 'vitest' const cliPath = fileURLToPath( new URL('../../packages/intent/dist/cli.mjs', import.meta.url), @@ -11,8 +14,9 @@ const coldStartBenchOptions = { time: 3_000, } -function runNode(args: Array): void { +function runNode(args: Array, cwd?: string): void { const result = spawnSync(process.execPath, args, { + cwd, stdio: 'ignore', timeout: 10_000, }) @@ -24,6 +28,27 @@ function runNode(args: Array): void { } describe('cold start', () => { + let root: string | undefined + function setup() { + if (root) return + root = mkdtempSync(join(tmpdir(), 'intent-prose-startup-')) + writeFileSync( + join(root, 'package.json'), + '{"name":"prose-library","version":"1.0.0"}\n', + ) + mkdirSync(join(root, 'skills', 'guide'), { recursive: true }) + writeFileSync( + join(root, 'skills', 'guide', 'SKILL.md'), + '---\nname: guide\ndescription: Use when reading the guide.\n---\nProse-only guidance.\n', + ) + } + function teardown() { + if (root) rmSync(root, { recursive: true, force: true }) + root = undefined + } + beforeAll(setup) + afterAll(teardown) + bench( 'empty node process (baseline)', () => { @@ -39,4 +64,13 @@ describe('cold start', () => { }, coldStartBenchOptions, ) + + bench( + 'intent validate prose-only skills', + () => { + setup() + runNode([cliPath, 'validate'], root) + }, + { ...coldStartBenchOptions, setup, teardown }, + ) }) diff --git a/packages/intent/src/commands/maintainer.ts b/packages/intent/src/commands/maintainer.ts index 383e959..abeeb74 100644 --- a/packages/intent/src/commands/maintainer.ts +++ b/packages/intent/src/commands/maintainer.ts @@ -19,6 +19,7 @@ import { } from '../maintainer/distribution.js' import { runSetupGithubActions } from '../setup/index.js' import { detectIntentCommandPackageManager } from '../shared/command-runner.js' +import { describeSkillExamples } from '../validate/blocks.js' import { getMetaDir } from './support.js' import { buildMaintainerGuidanceBlock, @@ -447,6 +448,12 @@ export async function runMaintainerCommand( for (const problem of status.problems) console.log(` ${problem}`) for (const path of status.staleFiles) console.log(` Run intent maintainer sync: ${path}`) + const examples = describeSkillExamples( + project.root, + review.items + .filter((item) => item.kind === 'skill' && !item.problems.length) + .map((item) => item.path), + ) for (const item of review.items) { const label = item.kind === 'skill' @@ -459,7 +466,10 @@ export async function runMaintainerCommand( : item.changedFiles.length ? `changed ${item.changedFiles.join(', ')}` : 'no recorded review' - console.log(` ${label} ${item.path}: ${detail}`) + const example = examples.get(item.path) + console.log( + ` ${label} ${item.path}: ${detail}${example ? `; ${example}` : ''}`, + ) } } if (action === 'check') { diff --git a/packages/intent/src/commands/validate.ts b/packages/intent/src/commands/validate.ts index a8f5ba0..cccaead 100644 --- a/packages/intent/src/commands/validate.ts +++ b/packages/intent/src/commands/validate.ts @@ -9,6 +9,7 @@ import { fail, isCliFailure } from '../shared/cli-error.js' import { resolveProjectContext } from '../core/project-context.js' import { findWorkspacePackages } from '../setup/workspace-patterns.js' import { createIntentFsCache } from '../discovery/fs-cache.js' +import { checkSkillBlocks } from '../validate/blocks.js' import { printWarnings } from './support.js' import type { ProjectContext } from '../core/project-context.js' @@ -462,6 +463,7 @@ async function runValidateCommandInternal( const errors: Array = [] const warnings: Array = [] + const skippedBlockChecks = new Set() const fixPlans: Array = [] const setVersionPlans: Array = [] let validatedCount = 0 @@ -482,6 +484,11 @@ async function runValidateCommandInternal( targetPath: skillsDir, }) + const checkedSkills: Array<{ + file: string + content: string + library: string | undefined + }> = [] for (const filePath of skillFiles) { const rel = relative(process.cwd(), filePath) const content = readFileSync(filePath, 'utf8') @@ -611,6 +618,12 @@ async function runValidateCommandInternal( ...collectAgentSkillSpecWarnings({ fm, rel }).map(formatWarning), ) + checkedSkills.push({ + file: rel, + content, + library: readScalarField(fm, 'library'), + }) + const lineCount = content.split(/\r?\n/).length if (lineCount > 500) { errors.push({ @@ -620,6 +633,43 @@ async function runValidateCommandInternal( } } + // Code blocks and links are checked against the owning package's own + // source, so a renamed export or option fails here with a skill line. + if (validateContext.packageRoot && checkedSkills.length) { + let packageName: string | undefined + try { + packageName = JSON.parse( + readFileSync(validateContext.targetPackageJsonPath!, 'utf8'), + ).name + } catch { + packageName = undefined + } + const byLibrary: Record = + Object.create(null) + for (const skill of checkedSkills) { + const library = skill.library ?? packageName + if (library) (byLibrary[library] ??= []).push(skill) + } + for (const [library, skills] of Object.entries(byLibrary)) { + const result = checkSkillBlocks({ + root: process.cwd(), + packageDir: validateContext.packageRoot, + library, + skills, + }) + if (result.skipped) skippedBlockChecks.add(result.skipped) + for (const finding of result.findings) { + if (finding.severity === 'error') + errors.push({ + file: `${finding.file}:${finding.line}`, + message: finding.message, + }) + else + warnings.push(`${finding.file}:${finding.line}: ${finding.message}`) + } + } + } + // In monorepos, _artifacts lives at the workspace root, not under each package's skills/ dir. const artifactsDir = join(skillsDir, '_artifacts') if (!validateContext.isMonorepo && existsSync(artifactsDir)) { @@ -668,6 +718,9 @@ async function runValidateCommandInternal( ) } + for (const reason of skippedBlockChecks) + warnings.push(`Skill code blocks were not typechecked: ${reason}`) + if (options.check) { for (const plan of fixPlans) { errors.push({ diff --git a/packages/intent/src/validate/blocks.ts b/packages/intent/src/validate/blocks.ts new file mode 100644 index 0000000..1eb9ecf --- /dev/null +++ b/packages/intent/src/validate/blocks.ts @@ -0,0 +1,433 @@ +import { execFileSync } from 'node:child_process' +import { existsSync, readFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { dirname, join, relative, resolve } from 'node:path' +import { resolveProjectContext } from '../core/project-context.js' +import { resolveWorkspacePackages } from '../setup/workspace-patterns.js' +import { parseFrontmatter, readScalarField } from '../shared/utils.js' +import type TS from 'typescript' + +interface SkillBlockFinding { + file: string + line: number + message: string + severity: 'error' | 'warning' +} + +export interface SkillBlockCheck { + blocks: number + findings: Array + // Set when the code blocks could not be typechecked; links are still checked. + skipped?: string +} + +interface CodeBlock { + file: string + line: number + code: string + // Virtual file extension: a plain ts block must not parse as JSX, or a + // generic arrow like (x: T) => x reads as an unclosed element. + extension: 'ts' | 'tsx' | 'js' | 'jsx' +} + +const codeFence = + /^ {0,3}(`{3,}|~{3,})[ \t]*([A-Za-z0-9_-]*)[^\n]*\n([\s\S]*?)\n {0,3}\1[ \t]*$/gm +const checkedLanguages = new Set([ + 'ts', + 'tsx', + 'typescript', + 'js', + 'jsx', + 'javascript', +]) +// A destination is either <...>, which may contain spaces, or a bare path. +const markdownLink = /\[[^\]]*\]\((?:<([^>]*)>|([^)\s]+))(?:\s+"[^"]*")?\)/g + +// Diagnostics that a deliberately partial example produces: names, modules, +// and globals the snippet leaves out. Everything else describes the library +// contract or a genuinely broken example. +const partialSnippetCodes = new Set([ + 1375, 2304, 2318, 2503, 2552, 2580, 2581, 2582, 2583, 2584, 2591, 2592, 2593, + 2602, 2686, 2688, 7006, 7026, 7031, 17004, +]) +const missingModuleCodes = new Set([2307, 2792]) + +function loadTypeScript(root: string): typeof TS | null { + for (const from of [join(root, 'package.json'), import.meta.url]) { + try { + return createRequire(from)('typescript') as typeof TS + } catch { + // Try the next location. + } + } + return null +} + +function extractCodeBlocks(file: string, content: string): Array { + const blocks: Array = [] + for (const match of content.matchAll(codeFence)) { + const language = match[2]!.toLowerCase() + if (!checkedLanguages.has(language)) continue + const line = content.slice(0, match.index).split('\n').length + 1 + const extension = + language === 'tsx' || language === 'jsx' + ? language + : language.startsWith('j') + ? 'js' + : 'ts' + blocks.push({ file, line, code: match[3]!, extension }) + } + return blocks +} + +function checkSkillLinks( + root: string, + file: string, + content: string, +): Array { + const findings: Array = [] + const absolute = resolve(root, file) + // Blank out fenced examples, keeping newlines so line numbers still match. + const prose = content.replace(codeFence, (block) => + block.replace(/[^\n]/g, ' '), + ) + for (const match of prose.matchAll(markdownLink)) { + const target = match[1] ?? match[2]! + if (/^[a-z][a-z0-9+.-]*:/i.test(target) || target.startsWith('#')) continue + const path = target.replace(/[#?].*$/, '') + if (!path) continue + if (!existsSync(resolve(dirname(absolute), path))) + findings.push({ + file, + line: content.slice(0, match.index).split('\n').length, + message: `Link target not found: ${target}`, + severity: 'error', + }) + } + return findings +} + +// The file that declares the library's public types. A declared entry that +// Git tracks is hand-written and used as-is; a missing or ignored one is +// build output, so the matching source file stands in for it. +function libraryEntry(packageDir: string): string | null { + let manifest: Record = {} + try { + manifest = JSON.parse( + readFileSync(join(packageDir, 'package.json'), 'utf8'), + ) + } catch { + // Fall through to the conventional source entry. + } + const exportsRoot = isRecord(manifest.exports) + ? (manifest.exports['.'] ?? manifest.exports) + : manifest.exports + const declared = [ + manifest.types, + manifest.typings, + isRecord(exportsRoot) ? exportsRoot.types : undefined, + isRecord(exportsRoot) && isRecord(exportsRoot.import) + ? exportsRoot.import.types + : undefined, + typeof exportsRoot === 'string' ? exportsRoot : undefined, + ].find((value): value is string => typeof value === 'string') + const candidates: Array = [] + if (declared) { + const path = resolve(packageDir, declared) + if (existsSync(path) && isTracked(packageDir, path)) return path + const name = declared + .split('/') + .at(-1)! + .replace(/\.d\.(c|m)?ts$/, '') + .replace(/\.(c|m)?[jt]sx?$/, '') + candidates.push( + `src/${name}.ts`, + `src/${name}.tsx`, + `src/${name}.d.ts`, + `src/${name}.d.cts`, + `src/${name}.d.mts`, + ) + } + candidates.push('src/index.ts', 'src/index.tsx', 'index.ts', 'index.d.ts') + for (const candidate of candidates) { + const path = resolve(packageDir, candidate) + if (existsSync(path)) return path + } + return null +} + +// Every workspace package mapped to its own entry, so an example that imports +// a sibling package (an adapter, a framework binding) is checked against it +// instead of silently resolving to nothing. +function workspacePaths(root: string): Record> { + const context = resolveProjectContext({ cwd: root }) + const workspaceRoot = context.workspaceRoot ?? root + const paths: Record> = {} + for (const dir of resolveWorkspacePackages( + workspaceRoot, + context.workspacePatterns, + )) { + let name: unknown + try { + name = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')).name + } catch { + continue + } + const entry = typeof name === 'string' ? libraryEntry(dir) : null + if (!entry) continue + paths[name as string] = [slash(entry)] + paths[`${name}/*`] = [ + slash(join(dirname(entry), '*')), + slash(join(dir, '*')), + ] + } + return paths +} + +function isTracked(packageDir: string, path: string): boolean { + try { + execFileSync( + 'git', + ['-c', 'core.fsmonitor=false', 'ls-files', '--error-unmatch', '--', path], + { cwd: packageDir, stdio: 'ignore' }, + ) + return true + } catch { + return false + } +} + +function packageName(packageDir: string): string | undefined { + try { + const name: unknown = JSON.parse( + readFileSync(join(packageDir, 'package.json'), 'utf8'), + ).name + return typeof name === 'string' ? name : undefined + } catch { + return undefined + } +} + +// TypeScript normalizes every path it hands back to forward slashes, so the +// virtual files and path mappings are keyed the same way on Windows. +const slash = (path: string) => path.replace(/\\/g, '/') + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function checkSkillBlocks( + options: { + root: string + packageDir: string + library: string + skills: Array<{ file: string; content: string }> + }, + ts?: typeof TS | null, +): SkillBlockCheck { + const { root, packageDir, library } = options + const findings = options.skills.flatMap((skill) => + checkSkillLinks(root, skill.file, skill.content), + ) + const blocks = options.skills.flatMap((skill) => + extractCodeBlocks(skill.file, skill.content), + ) + const result = (skipped?: string): SkillBlockCheck => ({ + blocks: blocks.length, + findings, + skipped, + }) + if (blocks.length === 0) return result() + // Prose and link validation do not need the TypeScript runtime. + if (ts === undefined) ts = loadTypeScript(root) + if (!ts) return result('TypeScript is not installed in this repository') + if (Number(ts.versionMajorMinor.split('.')[0]) < 5) + return result( + `TypeScript ${ts.version} is installed; 5.0 or newer is required`, + ) + const entry = libraryEntry(packageDir) + const ownsLibrary = packageName(packageDir) === library + if (ownsLibrary && !entry) + return result( + `no type entry found for ${library} in ${relative(root, packageDir) || '.'}`, + ) + + const virtualDir = slash(join(root, '.intent', 'skill-examples')) + const virtual = new Map() + blocks.forEach((block, index) => + virtual.set(`${virtualDir}/block-${index}.${block.extension}`, block), + ) + const compilerOptions: TS.CompilerOptions = { + noEmit: true, + strict: false, + skipLibCheck: true, + allowJs: true, + checkJs: true, + resolveJsonModule: true, + esModuleInterop: true, + allowSyntheticDefaultImports: true, + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + // Each fence is a standalone example, even when it has no imports. + moduleDetection: ts.ModuleDetectionKind.Force, + moduleResolution: ts.ModuleResolutionKind.Bundler, + jsx: ts.JsxEmit.Preserve, + lib: ['lib.esnext.d.ts', 'lib.dom.d.ts'], + paths: { + ...workspacePaths(root), + // A skill documenting another package (metadata.library) resolves that + // package through the workspace or node_modules, not this package's entry. + ...(ownsLibrary + ? { + [library]: [slash(entry!)], + [`${library}/*`]: [ + slash(join(dirname(entry!), '*')), + slash(join(packageDir, '*')), + ], + } + : {}), + }, + types: [], + } + const host = ts.createCompilerHost(compilerOptions, true) + const readFile = host.readFile.bind(host) + const fileExists = host.fileExists.bind(host) + host.fileExists = (path) => virtual.has(path) || fileExists(path) + host.readFile = (path) => virtual.get(path)?.code ?? readFile(path) + host.getSourceFile = (path, languageVersion) => { + const code = virtual.get(path)?.code ?? readFile(path) + return code === undefined + ? undefined + : ts.createSourceFile(path, code, languageVersion, true) + } + const program = ts.createProgram([...virtual.keys()], compilerOptions, host) + const checker = program.getTypeChecker() + const fromLibrary = (specifier: string) => + specifier === library || specifier.startsWith(`${library}/`) + + for (const [path, block] of virtual) { + const source = program.getSourceFile(path) + if (!source) continue + const at = (position: number) => + block.line + source.getLineAndCharacterOfPosition(position).line + for (const diagnostic of [ + ...program.getSyntacticDiagnostics(source), + ...program.getSemanticDiagnostics(source), + ]) { + if (partialSnippetCodes.has(diagnostic.code)) continue + const message = ts.flattenDiagnosticMessageText( + diagnostic.messageText, + ' ', + ) + if (missingModuleCodes.has(diagnostic.code)) { + const specifier = /Cannot find module '([^']+)'/.exec(message)?.[1] + if (!specifier || !fromLibrary(specifier)) continue + } + findings.push({ + file: block.file, + line: + diagnostic.start === undefined ? block.line : at(diagnostic.start), + message: `TS${diagnostic.code}: ${message}`, + severity: 'error', + }) + } + for (const statement of source.statements) { + if ( + !ts.isImportDeclaration(statement) || + !ts.isStringLiteral(statement.moduleSpecifier) || + !fromLibrary(statement.moduleSpecifier.text) + ) + continue + const bindings = statement.importClause?.namedBindings + if (!bindings || !ts.isNamedImports(bindings)) continue + for (const element of bindings.elements) { + let symbol = checker.getSymbolAtLocation(element.name) + if (symbol && symbol.flags & ts.SymbolFlags.Alias) + symbol = checker.getAliasedSymbol(symbol) + const tag = symbol + ?.getJsDocTags(checker) + .find((entry) => entry.name === 'deprecated') + if (!tag) continue + const detail = ts.displayPartsToString(tag.text).trim() + findings.push({ + file: block.file, + line: at(element.getStart(source)), + message: `${element.name.text} is deprecated${detail ? `: ${detail}` : ''}`, + severity: 'warning', + }) + } + } + } + return result() +} + +// One-line summary per skill for review items, in one program per package. +// Skills without code blocks, or whose blocks could not be checked, are left +// out of the result. +export function describeSkillExamples( + root: string, + files: Array, +): Map { + const groups = new Map< + string, + { packageDir: string; library: string; files: Array } + >() + for (const file of files) { + const absolute = resolve(root, file) + const { packageRoot } = resolveProjectContext({ + cwd: root, + targetPath: absolute, + }) + if (!packageRoot) continue + let library: unknown = readScalarField( + parseFrontmatter(absolute), + 'library', + ) + if (!library) { + try { + library = JSON.parse( + readFileSync(join(packageRoot, 'package.json'), 'utf8'), + ).name + } catch { + continue + } + } + if (typeof library !== 'string') continue + const key = `${packageRoot}\0${library}` + const group = groups.get(key) ?? { + packageDir: packageRoot, + library, + files: [], + } + group.files.push(file) + groups.set(key, group) + } + const summaries = new Map() + for (const group of groups.values()) { + const skills = group.files.map((file) => ({ + file, + content: readFileSync(resolve(root, file), 'utf8'), + })) + const result = checkSkillBlocks({ + root, + packageDir: group.packageDir, + library: group.library, + skills, + }) + if (result.skipped) continue + for (const skill of skills) { + if (!extractCodeBlocks(skill.file, skill.content).length) continue + const errors = result.findings.filter( + (finding) => + finding.file === skill.file && finding.severity === 'error', + ) + summaries.set( + skill.file, + errors.length + ? `${errors.length} example error(s), first at line ${errors[0]!.line}` + : 'examples still compile', + ) + } + } + return summaries +} diff --git a/packages/intent/tests/validate-blocks.test.ts b/packages/intent/tests/validate-blocks.test.ts new file mode 100644 index 0000000..8cde1c1 --- /dev/null +++ b/packages/intent/tests/validate-blocks.test.ts @@ -0,0 +1,458 @@ +import { execFileSync } from 'node:child_process' +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import { main } from '../src/cli.js' +import { checkSkillBlocks } from '../src/validate/blocks.js' + +// Typechecking examples against a real package takes longer than a unit test. +vi.setConfig({ testTimeout: 30_000 }) + +let root: string +let previousCwd: string + +function write(path: string, content: string) { + mkdirSync(dirname(join(root, path)), { recursive: true }) + writeFileSync(join(root, path), content) +} + +function skill(body: string) { + write( + 'skills/retries/SKILL.md', + `---\nname: retries\ndescription: Use when retrying requests.\nsources: [src/index.ts]\n---\n# Retries\n\n${body}`, + ) +} + +function check() { + return checkSkillBlocks({ + root, + packageDir: root, + library: '@acme/client', + skills: [ + { + file: 'skills/retries/SKILL.md', + content: readFileSync(join(root, 'skills/retries/SKILL.md'), 'utf8'), + }, + ], + }) +} + +beforeEach(() => { + previousCwd = process.cwd() + root = mkdtempSync(join(tmpdir(), 'intent-blocks-')) + process.chdir(root) + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.spyOn(console, 'error').mockImplementation(() => {}) + write('package.json', '{"name":"@acme/client","version":"1.0.0"}\n') + write( + 'src/index.ts', + [ + 'export interface RetryOptions { max: number }', + 'export function retry(fn: () => Promise, options: RetryOptions): Promise<{ ok: boolean }> {', + ' return fn().then(() => ({ ok: options.max > 0 }))', + '}', + '/** @deprecated Use retry. */', + 'export function legacyRetry(): void {}', + '', + ].join('\n'), + ) +}) +afterEach(() => { + process.chdir(previousCwd) + vi.restoreAllMocks() + rmSync(root, { recursive: true, force: true }) +}) + +it('accepts a partial example whose only gaps are names the snippet leaves out', () => { + skill( + '```ts\nimport { retry } from \'@acme/client\'\n\nconst result = await retry(() => fetchItems(), { max: 3 })\nresult.ok\n```\n\n```json\n{ "not": "checked" }\n```\n', + ) + const result = check() + expect(result.blocks).toBe(1) + expect(result.findings).toEqual([]) +}) + +it('parses a plain ts block as TypeScript rather than TSX', () => { + skill( + "```ts\nimport { retry } from '@acme/client'\nconst pick = (value: T) => value\nawait retry(() => Promise.resolve(), { max: pick(3) })\n```\n\n```tsx\nconst view =
{String(1)}
\n```\n", + ) + expect(check().findings).toEqual([]) +}) + +it('keeps declarations in separate examples independent', () => { + skill('```ts\nconst count = 1\n```\n\n```ts\nconst count = 2\n```\n') + expect(check().findings).toEqual([]) +}) + +it.each(['js', 'jsx'])( + 'checks library option types in %s examples', + (language) => { + skill( + `\`\`\`${language}\nimport { retry } from '@acme/client'\nretry(() => fetchItems(), { max: 'many' })\n\`\`\`\n`, + ) + expect(check().findings).toEqual([ + expect.objectContaining({ + line: 10, + message: expect.stringMatching(/TS2322/), + }), + ]) + }, +) + +it('reports a removed option, a missing export, and a broken example with the skill line', () => { + skill( + [ + 'Intro line.', + '', + '```ts', + "import { retry, backoff } from '@acme/client'", + '', + 'await retry(() => fetch("/x"), { attempts: 3 })', + '```', + '', + '```tsx', + 'const count: number = "three"', + '```', + '', + '```ts', + 'const broken = {', + '```', + '', + ].join('\n'), + ) + const findings = check().findings + expect(findings).toEqual([ + expect.objectContaining({ + line: 11, + severity: 'error', + message: expect.stringMatching(/TS2305: .*'backoff'/), + }), + expect.objectContaining({ + line: 13, + severity: 'error', + message: expect.stringMatching(/TS2353: .*'attempts'/), + }), + expect.objectContaining({ + line: 17, + severity: 'error', + message: expect.stringMatching(/TS2322/), + }), + // An example that does not parse is reported instead of passing unchecked. + expect.objectContaining({ + line: 21, + severity: 'error', + message: expect.stringMatching(/TS1005/), + }), + ]) +}) + +it('warns on deprecated imports and fails broken relative links', () => { + write('skills/retries/references/backoff.md', '# Backoff\n') + skill( + [ + 'See [backoff](), [again](<../retries/references/backoff.md>), and [missing](references/missing.md#top).', + 'External [docs](https://example.com/x) are not checked.', + '', + '```ts', + "import { legacyRetry } from '@acme/client'", + 'legacyRetry()', + '```', + '', + ].join('\n'), + ) + expect(check().findings).toEqual([ + expect.objectContaining({ + line: 8, + severity: 'error', + message: 'Link target not found: references/missing.md#top', + }), + expect.objectContaining({ + line: 12, + severity: 'warning', + message: 'legacyRetry is deprecated: Use retry.', + }), + ]) +}) + +it('revalidates unchanged skills after an imported source or link target changes', async () => { + write('src/index.ts', "export { retry } from './retry'\n") + write( + 'src/retry.ts', + 'export function retry(options: { max: number }): void {}\n', + ) + write('skills/retries/reference.md', '# Reference\n') + skill( + "See [reference](reference.md).\n\n```ts\nimport { retry } from '@acme/client'\nretry({ max: 3 })\n```\n", + ) + expect(await main(['validate'])).toBe(0) + write( + 'src/retry.ts', + 'export function retry(options: { max: string }): void {}\n', + ) + rmSync(join(root, 'skills/retries/reference.md')) + expect(await main(['validate'])).toBe(1) + const errors = vi.mocked(console.error).mock.calls.flat().join('\n') + expect(errors).toContain('TS2322') + expect(errors).toContain('Link target not found: reference.md') +}) + +it.each(['__proto__', 'constructor'])( + 'accepts %s as a library name without crashing', + async (library) => { + write('package.json', JSON.stringify({ name: library, version: '1.0.0' })) + skill( + `\`\`\`ts\nimport { retry } from '${library}'\nretry(() => fetchItems(), { max: 3 })\n\`\`\`\n`, + ) + expect(await main(['validate'])).toBe(0) + }, +) + +it('checks prose links without loading TypeScript', () => { + write('node_modules/typescript/package.json', '{"main":"index.cjs"}\n') + write( + 'node_modules/typescript/index.cjs', + "require('node:fs').writeFileSync('typescript-loaded', '')\n", + ) + skill('See [missing](missing.md).\n') + expect(check().findings).toEqual([ + expect.objectContaining({ message: 'Link target not found: missing.md' }), + ]) + expect(existsSync(join(root, 'typescript-loaded'))).toBe(false) +}) + +it('uses tracked hand-written declarations and maps build output back to source', () => { + execFileSync('git', ['-c', 'core.fsmonitor=false', 'init', '-q'], { + cwd: root, + }) + write('.gitignore', 'dist/\n') + write('dist/index.d.ts', 'export declare function retry(): void\n') + write('package.json', '{"name":"@acme/client","types":"dist/index.d.ts"}\n') + skill( + "```ts\nimport { retry } from '@acme/client'\nawait retry(() => Promise.resolve(), { max: 3 })\n```\n", + ) + // dist/ is ignored, so src/index.ts stands in and the call typechecks. + expect(check().findings).toEqual([]) + write('types/index.d.ts', 'export declare function retry(): void\n') + write('package.json', '{"name":"@acme/client","types":"types/index.d.ts"}\n') + execFileSync('git', ['-c', 'core.fsmonitor=false', 'add', 'types'], { + cwd: root, + }) + // A tracked declaration file is the public surface, and the call no longer fits it. + expect(check().findings).toEqual([ + expect.objectContaining({ + line: 10, + message: expect.stringMatching(/TS2554/), + }), + ]) +}) + +it('checks imports from sibling workspace packages against their own source', () => { + write('pnpm-workspace.yaml', 'packages:\n - packages/*\n') + write('packages/client/package.json', '{"name":"@acme/client"}\n') + write( + 'packages/client/src/index.ts', + "import type { Adapter } from '@acme/adapter'\nexport function run(options: { adapter: A; model: A['models'][number] }): void {}\n", + ) + write('packages/adapter/package.json', '{"name":"@acme/adapter"}\n') + write( + 'packages/adapter/src/index.ts', + "export interface Adapter { models: ReadonlyArray }\nexport function openai(): { models: readonly ['gpt-5'] } { return { models: ['gpt-5'] } }\n", + ) + write( + 'packages/client/skills/run/SKILL.md', + [ + '---', + 'name: run', + 'description: Use when running.', + '---', + '```ts', + "import { run } from '@acme/client'", + "import { openai, anthropic } from '@acme/adapter'", + "run({ adapter: openai(), model: 'gpt-9000' })", + '```', + '', + ].join('\n'), + ) + const findings = checkSkillBlocks({ + root, + packageDir: join(root, 'packages/client'), + library: '@acme/client', + skills: [ + { + file: 'packages/client/skills/run/SKILL.md', + content: readFileSync( + join(root, 'packages/client/skills/run/SKILL.md'), + 'utf8', + ), + }, + ], + }).findings + expect(findings).toEqual([ + expect.objectContaining({ + line: 7, + message: expect.stringMatching(/TS2305: .*'anthropic'/), + }), + expect.objectContaining({ + line: 8, + message: expect.stringMatching(/TS2322: .*gpt-9000/), + }), + ]) +}) + +it('ignores links inside fenced examples and checks a skill that documents a sibling package', () => { + write('pnpm-workspace.yaml', 'packages:\n - packages/*\n') + write('packages/client/package.json', '{"name":"@acme/client"}\n') + write('packages/client/src/index.ts', 'export const client = 1\n') + write('packages/adapter/package.json', '{"name":"@acme/adapter"}\n') + write('packages/adapter/src/index.ts', 'export function openai(): void {}\n') + write( + 'packages/client/skills/adapters/SKILL.md', + [ + '---', + 'name: adapters', + 'description: Use when choosing an adapter.', + 'metadata:', + ' library: "@acme/adapter"', + '---', + '```md', + 'A [link inside an example](does-not-exist.md) is not checked.', + '```', + '```ts', + "import { openai, gemini } from '@acme/adapter'", + '```', + '', + ].join('\n'), + ) + const findings = checkSkillBlocks({ + root, + packageDir: join(root, 'packages/client'), + library: '@acme/adapter', + skills: [ + { + file: 'packages/client/skills/adapters/SKILL.md', + content: readFileSync( + join(root, 'packages/client/skills/adapters/SKILL.md'), + 'utf8', + ), + }, + ], + }).findings + expect(findings).toEqual([ + expect.objectContaining({ + line: 11, + message: expect.stringMatching(/TS2305: .*'gemini'/), + }), + ]) +}) + +it('skips typechecking with a reason when TypeScript or a type entry is unavailable', () => { + skill("```ts\nimport { retry } from '@acme/client'\n```\n") + expect( + checkSkillBlocks( + { + root, + packageDir: root, + library: '@acme/client', + skills: [ + { + file: 'skills/retries/SKILL.md', + content: readFileSync( + join(root, 'skills/retries/SKILL.md'), + 'utf8', + ), + }, + ], + }, + null, + ).skipped, + ).toMatch(/TypeScript is not installed/) + expect( + checkSkillBlocks( + { + root, + packageDir: root, + library: '@acme/client', + skills: [ + { + file: 'skills/retries/SKILL.md', + content: read('skills/retries/SKILL.md'), + }, + ], + }, + { version: '4.9.5', versionMajorMinor: '4.9' } as never, + ).skipped, + ).toMatch(/TypeScript 4\.9\.5 is installed; 5\.0 or newer/) + rmSync(join(root, 'src'), { recursive: true }) + expect(check().skipped).toMatch(/no type entry found for @acme\/client/) +}) + +it('fails validate on a broken example and reports compile status on pending reviews', async () => { + skill( + "```ts\nimport { retry } from '@acme/client'\nawait retry(() => Promise.resolve(), { max: 'many' })\n```\n", + ) + expect(await main(['validate'])).toBe(1) + expect(vi.mocked(console.error).mock.calls.flat().join('\n')).toContain( + `${join('skills', 'retries', 'SKILL.md')}:10: TS2322`, + ) + skill( + "```ts\nimport { retry } from '@acme/client'\nawait retry(() => Promise.resolve(), { max: 3 })\n```\n", + ) + expect(await main(['validate'])).toBe(0) + execFileSync('git', ['-c', 'core.fsmonitor=false', 'init', '-q'], { + cwd: root, + }) + expect(await main(['maintainer', 'setup'])).toBe(0) + execFileSync('git', ['-c', 'core.fsmonitor=false', 'add', '.'], { cwd: root }) + execFileSync( + 'git', + [ + '-c', + 'core.fsmonitor=false', + '-c', + 'user.name=T', + '-c', + 'user.email=t@e', + 'commit', + '-qm', + 'init', + ], + { cwd: root }, + ) + write( + 'src/index.ts', + read('src/index.ts').replace('max: number', 'max: number; delay?: number'), + ) + execFileSync( + 'git', + [ + '-c', + 'core.fsmonitor=false', + '-c', + 'user.name=T', + '-c', + 'user.email=t@e', + 'commit', + '-qam', + 'add delay', + ], + { cwd: root }, + ) + vi.mocked(console.log).mockClear() + expect(await main(['maintainer', 'status'])).toBe(0) + expect(vi.mocked(console.log).mock.calls.flat().join('\n')).toContain( + 'Review skill skills/retries/SKILL.md: changed src/index.ts; examples still compile', + ) + expect(existsSync(join(root, '.intent/skill-examples'))).toBe(false) +}) + +function read(path: string) { + return readFileSync(join(root, path), 'utf8') +}