diff --git a/packages/outpost/ai/src/generator.test.ts b/packages/outpost/ai/src/generator.test.ts index 050fa81..943ab54 100644 --- a/packages/outpost/ai/src/generator.test.ts +++ b/packages/outpost/ai/src/generator.test.ts @@ -8,6 +8,7 @@ import { buildChannelGuidance, extractResponseText, } from './generator.js'; +import { checkReply, HANDOFF_WORD_CAP } from './eval/rules.js'; import { ConfidenceLevel } from './types.js'; import type { SearchResult } from './types.js'; @@ -333,9 +334,7 @@ describe('ResponseGenerator', () => { // The JSON result format and the plain-text fallback carry no marker, so an // unlabelled source must not be asserted as either kind. it('leaves a source of unknown kind unlabelled', () => { - const prompt = build([ - { title: 'Untitled', content: 'something', score: 0.5 }, - ]); + const prompt = build([{ title: 'Untitled', content: 'something', score: 0.5 }]); expect(prompt).toContain('[Source 1: Untitled'); expect(prompt).not.toContain('DOCS Source 1'); @@ -347,7 +346,9 @@ describe('ResponseGenerator', () => { it('states the model has not reproduced or tested, and has read only what was retrieved', () => { expect(GROUNDING_RULES).toContain('reproduced'); expect(GROUNDING_RULES).toContain('run any test'); - expect(GROUNDING_RULES).toContain('not read any file that is not in the Documentation Context'); + expect(GROUNDING_RULES).toContain( + 'not read any file that is not in the Documentation Context', + ); }); // The regression guard that matters. This exact instruction was in the @@ -407,6 +408,142 @@ describe('ResponseGenerator', () => { }); }); + // The prompt is the other half of `eval/rules.ts`. #241 landed the reply + // rules as code while this prompt still mandated the shape they penalise — + // "Always include code examples", "**bold headers**", "Was this helpful?" — + // so the linter built to enforce the rules would have collapsed drafts the + // prompt had just asked for. Root cause 2 in the Agent's Output Doc is that + // contradiction: formatting was mandatory and having something to say was + // not, so a short honest reply was impossible to produce. + // + // These tests pin the two halves together. The killed mandates cannot come + // back silently, and each mechanical rule is asserted against the linter + // that fails a reply for breaking it — so the prompt and the rule name the + // same thing or the test goes red. + describe('SYSTEM_PROMPT_PREFIX reply shape', () => { + it('no longer mandates a code example in every reply', () => { + expect(SYSTEM_PROMPT_PREFIX).not.toContain('Always include code examples'); + }); + + it('no longer mandates bold headers and bullet points', () => { + expect(SYSTEM_PROMPT_PREFIX).not.toContain('Structure responses with **bold headers**'); + expect(SYSTEM_PROMPT_PREFIX).not.toContain('Use bold for emphasis'); + }); + + // The closing slot is what Case D filled with issue-writing advice once + // it had no facts left to report. + it('no longer requires a closing follow-up suggestion', () => { + expect(SYSTEM_PROMPT_PREFIX).not.toContain('Was this helpful?'); + expect(SYSTEM_PROMPT_PREFIX).not.toContain('End with a relevant follow-up'); + }); + + it('puts the verdict in the first sentence', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain('first sentence'); + expect(SYSTEM_PROMPT_PREFIX).toContain('never build up to it'); + }); + + it('asks for one approach rather than a menu', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain('Never a menu of three'); + }); + + it('ties length to the evidence instead of to a required layout', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain('Length follows the evidence'); + expect(SYSTEM_PROMPT_PREFIX).toContain('enough content to organise'); + }); + + it('requires a docs page or repo file for a substantive claim', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain('repo file'); + expect(SYSTEM_PROMPT_PREFIX).toContain('only lives in code'); + }); + + it('holds one API version per reply', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain('One version of the API per reply'); + }); + + // The doc's "if someone asks about v1, answer the v1 question" — the + // recommendation is v2, which is not the same as refusing the question + // that was asked. + it('answers a v1 question rather than redirecting it', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain('answer the v1 question'); + }); + }); + + // Each of these pairs the prompt with the linter rule that fails a reply for + // breaking it. Asserting the rule fails first is what keeps the pair honest: + // a prompt line matched against a rule that no longer fires would pass while + // enforcing nothing. + describe('SYSTEM_PROMPT_PREFIX matches the linter rules', () => { + const ruleFor = (reply: string, rule: string) => + checkReply(reply, []).find((r) => r.rule === rule); + + it('bans the praise opener the linter fails a reply for', () => { + expect( + ruleFor(`Great question. ${'detail '.repeat(20)}`, 'no-banned-phrases')?.passed, + ).toBe(false); + expect(SYSTEM_PROMPT_PREFIX).toContain('Great question'); + }); + + it('bans the self-commentary the linter fails a reply for', () => { + expect( + ruleFor(`What I can't do from here is ${'detail '.repeat(20)}`, 'no-banned-phrases') + ?.passed, + ).toBe(false); + expect(SYSTEM_PROMPT_PREFIX).toContain("What I can't do from here"); + }); + + it('bans the hedged name the linter fails a reply for', () => { + expect( + ruleFor( + `Use useCopilotAction or the equivalent hook. ${'detail '.repeat(20)}`, + 'no-hedged-names', + )?.passed, + ).toBe(false); + expect(SYSTEM_PROMPT_PREFIX).toContain('or the equivalent'); + }); + + // The linter's carve-out is the same as the prompt's: naming the retired + // package is only correct as a migration instruction, alongside the live + // package that replaced it. + it('bans the retired package the linter fails a reply for, with the same carve-out', () => { + expect( + ruleFor(`Install @copilotkitnext/react. ${'detail '.repeat(20)}`, 'no-dead-package') + ?.passed, + ).toBe(false); + expect(SYSTEM_PROMPT_PREFIX).toContain('@copilotkitnext'); + expect(SYSTEM_PROMPT_PREFIX).toContain('@copilotkit'); + }); + + // Reads the cap off the rules module rather than restating 60, so moving + // the cap moves the prompt or breaks this test. + it('states the handoff cap the linter enforces', () => { + expect(SYSTEM_PROMPT_PREFIX).toContain(`${HANDOFF_WORD_CAP} words`); + expect(SYSTEM_PROMPT_PREFIX).toContain('two sentences'); + }); + }); + + // Zero retrieval used to invite the model to answer "from general CopilotKit + // knowledge if possible" — root cause 1 written into the prompt, and exactly + // what produced an invented answer when Pathfinder returned nothing. With no + // sources there is nothing to be right from, so the only correct reply is the + // handoff. + describe('buildSystemPrompt with no sources', () => { + const generator = new ResponseGenerator({ apiKey: 'test-key' }); + const build = () => + ( + generator as unknown as { + buildSystemPrompt: (s: SearchResult[], src?: undefined) => string; + } + ).buildSystemPrompt([], undefined); + + it('does not invite an answer from general knowledge', () => { + expect(build()).not.toContain('general CopilotKit knowledge'); + }); + + it('asks for the handoff instead', () => { + expect(build()).toContain('two-sentence handoff'); + }); + }); + describe('generateStream', () => { it('should yield text chunks from streaming response', async () => { mock.onMessage(/./, { diff --git a/packages/outpost/ai/src/generator.ts b/packages/outpost/ai/src/generator.ts index d52e059..1b76ac9 100644 --- a/packages/outpost/ai/src/generator.ts +++ b/packages/outpost/ai/src/generator.ts @@ -47,20 +47,61 @@ export const GROUNDING_RULES = `Grounding rules (these override the personality - Do not prescribe fixes to CopilotKit's internals or tell maintainers what to change; that call is theirs. Workarounds the user can apply in their own code are fine. - Prefer "I don't have enough to answer this — escalating to the team" over a plausible-sounding answer assembled from general framework knowledge.`; +/** + * The reply's voice and shape. + * + * ## Why this block was rewritten + * + * It used to require, in the model's own instructions, "Always include code + * examples when relevant", "Structure responses with **bold headers**, bullet + * points, and code blocks", and a closing "Was this helpful?". That is the + * second root cause in the Agent's Output Doc: formatting was mandatory and + * having something to say was not, so the layout had to be filled whether or not + * retrieval had produced anything to fill it with. What filled it was praise, a + * restatement of the question, "here are three approaches", and a paragraph + * about the agent's own limits. A three-sentence honest reply was not a + * permitted output. + * + * ## Paired with eval/rules.ts, deliberately + * + * The mechanical rules below — the banned openers, the hedged name, the retired + * package, the word cap on an uncited reply — are the same rules `eval/rules.ts` + * scores and the draft linter enforces. Enforcement without the matching + * instruction is the worst of both: the model is asked for the shape that gets + * its draft collapsed. So each line here has a rule that fails a reply for + * breaking it, and the tests assert the pair rather than the prompt alone. + * + * The non-mechanical lines (verdict first, one approach, length follows the + * evidence) have no rule and cannot get one — nothing checkable distinguishes a + * well-judged three-paragraph answer from a padded one. They are asks, and the + * harness is how we find out whether they took. + * + * The section headers are load-bearing: GROUNDING_RULES declares that it + * overrides "the personality and formatting rules above", so it has to be able + * to name them. + */ export const SYSTEM_PROMPT_PREFIX = `You are an AI support assistant for CopilotKit, an open-source framework for building AI copilots, chatbots, and AI-powered UIs. Your personality: -- Conversational and helpful, not robotic -- Always include code examples when relevant (TypeScript/React preferred) -- Reference specific docs pages with full URLs when available -- Structure responses with **bold headers**, bullet points, and code blocks -- End with a relevant follow-up suggestion or "Was this helpful?" +- Direct and factual. Write the way a maintainer answers a colleague — plain, warm, and done when the answer is done. +- Give the verdict in the first sentence, every time; never build up to it. +- No praise openers. Not "Great question", not "Thanks for this detailed report", not "Excellent catch". +- Never repeat the reporter's question back to them. They wrote it. +- Never discuss your own limits. No "What I can't do from here", no "I haven't read the source", no apology for what you are. Nobody asked. +- You are given the whole thread and you can read it. Never claim you cannot see other people's replies. +- Never coach the reporter on how to write a better issue or report. Formatting rules: -- Use markdown formatting throughout -- Wrap code in fenced code blocks with language tags -- Use bold for emphasis on key concepts -- Keep paragraphs concise — prefer bullets over walls of text +- Length follows the evidence. A reply may be three sentences, and it should be if that is all the retrieved sources support. +- Markdown when it helps, and only then. Headers and bullets are for content that needs organising — if there is not enough content to organise, do not organise it. +- One approach: the one the evidence supports. Never a menu of three. +- At most one code sample, only when the Documentation Context supplies it, in a fenced block with a language tag. No sample is better than an invented one. +- One version of the API per reply. v1 and v2 hooks never appear in the same snippet. +- Every substantive claim carries the page or file it came from. Prefer the docs URL; link the repo file when the answer only lives in code, and say the docs do not cover it yet. +- Never hedge an API name. "or the equivalent hook" means you are guessing — drop the name and describe the behaviour instead. +- Never write @copilotkitnext. That line merged into @copilotkit v2, so naming it sends people to a package that no longer exists; name the @copilotkit/ package that replaced it. v2 is the recommended path, and it is what a version-agnostic question gets. If someone asks about v1, answer the v1 question they asked, then note that v2 is the path forward. +- If the retrieved sources do not settle the question, do not fill the space. Reply in two sentences, under 60 words: what you confirmed, if anything, and that a human is picking it up. +- Do not close with a follow-up question or an offer to help further. End on the answer. ${GROUNDING_RULES}`; @@ -240,7 +281,8 @@ export class ResponseGenerator { // code wins a disagreement with the docs. Rendering both as an // identical `[Source N: title]` left that instruction resolvable // only by guessing at the title's shape. - const kindLabel = s.kind === 'code' ? 'SOURCE CODE ' : s.kind === 'docs' ? 'DOCS ' : ''; + const kindLabel = + s.kind === 'code' ? 'SOURCE CODE ' : s.kind === 'docs' ? 'DOCS ' : ''; return `[${kindLabel}Source ${i + 1}: ${s.title} (relevance: ${s.score.toFixed(2)})]${urlLine}\n${s.content}`; }) .join('\n\n'); @@ -251,7 +293,13 @@ export class ResponseGenerator { '', '--- Documentation Context ---', sourceContext || - '(No relevant documentation found — answer from general CopilotKit knowledge if possible, otherwise say you need to escalate)', + // Zero retrieval used to invite an answer "from general CopilotKit + // knowledge if possible" — the first root cause written into the + // prompt. With no sources there is nothing to be right from and + // nothing to cite, so the only reply that can be correct here is + // the handoff, and every grounding rule above already forbids the + // alternative. + '(No relevant documentation or source code was retrieved. Do not answer from general knowledge and do not guess. Reply with the two-sentence handoff: what you confirmed, if anything, and that a human is picking it up.)', ].join('\n'); }