Feat/response prompt rewrite

packages/outpost/ai/src/generator.test.ts

@@ -8,6 +8,7 @@ import {
     buildChannelGuidance,
     extractResponseText,
 } from './generator.js';
+import { checkReply, HANDOFF_WORD_CAP } from './eval/rules.js';
 import { ConfidenceLevel } from './types.js';
 import type { SearchResult } from './types.js';
 
@@ -333,9 +334,7 @@ describe('ResponseGenerator', () => {
         // The JSON result format and the plain-text fallback carry no marker, so an
         // unlabelled source must not be asserted as either kind.
         it('leaves a source of unknown kind unlabelled', () => {
-            const prompt = build([
-                { title: 'Untitled', content: 'something', score: 0.5 },
-            ]);
+            const prompt = build([{ title: 'Untitled', content: 'something', score: 0.5 }]);
 
             expect(prompt).toContain('[Source 1: Untitled');
             expect(prompt).not.toContain('DOCS Source 1');
@@ -347,7 +346,9 @@ describe('ResponseGenerator', () => {
         it('states the model has not reproduced or tested, and has read only what was retrieved', () => {
             expect(GROUNDING_RULES).toContain('reproduced');
             expect(GROUNDING_RULES).toContain('run any test');
-            expect(GROUNDING_RULES).toContain('not read any file that is not in the Documentation Context');
+            expect(GROUNDING_RULES).toContain(
+                'not read any file that is not in the Documentation Context',
+            );
         });
 
         // The regression guard that matters. This exact instruction was in the
@@ -407,6 +408,142 @@ describe('ResponseGenerator', () => {
         });
     });
 
+    // The prompt is the other half of `eval/rules.ts`. #241 landed the reply
+    // rules as code while this prompt still mandated the shape they penalise —
+    // "Always include code examples", "**bold headers**", "Was this helpful?" —
+    // so the linter built to enforce the rules would have collapsed drafts the
+    // prompt had just asked for. Root cause 2 in the Agent's Output Doc is that
+    // contradiction: formatting was mandatory and having something to say was
+    // not, so a short honest reply was impossible to produce.
+    //
+    // These tests pin the two halves together. The killed mandates cannot come
+    // back silently, and each mechanical rule is asserted against the linter
+    // that fails a reply for breaking it — so the prompt and the rule name the
+    // same thing or the test goes red.
+    describe('SYSTEM_PROMPT_PREFIX reply shape', () => {
+        it('no longer mandates a code example in every reply', () => {
+            expect(SYSTEM_PROMPT_PREFIX).not.toContain('Always include code examples');
+        });
+
+        it('no longer mandates bold headers and bullet points', () => {
+            expect(SYSTEM_PROMPT_PREFIX).not.toContain('Structure responses with **bold headers**');
+            expect(SYSTEM_PROMPT_PREFIX).not.toContain('Use bold for emphasis');
+        });
+
+        // The closing slot is what Case D filled with issue-writing advice once
+        // it had no facts left to report.
+        it('no longer requires a closing follow-up suggestion', () => {
+            expect(SYSTEM_PROMPT_PREFIX).not.toContain('Was this helpful?');
+            expect(SYSTEM_PROMPT_PREFIX).not.toContain('End with a relevant follow-up');
+        });
+
+        it('puts the verdict in the first sentence', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain('first sentence');
+            expect(SYSTEM_PROMPT_PREFIX).toContain('never build up to it');
+        });
+
+        it('asks for one approach rather than a menu', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain('Never a menu of three');
+        });
+
+        it('ties length to the evidence instead of to a required layout', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain('Length follows the evidence');
+            expect(SYSTEM_PROMPT_PREFIX).toContain('enough content to organise');
+        });
+
+        it('requires a docs page or repo file for a substantive claim', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain('repo file');
+            expect(SYSTEM_PROMPT_PREFIX).toContain('only lives in code');
+        });
+
+        it('holds one API version per reply', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain('One version of the API per reply');
+        });
+
+        // The doc's "if someone asks about v1, answer the v1 question" — the
+        // recommendation is v2, which is not the same as refusing the question
+        // that was asked.
+        it('answers a v1 question rather than redirecting it', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain('answer the v1 question');
+        });
+    });
+
+    // Each of these pairs the prompt with the linter rule that fails a reply for
+    // breaking it. Asserting the rule fails first is what keeps the pair honest:
+    // a prompt line matched against a rule that no longer fires would pass while
+    // enforcing nothing.
+    describe('SYSTEM_PROMPT_PREFIX matches the linter rules', () => {
+        const ruleFor = (reply: string, rule: string) =>
+            checkReply(reply, []).find((r) => r.rule === rule);
+
+        it('bans the praise opener the linter fails a reply for', () => {
+            expect(
+                ruleFor(`Great question. ${'detail '.repeat(20)}`, 'no-banned-phrases')?.passed,
+            ).toBe(false);
+            expect(SYSTEM_PROMPT_PREFIX).toContain('Great question');
+        });
+
+        it('bans the self-commentary the linter fails a reply for', () => {
+            expect(
+                ruleFor(`What I can't do from here is ${'detail '.repeat(20)}`, 'no-banned-phrases')
+                    ?.passed,
+            ).toBe(false);
+            expect(SYSTEM_PROMPT_PREFIX).toContain("What I can't do from here");
+        });
+
+        it('bans the hedged name the linter fails a reply for', () => {
+            expect(
+                ruleFor(
+                    `Use useCopilotAction or the equivalent hook. ${'detail '.repeat(20)}`,
+                    'no-hedged-names',
+                )?.passed,
+            ).toBe(false);
+            expect(SYSTEM_PROMPT_PREFIX).toContain('or the equivalent');
+        });
+
+        // The linter's carve-out is the same as the prompt's: naming the retired
+        // package is only correct as a migration instruction, alongside the live
+        // package that replaced it.
+        it('bans the retired package the linter fails a reply for, with the same carve-out', () => {
+            expect(
+                ruleFor(`Install @copilotkitnext/react. ${'detail '.repeat(20)}`, 'no-dead-package')
+                    ?.passed,
+            ).toBe(false);
+            expect(SYSTEM_PROMPT_PREFIX).toContain('@copilotkitnext');
+            expect(SYSTEM_PROMPT_PREFIX).toContain('@copilotkit');
+        });
+
+        // Reads the cap off the rules module rather than restating 60, so moving
+        // the cap moves the prompt or breaks this test.
+        it('states the handoff cap the linter enforces', () => {
+            expect(SYSTEM_PROMPT_PREFIX).toContain(`${HANDOFF_WORD_CAP} words`);
+            expect(SYSTEM_PROMPT_PREFIX).toContain('two sentences');
+        });
+    });
+
+    // Zero retrieval used to invite the model to answer "from general CopilotKit
+    // knowledge if possible" — root cause 1 written into the prompt, and exactly
+    // what produced an invented answer when Pathfinder returned nothing. With no
+    // sources there is nothing to be right from, so the only correct reply is the
+    // handoff.
+    describe('buildSystemPrompt with no sources', () => {
+        const generator = new ResponseGenerator({ apiKey: 'test-key' });
+        const build = () =>
+            (
+                generator as unknown as {
+                    buildSystemPrompt: (s: SearchResult[], src?: undefined) => string;
+                }
+            ).buildSystemPrompt([], undefined);
+
+        it('does not invite an answer from general knowledge', () => {
+            expect(build()).not.toContain('general CopilotKit knowledge');
+        });
+
+        it('asks for the handoff instead', () => {
+            expect(build()).toContain('two-sentence handoff');
+        });
+    });
+
     describe('generateStream', () => {
         it('should yield text chunks from streaming response', async () => {
             mock.onMessage(/./, {

packages/outpost/ai/src/generator.ts

@@ -47,20 +47,61 @@ export const GROUNDING_RULES = `Grounding rules (these override the personality
 - Do not prescribe fixes to CopilotKit's internals or tell maintainers what to change; that call is theirs. Workarounds the user can apply in their own code are fine.
 - Prefer "I don't have enough to answer this — escalating to the team" over a plausible-sounding answer assembled from general framework knowledge.`;
 
+/**
+ * The reply's voice and shape.
+ *
+ * ## Why this block was rewritten
+ *
+ * It used to require, in the model's own instructions, "Always include code
+ * examples when relevant", "Structure responses with **bold headers**, bullet
+ * points, and code blocks", and a closing "Was this helpful?". That is the
+ * second root cause in the Agent's Output Doc: formatting was mandatory and
+ * having something to say was not, so the layout had to be filled whether or not
+ * retrieval had produced anything to fill it with. What filled it was praise, a
+ * restatement of the question, "here are three approaches", and a paragraph
+ * about the agent's own limits. A three-sentence honest reply was not a
+ * permitted output.
+ *
+ * ## Paired with eval/rules.ts, deliberately
+ *
+ * The mechanical rules below — the banned openers, the hedged name, the retired
+ * package, the word cap on an uncited reply — are the same rules `eval/rules.ts`
+ * scores and the draft linter enforces. Enforcement without the matching
+ * instruction is the worst of both: the model is asked for the shape that gets
+ * its draft collapsed. So each line here has a rule that fails a reply for
+ * breaking it, and the tests assert the pair rather than the prompt alone.
+ *
+ * The non-mechanical lines (verdict first, one approach, length follows the
+ * evidence) have no rule and cannot get one — nothing checkable distinguishes a
+ * well-judged three-paragraph answer from a padded one. They are asks, and the
+ * harness is how we find out whether they took.
+ *
+ * The section headers are load-bearing: GROUNDING_RULES declares that it
+ * overrides "the personality and formatting rules above", so it has to be able
+ * to name them.
+ */
 export const SYSTEM_PROMPT_PREFIX = `You are an AI support assistant for CopilotKit, an open-source framework for building AI copilots, chatbots, and AI-powered UIs.
 
 Your personality:
-- Conversational and helpful, not robotic
-- Always include code examples when relevant (TypeScript/React preferred)
-- Reference specific docs pages with full URLs when available
-- Structure responses with **bold headers**, bullet points, and code blocks
-- End with a relevant follow-up suggestion or "Was this helpful?"
+- Direct and factual. Write the way a maintainer answers a colleague — plain, warm, and done when the answer is done.
+- Give the verdict in the first sentence, every time; never build up to it.
+- No praise openers. Not "Great question", not "Thanks for this detailed report", not "Excellent catch".
+- Never repeat the reporter's question back to them. They wrote it.
+- Never discuss your own limits. No "What I can't do from here", no "I haven't read the source", no apology for what you are. Nobody asked.
+- You are given the whole thread and you can read it. Never claim you cannot see other people's replies.
+- Never coach the reporter on how to write a better issue or report.
 
 Formatting rules:
-- Use markdown formatting throughout
-- Wrap code in fenced code blocks with language tags
-- Use bold for emphasis on key concepts
-- Keep paragraphs concise — prefer bullets over walls of text
+- Length follows the evidence. A reply may be three sentences, and it should be if that is all the retrieved sources support.
+- Markdown when it helps, and only then. Headers and bullets are for content that needs organising — if there is not enough content to organise, do not organise it.
+- One approach: the one the evidence supports. Never a menu of three.
+- At most one code sample, only when the Documentation Context supplies it, in a fenced block with a language tag. No sample is better than an invented one.
+- One version of the API per reply. v1 and v2 hooks never appear in the same snippet.
+- Every substantive claim carries the page or file it came from. Prefer the docs URL; link the repo file when the answer only lives in code, and say the docs do not cover it yet.
+- Never hedge an API name. "or the equivalent hook" means you are guessing — drop the name and describe the behaviour instead.
+- Never write @copilotkitnext. That line merged into @copilotkit v2, so naming it sends people to a package that no longer exists; name the @copilotkit/ package that replaced it. v2 is the recommended path, and it is what a version-agnostic question gets. If someone asks about v1, answer the v1 question they asked, then note that v2 is the path forward.
+- If the retrieved sources do not settle the question, do not fill the space. Reply in two sentences, under 60 words: what you confirmed, if anything, and that a human is picking it up.
+- Do not close with a follow-up question or an offer to help further. End on the answer.
 
 ${GROUNDING_RULES}`;
 
@@ -240,7 +281,8 @@ export class ResponseGenerator {
                 // code wins a disagreement with the docs. Rendering both as an
                 // identical `[Source N: title]` left that instruction resolvable
                 // only by guessing at the title's shape.
-                const kindLabel = s.kind === 'code' ? 'SOURCE CODE ' : s.kind === 'docs' ? 'DOCS ' : '';
+                const kindLabel =
+                    s.kind === 'code' ? 'SOURCE CODE ' : s.kind === 'docs' ? 'DOCS ' : '';
                 return `[${kindLabel}Source ${i + 1}: ${s.title} (relevance: ${s.score.toFixed(2)})]${urlLine}\n${s.content}`;
             })
             .join('\n\n');
@@ -251,7 +293,13 @@ export class ResponseGenerator {
             '',
             '--- Documentation Context ---',
             sourceContext ||
-                '(No relevant documentation found — answer from general CopilotKit knowledge if possible, otherwise say you need to escalate)',
+                // Zero retrieval used to invite an answer "from general CopilotKit
+                // knowledge if possible" — the first root cause written into the
+                // prompt. With no sources there is nothing to be right from and
+                // nothing to cite, so the only reply that can be correct here is
+                // the handoff, and every grounding rule above already forbids the
+                // alternative.
+                '(No relevant documentation or source code was retrieved. Do not answer from general knowledge and do not guess. Reply with the two-sentence handoff: what you confirmed, if anything, and that a human is picking it up.)',
         ].join('\n');
     }