From 92b7861732611c262afb565b94166f9c645aa7d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nathan=20=F0=9F=94=B6=20Tarbert?= <66887028+NathanTarbert@users.noreply.github.com> Date: Sat, 22 Aug 2026 07:27:35 -0400 Subject: [PATCH] fix(ai): strip an ID-selector prefix before the grounding lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `identifierSegments` split only on `.`, so an ID selector reached the grounding lookup with its prefix attached. Grounding is a substring check, so `#copilotKitPanel` could never match a source writing the bare `copilotKitPanel` — which is how the docs write it. Two such names then met SUPPRESS_AT_UNSOURCED_IDENTIFIERS and withheld a fully grounded answer, with nothing logged. Same failure class #222 was opened to close. The class-selector case passed only because `split('.')` happened to strip its prefix; `#` had no equivalent path. Now splits on `[.#]`, so the prefix is gone before both the shape check and the grounding lookup. Reproduced by execution before fixing: 'Target `#copilotKitPanel` and `#copilotKitSidebar` to reposition it.' against a source containing both bare names returned unsourcedIdentifiers for both, suppress: true, penalty 0.30. Three tests, added red: the prefix strip, the ID-and-class spellings of one name folding to a single key so one name cannot fill both threshold slots, and the suppression consequence itself. Refs #234 --- packages/outpost/ai/src/groundedness.test.ts | 33 ++++++++++++++++++++ packages/outpost/ai/src/groundedness.ts | 20 +++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/packages/outpost/ai/src/groundedness.test.ts b/packages/outpost/ai/src/groundedness.test.ts index c09b4de..de5e606 100644 --- a/packages/outpost/ai/src/groundedness.test.ts +++ b/packages/outpost/ai/src/groundedness.test.ts @@ -103,6 +103,27 @@ describe('extractCopilotKitIdentifiers', () => { ).toEqual(['copilot-chat', 'copilotSidebarPanel']); }); + // #234. The class-selector case above passed only because `split('.')` happened + // to strip the prefix; `#` had no such path, so an ID selector reached the + // grounding lookup with its prefix attached. Grounding is a substring check, + // so `#copilotkitpanel` could never match a source writing `copilotKitPanel` — + // the one spelling the docs actually use. + it('strips a leading ID-selector prefix from a backticked name', () => { + expect(extractCopilotKitIdentifiers('Target `#copilotKitPanel` to move it.')).toEqual([ + 'copilotKitPanel', + ]); + }); + + it('treats the ID and class spellings of one name as the same identifier', () => { + // Both forms case-fold to one key, so a single name cannot fill two of the + // two threshold slots on its own. + expect( + extractCopilotKitIdentifiers( + 'Give the node `#copilotKitPanel` and style `.copilotKitPanel`.', + ), + ).toEqual(['copilotKitPanel']); + }); + it('unwraps a JSX component that carries props', () => { // A model writing a fabricated component almost always gives it props, so // the prop-less-only form was the least likely spelling to appear. @@ -248,6 +269,18 @@ describe('extractCopilotKitIdentifiers', () => { }); describe('assessGroundedness', () => { + // #234, the consequence that matters: two ID selectors the sources DO contain + // suppressed a correct answer outright, with nothing logged. + it('does not suppress ID-selector names the sources actually contain', () => { + const result = assessGroundedness( + 'Target `#copilotKitPanel` and `#copilotKitSidebar` to reposition it.', + [source('Set copilotKitPanel and copilotKitSidebar on the wrapper nodes.')], + ); + expect(result.unsourcedIdentifiers).toEqual([]); + expect(result.suppress).toBe(false); + expect(result.penalty).toBe(0); + }); + it('gives a grounded answer no penalty and does not suppress it', () => { const response = 'You can replace the chat input with the `input` prop on the `CopilotChat` component. ' + diff --git a/packages/outpost/ai/src/groundedness.ts b/packages/outpost/ai/src/groundedness.ts index e068978..6313f9e 100644 --- a/packages/outpost/ai/src/groundedness.ts +++ b/packages/outpost/ai/src/groundedness.ts @@ -367,7 +367,25 @@ function identifierSegments(rawToken: string): string[] { // A dotted form names a member; report the segments that carry our name so the // grounding lookup compares something a source could plausibly contain. - return token.split('.').filter((segment) => segment && isCopilotKitIdentifier(segment)); + // + // Splitting on `#` as well as `.` is what strips a selector prefix before the + // lookup. `IDENTIFIER_PATH` admits either prefix, but only `.` used to be + // split, so an ID selector reached grounding as `#copilotKitPanel` — and + // grounding is a substring check, so it could never match a source writing the + // bare `copilotKitPanel`, which is how the docs write it. Two such names then + // met SUPPRESS_AT_UNSOURCED_IDENTIFIERS and withheld a correct answer. + // + // This closes the false-SUPPRESSION half only. The two prefixes still reach + // this function by different routes: `CSS_CLASS_PATTERN` finds a class + // selector anywhere in the response, deliberately including inside a ```css + // fence, while an ID selector arrives only via `BACKTICKED_PATTERN`, which + // cannot span a newline and so never matches inside a fence. So a fabricated + // `#copilotKitFake` in a css fence is still invisible where `.copilotKitFake` + // is caught. Fixing that means widening `CSS_CLASS_PATTERN` to `[.#]`, which + // ADDS suppression — the direction that withholds correct answers — and `#` + // carries traps `.` does not (`this.#copilotFoo` passes the lookbehind, since + // the character before `#` is `.`). That belongs in its own change. + return token.split(/[.#]/).filter((segment) => segment && isCopilotKitIdentifier(segment)); } /**