Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions packages/outpost/ai/src/groundedness.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,27 @@ describe('extractCopilotKitIdentifiers', () => {
).toEqual(['copilot-chat', 'copilotSidebarPanel']);
});

// #234. The class-selector case above passed only because `split('.')` happened
// to strip the prefix; `#` had no such path, so an ID selector reached the
// grounding lookup with its prefix attached. Grounding is a substring check,
// so `#copilotkitpanel` could never match a source writing `copilotKitPanel` —
// the one spelling the docs actually use.
it('strips a leading ID-selector prefix from a backticked name', () => {
expect(extractCopilotKitIdentifiers('Target `#copilotKitPanel` to move it.')).toEqual([
'copilotKitPanel',
]);
});

it('treats the ID and class spellings of one name as the same identifier', () => {
// Both forms case-fold to one key, so a single name cannot fill two of the
// two threshold slots on its own.
expect(
extractCopilotKitIdentifiers(
'Give the node `#copilotKitPanel` and style `.copilotKitPanel`.',
),
).toEqual(['copilotKitPanel']);
});

it('unwraps a JSX component that carries props', () => {
// A model writing a fabricated component almost always gives it props, so
// the prop-less-only form was the least likely spelling to appear.
Expand Down Expand Up @@ -248,6 +269,18 @@ describe('extractCopilotKitIdentifiers', () => {
});

describe('assessGroundedness', () => {
// #234, the consequence that matters: two ID selectors the sources DO contain
// suppressed a correct answer outright, with nothing logged.
it('does not suppress ID-selector names the sources actually contain', () => {
const result = assessGroundedness(
'Target `#copilotKitPanel` and `#copilotKitSidebar` to reposition it.',
[source('Set copilotKitPanel and copilotKitSidebar on the wrapper nodes.')],
);
expect(result.unsourcedIdentifiers).toEqual([]);
expect(result.suppress).toBe(false);
expect(result.penalty).toBe(0);
});

it('gives a grounded answer no penalty and does not suppress it', () => {
const response =
'You can replace the chat input with the `input` prop on the `CopilotChat` component. ' +
Expand Down
20 changes: 19 additions & 1 deletion packages/outpost/ai/src/groundedness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -367,7 +367,25 @@ function identifierSegments(rawToken: string): string[] {

// A dotted form names a member; report the segments that carry our name so the
// grounding lookup compares something a source could plausibly contain.
return token.split('.').filter((segment) => segment && isCopilotKitIdentifier(segment));
//
// Splitting on `#` as well as `.` is what strips a selector prefix before the
// lookup. `IDENTIFIER_PATH` admits either prefix, but only `.` used to be
// split, so an ID selector reached grounding as `#copilotKitPanel` — and
// grounding is a substring check, so it could never match a source writing the
// bare `copilotKitPanel`, which is how the docs write it. Two such names then
// met SUPPRESS_AT_UNSOURCED_IDENTIFIERS and withheld a correct answer.
//
// This closes the false-SUPPRESSION half only. The two prefixes still reach
// this function by different routes: `CSS_CLASS_PATTERN` finds a class
// selector anywhere in the response, deliberately including inside a ```css
// fence, while an ID selector arrives only via `BACKTICKED_PATTERN`, which
// cannot span a newline and so never matches inside a fence. So a fabricated
// `#copilotKitFake` in a css fence is still invisible where `.copilotKitFake`
// is caught. Fixing that means widening `CSS_CLASS_PATTERN` to `[.#]`, which
// ADDS suppression — the direction that withholds correct answers — and `#`
// carries traps `.` does not (`this.#copilotFoo` passes the lookbehind, since
// the character before `#` is `.`). That belongs in its own change.
return token.split(/[.#]/).filter((segment) => segment && isCopilotKitIdentifier(segment));
}

/**
Expand Down