diff --git a/CHANGELOG.md b/CHANGELOG.md index 025a6d0..7ce1766 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 10.1.0 — 2026-08-21 + +### Changed + +- `searchKnowledge` ranks its lexical list with Okapi BM25 instead of the hand-weighted substring scorer. A term that occurs in most pages is discounted by inverse document frequency, term frequency saturates, and a long page no longer outranks a short one by repetition. An exact title or path match, a title that contains the query, and a body that contains the query stay ahead of a bag-of-words match, so exact lookups keep their order. The hit shape, `normalizedScore`, `snippet`, `reasons`, the reciprocal-rank fusion with the link graph, and the path tie-break are unchanged. There is no option to select the previous scorer. +- The retrieval-eval retriever, the CLI `search` command, and `FileSystemSearchProvider` inherit the new ranking. The provider builds one lexical index per page index and drops both together on `refresh` or `invalidate()`. + +### Added + +- Add `buildKnowledgeLexicalIndex(pages, { tokenize, fieldBoosts })` and `scoreBm25(index, tokens, { k1, b })` in `src/lexical-index.ts`: a pure inverted index with field-boosted term frequencies, document lengths, average document length, and document count. No dependency and no native module, so the package stays importable at the edge. +- Add `tokenizeText`, the token stream that indexing and querying share; `tokenizeQuery` is its distinct-token form and moves to the same module, so one tokenizer serves both sides and the vocabularies cannot drift. +- Add `KNOWLEDGE_SEARCH_RETRIEVER_ID` (`bm25-rrf-v1`), the retriever identity to declare in a retrieval receipt minted from `searchKnowledge` results. +- `SearchKnowledgeOptions.lexicalIndex` accepts an index built from exactly the searched pages, for a caller that queries one page index repeatedly. A mismatched index is refused. + ## 10.0.0 — 2026-08-20 ### Breaking Changes diff --git a/README.md b/README.md index c1bc3c8..5f55023 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,9 @@ const search = createFileSystemSearchProvider({ root, index }) console.log(await search.search('How long is the refund window?', { limit: 3 })) ``` -The provider uses the package's local text search. +The provider ranks with BM25 over title, path, and body, keeps exact-title and phrase matches ahead of bag-of-words matches, and fuses that list with link and shared-source structure by reciprocal rank fusion. +It builds the lexical index once per page index and drops both together on `refresh` or `invalidate()`. +Declare `KNOWLEDGE_SEARCH_RETRIEVER_ID` (`bm25-rrf-v1`) as the retriever id when minting a retrieval receipt from these results. Pass `refresh: 'always'` to rebuild its index before every query, or call `invalidate()` after changing files. Use `asRetrievalEvalRetriever()` to send the same search path into retrieval tests. @@ -99,6 +101,7 @@ import { createKnowledgeUseReceipt, createKnowledgeVisibilitySnapshot, encodeKnowledgeVisibilitySnapshot, + KNOWLEDGE_SEARCH_RETRIEVER_ID, knowledgeVisibilityArtifactRef, } from '@tangle-network/agent-knowledge' @@ -109,7 +112,7 @@ await artifacts.put('artifact://run/visibility.json', bytes) const retrieval = createKnowledgeRetrievalReceipt({ runId, query: 'prior verifier obstruction', - retriever: { id: 'hybrid-search', version: '1.0.0', configDigest }, + retriever: { id: KNOWLEDGE_SEARCH_RETRIEVER_ID, version: '1.0.0', configDigest }, visibility, visibilityArtifact: knowledgeVisibilityArtifactRef({ uri: 'artifact://run/visibility.json', diff --git a/api-surface.json b/api-surface.json index 2b91dc1..4238302 100644 --- a/api-surface.json +++ b/api-surface.json @@ -68,6 +68,8 @@ "AgentMemoryWriteResult": "value", "ApplyWriteBlocksResult": "value", "AuditKnowledgeCitationsOptions": "value", + "Bm25Hit": "value", + "Bm25Options": "value", "BuildAgentMemorySequencesFromBenchmarkCasesOptions": "type", "BuildEvalKnowledgeBundleOptions": "value", "BuildKnowledgeRelationGraphInput": "value", @@ -143,6 +145,7 @@ "KB_STORE_DIR": "value", "KNOWLEDGE_EVENT_TYPES": "value", "KNOWLEDGE_RECEIPT_DIGEST_ALGORITHM": "value", + "KNOWLEDGE_SEARCH_RETRIEVER_ID": "value", "KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION": "value", "KbStore": "value", "KnowledgeAnswerBenchmarkCase": "type", @@ -232,6 +235,10 @@ "KnowledgeIndexSchema": "value", "KnowledgeInspection": "value", "KnowledgeLayout": "value", + "KnowledgeLexicalFieldBoosts": "value", + "KnowledgeLexicalIndex": "value", + "KnowledgeLexicalIndexOptions": "value", + "KnowledgeLexicalPosting": "value", "KnowledgeLintFinding": "value", "KnowledgeMemoryBenchmarkCase": "type", "KnowledgeMemoryBenchmarkTaskKind": "type", @@ -510,6 +517,7 @@ "buildKnowledgeBenchmarkScenarios": "value", "buildKnowledgeGraph": "value", "buildKnowledgeIndex": "value", + "buildKnowledgeLexicalIndex": "value", "buildKnowledgeRelationGraph": "value", "buildRetrievalBenchmarkCasesFromQrels": "value", "buildRetrievalEvalDispatch": "value", @@ -683,6 +691,7 @@ "runSerializedKnowledgeOptimization": "value", "runVerifiedResearchLoop": "value", "scenarioContentFingerprint": "value", + "scoreBm25": "value", "scoreKnowledgeBaseIndex": "value", "scoreKnowledgeBenchmarkArtifact": "value", "scoreMemoryBenchmarkArtifact": "value", @@ -709,6 +718,7 @@ "toRagasEvaluationRows": "value", "toTruLensRecords": "value", "tokenizeQuery": "value", + "tokenizeText": "value", "totalMaterialFacts": "value", "triageSource": "value", "validateKnowledgeIndex": "value", diff --git a/docs/knowledge-use-receipts.md b/docs/knowledge-use-receipts.md index ffc1e9c..7a1fe0d 100644 --- a/docs/knowledge-use-receipts.md +++ b/docs/knowledge-use-receipts.md @@ -80,7 +80,10 @@ The input takes the precomputed `visibility` snapshot. A result is accepted only ```ts import { canonicalCandidateDigest } from '@tangle-network/agent-interface' -import { createKnowledgeRetrievalReceipt } from '@tangle-network/agent-knowledge' +import { + createKnowledgeRetrievalReceipt, + KNOWLEDGE_SEARCH_RETRIEVER_ID, +} from '@tangle-network/agent-knowledge' const receipt = createKnowledgeRetrievalReceipt({ runId, @@ -89,9 +92,9 @@ const receipt = createKnowledgeRetrievalReceipt({ executionRef, query: 'prior obstruction calibrated verifier', retriever: { - id: 'inspectable-token-overlap', + id: KNOWLEDGE_SEARCH_RETRIEVER_ID, version: '1.0.0', - configDigest: canonicalCandidateDigest({ tokenizer: 'unicode-words', limit: 5 }), + configDigest: canonicalCandidateDigest({ k1: 1.2, b: 0.75, limit: 5 }), }, visibility, visibilityArtifact: artifact, diff --git a/package.json b/package.json index 406c7e6..f6d5518 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "10.0.0", + "version": "10.1.0", "description": "Build, search, evaluate, and improve source-backed knowledge bases.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { diff --git a/src/cli.ts b/src/cli.ts index 35d9cf9..5d1501a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -64,7 +64,7 @@ Commands: explain [--root .] [--json] Explain sources, links, inbound links, and related pages. search [--root .] [--pages-dir knowledge] [--limit 10] [--json] - Fast local token+graph search over the generated knowledge index. + Local BM25 and link-graph search (RRF fused) over the generated knowledge index. graph [--root .] [--format summary|json] Emit graph summary or JSON. lint [--root .] [--json] diff --git a/src/filesystem-search-provider.ts b/src/filesystem-search-provider.ts index fcef497..0b65d58 100644 --- a/src/filesystem-search-provider.ts +++ b/src/filesystem-search-provider.ts @@ -1,4 +1,5 @@ import { buildKnowledgeIndex } from './indexer' +import { buildKnowledgeLexicalIndex, type KnowledgeLexicalIndex } from './lexical-index' import { type KnowledgePagesOptions, normalizePagesDirectory } from './pages-directory' import type { RetrievalEvalRetriever, RetrievedKnowledgeHit } from './retrieval-eval' import { searchKnowledge } from './search' @@ -33,6 +34,7 @@ export class FileSystemSearchProvider { /** Root-relative directory the provider indexes. */ readonly pagesDirectory: string private index: KnowledgeIndex | undefined + private lexicalIndex: KnowledgeLexicalIndex | undefined private readonly defaultLimit: number private readonly refreshMode: 'manual' | 'always' @@ -47,6 +49,7 @@ export class FileSystemSearchProvider { async getIndex(options: FileSystemSearchOptions = {}): Promise { if (this.refreshMode === 'always' || options.refresh || !this.index) { this.index = await buildKnowledgeIndex(this.root, { pagesDirectory: this.pagesDirectory }) + this.lexicalIndex = undefined } return this.index } @@ -56,7 +59,11 @@ export class FileSystemSearchProvider { options: FileSystemSearchOptions = {}, ): Promise { const index = await this.getIndex(options) - return searchKnowledge(index, query, options.limit ?? this.defaultLimit) + if (!this.lexicalIndex) this.lexicalIndex = buildKnowledgeLexicalIndex(index.pages) + return searchKnowledge(index, query, { + limit: options.limit ?? this.defaultLimit, + lexicalIndex: this.lexicalIndex, + }) } async retrieve( @@ -78,6 +85,7 @@ export class FileSystemSearchProvider { invalidate(): void { this.index = undefined + this.lexicalIndex = undefined } } diff --git a/src/index.ts b/src/index.ts index 702e29a..6a5b6e6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -29,6 +29,7 @@ export * from './investment-thesis-task' export * from './kb-improvement' export * from './kb-store' export * from './knowledge-use-receipts' +export * from './lexical-index' export * from './lint' export * from './material-facts-metric' export * from './memory/index' diff --git a/src/lexical-index.test.ts b/src/lexical-index.test.ts new file mode 100644 index 0000000..93e50d5 --- /dev/null +++ b/src/lexical-index.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest' +import { buildKnowledgeLexicalIndex, scoreBm25, tokenizeQuery, tokenizeText } from './lexical-index' +import type { KnowledgePage } from './types' + +function page(id: string, title: string, text: string, path = `knowledge/${id}.md`): KnowledgePage { + return { id, path, title, text, frontmatter: {}, sourceIds: [], tags: [], outLinks: [] } +} + +describe('lexical tokenization', () => { + it('gives the query exactly the distinct terms the index stores, including CJK bigrams', () => { + const indexed = tokenizeText('机器学习 机器学习 flash attention attention') + + expect(tokenizeQuery('机器学习 机器学习 flash attention attention')).toEqual([ + ...new Set(indexed), + ]) + expect(indexed).toContain('机器') + expect(indexed.filter((token) => token === 'attention')).toHaveLength(2) + }) +}) + +describe('scoreBm25', () => { + it('ranks a page holding a rare term above pages holding only a corpus-wide term', () => { + const common = 'research ' + const index = buildKnowledgeLexicalIndex([ + page('rare', 'Page a', `${common.repeat(8)} obstruction`), + page('common-heavy', 'Page b', common.repeat(30)), + page('common-c', 'Page c', common.repeat(8)), + page('common-d', 'Page d', common.repeat(8)), + ]) + + expect(scoreBm25(index, tokenizeQuery('research obstruction'))[0]?.page.id).toBe('rare') + expect(scoreBm25(index, ['obstruction'])[0]!.score).toBeGreaterThan( + scoreBm25(index, ['research']).find((hit) => hit.page.id === 'common-heavy')!.score, + ) + }) + + it('ranks the shorter page first at equal term frequency', () => { + const index = buildKnowledgeLexicalIndex([ + page('short', 'Short', 'tiling tiling'), + page('long', 'Long', `tiling tiling ${'padding '.repeat(40)}`), + ]) + + expect(scoreBm25(index, ['tiling']).map((hit) => hit.page.id)).toEqual(['short', 'long']) + }) + + it('saturates term frequency so repetition cannot dominate the ranking', () => { + const index = buildKnowledgeLexicalIndex([ + page('once', 'Once', 'tiling'), + page('often', 'Often', 'tiling '.repeat(50)), + ]) + const [often, once] = scoreBm25(index, ['tiling']) + + expect(often!.page.id).toBe('often') + expect(often!.score / once!.score).toBeLessThan(3) + }) +}) diff --git a/src/lexical-index.ts b/src/lexical-index.ts new file mode 100644 index 0000000..bff278a --- /dev/null +++ b/src/lexical-index.ts @@ -0,0 +1,203 @@ +import type { KnowledgePage } from './types' + +const STOP_WORDS = new Set([ + 'the', + 'is', + 'a', + 'an', + 'what', + 'how', + 'are', + 'was', + 'were', + 'to', + 'for', + 'of', + 'with', + 'by', + 'in', + 'on', + 'and', +]) + +/** + * The token stream of one text: lower-cased, split on whitespace and + * punctuation, single characters and stop words removed, and a CJK run + * expanded into its bigrams and characters. Repeats are kept so a term + * frequency can be counted. Indexing and querying share this function, so the + * two vocabularies cannot drift. + */ +export function tokenizeText(text: string): string[] { + const raw = text + .toLowerCase() + .split(/[\s,，。！？、；：""''（）()\-_/\\·~～…]+/) + .filter((token) => token.length > 1 && !STOP_WORDS.has(token)) + const tokens: string[] = [] + for (const token of raw) { + if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(token) && token.length > 2) { + const chars = [...token] + for (let i = 0; i < chars.length - 1; i++) tokens.push(chars[i]! + chars[i + 1]!) + tokens.push(...chars) + } + tokens.push(token) + } + return tokens +} + +/** The distinct query terms, in first-occurrence order. */ +export function tokenizeQuery(query: string): string[] { + return [...new Set(tokenizeText(query))] +} + +export interface KnowledgeLexicalFieldBoosts { + /** Multiplier for a term occurrence in the page title. Defaults to 3. */ + title?: number + /** Multiplier for a term occurrence in the page path without its extension. Defaults to 2. */ + path?: number + /** Multiplier for a term occurrence in the page body. Defaults to 1. */ + text?: number +} + +export interface KnowledgeLexicalIndexOptions { + /** Token stream of one text. Defaults to `tokenizeText`. */ + tokenize?: (text: string) => string[] + fieldBoosts?: KnowledgeLexicalFieldBoosts +} + +export interface KnowledgeLexicalPosting { + /** Position of the page in `KnowledgeLexicalIndex.pages`. */ + ordinal: number + /** Field-boosted term frequency in that page. */ + tf: number +} + +/** + * Inverted index over a fixed page list. + * + * Ordinals are positions in `pages`. `documentLengths` are field-boosted token + * counts, so length normalization and term frequency use one scale. + */ +export interface KnowledgeLexicalIndex { + readonly pages: readonly KnowledgePage[] + readonly postings: ReadonlyMap + readonly documentLengths: readonly number[] + readonly averageDocumentLength: number + readonly documentCount: number + readonly tokenize: (text: string) => string[] + readonly fieldBoosts: Readonly> +} + +export interface Bm25Options { + /** Term-frequency saturation. Defaults to 1.2. */ + k1?: number + /** Length-normalization strength in [0, 1]. Defaults to 0.75. */ + b?: number +} + +export interface Bm25Hit { + page: KnowledgePage + score: number +} + +const DEFAULT_FIELD_BOOSTS: Readonly> = Object.freeze({ + title: 3, + path: 2, + text: 1, +}) + +export function buildKnowledgeLexicalIndex( + pages: readonly KnowledgePage[], + options: KnowledgeLexicalIndexOptions = {}, +): KnowledgeLexicalIndex { + const tokenize = options.tokenize ?? tokenizeText + const fieldBoosts = resolveFieldBoosts(options.fieldBoosts) + const postings = new Map() + const documentLengths: number[] = [] + let totalLength = 0 + + pages.forEach((page, ordinal) => { + const frequencies = new Map() + let length = 0 + for (const [text, boost] of [ + [page.title, fieldBoosts.title], + [page.path.replace(/\.md$/, ''), fieldBoosts.path], + [page.text, fieldBoosts.text], + ] as const) { + if (boost === 0) continue + for (const token of tokenize(text)) { + frequencies.set(token, (frequencies.get(token) ?? 0) + boost) + length += boost + } + } + documentLengths.push(length) + totalLength += length + for (const [term, tf] of frequencies) { + let list = postings.get(term) + if (!list) { + list = [] + postings.set(term, list) + } + list.push({ ordinal, tf }) + } + }) + + return { + pages, + postings, + documentLengths, + averageDocumentLength: pages.length > 0 ? totalLength / pages.length : 0, + documentCount: pages.length, + tokenize, + fieldBoosts, + } +} + +/** + * Okapi BM25 over the distinct query terms, with the Lucene inverse document + * frequency `ln(1 + (N - df + 0.5) / (df + 0.5))`, which is positive for every + * indexed term. Pages with no matching term are absent. The result is ordered + * by score, then by path, so it does not depend on page order. + */ +export function scoreBm25( + index: KnowledgeLexicalIndex, + tokens: readonly string[], + options: Bm25Options = {}, +): Bm25Hit[] { + const k1 = options.k1 ?? 1.2 + const b = options.b ?? 0.75 + if (!Number.isFinite(k1) || k1 < 0) throw new Error(`bm25 k1 must be >= 0, got ${String(k1)}`) + if (!Number.isFinite(b) || b < 0 || b > 1) { + throw new Error(`bm25 b must lie in [0, 1], got ${String(b)}`) + } + const scores = new Map() + const { documentCount, averageDocumentLength, documentLengths } = index + for (const term of new Set(tokens)) { + const list = index.postings.get(term) + if (!list) continue + const df = list.length + const idf = Math.log(1 + (documentCount - df + 0.5) / (df + 0.5)) + for (const { ordinal, tf } of list) { + const lengthRatio = + averageDocumentLength > 0 ? documentLengths[ordinal]! / averageDocumentLength : 1 + const saturated = (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * lengthRatio)) + scores.set(ordinal, (scores.get(ordinal) ?? 0) + idf * saturated) + } + } + return [...scores.entries()] + .map(([ordinal, score]) => ({ page: index.pages[ordinal]!, score })) + .sort( + (left, right) => right.score - left.score || left.page.path.localeCompare(right.page.path), + ) +} + +function resolveFieldBoosts( + boosts: KnowledgeLexicalFieldBoosts | undefined, +): Readonly> { + const resolved = { ...DEFAULT_FIELD_BOOSTS, ...boosts } + for (const [field, boost] of Object.entries(resolved)) { + if (!Number.isFinite(boost) || boost < 0) { + throw new Error(`lexical field boost ${field} must be >= 0, got ${String(boost)}`) + } + } + return Object.freeze(resolved) +} diff --git a/src/search.test.ts b/src/search.test.ts index aa25664..a1118f2 100644 --- a/src/search.test.ts +++ b/src/search.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { buildKnowledgeLexicalIndex } from './lexical-index' import { type SearchKnowledgeOptions, searchKnowledge } from './search' import type { KnowledgeIndex, KnowledgePage } from './types' @@ -88,3 +89,83 @@ describe('searchKnowledge filters', () => { ) }) }) + +describe('searchKnowledge ranking', () => { + it('keeps an exact title match ahead of a page that repeats the query terms', () => { + const pages = [ + page( + 'benchmarks', + 'Flash Attention benchmarks', + 'flash attention numbers measured again. '.repeat(20), + ), + page('exact', 'Flash Attention', 'IO aware attention.'), + ] + + expect( + searchKnowledge(index(pages), 'Flash Attention', 2).map((hit) => hit.citationId), + ).toEqual(['exact', 'benchmarks']) + }) + + it('returns identical hits when the indexed pages are reordered', () => { + const pages = [ + page('with-rare', 'Verifier notes', 'research verifier obstruction'), + page('common-heavy', 'Research log', 'research '.repeat(40)), + page('linked', 'Linked page', 'verifier prose', { outLinks: ['with-rare'] }), + ] + + const forward = searchKnowledge(index(pages), 'verifier obstruction', 5) + const reverse = searchKnowledge(index([...pages].reverse()), 'verifier obstruction', 5) + + expect(reverse.map((hit) => [hit.citationId, hit.score, hit.rank])).toEqual( + forward.map((hit) => [hit.citationId, hit.score, hit.rank]), + ) + }) + + it('recalls every member of a near-duplicate cluster ahead of unrelated prose', () => { + const repeated = + 'A verified research page explains the mechanism, records the experiment, names the evidence, and preserves the result for later agents. ' + const pages = [ + page('copy-a', 'Same title', repeated.repeat(2)), + page('copy-b', 'Same title', repeated.repeat(2)), + page('revision', 'Same title', `${repeated.repeat(2)} The decisive measured value moved.`), + page( + 'unrelated', + 'Kitchen', + 'A kitchen inventory lists pans, knives, towels, plates, and groceries. '.repeat(3), + ), + ] + + const hits = searchKnowledge(index(pages), repeated.trim(), 4) + + expect( + hits + .slice(0, 3) + .map((hit) => hit.citationId) + .sort(), + ).toEqual(['copy-a', 'copy-b', 'revision']) + expect(hits.map((hit) => hit.citationId)).not.toContain('unrelated') + }) + + it('scores a filtered search against a whole-corpus lexical index and refuses a foreign one', () => { + const pages = [ + page('prior-alpha', 'Alpha exact prior', 'alpha alpha alpha', { kind: 'prior' }), + page('finding-alpha', 'Alpha measured finding', 'alpha measurement', { kind: 'finding' }), + ] + const searched = index(pages) + const lexicalIndex = buildKnowledgeLexicalIndex(searched.pages) + + expect(searchKnowledge(searched, 'alpha', { limit: 2, lexicalIndex })).toEqual( + searchKnowledge(searched, 'alpha', 2), + ) + expect( + searchKnowledge(searched, 'alpha', { kinds: ['finding'], lexicalIndex }).map( + (hit) => hit.citationId, + ), + ).toEqual(['finding-alpha']) + expect(() => + searchKnowledge(searched, 'alpha', { + lexicalIndex: buildKnowledgeLexicalIndex([pages[0]!]), + }), + ).toThrow(/lexical index/) + }) +}) diff --git a/src/search.ts b/src/search.ts index 0fc64f9..3e165a7 100644 --- a/src/search.ts +++ b/src/search.ts @@ -1,25 +1,15 @@ +import { buildKnowledgeLexicalIndex, type KnowledgeLexicalIndex, scoreBm25 } from './lexical-index' import type { KnowledgeId, KnowledgeIndex, KnowledgePage, KnowledgeSearchResult } from './types' const RRF_K = 60 -const STOP_WORDS = new Set([ - 'the', - 'is', - 'a', - 'an', - 'what', - 'how', - 'are', - 'was', - 'were', - 'to', - 'for', - 'of', - 'with', - 'by', - 'in', - 'on', - 'and', -]) + +/** + * Identity of the ranking `searchKnowledge` performs: BM25 over title, path, + * and body, exact-title and phrase matches ahead of bag-of-words matches, and + * reciprocal rank fusion with link and shared-source structure. Declare it as + * the retriever id of a retrieval receipt minted from these results. + */ +export const KNOWLEDGE_SEARCH_RETRIEVER_ID = 'bm25-rrf-v1' export interface SearchKnowledgeOptions { /** Maximum results returned. Defaults to 10. */ @@ -32,6 +22,11 @@ export interface SearchKnowledgeOptions { kinds?: readonly string[] /** Additional caller-owned filter, applied before either ranking stage. */ predicate?: (page: KnowledgePage) => boolean + /** + * A lexical index built from exactly `index.pages`, supplied by a caller that + * searches one index repeatedly. When absent, one is built for this call. + */ + lexicalIndex?: KnowledgeLexicalIndex } /** @@ -70,9 +65,15 @@ export function searchKnowledge( } const pages = filterPages(index.pages, options) - const tokenRanked = rankByTokens(pages, trimmed) - const graphRanked = rankByGraph(pages, tokenRanked) - const scores = reciprocalRankFusion([tokenRanked.map((p) => p.id), graphRanked.map((p) => p.id)]) + const lexicalIndex = options.lexicalIndex + ? assertLexicalIndexMatches(options.lexicalIndex, index) + : buildKnowledgeLexicalIndex(index.pages) + const lexicalRanked = rankLexical(pages, trimmed, lexicalIndex) + const graphRanked = rankByGraph(pages, lexicalRanked) + const scores = reciprocalRankFusion([ + lexicalRanked.map((p) => p.id), + graphRanked.map((p) => p.id), + ]) const byId = new Map(pages.map((page) => [page.id, page])) const ranked = [...scores.entries()] @@ -99,23 +100,6 @@ export function searchKnowledge( })) } -export function tokenizeQuery(query: string): string[] { - const raw = query - .toLowerCase() - .split(/[\s,，。！？、；：""''（）()\-_/\\·~～…]+/) - .filter((token) => token.length > 1 && !STOP_WORDS.has(token)) - const tokens: string[] = [] - for (const token of raw) { - if (/[\u4e00-\u9fff\u3400-\u4dbf]/.test(token) && token.length > 2) { - const chars = [...token] - for (let i = 0; i < chars.length - 1; i++) tokens.push(chars[i]! + chars[i + 1]!) - tokens.push(...chars) - } - tokens.push(token) - } - return [...new Set(tokens)] -} - export function reciprocalRankFusion(rankLists: string[][], k = RRF_K): Map { const scores = new Map() for (const list of rankLists) { @@ -142,26 +126,67 @@ function filterPages(pages: KnowledgePage[], options: SearchKnowledgeOptions): K }) } -function rankByTokens(pages: KnowledgePage[], query: string): KnowledgePage[] { - const tokens = tokenizeQuery(query) - const effective = tokens.length > 0 ? tokens : [query.toLowerCase()] +function assertLexicalIndexMatches( + lexicalIndex: KnowledgeLexicalIndex, + index: KnowledgeIndex, +): KnowledgeLexicalIndex { + if ( + lexicalIndex.pages.length !== index.pages.length || + lexicalIndex.pages.some((page, ordinal) => page !== index.pages[ordinal]) + ) { + throw new Error('lexical index was not built from the pages of the searched index') + } + return lexicalIndex +} + +/** + * The lexical rank list. An exact title or path match outranks a title that + * contains the query, which outranks a body that contains the query, which + * outranks a bag-of-words match; BM25 orders pages inside each of those tiers. + * The tiers are an ordering, not a score, so a strong bag-of-words page cannot + * overtake an exact match by term repetition alone. + */ +function rankLexical( + pages: KnowledgePage[], + query: string, + lexicalIndex: KnowledgeLexicalIndex, +): KnowledgePage[] { + const tokens = [...new Set(lexicalIndex.tokenize(query))] + const bm25 = new Map(scoreBm25(lexicalIndex, tokens).map((hit) => [hit.page, hit.score])) + const phrase = query.toLowerCase() return pages - .map((page) => ({ page, score: tokenScore(page, query, effective) })) - .filter((item) => item.score > 0) - .sort((a, b) => b.score - a.score || a.page.path.localeCompare(b.page.path)) + .flatMap((page) => { + const score = bm25.get(page) ?? 0 + // A query that tokenizes to nothing (stop words, single characters) can + // still match as a phrase; otherwise only pages with a scored term are + // candidates, and every phrase match is one of them. + if (score === 0 && tokens.length > 0) return [] + const tier = phraseTier(page, phrase) + if (score === 0 && tier === 0) return [] + return [{ page, tier, score }] + }) + .sort((a, b) => b.tier - a.tier || b.score - a.score || a.page.path.localeCompare(b.page.path)) .map((item) => item.page) } -function rankByGraph(pages: KnowledgePage[], tokenRanked: KnowledgePage[]): KnowledgePage[] { - if (tokenRanked.length === 0) return [] - const seeds = new Set(tokenRanked.slice(0, 5).map((page) => page.id)) +function phraseTier(page: KnowledgePage, phrase: string): number { + const title = page.title.toLowerCase() + if (title === phrase || page.path.toLowerCase().endsWith(`${phrase}.md`)) return 3 + if (title.includes(phrase)) return 2 + if (page.text.toLowerCase().includes(phrase)) return 1 + return 0 +} + +function rankByGraph(pages: KnowledgePage[], lexicalRanked: KnowledgePage[]): KnowledgePage[] { + if (lexicalRanked.length === 0) return [] + const seeds = new Set(lexicalRanked.slice(0, 5).map((page) => page.id)) return pages .map((page) => ({ page, score: page.outLinks.filter((link) => seeds.has(link)).length + page.sourceIds.filter((source) => - tokenRanked.some((seed) => seed.sourceIds.includes(source)), + lexicalRanked.some((seed) => seed.sourceIds.includes(source)), ).length, })) .filter((item) => item.score > 0) @@ -169,23 +194,6 @@ function rankByGraph(pages: KnowledgePage[], tokenRanked: KnowledgePage[]): Know .map((item) => item.page) } -function tokenScore(page: KnowledgePage, query: string, tokens: string[]): number { - const title = page.title.toLowerCase() - const path = page.path.toLowerCase() - const body = page.text.toLowerCase() - const phrase = query.toLowerCase() - let score = 0 - if (path.endsWith(`${phrase}.md`) || title === phrase) score += 200 - if (title.includes(phrase)) score += 50 - if (body.includes(phrase)) score += 20 - for (const token of tokens) { - if (title.includes(token)) score += 5 - if (body.includes(token)) score += 1 - if (path.includes(token)) score += 3 - } - return score -} - function buildSnippet(text: string, query: string): string { const compact = text.replace(/\s+/g, ' ').trim() const idx = compact.toLowerCase().indexOf(query.toLowerCase())