feat(search): rank with BM25 over a real lexical index

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 10.1.0 — 2026-08-21
+
+### Changed
+
+- `searchKnowledge` ranks its lexical list with Okapi BM25 instead of the hand-weighted substring scorer. A term that occurs in most pages is discounted by inverse document frequency, term frequency saturates, and a long page no longer outranks a short one by repetition. An exact title or path match, a title that contains the query, and a body that contains the query stay ahead of a bag-of-words match, so exact lookups keep their order. The hit shape, `normalizedScore`, `snippet`, `reasons`, the reciprocal-rank fusion with the link graph, and the path tie-break are unchanged. There is no option to select the previous scorer.
+- The retrieval-eval retriever, the CLI `search` command, and `FileSystemSearchProvider` inherit the new ranking. The provider builds one lexical index per page index and drops both together on `refresh` or `invalidate()`.
+
+### Added
+
+- Add `buildKnowledgeLexicalIndex(pages, { tokenize, fieldBoosts })` and `scoreBm25(index, tokens, { k1, b })` in `src/lexical-index.ts`: a pure inverted index with field-boosted term frequencies, document lengths, average document length, and document count. No dependency and no native module, so the package stays importable at the edge.
+- Add `tokenizeText`, the token stream that indexing and querying share; `tokenizeQuery` is its distinct-token form and moves to the same module, so one tokenizer serves both sides and the vocabularies cannot drift.
+- Add `KNOWLEDGE_SEARCH_RETRIEVER_ID` (`bm25-rrf-v1`), the retriever identity to declare in a retrieval receipt minted from `searchKnowledge` results.
+- `SearchKnowledgeOptions.lexicalIndex` accepts an index built from exactly the searched pages, for a caller that queries one page index repeatedly. A mismatched index is refused.
+
 ## 10.0.0 — 2026-08-20
 
 ### Breaking Changes

README.md

@@ -78,7 +78,9 @@ const search = createFileSystemSearchProvider({ root, index })
 console.log(await search.search('How long is the refund window?', { limit: 3 }))
 ```
 
-The provider uses the package's local text search.
+The provider ranks with BM25 over title, path, and body, keeps exact-title and phrase matches ahead of bag-of-words matches, and fuses that list with link and shared-source structure by reciprocal rank fusion.
+It builds the lexical index once per page index and drops both together on `refresh` or `invalidate()`.
+Declare `KNOWLEDGE_SEARCH_RETRIEVER_ID` (`bm25-rrf-v1`) as the retriever id when minting a retrieval receipt from these results.
 Pass `refresh: 'always'` to rebuild its index before every query, or call `invalidate()` after changing files.
 Use `asRetrievalEvalRetriever()` to send the same search path into retrieval tests.
 
@@ -99,6 +101,7 @@ import {
   createKnowledgeUseReceipt,
   createKnowledgeVisibilitySnapshot,
   encodeKnowledgeVisibilitySnapshot,
+  KNOWLEDGE_SEARCH_RETRIEVER_ID,
   knowledgeVisibilityArtifactRef,
 } from '@tangle-network/agent-knowledge'
 
@@ -109,7 +112,7 @@ await artifacts.put('artifact://run/visibility.json', bytes)
 const retrieval = createKnowledgeRetrievalReceipt({
   runId,
   query: 'prior verifier obstruction',
-  retriever: { id: 'hybrid-search', version: '1.0.0', configDigest },
+  retriever: { id: KNOWLEDGE_SEARCH_RETRIEVER_ID, version: '1.0.0', configDigest },
   visibility,
   visibilityArtifact: knowledgeVisibilityArtifactRef({
     uri: 'artifact://run/visibility.json',

api-surface.json

@@ -68,6 +68,8 @@
       "AgentMemoryWriteResult": "value",
       "ApplyWriteBlocksResult": "value",
       "AuditKnowledgeCitationsOptions": "value",
+      "Bm25Hit": "value",
+      "Bm25Options": "value",
       "BuildAgentMemorySequencesFromBenchmarkCasesOptions": "type",
       "BuildEvalKnowledgeBundleOptions": "value",
       "BuildKnowledgeRelationGraphInput": "value",
@@ -143,6 +145,7 @@
       "KB_STORE_DIR": "value",
       "KNOWLEDGE_EVENT_TYPES": "value",
       "KNOWLEDGE_RECEIPT_DIGEST_ALGORITHM": "value",
+      "KNOWLEDGE_SEARCH_RETRIEVER_ID": "value",
       "KNOWLEDGE_USE_RECEIPT_SCHEMA_VERSION": "value",
       "KbStore": "value",
       "KnowledgeAnswerBenchmarkCase": "type",
@@ -232,6 +235,10 @@
       "KnowledgeIndexSchema": "value",
       "KnowledgeInspection": "value",
       "KnowledgeLayout": "value",
+      "KnowledgeLexicalFieldBoosts": "value",
+      "KnowledgeLexicalIndex": "value",
+      "KnowledgeLexicalIndexOptions": "value",
+      "KnowledgeLexicalPosting": "value",
       "KnowledgeLintFinding": "value",
       "KnowledgeMemoryBenchmarkCase": "type",
       "KnowledgeMemoryBenchmarkTaskKind": "type",
@@ -510,6 +517,7 @@
       "buildKnowledgeBenchmarkScenarios": "value",
       "buildKnowledgeGraph": "value",
       "buildKnowledgeIndex": "value",
+      "buildKnowledgeLexicalIndex": "value",
       "buildKnowledgeRelationGraph": "value",
       "buildRetrievalBenchmarkCasesFromQrels": "value",
       "buildRetrievalEvalDispatch": "value",
@@ -683,6 +691,7 @@
       "runSerializedKnowledgeOptimization": "value",
       "runVerifiedResearchLoop": "value",
       "scenarioContentFingerprint": "value",
+      "scoreBm25": "value",
       "scoreKnowledgeBaseIndex": "value",
       "scoreKnowledgeBenchmarkArtifact": "value",
       "scoreMemoryBenchmarkArtifact": "value",
@@ -709,6 +718,7 @@
       "toRagasEvaluationRows": "value",
       "toTruLensRecords": "value",
       "tokenizeQuery": "value",
+      "tokenizeText": "value",
       "totalMaterialFacts": "value",
       "triageSource": "value",
       "validateKnowledgeIndex": "value",

docs/knowledge-use-receipts.md

@@ -80,7 +80,10 @@ The input takes the precomputed `visibility` snapshot. A result is accepted only
 
 ```ts
 import { canonicalCandidateDigest } from '@tangle-network/agent-interface'
-import { createKnowledgeRetrievalReceipt } from '@tangle-network/agent-knowledge'
+import {
+  createKnowledgeRetrievalReceipt,
+  KNOWLEDGE_SEARCH_RETRIEVER_ID,
+} from '@tangle-network/agent-knowledge'
 
 const receipt = createKnowledgeRetrievalReceipt({
   runId,
@@ -89,9 +92,9 @@ const receipt = createKnowledgeRetrievalReceipt({
   executionRef,
   query: 'prior obstruction calibrated verifier',
   retriever: {
-    id: 'inspectable-token-overlap',
+    id: KNOWLEDGE_SEARCH_RETRIEVER_ID,
     version: '1.0.0',
-    configDigest: canonicalCandidateDigest({ tokenizer: 'unicode-words', limit: 5 }),
+    configDigest: canonicalCandidateDigest({ k1: 1.2, b: 0.75, limit: 5 }),
   },
   visibility,
   visibilityArtifact: artifact,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-knowledge",
-  "version": "10.0.0",
+  "version": "10.1.0",
   "description": "Build, search, evaluate, and improve source-backed knowledge bases.",
   "homepage": "https://github.com/tangle-network/agent-knowledge#readme",
   "repository": {

src/cli.ts

@@ -64,7 +64,7 @@ Commands:
   explain <page|id|query> [--root .] [--json]
       Explain sources, links, inbound links, and related pages.
   search <query> [--root .] [--pages-dir knowledge] [--limit 10] [--json]
-      Fast local token+graph search over the generated knowledge index.
+      Local BM25 and link-graph search (RRF fused) over the generated knowledge index.
   graph [--root .] [--format summary|json]
       Emit graph summary or JSON.
   lint [--root .] [--json]

src/filesystem-search-provider.ts

@@ -1,4 +1,5 @@
 import { buildKnowledgeIndex } from './indexer'
+import { buildKnowledgeLexicalIndex, type KnowledgeLexicalIndex } from './lexical-index'
 import { type KnowledgePagesOptions, normalizePagesDirectory } from './pages-directory'
 import type { RetrievalEvalRetriever, RetrievedKnowledgeHit } from './retrieval-eval'
 import { searchKnowledge } from './search'
@@ -33,6 +34,7 @@ export class FileSystemSearchProvider {
   /** Root-relative directory the provider indexes. */
   readonly pagesDirectory: string
   private index: KnowledgeIndex | undefined
+  private lexicalIndex: KnowledgeLexicalIndex | undefined
   private readonly defaultLimit: number
   private readonly refreshMode: 'manual' | 'always'
 
@@ -47,6 +49,7 @@ export class FileSystemSearchProvider {
   async getIndex(options: FileSystemSearchOptions = {}): Promise<KnowledgeIndex> {
     if (this.refreshMode === 'always' || options.refresh || !this.index) {
       this.index = await buildKnowledgeIndex(this.root, { pagesDirectory: this.pagesDirectory })
+      this.lexicalIndex = undefined
     }
     return this.index
   }
@@ -56,7 +59,11 @@ export class FileSystemSearchProvider {
     options: FileSystemSearchOptions = {},
   ): Promise<KnowledgeSearchResult[]> {
     const index = await this.getIndex(options)
-    return searchKnowledge(index, query, options.limit ?? this.defaultLimit)
+    if (!this.lexicalIndex) this.lexicalIndex = buildKnowledgeLexicalIndex(index.pages)
+    return searchKnowledge(index, query, {
+      limit: options.limit ?? this.defaultLimit,
+      lexicalIndex: this.lexicalIndex,
+    })
   }
 
   async retrieve(
@@ -78,6 +85,7 @@ export class FileSystemSearchProvider {
 
   invalidate(): void {
     this.index = undefined
+    this.lexicalIndex = undefined
   }
 }
 

src/index.ts

@@ -29,6 +29,7 @@ export * from './investment-thesis-task'
 export * from './kb-improvement'
 export * from './kb-store'
 export * from './knowledge-use-receipts'
+export * from './lexical-index'
 export * from './lint'
 export * from './material-facts-metric'
 export * from './memory/index'

src/lexical-index.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest'
+import { buildKnowledgeLexicalIndex, scoreBm25, tokenizeQuery, tokenizeText } from './lexical-index'
+import type { KnowledgePage } from './types'
+
+function page(id: string, title: string, text: string, path = `knowledge/${id}.md`): KnowledgePage {
+  return { id, path, title, text, frontmatter: {}, sourceIds: [], tags: [], outLinks: [] }
+}
+
+describe('lexical tokenization', () => {
+  it('gives the query exactly the distinct terms the index stores, including CJK bigrams', () => {
+    const indexed = tokenizeText('机器学习 机器学习 flash attention attention')
+
+    expect(tokenizeQuery('机器学习 机器学习 flash attention attention')).toEqual([
+      ...new Set(indexed),
+    ])
+    expect(indexed).toContain('机器')
+    expect(indexed.filter((token) => token === 'attention')).toHaveLength(2)
+  })
+})
+
+describe('scoreBm25', () => {
+  it('ranks a page holding a rare term above pages holding only a corpus-wide term', () => {
+    const common = 'research '
+    const index = buildKnowledgeLexicalIndex([
+      page('rare', 'Page a', `${common.repeat(8)} obstruction`),
+      page('common-heavy', 'Page b', common.repeat(30)),
+      page('common-c', 'Page c', common.repeat(8)),
+      page('common-d', 'Page d', common.repeat(8)),
+    ])
+
+    expect(scoreBm25(index, tokenizeQuery('research obstruction'))[0]?.page.id).toBe('rare')
+    expect(scoreBm25(index, ['obstruction'])[0]!.score).toBeGreaterThan(
+      scoreBm25(index, ['research']).find((hit) => hit.page.id === 'common-heavy')!.score,
+    )
+  })
+
+  it('ranks the shorter page first at equal term frequency', () => {
+    const index = buildKnowledgeLexicalIndex([
+      page('short', 'Short', 'tiling tiling'),
+      page('long', 'Long', `tiling tiling ${'padding '.repeat(40)}`),
+    ])
+
+    expect(scoreBm25(index, ['tiling']).map((hit) => hit.page.id)).toEqual(['short', 'long'])
+  })
+
+  it('saturates term frequency so repetition cannot dominate the ranking', () => {
+    const index = buildKnowledgeLexicalIndex([
+      page('once', 'Once', 'tiling'),
+      page('often', 'Often', 'tiling '.repeat(50)),
+    ])
+    const [often, once] = scoreBm25(index, ['tiling'])
+
+    expect(often!.page.id).toBe('often')
+    expect(often!.score / once!.score).toBeLessThan(3)
+  })
+})