From 74dd5d4e5a3c7a5f16c710930270b6e99a0f8464 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 9 Aug 2026 02:39:55 +0000 Subject: [PATCH 1/6] feat(tools): add dependency-free BM25 ranker --- src/services/tools/Bm25Ranker.ts | 101 ++++++++++++++++++ .../tools/__tests__/Bm25Ranker.spec.ts | 72 +++++++++++++ src/services/tools/types.ts | 9 ++ 3 files changed, 182 insertions(+) create mode 100644 src/services/tools/Bm25Ranker.ts create mode 100644 src/services/tools/__tests__/Bm25Ranker.spec.ts create mode 100644 src/services/tools/types.ts diff --git a/src/services/tools/Bm25Ranker.ts b/src/services/tools/Bm25Ranker.ts new file mode 100644 index 0000000000..c90f5a874d --- /dev/null +++ b/src/services/tools/Bm25Ranker.ts @@ -0,0 +1,101 @@ +import { Ranker, ToolDoc } from "./types" + +type IndexedDocument = { + item: ToolDoc + termFrequencies: Map + length: number + index: number +} + +export class Bm25Ranker implements Ranker { + private readonly k1 = 1.5 + private readonly b = 0.75 + private indexedItems: ToolDoc[] | undefined + private documents: IndexedDocument[] = [] + private documentFrequency = new Map() + private averageDocumentLength = 0 + + rank(query: string, items: ToolDoc[], k: number): ToolDoc[] { + const queryTerms = tokenize(query) + + if (queryTerms.length === 0 || items.length === 0 || k <= 0) { + return [] + } + + this.ensureIndex(items) + + const scores = this.documents + .map((document) => ({ + document, + score: this.score(document, new Set(queryTerms)), + })) + .filter(({ score }) => score > 0) + + scores.sort((left, right) => right.score - left.score || left.document.index - right.document.index) + + return scores.slice(0, k).map(({ document }) => document.item) + } + + private ensureIndex(items: ToolDoc[]): void { + if (this.indexedItems === items) { + return + } + + this.indexedItems = items + this.documentFrequency = new Map() + this.documents = items.map((item, index) => { + const termFrequencies = countTerms(item) + + for (const term of termFrequencies.keys()) { + this.documentFrequency.set(term, (this.documentFrequency.get(term) ?? 0) + 1) + } + + return { + item, + termFrequencies, + length: Array.from(termFrequencies.values()).reduce((total, count) => total + count, 0), + index, + } + }) + this.averageDocumentLength = + this.documents.reduce((total, document) => total + document.length, 0) / this.documents.length + } + + private score(document: IndexedDocument, queryTerms: Set): number { + let score = 0 + const documentCount = this.documents.length + + for (const term of queryTerms) { + const termFrequency = document.termFrequencies.get(term) + + if (!termFrequency) { + continue + } + + const documentFrequency = this.documentFrequency.get(term) ?? 0 + const inverseDocumentFrequency = Math.log( + 1 + (documentCount - documentFrequency + 0.5) / (documentFrequency + 0.5), + ) + const normalization = this.k1 * (1 - this.b + this.b * (document.length / this.averageDocumentLength)) + + score += inverseDocumentFrequency * ((termFrequency * (this.k1 + 1)) / (termFrequency + normalization)) + } + + return score + } +} + +function tokenize(value: string): string[] { + return value.toLowerCase().split(/\W+/).filter(Boolean) +} + +function countTerms(item: ToolDoc): Map { + const frequencies = new Map() + const text = `${item.serverName} ${item.toolName} ${item.description}` + + for (const term of tokenize(text)) { + frequencies.set(term, (frequencies.get(term) ?? 0) + 1) + } + + return frequencies +} diff --git a/src/services/tools/__tests__/Bm25Ranker.spec.ts b/src/services/tools/__tests__/Bm25Ranker.spec.ts new file mode 100644 index 0000000000..f93ffb9727 --- /dev/null +++ b/src/services/tools/__tests__/Bm25Ranker.spec.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest" +import { Bm25Ranker } from "../Bm25Ranker" +import { ToolDoc } from "../types" + +const tool = (serverName: string, toolName: string, description: string): ToolDoc => ({ + serverName, + toolName, + description, +}) + +describe("Bm25Ranker", () => { + it("ranks matching MCP tools by BM25 relevance", () => { + const items = [ + tool("slack", "postMessage", "Send a Slack message"), + tool("jira", "createIssue", "Send messages and create Jira issues"), + tool("postgres", "query", "Send data and query a Postgres database"), + ] + + expect(new Bm25Ranker().rank("send slack message", items, 3)).toEqual([items[0], items[1], items[2]]) + }) + + it("returns an empty result for empty queries, empty items, and no matches", () => { + const ranker = new Bm25Ranker() + const items = [tool("slack", "postMessage", "Send a message")] + + expect(ranker.rank(" ", items, 10)).toEqual([]) + expect(ranker.rank("message", [], 10)).toEqual([]) + expect(ranker.rank("unrelated", items, 10)).toEqual([]) + }) + + it("truncates results to the requested top-k", () => { + const items = [ + tool("one", "search", "Search records"), + tool("two", "search", "Search records"), + tool("three", "search", "Search records"), + ] + + expect(new Bm25Ranker().rank("search", items, 2)).toEqual(items.slice(0, 2)) + expect(new Bm25Ranker().rank("search", items, 0)).toEqual([]) + }) + + it("preserves input order for equal scores", () => { + const items = [tool("first", "lookup", "Find a record"), tool("second", "lookup", "Find a record")] + + expect(new Bm25Ranker().rank("record", items, 10)).toEqual(items) + }) + + it("tokenizes case-insensitively around punctuation", () => { + const matching = tool("Slack-Server", "POST.Message", "Send a message") + const nonMatching = tool("calendar", "createEvent", "Create a calendar event") + + expect(new Bm25Ranker().rank("SLACK post message", [nonMatching, matching], 10)).toEqual([matching]) + }) + + it("handles tools with empty descriptions", () => { + const item = tool("GitHub", "listRepos", "") + + expect(new Bm25Ranker().rank("github repos", [item], 10)).toEqual([item]) + }) + + it("rebuilds only when the items array reference changes", () => { + const items = [tool("slack", "send", "Send a message")] + const ranker = new Bm25Ranker() + + ranker.rank("send", items, 10) + items.push(tool("jira", "create", "Create an issue")) + expect(ranker.rank("issue", items, 10)).toEqual([]) + + const newItems = items.slice() + expect(ranker.rank("issue", newItems, 10)).toEqual([newItems[1]]) + }) +}) diff --git a/src/services/tools/types.ts b/src/services/tools/types.ts new file mode 100644 index 0000000000..86170c8aa8 --- /dev/null +++ b/src/services/tools/types.ts @@ -0,0 +1,9 @@ +export type ToolDoc = { + serverName: string + toolName: string + description: string +} + +export interface Ranker { + rank(query: string, items: ToolDoc[], k: number): ToolDoc[] +} From 704a32116e5140780fe99ff136eb2e9bda7afc01 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 14 Oct 2026 22:16:34 -0400 Subject: [PATCH 2/6] fix(tools): split underscores in BM25 tokenizer --- src/services/tools/Bm25Ranker.ts | 5 ++++- src/services/tools/__tests__/Bm25Ranker.spec.ts | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/services/tools/Bm25Ranker.ts b/src/services/tools/Bm25Ranker.ts index c90f5a874d..3ac604b02f 100644 --- a/src/services/tools/Bm25Ranker.ts +++ b/src/services/tools/Bm25Ranker.ts @@ -86,7 +86,10 @@ export class Bm25Ranker implements Ranker { } function tokenize(value: string): string[] { - return value.toLowerCase().split(/\W+/).filter(Boolean) + return value + .toLowerCase() + .split(/[\W_]+/) + .filter(Boolean) } function countTerms(item: ToolDoc): Map { diff --git a/src/services/tools/__tests__/Bm25Ranker.spec.ts b/src/services/tools/__tests__/Bm25Ranker.spec.ts index f93ffb9727..480ff80843 100644 --- a/src/services/tools/__tests__/Bm25Ranker.spec.ts +++ b/src/services/tools/__tests__/Bm25Ranker.spec.ts @@ -52,6 +52,13 @@ describe("Bm25Ranker", () => { expect(new Bm25Ranker().rank("SLACK post message", [nonMatching, matching], 10)).toEqual([matching]) }) + it("splits underscore-separated tool names into separate terms", () => { + const matching = tool("github", "create_issue", "Open a ticket") + const nonMatching = tool("calendar", "list_events", "Show upcoming events") + + expect(new Bm25Ranker().rank("create issue", [nonMatching, matching], 10)).toEqual([matching]) + }) + it("handles tools with empty descriptions", () => { const item = tool("GitHub", "listRepos", "") From b9c74fced9489c6cf9442e670da576d32e49b2f0 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 17 Oct 2026 03:01:53 +0000 Subject: [PATCH 3/6] fix: preserve Unicode letters and numbers in MCP search tokens --- src/services/tools/Bm25Ranker.ts | 2 +- src/services/tools/__tests__/Bm25Ranker.spec.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/services/tools/Bm25Ranker.ts b/src/services/tools/Bm25Ranker.ts index 3ac604b02f..221f609c53 100644 --- a/src/services/tools/Bm25Ranker.ts +++ b/src/services/tools/Bm25Ranker.ts @@ -88,7 +88,7 @@ export class Bm25Ranker implements Ranker { function tokenize(value: string): string[] { return value .toLowerCase() - .split(/[\W_]+/) + .split(/[^\p{L}\p{N}]+/u) .filter(Boolean) } diff --git a/src/services/tools/__tests__/Bm25Ranker.spec.ts b/src/services/tools/__tests__/Bm25Ranker.spec.ts index 480ff80843..07827c3aeb 100644 --- a/src/services/tools/__tests__/Bm25Ranker.spec.ts +++ b/src/services/tools/__tests__/Bm25Ranker.spec.ts @@ -59,6 +59,14 @@ describe("Bm25Ranker", () => { expect(new Bm25Ranker().rank("create issue", [nonMatching, matching], 10)).toEqual([matching]) }) + it("preserves Unicode letters and numbers in tokens", () => { + const russian = tool("погода-сервер", "прогноз", "Прогноз погоды") + const arabic = tool("أدوات", "مقياس", "يدعم ٣ بروتوكولات") + + expect(new Bm25Ranker().rank("погода", [arabic, russian], 10)).toEqual([russian]) + expect(new Bm25Ranker().rank("٣", [russian, arabic], 10)).toEqual([arabic]) + }) + it("handles tools with empty descriptions", () => { const item = tool("GitHub", "listRepos", "") From 3ebe6533eacf44ea4ff6bde964d4e746e6bb1353 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 18 Oct 2026 03:44:01 +0000 Subject: [PATCH 4/6] fix(tools): normalize Unicode in BM25 tokenizer --- src/services/tools/Bm25Ranker.ts | 3 ++- src/services/tools/__tests__/Bm25Ranker.spec.ts | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/services/tools/Bm25Ranker.ts b/src/services/tools/Bm25Ranker.ts index 221f609c53..b82ad6975a 100644 --- a/src/services/tools/Bm25Ranker.ts +++ b/src/services/tools/Bm25Ranker.ts @@ -87,8 +87,9 @@ export class Bm25Ranker implements Ranker { function tokenize(value: string): string[] { return value + .normalize("NFC") .toLowerCase() - .split(/[^\p{L}\p{N}]+/u) + .split(/[^\p{L}\p{M}\p{N}]+/u) .filter(Boolean) } diff --git a/src/services/tools/__tests__/Bm25Ranker.spec.ts b/src/services/tools/__tests__/Bm25Ranker.spec.ts index 07827c3aeb..fb95409668 100644 --- a/src/services/tools/__tests__/Bm25Ranker.spec.ts +++ b/src/services/tools/__tests__/Bm25Ranker.spec.ts @@ -67,6 +67,14 @@ describe("Bm25Ranker", () => { expect(new Bm25Ranker().rank("٣", [russian, arabic], 10)).toEqual([arabic]) }) + it("matches composed and decomposed forms of the same accented character", () => { + const decomposed = tool("café", "menu", "Liste du café") + const other = tool("calendar", "events", "Show events") + + expect(new Bm25Ranker().rank("café", [other, decomposed], 10)).toEqual([decomposed]) + expect(new Bm25Ranker().rank("café", [other, tool("café", "menu", "Menu")], 10)).toHaveLength(1) + }) + it("handles tools with empty descriptions", () => { const item = tool("GitHub", "listRepos", "") From 45f1b7f84307cb3dc5bc0d66afd1c9177312c7de Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 19 Oct 2026 22:07:17 +0000 Subject: [PATCH 5/6] fix(knip): exclude unused BM25 ranker from production check --- knip.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/knip.json b/knip.json index 6f57a2fe54..2a90f90c30 100644 --- a/knip.json +++ b/knip.json @@ -27,6 +27,8 @@ "!utils/WorkspacePathResolver.ts!", "!test-utils/**!", "!utils/versionToken.ts!", + "!services/tools/Bm25Ranker.ts!", + "!services/tools/types.ts!", "!core/tools/helpers/toolResultFormatting.ts!" ], "ignoreDependencies": [ From d2fd7a52142294a6687fea33666ce62afd1b878d Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 1 Nov 2026 01:32:30 +0000 Subject: [PATCH 6/6] test(tools): strengthen BM25 ranking and Unicode assertions --- src/services/tools/__tests__/Bm25Ranker.spec.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/services/tools/__tests__/Bm25Ranker.spec.ts b/src/services/tools/__tests__/Bm25Ranker.spec.ts index fb95409668..24938e5113 100644 --- a/src/services/tools/__tests__/Bm25Ranker.spec.ts +++ b/src/services/tools/__tests__/Bm25Ranker.spec.ts @@ -11,12 +11,12 @@ const tool = (serverName: string, toolName: string, description: string): ToolDo describe("Bm25Ranker", () => { it("ranks matching MCP tools by BM25 relevance", () => { const items = [ - tool("slack", "postMessage", "Send a Slack message"), - tool("jira", "createIssue", "Send messages and create Jira issues"), tool("postgres", "query", "Send data and query a Postgres database"), + tool("jira", "createIssue", "Send messages and create Jira issues"), + tool("slack", "postMessage", "Send a Slack message"), ] - expect(new Bm25Ranker().rank("send slack message", items, 3)).toEqual([items[0], items[1], items[2]]) + expect(new Bm25Ranker().rank("send slack message", items, 3)).toEqual([items[2], items[1], items[0]]) }) it("returns an empty result for empty queries, empty items, and no matches", () => { @@ -69,10 +69,11 @@ describe("Bm25Ranker", () => { it("matches composed and decomposed forms of the same accented character", () => { const decomposed = tool("café", "menu", "Liste du café") + const composed = tool("café", "menu", "Menu") const other = tool("calendar", "events", "Show events") expect(new Bm25Ranker().rank("café", [other, decomposed], 10)).toEqual([decomposed]) - expect(new Bm25Ranker().rank("café", [other, tool("café", "menu", "Menu")], 10)).toHaveLength(1) + expect(new Bm25Ranker().rank("café", [other, composed], 10)).toEqual([composed]) }) it("handles tools with empty descriptions", () => {