From 01b5a2ec26e1f06679a8a13eb42facd0d1b16f28 Mon Sep 17 00:00:00 2001
From: can1357 <me@can.ac>
Date: Mon, 11 May 2026 00:53:50 +0200
Subject: [PATCH] feat(coding-agent/tool-discovery): improved tool discovery
 tokenization and scoring

- Normalized tool discovery token parsing with Unicode decomposition, removing combining marks and splitting acronym and camel-case boundaries for better tokenization.
- Expanded token separators to treat any non-letter or non-digit character as a boundary during indexing.
- Added a BM25_DELTA term to the BM25 scoring calculation to adjust relevance scoring for matching terms.
---
 .../src/tool-discovery/tool-index.ts          | 29 ++++++++++++++-----
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/packages/coding-agent/src/tool-discovery/tool-index.ts b/packages/coding-agent/src/tool-discovery/tool-index.ts
index b2ffdc5f8..71662ddb5 100644
--- a/packages/coding-agent/src/tool-discovery/tool-index.ts
+++ b/packages/coding-agent/src/tool-discovery/tool-index.ts
@@ -89,6 +89,7 @@ export interface DiscoverableMCPSearchResult {
 
 const BM25_K1 = 1.2;
 const BM25_B = 0.75;
+const BM25_DELTA = 1.0;
 const FIELD_WEIGHTS = {
 	name: 6,
 	label: 4,
@@ -112,13 +113,24 @@ function getSchemaPropertyKeys(parameters: unknown): string[] {
 }
 
 function tokenize(value: string): string[] {
-	return value
-		.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
-		.replace(/[^a-zA-Z0-9]+/g, " ")
-		.toLowerCase()
-		.trim()
-		.split(/\s+/)
-		.filter(token => token.length > 0);
+	return (
+		value
+			.normalize("NFKD")
+			// Drop combining marks (accents) so "café" → "cafe".
+			.replace(/\p{M}+/gu, "")
+			// Split ACRONYMBoundary: "MCPTool" → "MCP Tool".
+			.replace(/(\p{Lu}+)(\p{Lu}\p{Ll})/gu, "$1 $2")
+			// Split camelCase / digit→letter: "fooBar" → "foo Bar", "v2Beta" → "v2 Beta".
+			.replace(/(\p{Ll}|\p{N})(\p{Lu})/gu, "$1 $2")
+			// Everything that isn't a letter or digit becomes a separator. This subsumes markdown
+			// punctuation (`|*_`#-~>[]()`), box-drawing glyphs (─│┌), em/en dashes, smart quotes,
+			// zero-width spaces, NBSPs, etc.
+			.replace(/[^\p{L}\p{N}]+/gu, " ")
+			.toLowerCase()
+			.trim()
+			.split(/\s+/)
+			.filter(token => token.length > 0)
+	);
 }
 
 function addWeightedTokens(termFrequencies: Map<string, number>, value: string | undefined, weight: number): void {
@@ -274,7 +286,8 @@ export function searchDiscoverableTools(
 				const documentFrequency = index.documentFrequencies.get(token) ?? 0;
 				const idf = Math.log(1 + (index.documents.length - documentFrequency + 0.5) / (documentFrequency + 0.5));
 				const normalization = BM25_K1 * (1 - BM25_B + BM25_B * (document.length / index.averageLength));
-				score += queryTermCount * idf * ((termFrequency * (BM25_K1 + 1)) / (termFrequency + normalization));
+				score +=
+					queryTermCount * idf * ((termFrequency * (BM25_K1 + 1)) / (termFrequency + normalization) + BM25_DELTA);
 			}
 			return { tool: document.tool, score };
 		})