From 28e71e31a21a09d14e83ca335ac5745ae1e64216 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 30 Oct 2026 22:33:16 +0000 Subject: [PATCH] fix: triage quick wins for chunk cap, map, and path search Clarify searchable chunk limits, exclude vendored bundles from bounded map, and boost keyword hits for file-like query tokens. Adds regression tests. --- README.md | 2 +- docs/capabilities.md | 2 +- src/cli.ts | 8 +-- src/core/codebase-map.ts | 6 +- src/core/indexer.ts | 36 ++++++++-- src/core/search.ts | 75 +++++++++++++++++++-- src/index.ts | 8 +-- src/types/index.ts | 4 ++ tests/codebase-map.test.ts | 27 ++++++++ tests/indexer-chunk-limit-reporting.test.ts | 51 ++++++++++++++ tests/search-path-token-boost.test.ts | 23 +++++++ 11 files changed, 219 insertions(+), 23 deletions(-) create mode 100644 tests/indexer-chunk-limit-reporting.test.ts create mode 100644 tests/search-path-token-boost.test.ts diff --git a/README.md b/README.md index d93d577..a61facd 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ The method and failures are documented so the measurements can be inspected with - The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. - Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. - Dedicated Angular, React, and Next.js analyzers ship in published 2.2.0; NestJS support is planned for unpublished 2.4.0. Other projects use the generic analyzer and the language parsers available for that stack. -- The default searchable-chunk limit is 5,000 per project. Customizing this via `projects[].parsing.maxChunks` in `~/.codebase-context/config.json` requires 2.4.0 (not yet published). +- The default searchable-chunk limit is 5,000 per project. Raise it with `projects[].parsing.maxChunks` in `~/.codebase-context/config.json` (see [capabilities](docs/capabilities.md)). - The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. ## Reference diff --git a/docs/capabilities.md b/docs/capabilities.md index 49fc4f9..8211c65 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -22,7 +22,7 @@ Per-project config overrides supported today: - `projects[].excludePatterns`: merged with the built-in exclusion set for that project at index time - `projects[].analyzerHints.analyzer`: prefers a registered analyzer by name for that project and falls back safely when the name is missing or invalid - `projects[].analyzerHints.extensions`: adds project-local source extensions for indexing and auto-refresh watching without changing defaults for other projects -- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project (requires 2.4.0, not yet published) +- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project (default 5000; configurable via `~/.codebase-context/config.json`) Copy-pasteable client config templates are shipped in the package: diff --git a/src/cli.ts b/src/cli.ts index 345eaec..2121e73 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,7 +9,7 @@ import path from 'path'; import { promises as fs } from 'fs'; import { KEYWORD_INDEX_FILENAME } from './constants/codebase-context.js'; import { makePaths } from './project-state.js'; -import { CodebaseIndexer } from './core/indexer.js'; +import { CodebaseIndexer, formatIndexingCompleteLine } from './core/indexer.js'; import { dispatchTool } from './tools/index.js'; import type { ToolContext } from './tools/index.js'; import type { IndexState } from './tools/types.js'; @@ -155,11 +155,7 @@ async function initToolContext(): Promise { indexState.lastIndexed = new Date(); indexState.stats = stats; - console.error( - `Complete: ${stats.indexedFiles} files, ${stats.totalChunks} chunks in ${( - stats.duration / 1000 - ).toFixed(2)}s` - ); + console.error(formatIndexingCompleteLine(stats)); } catch (error) { indexState.status = 'error'; indexState.error = error instanceof Error ? error.message : String(error); diff --git a/src/core/codebase-map.ts b/src/core/codebase-map.ts index 87b491f..8ae5b05 100644 --- a/src/core/codebase-map.ts +++ b/src/core/codebase-map.ts @@ -78,6 +78,9 @@ const MAP_EXCLUDED_PATH_PATTERNS = [ /(?:^|\/)(?:fixtures?|__fixtures__)(?:\/|$)/i, /(?:^|\/)(?:generated|__generated__)(?:\/|$)/i, /(?:^|\/)[^/]*\.(?:generated|gen|min)\.[^/]+$/i, + /(?:^|\/)raw(?:\/|$)/i, + /(?:^|\/)live-from-ai(?:\/|$)/i, + /[^/]+-[A-Za-z0-9]{6,}\.(?:js|css|mjs|cjs)$/i, /\.snap$/i ] as const; @@ -180,6 +183,7 @@ async function buildPinnedCodebaseMap( const entrypoints: string[] = []; for (const [file, imports] of Object.entries(graphImports)) { if (imports.length === 0) continue; // no imports — not an entrypoint + if (!isMapEligiblePath(file, mode)) continue; const importers = graphImportedBy[file]; if (!importers || importers.length === 0) { entrypoints.push(file); @@ -193,7 +197,7 @@ async function buildPinnedCodebaseMap( graphImportedBy ).map(([file, importers]) => ({ file, count: importers.length })); const hubFiles: string[] = sortByCountThenAlpha( - importedByCounts, + importedByCounts.filter(({ file }) => isMapEligiblePath(file, mode)), (x) => x.count, (x) => x.file ) diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 6d9ed37..d3a04e9 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -141,6 +141,8 @@ export interface IndexerOptions { interface PersistedIndexingStats { indexedFiles: number; totalChunks: number; + searchableChunks?: number; + chunkLimitApplied?: boolean; totalFiles: number; generatedAt: string; } @@ -956,7 +958,9 @@ export class CodebaseIndexer { let chunksToEmbed = chunksForEmbedding; if (chunksForEmbedding.length > maxChunks) { console.warn( - `WARNING: ${chunksForEmbedding.length} chunks exceed limit. Indexing first ${maxChunks} chunks.` + `WARNING: ${chunksForEmbedding.length} chunks exceed the searchable limit (${maxChunks}). ` + + `Only the first ${maxChunks} chunks are indexed for search. ` + + `Raise projects[].parsing.maxChunks in ~/.codebase-context/config.json for this repo root.` ); chunksToEmbed = chunksForEmbedding.slice(0, maxChunks); } @@ -1098,8 +1102,10 @@ export class CodebaseIndexer { // Keyword index always uses ALL chunks (full regen) const indexPath = path.join(generationDirectory, KEYWORD_INDEX_FILENAME); // Memory safety: cap keyword index too - const keywordChunks = - allChunks.length > maxChunks ? allChunks.slice(0, maxChunks) : allChunks; + const chunkLimitApplied = allChunks.length > maxChunks; + const keywordChunks = chunkLimitApplied ? allChunks.slice(0, maxChunks) : allChunks; + stats.searchableChunks = keywordChunks.length; + stats.chunkLimitApplied = chunkLimitApplied; await fs.writeFile( indexPath, JSON.stringify({ @@ -1206,6 +1212,8 @@ export class CodebaseIndexer { const persistedStats: PersistedIndexingStats = { indexedFiles: stats.indexedFiles, totalChunks: stats.totalChunks, + searchableChunks: stats.searchableChunks, + chunkLimitApplied: stats.chunkLimitApplied, totalFiles: stats.totalFiles, generatedAt }; @@ -1264,7 +1272,7 @@ export class CodebaseIndexer { ); } else { console.error(`Indexing complete in ${stats.duration}ms`); - console.error(`Indexed ${stats.indexedFiles} files, ${stats.totalChunks} chunks`); + console.error(formatIndexingProgressSummary(stats)); } return stats; @@ -1589,3 +1597,23 @@ export class CodebaseIndexer { return { ...this.progress }; } } + +export function formatIndexingProgressSummary(stats: IndexingStats): string { + const searchable = stats.searchableChunks ?? stats.totalChunks; + if (stats.chunkLimitApplied && searchable < stats.totalChunks) { + return ( + `Indexed ${stats.indexedFiles} files, ${stats.totalChunks} chunks analyzed, ` + + `${searchable} chunks searchable (limit applied)` + ); + } + return `Indexed ${stats.indexedFiles} files, ${stats.totalChunks} chunks`; +} + +export function formatIndexingCompleteLine(stats: IndexingStats): string { + const searchable = stats.searchableChunks ?? stats.totalChunks; + const durationSec = (stats.duration / 1000).toFixed(2); + if (stats.chunkLimitApplied && searchable < stats.totalChunks) { + return `Complete: ${stats.indexedFiles} files, ${searchable}/${stats.totalChunks} chunks searchable in ${durationSec}s`; + } + return `Complete: ${stats.indexedFiles} files, ${stats.totalChunks} chunks in ${durationSec}s`; +} diff --git a/src/core/search.ts b/src/core/search.ts index 743a3b2..ce1d72b 100644 --- a/src/core/search.ts +++ b/src/core/search.ts @@ -39,6 +39,24 @@ import { VECTOR_DB_DIRNAME } from '../constants/codebase-context.js'; +/** File/path-like tokens in a natural-language query (e.g. NOW.md, check-agent-instructions.mjs). */ +export function extractQueryPathTokens(query: string): string[] { + const tokens = new Set(); + for (const match of query.matchAll(/\b([A-Za-z0-9][A-Za-z0-9._-]*\.(?:md|mdx|txt))\b/g)) { + tokens.add(match[1].toLowerCase()); + } + for (const match of query.matchAll(/\b([a-z][a-z0-9-]*(?:-[a-z0-9]+)+)\.(?:mjs|cjs|js|ts)\b/gi)) { + tokens.add(match[0].toLowerCase()); + tokens.add(match[1].toLowerCase()); + } + for (const match of query.matchAll(/\b([a-z][a-z0-9-]*(?:-[a-z0-9]+){2,})\b/gi)) { + if (match[1].length >= 12) { + tokens.add(match[1].toLowerCase()); + } + } + return [...tokens]; +} + export interface SearchOptions { useSemanticSearch?: boolean; useKeywordSearch?: boolean; @@ -1941,11 +1959,23 @@ export class CodebaseSearcher { } if (fuseResults.length === 0 || (fuseResults[0].score ?? 0) > KEYWORD_FUSE_OPTIONS.threshold!) { - const rescue = await this.rescueKeywordMatches(query, limit, filters, validation); - const seen = new Set(rescue.map(({ item }) => item.id)); - fuseResults = [...rescue, ...fuseResults.filter(({ item }) => !seen.has(item.id))].sort( - (a, b) => (a.score ?? 0) - (b.score ?? 0) + const pathRescue: FuseResult[] = this.matchChunksByPathTokens(query, filters).map( + (chunk, refIndex) => ({ + item: chunk, + refIndex, + score: 0 + }) ); + const rescue = await this.rescueKeywordMatches(query, limit, filters, validation); + const seen = new Set([ + ...pathRescue.map(({ item }) => item.id), + ...rescue.map(({ item }) => item.id) + ]); + fuseResults = [ + ...pathRescue, + ...rescue, + ...fuseResults.filter(({ item }) => !seen.has(item.id)) + ].sort((a, b) => (a.score ?? 0) - (b.score ?? 0)); } const current = await this.selectCurrentCandidates( fuseResults, @@ -1984,6 +2014,20 @@ export class CodebaseSearcher { score = Math.min(1.0, score + 0.1); } + for (const token of extractQueryPathTokens(query)) { + const base = path.basename(relativePathLower); + const stem = base.replace(/\.[^.]+$/, ''); + if ( + base === token || + relativePathLower.endsWith(`/${token}`) || + stem === token.replace(/\.[^.]+$/, '') + ) { + score = Math.min(1.0, score + 0.45); + } else if (stem.includes(token) || relativePathLower.includes(token)) { + score = Math.min(1.0, score + 0.25); + } + } + return { chunk, score @@ -1996,6 +2040,29 @@ export class CodebaseSearcher { ); } + private matchChunksByPathTokens(query: string, filters?: SearchFilters): CodeChunk[] { + const tokens = extractQueryPathTokens(query); + if (tokens.length === 0) { + return []; + } + return this.chunks.filter((item) => { + if (!this.matchesSearchFilters(item, filters)) { + return false; + } + const relativePathLower = item.relativePath.toLowerCase(); + const base = path.basename(relativePathLower); + const stem = base.replace(/\.[^.]+$/, ''); + return tokens.some( + (token) => + base === token || + relativePathLower.endsWith(`/${token}`) || + stem === token.replace(/\.[^.]+$/, '') || + stem.includes(token) || + relativePathLower.includes(token) + ); + }); + } + private generateRelevanceReason(chunk: CodeChunk, query: string): string { const reasons: string[] = []; diff --git a/src/index.ts b/src/index.ts index a3bd53b..3373378 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,7 +13,7 @@ import { fileURLToPath } from 'url'; import type { Server } from '@modelcontextprotocol/sdk/server/index.js'; import type { ProjectConfig } from './server/config.js'; import type { Resource } from '@modelcontextprotocol/sdk/types.js'; -import { CodebaseIndexer } from './core/indexer.js'; +import { CodebaseIndexer, formatIndexingCompleteLine } from './core/indexer.js'; import { analyzerRegistry } from './core/analyzer-registry.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; @@ -1389,11 +1389,7 @@ async function performIndexingOnce( project.indexState.lastIndexed = new Date(); project.indexState.stats = stats; - console.error( - `Complete: ${stats.indexedFiles} files, ${stats.totalChunks} chunks in ${( - stats.duration / 1000 - ).toFixed(2)}s` - ); + console.error(formatIndexingCompleteLine(stats)); // Auto-extract memories from git history (non-blocking, best-effort) try { diff --git a/src/types/index.ts b/src/types/index.ts index 619b5e9..230e603 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -433,6 +433,10 @@ export interface IndexingStats { indexedFiles: number; skippedFiles: number; totalChunks: number; + /** Chunks written to the keyword index (may be lower than totalChunks when capped). */ + searchableChunks?: number; + /** True when parsing.maxChunks (default 5000) truncated the searchable index. */ + chunkLimitApplied?: boolean; totalLines: number; duration: number; // milliseconds avgChunkSize: number; diff --git a/tests/codebase-map.test.ts b/tests/codebase-map.test.ts index 10f5b49..e390c99 100644 --- a/tests/codebase-map.test.ts +++ b/tests/codebase-map.test.ts @@ -126,6 +126,33 @@ describe('buildCodebaseMap', () => { expect(map.architecture.entrypoints).toEqual(['src/cli.ts', 'src/index.ts']); }); + it('excludes vendored raw bundles from bounded map entrypoints and layers', async () => { + const rootPath = await createTempMapProject({ + graph: { + imports: { + 'live-from-ai/raw/index-BMtpUqT3.js': ['live-from-ai/raw/vendor.js'], + 'NOW.md': [], + 'scripts/context-route.mjs': ['NOW.md'] + }, + importedBy: { + 'live-from-ai/raw/index-BMtpUqT3.js': [], + 'NOW.md': ['scripts/context-route.mjs'], + 'scripts/context-route.mjs': [] + }, + exports: {} + } + }); + try { + const project = createProjectState(rootPath); + const map = await buildCodebaseMap(project); + expect(map.architecture.layers.map((layer) => layer.name)).not.toContain('live-from-ai'); + expect(map.architecture.entrypoints.some((ep) => ep.includes('index-BMtpUqT3'))).toBe(false); + expect(map.architecture.entrypoints).toContain('scripts/context-route.mjs'); + } finally { + await removeTempMapProject(rootPath); + } + }); + it('derives hub files: top 5 by importedBy count, sorted count-desc then alpha', async () => { const project = createProjectState(FIXTURE_ROOT); const map = await buildCodebaseMap(project, { mode: 'full' }); diff --git a/tests/indexer-chunk-limit-reporting.test.ts b/tests/indexer-chunk-limit-reporting.test.ts new file mode 100644 index 0000000..0156b0a --- /dev/null +++ b/tests/indexer-chunk-limit-reporting.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + formatIndexingCompleteLine, + formatIndexingProgressSummary +} from '../src/core/indexer.js'; +import type { IndexingStats } from '../src/types/index.js'; + +function baseStats(overrides: Partial = {}): IndexingStats { + return { + totalFiles: 100, + indexedFiles: 90, + skippedFiles: 10, + totalChunks: 88424, + searchableChunks: 5000, + chunkLimitApplied: true, + totalLines: 1, + duration: 2500, + avgChunkSize: 100, + componentsByType: {}, + componentsByLayer: { + core: 0, + feature: 0, + shared: 0, + infrastructure: 0, + unknown: 0 + }, + errors: [], + startedAt: new Date(), + ...overrides + }; +} + +describe('indexer chunk limit reporting', () => { + it('formatIndexingProgressSummary distinguishes analyzed vs searchable counts', () => { + const line = formatIndexingProgressSummary(baseStats()); + expect(line).toContain('88424 chunks analyzed'); + expect(line).toContain('5000 chunks searchable'); + }); + + it('formatIndexingCompleteLine shows searchable fraction when capped', () => { + const line = formatIndexingCompleteLine(baseStats()); + expect(line).toBe('Complete: 90 files, 5000/88424 chunks searchable in 2.50s'); + }); + + it('formatIndexingCompleteLine stays compact when no cap applied', () => { + const line = formatIndexingCompleteLine( + baseStats({ totalChunks: 120, searchableChunks: 120, chunkLimitApplied: false }) + ); + expect(line).toBe('Complete: 90 files, 120 chunks in 2.50s'); + }); +}); diff --git a/tests/search-path-token-boost.test.ts b/tests/search-path-token-boost.test.ts new file mode 100644 index 0000000..74a0ff2 --- /dev/null +++ b/tests/search-path-token-boost.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { extractQueryPathTokens } from '../src/core/search.js'; + +describe('extractQueryPathTokens', () => { + it('extracts markdown filenames from natural-language queries', () => { + const tokens = extractQueryPathTokens( + 'NOW.md live root entrypoint current state for routing' + ); + expect(tokens).toContain('now.md'); + }); + + it('extracts script paths and hyphenated stems', () => { + const tokens = extractQueryPathTokens( + 'what must I check before editing AGENTS.md protected agent instructions' + ); + expect(tokens).toContain('agents.md'); + }); + + it('extracts explicit script module names', () => { + const tokens = extractQueryPathTokens('run check-agent-instructions before commit'); + expect(tokens).toContain('check-agent-instructions'); + }); +});