diff --git a/.gitignore b/.gitignore index cee0f89..af67add 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,8 @@ repos/ # Local benchmark/lab outputs benchmark-runs/ +benchmark-repos/ +.aider.tags.cache.v4/ /outputs/ /results/*/raw/ /results/*/local/ diff --git a/docs/benchmark.md b/docs/benchmark.md index 73d073b..a9ae0fb 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -1,5 +1,53 @@ # What our local code-context experiments show +## Reproducible niche benchmark (prior art colleagues) + +This section reports a **frozen, rerunnable** comparison on **18 tasks** (team-convention lookup, edit-preflight navigation, symbol retrieval) across **three public repos**: [angular-spotify](https://github.com/trungk18/angular-spotify) (TypeScript), [Excalidraw](https://github.com/excalidraw/excalidraw) (TypeScript), and [FastAPI](https://github.com/fastapi/fastapi) (Python). Tasks and expected patterns live in `tests/fixtures/niche-benchmark-manifest.json`; comparator credits and rules live in `tests/fixtures/niche-benchmark-protocol.json`. Full machine output is checked in at `results/niche-benchmark-evidence.json` (generated **2026-10-30** on Node **v22.23.4**). + +**Primary accuracy metric:** top-3 file hit rate (expected path substring appears in the first three ranked paths returned for the same natural-language query). + +**Efficiency metrics:** estimated tokens returned (UTF-8 bytes ÷ 4) and query latency; index/setup time is reported separately in the evidence JSON. + +### One-command rerun + +```bash +pnpm install && pnpm run build +npm run benchmark:niche +# or: node scripts/run-niche-benchmark.mjs --validate-fixtures +``` + +Optional: `--lanes codebase-context,jCodeMunch,aider-repo-map,repomix,ripgrep` and `--output results/niche-benchmark-evidence.json`. + +**Prerequisites (not installed via npm publish):** `git`, `rg`, `python3` with `jcodemunch-mcp` and `aider-chat`, and network access to clone the three pinned repos into `benchmark-repos/`. + +### Aggregate results (18 tasks) + +| Colleague tool | Top-3 hit rate | Avg est. tokens | Avg query ms | Avg index ms | +| --- | ---: | ---: | ---: | ---: | +| **Codebase Context** | **50.0%** | 642 | 928 | 673 | +| [Aider](https://github.com/Aider-AI/aider) repo map | 22.2% | **95** | 1442 | 350 | +| [Repomix](https://github.com/yamadashy/repomix) (pack + section rank) | 11.1% | 5,559,174 | **114** | 286 | +| [jCodeMunch](https://github.com/jgravelle/jcodemunch-mcp) | 5.6% | **202** | 701 | **69** | +| [ripgrep](https://github.com/BurntSushi/ripgrep) (lexical baseline) | 5.6% | 5,001* | **76** | 0 | + +\*Ripgrep and Repomix token figures include large truncated raw payloads in this harness; treat them as upper-bound context cost, not model-ready bundles. + +### Where Codebase Context loses (same run) + +- **Team-convention file ranking:** 0/6 top-3 file hits — `patterns` answers matched expected **signals** (for example `dependencyInjection`, `stateManagement`) but did not return ranked repo paths, so file-based scoring stays at zero. **Aider repo map** reached **33%** top-3 on the same team-convention slice. +- **Payload size vs Aider / jCodeMunch:** median-style averages **642** estimated tokens vs **95** (Aider) and **202** (jCodeMunch) on the same queries. +- **Query latency vs ripgrep:** **928 ms** average query time vs **76 ms** for the ripgrep baseline (no index). +- **Edit-preflight slice:** **67%** top-3 here, but still below perfect; broad unsafe tasks are out of scope for this table (see edit-preflight fixtures for abstention tests). +- **Cross-repo variance:** symbol-retrieval top-3 was **83%** on the two TypeScript repos but **50%** on FastAPI in this run (see `byNiche` in the evidence JSON). + +### What this does not claim + +- No patch correctness, agent productivity, or npm-package performance guarantees. +- Repomix is not a search engine; the lane documents pack-generation cost plus section ranking. +- Earlier ContextBench / discovery families below remain separate evidence; they are not merged into this table. + +--- + This report describes local evidence from five ways of giving an AI coding agent context: Raw-native, Codebase Context, jCodeMunch, Repowise, and context-mode. The evidence is split into separate families because retrieval, repeated runs, full-agent observations, and a metered replay do not answer the same question. The families are not pooled into one score. The clearest current finding is narrow: in this fixed retrieval setup, Codebase Context recovered 25.7% of expected gold files at 11.5% file precision, while fresh indexing had material local cost. These measurements do not show better completed coding tasks. diff --git a/package.json b/package.json index a855889..efbbda6 100644 --- a/package.json +++ b/package.json @@ -132,6 +132,7 @@ "type-check": "tsc --noEmit", "mcp:inspect": "npx -y @modelcontextprotocol/inspector node dist/index.js .", "eval": "pnpm run build && node scripts/run-eval.mjs", + "benchmark:niche": "node scripts/run-niche-benchmark.mjs", "prepare": "husky" }, "dependencies": { diff --git a/results/niche-benchmark-evidence.json b/results/niche-benchmark-evidence.json new file mode 100644 index 0000000..a058157 --- /dev/null +++ b/results/niche-benchmark-evidence.json @@ -0,0 +1,1885 @@ +{ + "artifactType": "niche-prior-art-benchmark-evidence", + "generatedAt": "2026-10-30T22:39:55.772Z", + "protocol": { + "name": "niche-prior-art-benchmark", + "frozenDate": "2026-10-30", + "manifestFrozenDate": "2026-10-30" + }, + "runMeta": { + "manifestPath": "/workspace/tests/fixtures/niche-benchmark-manifest.json", + "protocolPath": "/workspace/tests/fixtures/niche-benchmark-protocol.json", + "manifestSha256": "9dd48fb64b7e81e6a16e694be031134f7a1506ff9a259b4cd084ab44827c704c", + "protocolSha256": "0b666b5ff5ed86f9a31cd0cc2857ef7dcd233ddf9c85b93091eb92488522aed0", + "nodeVersion": "v22.23.4", + "lanes": [ + "codebase-context", + "jCodeMunch", + "aider-repo-map", + "repomix", + "ripgrep" + ], + "repoCheckouts": { + "angular-spotify": { + "ref": "ff9efa765c53cfde78c9a172c62d515ae8ef9fe0", + "path": "/workspace/benchmark-repos/angular-spotify" + }, + "excalidraw": { + "ref": "e18c1dd213000dde0ae94ef7eb00aab537b39708", + "path": "/workspace/benchmark-repos/excalidraw" + }, + "fastapi": { + "ref": "628c34e0cae200564d191c95d7edea78c88c4b5e", + "path": "/workspace/benchmark-repos/fastapi" + } + } + }, + "comparators": [ + { + "id": "codebase-context", + "displayName": "Codebase Context", + "credit": "This repository (local dist CLI / MCP surfaces).", + "projectUrl": "https://github.com/PatrickSys/codebase-context", + "status": "completed", + "tasksTotal": 18, + "tasksCompleted": 18, + "tasksFailed": 0, + "top3HitRate": 0.5, + "averageUsefulness": 0.6666666666666666, + "averageEstimatedTokens": 641.7222222222222, + "averageQueryElapsedMs": 927.6111111111111, + "averageIndexElapsedMs": 672.7777777777779, + "byNiche": { + "team-convention": { + "tasks": 6, + "top3HitRate": 0 + }, + "edit-preflight": { + "tasks": 6, + "top3HitRate": 0.6666666666666666 + }, + "symbol-retrieval": { + "tasks": 6, + "top3HitRate": 0.8333333333333334 + } + }, + "taskResults": [ + { + "taskId": "as-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 1, + "matchedSignals": [ + "dependencyInjection" + ], + "payloadBytes": 1295, + "estimatedTokens": 324, + "queryElapsedMs": 518, + "indexElapsedMs": 512.5, + "topFiles": [] + }, + { + "taskId": "as-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 1, + "matchedSignals": [ + "stateManagement" + ], + "payloadBytes": 1220, + "estimatedTokens": 305, + "queryElapsedMs": 492, + "indexElapsedMs": 512.5, + "topFiles": [] + }, + { + "taskId": "as-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 4017, + "estimatedTokens": 1005, + "queryElapsedMs": 978, + "indexElapsedMs": 512.5, + "topFiles": [ + "libs/web/auth/data-access/src/lib/store/auth.store.ts", + "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "libs/web/visualizer/data-access/src/lib/audio-context.ts", + "libs/web/visualizer/data-access/src/lib/particles-renderer.ts", + "libs/web/visualizer/data-access/src/lib/sound-lines-renderer.ts" + ] + }, + { + "taskId": "as-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": true, + "firstRelevantHit": 2, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 4114, + "estimatedTokens": 1029, + "queryElapsedMs": 719, + "indexElapsedMs": 512.5, + "topFiles": [ + "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "libs/web/shell/ui/player-controls/src/lib/player-controls.component.ts", + "libs/web/shared/ui/media-summary/src/lib/media-summary.component.ts", + "libs/web/shell/ui/player-playback/src/lib/player-playback.component.ts", + "libs/web/lyrics/ui/lyrics-pip/src/lib/lyrics-pip.component.ts" + ] + }, + { + "taskId": "as-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 2420, + "estimatedTokens": 605, + "queryElapsedMs": 645, + "indexElapsedMs": 512.5, + "topFiles": [ + "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "libs/web/auth/util/src/lib/interceptors/unauthorized.interceptor.ts", + "libs/web/future-responsive/src/lib/albums.mock.json", + "libs/web/container-queries/src/lib/albums.mock.json", + "tsconfig.base.json" + ] + }, + { + "taskId": "as-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 2834, + "estimatedTokens": 709, + "queryElapsedMs": 906, + "indexElapsedMs": 512.5, + "topFiles": [ + "libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts", + "libs/web/auth/data-access/src/lib/store/auth.store.ts", + "libs/web/shared/data-access/store/src/lib/ui/ui-store.ts", + "libs/web/auth/util/src/lib/interceptors/unauthorized.interceptor.ts", + "libs/web/auth/ui/unauthorized-modal/src/lib/unauthorized-modal.component.html" + ] + }, + { + "taskId": "ex-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 1, + "matchedSignals": [ + "stateManagement" + ], + "payloadBytes": 1124, + "estimatedTokens": 281, + "queryElapsedMs": 500, + "indexElapsedMs": 1122, + "topFiles": [] + }, + { + "taskId": "ex-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 51, + "estimatedTokens": 13, + "queryElapsedMs": 491, + "indexElapsedMs": 1122, + "topFiles": [] + }, + { + "taskId": "ex-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 4389, + "estimatedTokens": 1098, + "queryElapsedMs": 1336, + "indexElapsedMs": 1122, + "topFiles": [ + "packages/excalidraw/history.ts", + "excalidraw-app/data/FileManager.ts", + "excalidraw-app/app_constants.ts", + "excalidraw-app/collab/Collab.tsx", + "excalidraw-app/App.tsx" + ] + }, + { + "taskId": "ex-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": true, + "firstRelevantHit": 3, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 4537, + "estimatedTokens": 1135, + "queryElapsedMs": 1261, + "indexElapsedMs": 1122, + "topFiles": [ + "packages/excalidraw/components/App.tsx", + "packages/element/src/types.ts", + "packages/excalidraw/data/blob.ts", + "packages/excalidraw/types.ts", + "excalidraw-app/data/firebase.ts" + ] + }, + { + "taskId": "ex-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 3763, + "estimatedTokens": 941, + "queryElapsedMs": 1224, + "indexElapsedMs": 1122, + "topFiles": [ + "packages/math/src/types.ts", + "packages/utils/src/withinBounds.ts", + "packages/excalidraw/clipboard.ts", + "excalidraw-app/data/index.ts", + "packages/excalidraw/tests/test-utils.ts" + ] + }, + { + "taskId": "ex-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 2, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 3568, + "estimatedTokens": 892, + "queryElapsedMs": 1488, + "indexElapsedMs": 1122, + "topFiles": [ + "packages/element/src/types.ts", + "packages/excalidraw/data/blob.ts", + "excalidraw-app/data/firebase.ts", + "packages/excalidraw/data/index.ts", + "packages/excalidraw/data/image.ts" + ] + }, + { + "taskId": "fa-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 51, + "estimatedTokens": 13, + "queryElapsedMs": 477, + "indexElapsedMs": 383.8333333333333, + "topFiles": [] + }, + { + "taskId": "fa-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 51, + "estimatedTokens": 13, + "queryElapsedMs": 452, + "indexElapsedMs": 383.8333333333333, + "topFiles": [] + }, + { + "taskId": "fa-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 3144, + "estimatedTokens": 786, + "queryElapsedMs": 913, + "indexElapsedMs": 383.8333333333333, + "topFiles": [ + "fastapi/security/oauth2.py", + "docs_src/security/tutorial005_py39.py", + "docs_src/security/tutorial005_py310.py", + "fastapi/security/http.py", + "fastapi/applications.py" + ] + }, + { + "taskId": "fa-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 3476, + "estimatedTokens": 869, + "queryElapsedMs": 1313, + "indexElapsedMs": 383.8333333333333, + "topFiles": [ + "tests/test_response_model_as_return_annotation.py", + "tests/test_tuples.py", + "tests/test_schema_extra_examples.py", + "tests/test_response_by_alias.py", + "tests/test_response_model_sub_types.py" + ] + }, + { + "taskId": "fa-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 2727, + "estimatedTokens": 682, + "queryElapsedMs": 1520, + "indexElapsedMs": 383.8333333333333, + "topFiles": [ + "fastapi/routing.py", + "fastapi/utils.py", + "tests/test_multi_body_errors.py", + "scripts/contributors.py", + "tests/test_router_events.py" + ] + }, + { + "taskId": "fa-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 3403, + "estimatedTokens": 851, + "queryElapsedMs": 1464, + "indexElapsedMs": 383.8333333333333, + "topFiles": [ + "tests/test_starlette_exception.py", + "tests/test_security_http_basic_optional.py", + "tests/test_multi_body_errors.py", + "tests/test_webhooks_security.py", + "tests/test_union_body.py" + ] + } + ] + }, + { + "id": "jCodeMunch", + "displayName": "jCodeMunch", + "credit": "jCodeMunch MCP symbol search and repo indexing (jcodemunch-mcp on PyPI).", + "projectUrl": "https://github.com/jgrahamc/jcodemunch", + "status": "completed", + "tasksTotal": 18, + "tasksCompleted": 18, + "tasksFailed": 0, + "top3HitRate": 0.05555555555555555, + "averageUsefulness": 0.1111111111111111, + "averageEstimatedTokens": 202.38888888888889, + "averageQueryElapsedMs": 701.4444444444445, + "averageIndexElapsedMs": 69.27777777777777, + "byNiche": { + "team-convention": { + "tasks": 6, + "top3HitRate": 0 + }, + "edit-preflight": { + "tasks": 6, + "top3HitRate": 0 + }, + "symbol-retrieval": { + "tasks": 6, + "top3HitRate": 0.16666666666666666 + } + }, + "taskResults": [ + { + "taskId": "as-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 787, + "estimatedTokens": 197, + "queryElapsedMs": 221, + "indexElapsedMs": 55.166666666666664, + "topFiles": [ + "constant" + ] + }, + { + "taskId": "as-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 1141, + "estimatedTokens": 286, + "queryElapsedMs": 222, + "indexElapsedMs": 55.166666666666664, + "topFiles": [ + "method", + "constant", + "type" + ] + }, + { + "taskId": "as-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 905, + "estimatedTokens": 227, + "queryElapsedMs": 225, + "indexElapsedMs": 55.166666666666664, + "topFiles": [ + "type", + "method" + ] + }, + { + "taskId": "as-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 1159, + "estimatedTokens": 290, + "queryElapsedMs": 222, + "indexElapsedMs": 55.166666666666664, + "topFiles": [ + "method" + ] + }, + { + "taskId": "as-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 1072, + "estimatedTokens": 268, + "queryElapsedMs": 221, + "indexElapsedMs": 55.166666666666664, + "topFiles": [ + "method" + ] + }, + { + "taskId": "as-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 905, + "estimatedTokens": 227, + "queryElapsedMs": 224, + "indexElapsedMs": 55.166666666666664, + "topFiles": [ + "type", + "method" + ] + }, + { + "taskId": "ex-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 947, + "estimatedTokens": 237, + "queryElapsedMs": 1036, + "indexElapsedMs": 50.833333333333336, + "topFiles": [ + "type", + "function" + ] + }, + { + "taskId": "ex-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 818, + "estimatedTokens": 205, + "queryElapsedMs": 1022, + "indexElapsedMs": 50.833333333333336, + "topFiles": [ + "function" + ] + }, + { + "taskId": "ex-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 797, + "estimatedTokens": 200, + "queryElapsedMs": 1195, + "indexElapsedMs": 50.833333333333336, + "topFiles": [ + "function", + "method", + "type" + ] + }, + { + "taskId": "ex-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 772, + "estimatedTokens": 193, + "queryElapsedMs": 1123, + "indexElapsedMs": 50.833333333333336, + "topFiles": [ + "method", + "function" + ] + }, + { + "taskId": "ex-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 592, + "estimatedTokens": 148, + "queryElapsedMs": 1081, + "indexElapsedMs": 50.833333333333336, + "topFiles": [ + "type" + ] + }, + { + "taskId": "ex-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 626, + "estimatedTokens": 157, + "queryElapsedMs": 1201, + "indexElapsedMs": 50.833333333333336, + "topFiles": [ + "function" + ] + }, + { + "taskId": "fa-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 850, + "estimatedTokens": 213, + "queryElapsedMs": 733, + "indexElapsedMs": 101.83333333333333, + "topFiles": [ + "method" + ] + }, + { + "taskId": "fa-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 1, + "matchedSignals": [ + "APIRouter", + "router" + ], + "payloadBytes": 510, + "estimatedTokens": 128, + "queryElapsedMs": 939, + "indexElapsedMs": 101.83333333333333, + "topFiles": [ + "method" + ] + }, + { + "taskId": "fa-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 721, + "estimatedTokens": 181, + "queryElapsedMs": 760, + "indexElapsedMs": 101.83333333333333, + "topFiles": [ + "class", + "function" + ] + }, + { + "taskId": "fa-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 496, + "estimatedTokens": 124, + "queryElapsedMs": 764, + "indexElapsedMs": 101.83333333333333, + "topFiles": [ + "class", + "method" + ] + }, + { + "taskId": "fa-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 634, + "estimatedTokens": 159, + "queryElapsedMs": 706, + "indexElapsedMs": 101.83333333333333, + "topFiles": [ + "function", + "field", + "class" + ] + }, + { + "taskId": "fa-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 810, + "estimatedTokens": 203, + "queryElapsedMs": 731, + "indexElapsedMs": 101.83333333333333, + "topFiles": [ + "function" + ] + } + ] + }, + { + "id": "aider-repo-map", + "displayName": "Aider repo map", + "credit": "Paul Gauthier's Aider RepoMap ranking (aider-chat on PyPI). Query tokens are passed as mentioned identifiers; there is no separate natural-language search API.", + "projectUrl": "https://github.com/Aider-AI/aider", + "status": "completed", + "tasksTotal": 18, + "tasksCompleted": 18, + "tasksFailed": 0, + "top3HitRate": 0.2222222222222222, + "averageUsefulness": 0.16666666666666666, + "averageEstimatedTokens": 95.05555555555556, + "averageQueryElapsedMs": 1441.8654441833496, + "averageIndexElapsedMs": 349.8888888888889, + "byNiche": { + "team-convention": { + "tasks": 6, + "top3HitRate": 0.3333333333333333 + }, + "edit-preflight": { + "tasks": 6, + "top3HitRate": 0 + }, + "symbol-retrieval": { + "tasks": 6, + "top3HitRate": 0.3333333333333333 + } + }, + "taskResults": [ + { + "taskId": "as-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 468, + "estimatedTokens": 117, + "queryElapsedMs": 1425.365924835205, + "indexElapsedMs": 351.3333333333333, + "topFiles": [ + "libs/web/shared/app-config/src/lib/app.config.ts", + "libs/web/shared/data-access/models/src/lib/api/play-request.ts", + "libs/web/shared/data-access/models/src/lib/api/spotify-api-params.ts", + "libs/web/visualizer/data-access/src/lib/visualizer-renderer.interface.ts", + "libs/web/shared/data-access/models/src/lib/api/track-extended.ts" + ] + }, + { + "taskId": "as-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 505, + "estimatedTokens": 127, + "queryElapsedMs": 1442.3494338989258, + "indexElapsedMs": 351.3333333333333, + "topFiles": [ + "libs/web/shared/app-config/src/lib/app.config.ts", + "libs/web/shared/data-access/models/src/lib/api/play-request.ts", + "libs/web/shared/data-access/models/src/lib/api/spotify-api-params.ts", + "libs/web/visualizer/data-access/src/lib/visualizer-renderer.interface.ts", + "libs/web/shared/data-access/models/src/lib/api/track-extended.ts" + ] + }, + { + "taskId": "as-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 530, + "estimatedTokens": 133, + "queryElapsedMs": 1495.1844215393066, + "indexElapsedMs": 351.3333333333333, + "topFiles": [ + "libs/web/shared/app-config/src/lib/app.config.ts", + "libs/web/shared/data-access/models/src/lib/api/play-request.ts", + "libs/web/shared/data-access/models/src/lib/api/spotify-api-params.ts", + "libs/web/visualizer/data-access/src/lib/visualizer-renderer.interface.ts", + "libs/web/shared/data-access/models/src/lib/api/track-extended.ts" + ] + }, + { + "taskId": "as-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 488, + "estimatedTokens": 122, + "queryElapsedMs": 1396.5404033660889, + "indexElapsedMs": 351.3333333333333, + "topFiles": [ + "libs/web/shared/app-config/src/lib/app.config.ts", + "libs/web/shared/data-access/models/src/lib/api/play-request.ts", + "libs/web/shared/data-access/models/src/lib/api/spotify-api-params.ts", + "libs/web/visualizer/data-access/src/lib/visualizer-renderer.interface.ts", + "libs/web/shared/data-access/models/src/lib/api/track-extended.ts" + ] + }, + { + "taskId": "as-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 460, + "estimatedTokens": 115, + "queryElapsedMs": 1381.8109035491943, + "indexElapsedMs": 351.3333333333333, + "topFiles": [ + "libs/web/shared/app-config/src/lib/app.config.ts", + "libs/web/shared/data-access/models/src/lib/api/play-request.ts", + "libs/web/shared/data-access/models/src/lib/api/spotify-api-params.ts", + "libs/web/visualizer/data-access/src/lib/visualizer-renderer.interface.ts", + "libs/web/shared/data-access/models/src/lib/api/track-extended.ts" + ] + }, + { + "taskId": "as-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 496, + "estimatedTokens": 124, + "queryElapsedMs": 1579.0975093841553, + "indexElapsedMs": 351.3333333333333, + "topFiles": [ + "libs/web/shared/app-config/src/lib/app.config.ts", + "libs/web/shared/data-access/models/src/lib/api/play-request.ts", + "libs/web/shared/data-access/models/src/lib/api/spotify-api-params.ts", + "libs/web/visualizer/data-access/src/lib/visualizer-renderer.interface.ts", + "libs/web/shared/data-access/models/src/lib/api/track-extended.ts" + ] + }, + { + "taskId": "ex-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 335, + "estimatedTokens": 84, + "queryElapsedMs": 1385.5478763580322, + "indexElapsedMs": 333.8333333333333, + "topFiles": [ + "packages/excalidraw/charts/charts.types.ts", + "packages/excalidraw/global.d.ts", + "packages/math/src/types.ts", + "packages/element/src/types.ts", + "packages/excalidraw/types.ts" + ] + }, + { + "taskId": "ex-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": true, + "firstRelevantHit": 3, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 364, + "estimatedTokens": 91, + "queryElapsedMs": 1377.885341644287, + "indexElapsedMs": 333.8333333333333, + "topFiles": [ + "packages/excalidraw/charts/charts.types.ts", + "packages/excalidraw/global.d.ts", + "packages/math/src/types.ts", + "packages/element/src/types.ts", + "packages/excalidraw/types.ts" + ] + }, + { + "taskId": "ex-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 331, + "estimatedTokens": 83, + "queryElapsedMs": 1376.3632774353027, + "indexElapsedMs": 333.8333333333333, + "topFiles": [ + "packages/excalidraw/types.ts", + "packages/math/src/types.ts", + "packages/excalidraw/charts/charts.types.ts", + "packages/element/src/types.ts", + "packages/common/src/editorInterface.ts" + ] + }, + { + "taskId": "ex-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 348, + "estimatedTokens": 87, + "queryElapsedMs": 1386.0201835632324, + "indexElapsedMs": 333.8333333333333, + "topFiles": [ + "packages/excalidraw/types.ts", + "packages/math/src/types.ts", + "packages/excalidraw/charts/charts.types.ts", + "packages/element/src/types.ts", + "packages/common/src/editorInterface.ts" + ] + }, + { + "taskId": "ex-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 308, + "estimatedTokens": 77, + "queryElapsedMs": 1516.0613059997559, + "indexElapsedMs": 333.8333333333333, + "topFiles": [ + "packages/math/src/types.ts", + "packages/element/src/types.ts", + "packages/excalidraw/tests/helpers/ui.ts", + "packages/common/src/bounds.ts", + "packages/excalidraw/types.ts" + ] + }, + { + "taskId": "ex-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 334, + "estimatedTokens": 84, + "queryElapsedMs": 1381.5414905548096, + "indexElapsedMs": 333.8333333333333, + "topFiles": [ + "packages/excalidraw/types.ts", + "packages/math/src/types.ts", + "packages/excalidraw/charts/charts.types.ts", + "packages/element/src/types.ts", + "packages/common/src/editorInterface.ts" + ] + }, + { + "taskId": "fa-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0.5, + "matchedSignals": [ + "dependency" + ], + "payloadBytes": 307, + "estimatedTokens": 77, + "queryElapsedMs": 1467.3211574554443, + "indexElapsedMs": 364.5, + "topFiles": [ + "fastapi/applications.py", + "fastapi/encoders.py", + "fastapi/routing.py", + "fastapi/_compat.py", + "docs/en/docs/js/termynal.js" + ] + }, + { + "taskId": "fa-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 0.5, + "matchedSignals": [ + "router" + ], + "payloadBytes": 292, + "estimatedTokens": 73, + "queryElapsedMs": 1499.6511936187744, + "indexElapsedMs": 364.5, + "topFiles": [ + "fastapi/routing.py", + "fastapi/exceptions.py", + "fastapi/encoders.py", + "fastapi/_compat.py", + "fastapi/utils.py" + ] + }, + { + "taskId": "fa-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 308, + "estimatedTokens": 77, + "queryElapsedMs": 1510.103464126587, + "indexElapsedMs": 364.5, + "topFiles": [ + "fastapi/encoders.py", + "fastapi/applications.py", + "docs/en/docs/js/termynal.js", + "fastapi/_compat.py", + "fastapi/exceptions.py" + ] + }, + { + "taskId": "fa-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 317, + "estimatedTokens": 80, + "queryElapsedMs": 1470.5393314361572, + "indexElapsedMs": 364.5, + "topFiles": [ + "fastapi/dependencies/models.py", + "fastapi/_compat.py", + "fastapi/exceptions.py", + "docs/en/docs/js/termynal.js", + "fastapi/encoders.py" + ] + }, + { + "taskId": "fa-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 3, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 320, + "estimatedTokens": 80, + "queryElapsedMs": 1429.5134544372559, + "indexElapsedMs": 364.5, + "topFiles": [ + "fastapi/encoders.py", + "fastapi/applications.py", + "fastapi/routing.py", + "tests/test_additional_properties_bool.py", + "fastapi/_compat.py" + ] + }, + { + "taskId": "fa-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": 5, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 317, + "estimatedTokens": 80, + "queryElapsedMs": 1432.6813220977783, + "indexElapsedMs": 364.5, + "topFiles": [ + "fastapi/encoders.py", + "fastapi/applications.py", + "docs/en/docs/js/termynal.js", + "fastapi/_compat.py", + "fastapi/exceptions.py" + ] + } + ] + }, + { + "id": "repomix", + "displayName": "Repomix", + "credit": "Repomix repository packing (repomix on npm). Repomix does not expose query-specific retrieval; the lane charges pack generation to index time and ranks files by scanning pack sections for query tokens.", + "projectUrl": "https://github.com/yamadashy/repomix", + "status": "completed", + "tasksTotal": 18, + "tasksCompleted": 18, + "tasksFailed": 0, + "top3HitRate": 0.1111111111111111, + "averageUsefulness": 0.1111111111111111, + "averageEstimatedTokens": 5559173.666666667, + "averageQueryElapsedMs": 113.88888888888889, + "averageIndexElapsedMs": 286.27777777777777, + "byNiche": { + "team-convention": { + "tasks": 6, + "top3HitRate": 0 + }, + "edit-preflight": { + "tasks": 6, + "top3HitRate": 0.16666666666666666 + }, + "symbol-retrieval": { + "tasks": 6, + "top3HitRate": 0.16666666666666666 + } + }, + "taskResults": [ + { + "taskId": "as-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 1847698, + "queryElapsedMs": 26, + "indexElapsedMs": 178.66666666666666, + "topFiles": [ + ".codebase-context/.generations/a2cd3546-363b-4787-910e-11cce05901af/index/index-build.json" + ] + }, + { + "taskId": "as-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 1847698, + "queryElapsedMs": 28, + "indexElapsedMs": 178.66666666666666, + "topFiles": [ + ".codebase-context/.generations/a2cd3546-363b-4787-910e-11cce05901af/index/index-build.json" + ] + }, + { + "taskId": "as-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 1847698, + "queryElapsedMs": 21, + "indexElapsedMs": 178.66666666666666, + "topFiles": [ + ".codebase-context/.generations/a2cd3546-363b-4787-910e-11cce05901af/index/index-build.json" + ] + }, + { + "taskId": "as-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 1847698, + "queryElapsedMs": 21, + "indexElapsedMs": 178.66666666666666, + "topFiles": [ + ".codebase-context/.generations/a2cd3546-363b-4787-910e-11cce05901af/index/index-build.json" + ] + }, + { + "taskId": "as-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 1847698, + "queryElapsedMs": 14, + "indexElapsedMs": 178.66666666666666, + "topFiles": [ + ".codebase-context/.generations/a2cd3546-363b-4787-910e-11cce05901af/index/index-build.json" + ] + }, + { + "taskId": "as-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 1847698, + "queryElapsedMs": 18, + "indexElapsedMs": 178.66666666666666, + "topFiles": [ + ".codebase-context/.generations/a2cd3546-363b-4787-910e-11cce05901af/index/index-build.json" + ] + }, + { + "taskId": "ex-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 8754051, + "queryElapsedMs": 235, + "indexElapsedMs": 346.6666666666667, + "topFiles": [ + ".codebase-context/.generations/7ef9aa97-4e56-4dbe-aaa5-ba2bfb795092/index/index-build.json" + ] + }, + { + "taskId": "ex-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 8754051, + "queryElapsedMs": 235, + "indexElapsedMs": 346.6666666666667, + "topFiles": [ + ".codebase-context/.generations/7ef9aa97-4e56-4dbe-aaa5-ba2bfb795092/index/index-build.json" + ] + }, + { + "taskId": "ex-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 8754051, + "queryElapsedMs": 162, + "indexElapsedMs": 346.6666666666667, + "topFiles": [ + ".codebase-context/.generations/7ef9aa97-4e56-4dbe-aaa5-ba2bfb795092/index/index-build.json" + ] + }, + { + "taskId": "ex-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 8754051, + "queryElapsedMs": 194, + "indexElapsedMs": 346.6666666666667, + "topFiles": [ + ".codebase-context/.generations/7ef9aa97-4e56-4dbe-aaa5-ba2bfb795092/index/index-build.json" + ] + }, + { + "taskId": "ex-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 8754051, + "queryElapsedMs": 187, + "indexElapsedMs": 346.6666666666667, + "topFiles": [ + ".codebase-context/.generations/7ef9aa97-4e56-4dbe-aaa5-ba2bfb795092/index/index-build.json" + ] + }, + { + "taskId": "ex-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": true, + "firstRelevantHit": 1, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 8754051, + "queryElapsedMs": 189, + "indexElapsedMs": 346.6666666666667, + "topFiles": [ + ".codebase-context/.generations/7ef9aa97-4e56-4dbe-aaa5-ba2bfb795092/index/index-build.json" + ] + }, + { + "taskId": "fa-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 6075772, + "queryElapsedMs": 133, + "indexElapsedMs": 333.5, + "topFiles": [ + ".codebase-context/.generations/93a21abb-70cf-4b9c-8b23-c234ecf4165f/index/index-build.json" + ] + }, + { + "taskId": "fa-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 6075772, + "queryElapsedMs": 111, + "indexElapsedMs": 333.5, + "topFiles": [ + ".codebase-context/.generations/93a21abb-70cf-4b9c-8b23-c234ecf4165f/index/index-build.json" + ] + }, + { + "taskId": "fa-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 6075772, + "queryElapsedMs": 121, + "indexElapsedMs": 333.5, + "topFiles": [ + ".codebase-context/.generations/93a21abb-70cf-4b9c-8b23-c234ecf4165f/index/index-build.json" + ] + }, + { + "taskId": "fa-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 6075772, + "queryElapsedMs": 113, + "indexElapsedMs": 333.5, + "topFiles": [ + ".codebase-context/.generations/93a21abb-70cf-4b9c-8b23-c234ecf4165f/index/index-build.json" + ] + }, + { + "taskId": "fa-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 6075772, + "queryElapsedMs": 122, + "indexElapsedMs": 333.5, + "topFiles": [ + ".codebase-context/.generations/93a21abb-70cf-4b9c-8b23-c234ecf4165f/index/index-build.json" + ] + }, + { + "taskId": "fa-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 50000, + "estimatedTokens": 6075772, + "queryElapsedMs": 120, + "indexElapsedMs": 333.5, + "topFiles": [ + ".codebase-context/.generations/93a21abb-70cf-4b9c-8b23-c234ecf4165f/index/index-build.json" + ] + } + ] + }, + { + "id": "ripgrep", + "displayName": "ripgrep (lexical baseline)", + "credit": "BurntSushi ripgrep JSON search over the repo checkout. No index; measures lexical retrieval only.", + "projectUrl": "https://github.com/BurntSushi/ripgrep", + "status": "completed", + "tasksTotal": 18, + "tasksCompleted": 18, + "tasksFailed": 0, + "top3HitRate": 0.05555555555555555, + "averageUsefulness": 0.1111111111111111, + "averageEstimatedTokens": 5000.5, + "averageQueryElapsedMs": 76.16666666666667, + "averageIndexElapsedMs": 0, + "byNiche": { + "team-convention": { + "tasks": 6, + "top3HitRate": 0 + }, + "edit-preflight": { + "tasks": 6, + "top3HitRate": 0.16666666666666666 + }, + "symbol-retrieval": { + "tasks": 6, + "top3HitRate": 0 + } + }, + "taskResults": [ + { + "taskId": "as-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20008, + "estimatedTokens": 5002, + "queryElapsedMs": 24, + "indexElapsedMs": 0, + "topFiles": [ + "apps/angular-spotify/src/polyfills.ts", + "README.md", + "yarn.lock", + "docs/plans/2026-03-07-infinite-scroll-pagination-design.md", + "docs/plans/2026-03-01-visualizer-selector-design.md" + ] + }, + { + "taskId": "as-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 23, + "indexElapsedMs": 0, + "topFiles": [ + "apps/angular-spotify/src/environments/environment.ts", + "apps/angular-spotify/src/app/app.component.spec.ts", + "apps/angular-spotify/src/polyfills.ts", + ".codebase-context/.generations/e369989f-3d39-444e-a743-4ed8fb5ce7f5/intelligence.json", + "README.md" + ] + }, + { + "taskId": "as-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 33, + "indexElapsedMs": 0, + "topFiles": [ + "tsconfig.base.json", + "apps/angular-spotify/ngsw-config.json", + "apps/angular-spotify/project.json", + "apps/angular-spotify/src/environments/environment.prod.ts", + "apps/angular-spotify/src/environments/environment.ts" + ] + }, + { + "taskId": "as-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 26, + "indexElapsedMs": 0, + "topFiles": [ + "tsconfig.base.json", + "apps/angular-spotify/src/environments/environment.ts", + "apps/angular-spotify/src/manifest.webmanifest", + "apps/angular-spotify/src/index.prod.html", + "apps/angular-spotify/src/app/app.component.spec.ts" + ] + }, + { + "taskId": "as-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20008, + "estimatedTokens": 5002, + "queryElapsedMs": 22, + "indexElapsedMs": 0, + "topFiles": [ + "tsconfig.base.json", + ".codebase-context/.generations/e369989f-3d39-444e-a743-4ed8fb5ce7f5/intelligence.json", + "README.md", + "migrations.json", + "docs/plans/2026-03-07-infinite-scroll-pagination-design.md" + ] + }, + { + "taskId": "as-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 24, + "indexElapsedMs": 0, + "topFiles": [ + "tsconfig.base.json", + "apps/angular-spotify/src/environments/environment.prod.ts", + "apps/angular-spotify/src/environments/environment.ts", + "apps/angular-spotify/src/polyfills.ts", + "apps/angular-spotify/src/custom/ant/_button.scss" + ] + }, + { + "taskId": "ex-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20006, + "estimatedTokens": 5002, + "queryElapsedMs": 84, + "indexElapsedMs": 0, + "topFiles": [ + "vitest.config.mts", + ".github/copilot-instructions.md", + "yarn.lock", + ".eslintrc.json", + "examples/with-script-in-browser/components/CustomFooter.tsx" + ] + }, + { + "taskId": "ex-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": 5, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 93, + "indexElapsedMs": 0, + "topFiles": [ + ".env.development", + ".eslintrc.json", + "scripts/release.js", + "scripts/build-node.js", + "excalidraw-app/collab/Collab.tsx" + ] + }, + { + "taskId": "ex-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20002, + "estimatedTokens": 5001, + "queryElapsedMs": 105, + "indexElapsedMs": 0, + "topFiles": [ + "examples/with-nextjs/README.md", + "examples/with-script-in-browser/index.tsx", + ".eslintrc.json", + "README.md", + "examples/with-script-in-browser/components/ExampleApp.tsx" + ] + }, + { + "taskId": "ex-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": true, + "firstRelevantHit": 3, + "usefulnessScore": 1, + "matchedSignals": [], + "payloadBytes": 20004, + "estimatedTokens": 5001, + "queryElapsedMs": 119, + "indexElapsedMs": 0, + "topFiles": [ + ".dockerignore", + ".gitignore", + ".eslintrc.json", + "scripts/release.js", + "scripts/buildWasm.js" + ] + }, + { + "taskId": "ex-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 158, + "indexElapsedMs": 0, + "topFiles": [ + ".gitignore", + "examples/with-nextjs/.gitignore", + "examples/with-nextjs/next.config.js", + "examples/with-nextjs/tsconfig.json", + "examples/with-nextjs/yarn.lock" + ] + }, + { + "taskId": "ex-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 110, + "indexElapsedMs": 0, + "topFiles": [ + ".gitignore", + "examples/with-nextjs/next.config.js", + "examples/with-nextjs/src/pages/excalidraw-in-pages.tsx", + "examples/with-nextjs/src/excalidrawWrapper.tsx", + "examples/with-nextjs/src/app/layout.tsx" + ] + }, + { + "taskId": "fa-team-01", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0.5, + "matchedSignals": [ + "dependency" + ], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 99, + "indexElapsedMs": 0, + "topFiles": [ + "requirements-docs.txt", + "pyproject.toml", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/index/index-build.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/intelligence.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/relationships.json" + ] + }, + { + "taskId": "fa-team-02", + "status": "completed", + "niche": "team-convention", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0.5, + "matchedSignals": [ + "router" + ], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 105, + "indexElapsedMs": 0, + "topFiles": [ + "pyproject.toml", + "requirements-docs-tests.txt", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/intelligence.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/health.json", + "LICENSE" + ] + }, + { + "taskId": "fa-ep-01", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20002, + "estimatedTokens": 5001, + "queryElapsedMs": 82, + "indexElapsedMs": 0, + "topFiles": [ + "pyproject.toml", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/intelligence.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/relationships.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/health.json", + "LICENSE" + ] + }, + { + "taskId": "fa-ep-02", + "status": "completed", + "niche": "edit-preflight", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 96, + "indexElapsedMs": 0, + "topFiles": [ + "requirements-docs.txt", + "pyproject.toml", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/index/index-build.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/intelligence.json", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/relationships.json" + ] + }, + { + "taskId": "fa-sym-01", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 76, + "indexElapsedMs": 0, + "topFiles": [ + "requirements-docs.txt", + "pyproject.toml", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/health.json", + "LICENSE", + "docs_src/body/tutorial001_py310.py" + ] + }, + { + "taskId": "fa-sym-02", + "status": "completed", + "niche": "symbol-retrieval", + "top3Hit": false, + "firstRelevantHit": null, + "usefulnessScore": 0, + "matchedSignals": [], + "payloadBytes": 20000, + "estimatedTokens": 5000, + "queryElapsedMs": 92, + "indexElapsedMs": 0, + "topFiles": [ + "pyproject.toml", + "requirements-docs-tests.txt", + "requirements-docs-insiders.txt", + ".codebase-context/.generations/b4709add-8981-4a99-af8d-5bce4ff4b916/health.json", + "scripts/notify_translations.py" + ] + } + ] + } + ], + "summaryTable": [ + { + "laneId": "codebase-context", + "status": "completed", + "top3HitRate": 0.5, + "averageEstimatedTokens": 641.7222222222222, + "averageQueryElapsedMs": 927.6111111111111, + "averageIndexElapsedMs": 672.7777777777779 + }, + { + "laneId": "jCodeMunch", + "status": "completed", + "top3HitRate": 0.05555555555555555, + "averageEstimatedTokens": 202.38888888888889, + "averageQueryElapsedMs": 701.4444444444445, + "averageIndexElapsedMs": 69.27777777777777 + }, + { + "laneId": "aider-repo-map", + "status": "completed", + "top3HitRate": 0.2222222222222222, + "averageEstimatedTokens": 95.05555555555556, + "averageQueryElapsedMs": 1441.8654441833496, + "averageIndexElapsedMs": 349.8888888888889 + }, + { + "laneId": "repomix", + "status": "completed", + "top3HitRate": 0.1111111111111111, + "averageEstimatedTokens": 5559173.666666667, + "averageQueryElapsedMs": 113.88888888888889, + "averageIndexElapsedMs": 286.27777777777777 + }, + { + "laneId": "ripgrep", + "status": "completed", + "top3HitRate": 0.05555555555555555, + "averageEstimatedTokens": 5000.5, + "averageQueryElapsedMs": 76.16666666666667, + "averageIndexElapsedMs": 0 + } + ] +} diff --git a/scripts/lib/niche-benchmark/aider-repomap-query.py b/scripts/lib/niche-benchmark/aider-repomap-query.py new file mode 100644 index 0000000..a366133 --- /dev/null +++ b/scripts/lib/niche-benchmark/aider-repomap-query.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Aider RepoMap lane for the niche prior-art benchmark.""" +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time + + +def tokenize(text: str) -> list[str]: + return [ + token + for token in re.findall(r"[A-Za-z_][A-Za-z0-9_+#.-]{2,}", text) + if token.lower() + not in {"the", "and", "for", "with", "from", "that", "this", "what", "how", "does", "are"} + ] + + +def git_tracked_files(root: str) -> list[str]: + output = subprocess.check_output(["git", "ls-files"], cwd=root, text=True) + return [line.strip() for line in output.splitlines() if line.strip()] + + +def ranked_files(root: str, query: str, limit: int) -> tuple[list[str], str, float]: + from aider.io import InputOutput + from aider.models import Model + from aider.repomap import RepoMap + + started = time.time() + io = InputOutput(yes=True) + model = Model("gpt-4o-mini") + repo_map = RepoMap(map_tokens=4096, root=root, main_model=model, io=io) + tracked = git_tracked_files(root) + source_files = [ + path + for path in tracked + if any( + path.endswith(ext) + for ext in (".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".rs", ".java", ".cs", ".cpp", ".c", ".h") + ) + ] + mentioned = tokenize(query) + tags = repo_map.get_ranked_tags([], source_files, [], mentioned) + top_files: list[str] = [] + for tag in tags: + rel = getattr(tag, "rel_fname", None) or getattr(tag, "fname", None) + if not isinstance(rel, str): + continue + normalized = rel.replace("\\", "/").lstrip("./") + if normalized and normalized not in top_files: + top_files.append(normalized) + if len(top_files) >= limit: + break + payload = json.dumps( + {"mentionedIdents": mentioned, "rankedTags": len(tags), "topFiles": top_files}, + indent=2, + ) + return top_files, payload, (time.time() - started) * 1000 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", required=True) + parser.add_argument("--query", default="") + parser.add_argument("--limit", type=int, default=5) + parser.add_argument("--warm-index", action="store_true") + args = parser.parse_args() + + root = os.path.abspath(args.root) + if not os.path.isdir(os.path.join(root, ".git")): + print(json.dumps({"error": "missing_git_metadata", "topFiles": [], "payload": ""})) + return 2 + + try: + if args.warm_index: + _, _, elapsed_ms = ranked_files(root, "warm index scan", min(args.limit, 3)) + print(json.dumps({"status": "indexed", "indexElapsedMs": elapsed_ms})) + return 0 + + top_files, payload, elapsed_ms = ranked_files(root, args.query, args.limit) + print( + json.dumps( + { + "topFiles": top_files, + "payload": payload, + "queryElapsedMs": elapsed_ms, + "indexElapsedMs": 0, + } + ) + ) + return 0 + except Exception as exc: # noqa: BLE001 - benchmark boundary + print(json.dumps({"error": str(exc), "topFiles": [], "payload": ""})) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/lib/niche-benchmark/lanes.mjs b/scripts/lib/niche-benchmark/lanes.mjs new file mode 100644 index 0000000..b4b296f --- /dev/null +++ b/scripts/lib/niche-benchmark/lanes.mjs @@ -0,0 +1,408 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { withManagedStdioClientSession } from '../managed-mcp-session.mjs'; +import { + countUtf8Bytes, + estimateTokens, + normalizeRelativePath, + scoreTaskResult +} from './scoring.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const aiderScript = path.join(__dirname, 'aider-repomap-query.py'); +const pythonCmd = process.platform === 'win32' ? 'python' : 'python3'; + +function runCommand(command, args, options = {}) { + const started = Date.now(); + const result = spawnSync(command, args, { + encoding: 'utf8', + maxBuffer: 256 * 1024 * 1024, + timeout: options.timeoutMs ?? 300_000, + cwd: options.cwd ?? process.cwd(), + env: options.env ?? process.env + }); + return { + status: typeof result.status === 'number' ? result.status : 1, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + durationMs: Date.now() - started, + error: result.error?.message ?? null + }; +} + +function extractSearchFilesFromJson(value) { + const topFiles = []; + if (!value || typeof value !== 'object') return topFiles; + const results = Array.isArray(value.results) ? value.results : []; + for (const entry of results) { + const normalized = normalizeRelativePath(entry.file ?? entry.path ?? ''); + if (normalized && !topFiles.includes(normalized)) topFiles.push(normalized); + } + return topFiles; +} + +function codebaseContextCommand(projectRoot) { + const distIndex = path.join(projectRoot, 'dist', 'index.js'); + if (existsSync(distIndex)) { + return { command: process.execPath, args: [distIndex] }; + } + return { + command: process.platform === 'win32' ? 'npm.cmd' : 'npm', + args: ['exec', '--', 'codebase-context'] + }; +} + +export function ensureRepoCheckout(repo, projectRoot) { + const cloneRoot = path.join(projectRoot, 'benchmark-repos'); + const checkout = path.join(cloneRoot, repo.id); + mkdirSync(cloneRoot, { recursive: true }); + if (!existsSync(path.join(checkout, '.git'))) { + const clone = runCommand('git', ['clone', '--filter=blob:none', repo.repositoryUrl, checkout], { + timeoutMs: 600_000 + }); + if (clone.status !== 0) { + throw new Error(`git clone failed for ${repo.id}: ${clone.stderr || clone.error}`); + } + } + const fetch = runCommand( + 'git', + ['fetch', '--depth', '1', 'origin', repo.repositoryRef], + { cwd: checkout, timeoutMs: 600_000 } + ); + if (fetch.status !== 0) { + throw new Error(`git fetch failed for ${repo.id}: ${fetch.stderr || fetch.error}`); + } + const checkoutRun = runCommand('git', ['checkout', '--force', repo.repositoryRef], { + cwd: checkout, + timeoutMs: 120_000 + }); + if (checkoutRun.status !== 0) { + throw new Error(`git checkout failed for ${repo.id}: ${checkoutRun.stderr || checkoutRun.error}`); + } + return checkout; +} + +export async function warmCodebaseContextIndex(repoPath, projectRoot) { + const base = codebaseContextCommand(projectRoot); + const env = { + ...process.env, + CODEBASE_ROOT: repoPath, + CODEBASE_CONTEXT_ASCII: '1' + }; + return runCommand( + base.command, + [...base.args, 'reindex', '--no-embeddings', '--json'], + { env, timeoutMs: 900_000 } + ); +} + +export async function runCodebaseContextTask(task, repoPath, projectRoot, indexElapsedMs) { + const base = codebaseContextCommand(projectRoot); + const env = { + ...process.env, + CODEBASE_ROOT: repoPath, + CODEBASE_CONTEXT_ASCII: '1' + }; + const cc = task.codebaseContext; + const args = [...base.args, cc.command]; + if (cc.command === 'search') { + args.push('--query', task.query); + } + args.push(...cc.args); + const started = Date.now(); + const result = runCommand(base.command, args, { env, timeoutMs: 180_000 }); + const queryElapsedMs = Date.now() - started; + let payload = `${result.stdout}\n${result.stderr}`.trim(); + let topFiles = []; + if (result.status === 0) { + try { + const parsed = JSON.parse(result.stdout); + payload = JSON.stringify(parsed); + topFiles = extractSearchFilesFromJson(parsed).map((filePath) => { + const resolvedRepo = path.resolve(repoPath); + const resolvedFile = path.resolve(filePath); + if (resolvedFile.startsWith(resolvedRepo + path.sep)) { + return normalizeRelativePath(path.relative(resolvedRepo, resolvedFile)); + } + return normalizeRelativePath(filePath); + }); + if (topFiles.length === 0 && Array.isArray(parsed.patterns)) { + for (const pattern of parsed.patterns) { + const example = normalizeRelativePath(pattern?.bestExample ?? pattern?.example ?? ''); + if (example) topFiles.push(example); + } + } + } catch { + topFiles = []; + } + } + const score = scoreTaskResult(task, { payload, topFiles }); + return { + status: result.status === 0 ? 'completed' : 'tool_error', + niche: task.niche, + ...score, + payloadBytes: countUtf8Bytes(payload), + estimatedTokens: estimateTokens(countUtf8Bytes(payload)), + queryElapsedMs, + indexElapsedMs: indexElapsedMs ?? 0, + topFiles + }; +} + +function tokenizeQuery(query) { + return query + .toLowerCase() + .split(/[^a-z0-9_+#./-]+/i) + .map((token) => token.trim()) + .filter((token) => token.length > 2); +} + +export function runRipgrepTask(task, repoPath, indexElapsedMs = 0) { + const tokens = tokenizeQuery(task.query); + const pattern = tokens.slice(0, 8).join('|') || task.query.slice(0, 80); + const started = Date.now(); + const result = runCommand( + process.env.RG_PATH || 'rg', + ['--json', '-n', '--hidden', '--glob', '!.git/**', '--glob', '!node_modules/**', '--glob', '!dist/**', pattern, '.'], + { cwd: repoPath, timeoutMs: 120_000 } + ); + const queryElapsedMs = Date.now() - started; + const topFiles = []; + for (const line of result.stdout.split('\n')) { + if (!line.trim()) continue; + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + if (event.type !== 'match') continue; + const normalized = normalizeRelativePath(event.data?.path?.text ?? ''); + if (normalized && !topFiles.includes(normalized)) topFiles.push(normalized); + if (topFiles.length >= 5) break; + } + const payload = result.stdout.slice(0, 20_000); + const score = scoreTaskResult(task, { payload, topFiles }); + return { + status: result.status === 0 || result.status === 1 ? 'completed' : 'tool_error', + niche: task.niche, + ...score, + payloadBytes: countUtf8Bytes(payload), + estimatedTokens: estimateTokens(countUtf8Bytes(payload)), + queryElapsedMs, + indexElapsedMs, + topFiles + }; +} + +function parseJCodeMunchPayload(result) { + const textParts = Array.isArray(result?.content) + ? result.content + .map((item) => (typeof item?.text === 'string' ? item.text.trim() : '')) + .filter(Boolean) + : []; + const payload = textParts.join('\n'); + let topFiles = []; + try { + const parsed = JSON.parse(payload); + const results = Array.isArray(parsed.results) ? parsed.results : []; + for (const entry of results) { + const normalized = normalizeRelativePath(entry.file ?? entry.path ?? ''); + if (normalized && !topFiles.includes(normalized)) topFiles.push(normalized); + } + } catch { + for (const line of payload.split('\n')) { + if (!line.startsWith('s,')) continue; + const columns = line.split(','); + const fileColumn = columns[3] ?? columns[4]; + const normalized = normalizeRelativePath(fileColumn ?? ''); + if (normalized && !topFiles.includes(normalized)) topFiles.push(normalized); + } + } + return { payload, topFiles }; +} + +export async function warmJCodeMunchRepo(repoPath) { + return withManagedStdioClientSession( + { + serverCommand: pythonCmd, + serverArgs: ['-m', 'jcodemunch_mcp.server', 'serve', '--transport', 'stdio', '--log-level', 'ERROR'], + serverEnv: { ...process.env, JCODEMUNCH_USE_AI_SUMMARIES: 'false' }, + cwd: repoPath, + connectTimeoutMs: 30_000 + }, + async ({ client }) => { + await new Promise((resolve) => setTimeout(resolve, 2_000)); + const started = Date.now(); + const indexResult = await client.callTool({ + name: 'index_folder', + arguments: { + path: path.resolve(repoPath), + use_ai_summaries: false, + incremental: true, + follow_symlinks: false, + extra_ignore_patterns: ['.codebase-context/**', 'node_modules/**'] + } + }); + const { payload } = parseJCodeMunchPayload(indexResult); + let repoName = null; + try { + const parsed = JSON.parse(payload || '{}'); + repoName = typeof parsed.repo === 'string' ? parsed.repo : parsed.name ?? null; + } catch { + repoName = null; + } + return { repoName, indexElapsedMs: Date.now() - started }; + } + ); +} + +export async function runJCodeMunchTask(task, repoPath, repoName, indexElapsedMs) { + if (!repoName) { + return { + status: 'setup_failed', + niche: task.niche, + top3Hit: false, + usefulnessScore: 0, + queryElapsedMs: 0, + indexElapsedMs, + payloadBytes: 0, + estimatedTokens: 0, + topFiles: [] + }; + } + return withManagedStdioClientSession( + { + serverCommand: pythonCmd, + serverArgs: ['-m', 'jcodemunch_mcp.server', 'serve', '--transport', 'stdio', '--log-level', 'ERROR'], + serverEnv: { ...process.env, JCODEMUNCH_USE_AI_SUMMARIES: 'false' }, + cwd: repoPath, + connectTimeoutMs: 30_000 + }, + async ({ client }) => { + await new Promise((resolve) => setTimeout(resolve, 1_000)); + const started = Date.now(); + const result = await client.callTool({ + name: 'search_symbols', + arguments: { + repo: repoName, + query: task.query, + max_results: 5, + detail_level: 'compact', + semantic: false + } + }); + const queryElapsedMs = Date.now() - started; + const { payload, topFiles } = parseJCodeMunchPayload(result); + const score = scoreTaskResult(task, { payload, topFiles }); + return { + status: 'completed', + niche: task.niche, + ...score, + payloadBytes: countUtf8Bytes(payload), + estimatedTokens: estimateTokens(countUtf8Bytes(payload)), + queryElapsedMs, + indexElapsedMs, + topFiles + }; + } + ); +} + +export function warmAiderRepo(repoPath) { + return runCommand( + pythonCmd, + [aiderScript, '--root', repoPath, '--warm-index', '--limit', '3'], + { cwd: repoPath, timeoutMs: 900_000 } + ); +} + +export function runAiderTask(task, repoPath, indexElapsedMs) { + const result = runCommand( + pythonCmd, + [aiderScript, '--root', repoPath, '--query', task.query, '--limit', '5'], + { cwd: repoPath, timeoutMs: 300_000 } + ); + let topFiles = []; + let payload = result.stdout; + try { + const parsed = JSON.parse(result.stdout); + topFiles = Array.isArray(parsed.topFiles) ? parsed.topFiles : []; + payload = parsed.payload ?? result.stdout; + } catch { + topFiles = []; + } + const score = scoreTaskResult(task, { payload, topFiles }); + return { + status: result.status === 0 ? 'completed' : 'tool_error', + niche: task.niche, + ...score, + payloadBytes: countUtf8Bytes(payload), + estimatedTokens: estimateTokens(countUtf8Bytes(payload)), + queryElapsedMs: (() => { + try { + return JSON.parse(result.stdout || '{}').queryElapsedMs ?? result.durationMs; + } catch { + return result.durationMs; + } + })(), + indexElapsedMs, + topFiles + }; +} + +function rankRepomixPack(packText, query, limit) { + const tokens = tokenizeQuery(query); + const sections = packText.split(/\n={32}\n/); + const scores = new Map(); + for (const section of sections) { + const fileMatch = section.match(/^File:\s+(.+)$/m); + if (!fileMatch) continue; + const filePath = fileMatch[1].trim(); + const lower = section.toLowerCase(); + let score = 0; + for (const token of tokens) { + if (lower.includes(token)) score += 1; + } + if (score > 0) scores.set(filePath, score); + } + return [...scores.entries()] + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, limit) + .map(([filePath]) => filePath.replace(/\\/g, '/')); +} + +export function warmRepomixRepo(repoPath) { + const started = Date.now(); + const result = runCommand( + process.platform === 'win32' ? 'npx.cmd' : 'npx', + ['--yes', 'repomix', '--stdout', '--quiet', '--style', 'plain'], + { cwd: repoPath, timeoutMs: 900_000 } + ); + return { + packText: result.stdout, + indexElapsedMs: Date.now() - started, + status: result.status === 0 ? 'completed' : 'setup_failed', + error: result.status === 0 ? null : result.stderr || result.error + }; +} + +export function runRepomixTask(task, packText, indexElapsedMs) { + const started = Date.now(); + const topFiles = rankRepomixPack(packText, task.query, 5); + const payload = topFiles.join('\n'); + const queryElapsedMs = Date.now() - started; + const score = scoreTaskResult(task, { payload, topFiles }); + return { + status: packText ? 'completed' : 'setup_failed', + niche: task.niche, + ...score, + payloadBytes: countUtf8Bytes(packText.slice(0, 50_000)), + estimatedTokens: estimateTokens(countUtf8Bytes(packText)), + queryElapsedMs, + indexElapsedMs, + topFiles + }; +} diff --git a/scripts/lib/niche-benchmark/scoring.mjs b/scripts/lib/niche-benchmark/scoring.mjs new file mode 100644 index 0000000..c3ad84c --- /dev/null +++ b/scripts/lib/niche-benchmark/scoring.mjs @@ -0,0 +1,134 @@ +export function normalizeText(value) { + return String(value ?? '') + .toLowerCase() + .replace(/\\/g, '/'); +} + +export function normalizeRelativePath(candidate) { + if (typeof candidate !== 'string') return null; + const trimmed = candidate.trim().replace(/^["']|["']$/g, ''); + if (!trimmed) return null; + const normalized = trimmed.replace(/\\/g, '/').replace(/^\.\//, ''); + if (/^[A-Za-z]:\//.test(normalized)) { + return normalized.replace(/^[A-Za-z]:\//, ''); + } + return normalized.split(':')[0]; +} + +export function matchesPatterns(candidate, patterns) { + if (!patterns?.length) return false; + const normalizedCandidate = normalizeText(candidate); + return patterns.some((pattern) => normalizedCandidate.includes(normalizeText(pattern))); +} + +export function firstRelevantHit(topFiles, patterns) { + if (!patterns?.length) return null; + for (let index = 0; index < topFiles.length; index++) { + if (matchesPatterns(topFiles[index], patterns)) return index + 1; + } + return null; +} + +export function scoreTaskResult(task, { payload = '', topFiles = [] }) { + const hit = firstRelevantHit(topFiles, task.expectedFilePatterns); + const top3Hit = hit !== null && hit <= 3; + const normalizedPayload = normalizeText(payload); + const matchedSignals = (task.expectedSignals ?? []).filter((signal) => + normalizedPayload.includes(normalizeText(signal)) + ); + const usefulnessScore = + task.expectedSignals?.length > 0 + ? matchedSignals.length / Math.max(task.expectedSignals.length, 1) + : top3Hit + ? 1 + : 0; + + return { + top3Hit, + firstRelevantHit: hit, + usefulnessScore, + matchedSignals + }; +} + +export function countUtf8Bytes(str) { + return Buffer.byteLength(String(str ?? ''), 'utf8'); +} + +export function estimateTokens(bytes) { + return Math.ceil(bytes / 4); +} + +export function aggregateLaneResults(taskResults) { + const completed = taskResults.filter((result) => result.status === 'completed'); + const n = taskResults.length; + const top3Hits = completed.filter((result) => result.top3Hit).length; + const avgUsefulness = + completed.length > 0 + ? completed.reduce((sum, result) => sum + result.usefulnessScore, 0) / completed.length + : null; + const avgTokens = + completed.length > 0 + ? completed.reduce((sum, result) => sum + result.estimatedTokens, 0) / completed.length + : null; + const avgQueryMs = + completed.length > 0 + ? completed.reduce((sum, result) => sum + result.queryElapsedMs, 0) / completed.length + : null; + const avgIndexMs = + completed.length > 0 + ? completed.reduce((sum, result) => sum + (result.indexElapsedMs ?? 0), 0) / completed.length + : null; + + const byNiche = {}; + for (const niche of ['team-convention', 'edit-preflight', 'symbol-retrieval']) { + const nicheRows = completed.filter((result) => result.niche === niche); + byNiche[niche] = { + tasks: nicheRows.length, + top3HitRate: nicheRows.length > 0 ? nicheRows.filter((row) => row.top3Hit).length / nicheRows.length : null + }; + } + + const failed = taskResults.filter((result) => result.status !== 'completed'); + return { + status: completed.length === 0 ? 'setup_failed' : failed.length > 0 ? 'partial' : 'completed', + tasksTotal: n, + tasksCompleted: completed.length, + tasksFailed: failed.length, + top3HitRate: completed.length > 0 ? top3Hits / completed.length : null, + averageUsefulness: avgUsefulness, + averageEstimatedTokens: avgTokens, + averageQueryElapsedMs: avgQueryMs, + averageIndexElapsedMs: avgIndexMs, + byNiche, + taskResults + }; +} + +export function aggregateEvidence(laneEvidence, manifest, protocol, runMeta) { + return { + artifactType: 'niche-prior-art-benchmark-evidence', + generatedAt: new Date().toISOString(), + protocol: { + name: protocol.name, + frozenDate: protocol.frozenDate, + manifestFrozenDate: manifest.frozenDate + }, + runMeta, + comparators: protocol.comparators.map((comparator) => ({ + id: comparator.id, + displayName: comparator.displayName, + credit: comparator.credit, + projectUrl: comparator.projectUrl, + ...(laneEvidence[comparator.id] ?? { status: 'not_run' }) + })), + summaryTable: Object.entries(laneEvidence).map(([laneId, lane]) => ({ + laneId, + status: lane.status, + top3HitRate: lane.top3HitRate, + averageEstimatedTokens: lane.averageEstimatedTokens, + averageQueryElapsedMs: lane.averageQueryElapsedMs, + averageIndexElapsedMs: lane.averageIndexElapsedMs + })) + }; +} diff --git a/scripts/run-niche-benchmark.mjs b/scripts/run-niche-benchmark.mjs new file mode 100644 index 0000000..06200d3 --- /dev/null +++ b/scripts/run-niche-benchmark.mjs @@ -0,0 +1,283 @@ +#!/usr/bin/env node +/** + * One-command niche prior-art benchmark runner. + * + * Usage: + * node scripts/run-niche-benchmark.mjs + * node scripts/run-niche-benchmark.mjs --lanes codebase-context,ripgrep --output results/niche-benchmark-evidence.json + * node scripts/run-niche-benchmark.mjs --validate-fixtures + */ + +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; +import { spawnSync } from 'node:child_process'; +import { + ensureRepoCheckout, + runAiderTask, + runCodebaseContextTask, + runJCodeMunchTask, + runRepomixTask, + runRipgrepTask, + warmAiderRepo, + warmCodebaseContextIndex, + warmJCodeMunchRepo, + warmRepomixRepo +} from './lib/niche-benchmark/lanes.mjs'; +import { aggregateEvidence, aggregateLaneResults } from './lib/niche-benchmark/scoring.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const projectRoot = path.join(__dirname, '..'); +const defaultManifest = path.join(projectRoot, 'tests', 'fixtures', 'niche-benchmark-manifest.json'); +const defaultProtocol = path.join(projectRoot, 'tests', 'fixtures', 'niche-benchmark-protocol.json'); +const defaultOutput = path.join(projectRoot, 'results', 'niche-benchmark-evidence.json'); + +const ALL_LANES = ['codebase-context', 'jCodeMunch', 'aider-repo-map', 'repomix', 'ripgrep']; + +function sha256File(filePath) { + return createHash('sha256').update(readFileSync(filePath)).digest('hex'); +} + +function loadJson(filePath) { + return JSON.parse(readFileSync(filePath, 'utf8')); +} + +function ensureBuilt() { + const distIndex = path.join(projectRoot, 'dist', 'index.js'); + if (existsSync(distIndex)) return; + console.log('Building codebase-context (dist missing)...'); + const build = spawnSync('pnpm', ['run', 'build'], { + cwd: projectRoot, + stdio: 'inherit', + encoding: 'utf8' + }); + if (build.status !== 0) { + throw new Error('pnpm run build failed; dist/index.js is required for the codebase-context lane'); + } +} + +function validateFixtures(manifestPath, protocolPath) { + const manifest = loadJson(manifestPath); + const protocol = loadJson(protocolPath); + const errors = []; + if (manifest.tasks.length !== 18) errors.push(`expected 18 tasks, found ${manifest.tasks.length}`); + if (manifest.repos.length !== 3) errors.push(`expected 3 repos, found ${manifest.repos.length}`); + const nicheCounts = {}; + for (const task of manifest.tasks) { + nicheCounts[task.niche] = (nicheCounts[task.niche] ?? 0) + 1; + } + for (const niche of protocol.niches) { + if (nicheCounts[niche] !== 6) { + errors.push(`expected 6 ${niche} tasks, found ${nicheCounts[niche] ?? 0}`); + } + } + const repoIds = new Set(manifest.repos.map((repo) => repo.id)); + for (const task of manifest.tasks) { + if (!repoIds.has(task.repoId)) errors.push(`task ${task.id} references unknown repo ${task.repoId}`); + } + if (errors.length > 0) { + console.error(errors.join('\n')); + process.exit(1); + } + console.log('Fixture validation passed.'); + console.log(` manifest: ${manifestPath}`); + console.log(` protocol: ${protocolPath}`); + console.log(` manifest sha256: ${sha256File(manifestPath)}`); + console.log(` protocol sha256: ${sha256File(protocolPath)}`); +} + +async function runLane(laneId, manifest, repoCheckouts) { + const tasksByRepo = manifest.repos.map((repo) => ({ + repo, + checkout: repoCheckouts[repo.id], + tasks: manifest.tasks.filter((task) => task.repoId === repo.id) + })); + + const taskResults = []; + + if (laneId === 'ripgrep') { + for (const group of tasksByRepo) { + for (const task of group.tasks) { + taskResults.push({ + taskId: task.id, + ...runRipgrepTask(task, group.checkout, 0) + }); + } + } + return aggregateLaneResults(taskResults.map((row) => ({ ...row, status: row.status ?? 'completed' }))); + } + + for (const group of tasksByRepo) { + let indexElapsedMs = 0; + let repomixPack = ''; + let jcodemunchRepo = null; + + if (laneId === 'codebase-context') { + const index = await warmCodebaseContextIndex(group.checkout, projectRoot); + indexElapsedMs = index.durationMs; + if (index.status !== 0) { + return { + status: 'setup_failed', + reason: `codebase-context reindex failed on ${group.repo.id}`, + taskResults: [] + }; + } + } else if (laneId === 'jCodeMunch') { + try { + const warmed = await warmJCodeMunchRepo(group.checkout); + jcodemunchRepo = warmed.repoName; + indexElapsedMs = warmed.indexElapsedMs; + } catch (error) { + return { + status: 'setup_failed', + reason: `jCodeMunch setup failed on ${group.repo.id}: ${error.message}`, + taskResults: [] + }; + } + } else if (laneId === 'aider-repo-map') { + const warm = warmAiderRepo(group.checkout); + indexElapsedMs = warm.durationMs; + if (warm.status !== 0) { + return { + status: 'setup_failed', + reason: `Aider repo map warm index failed on ${group.repo.id}`, + taskResults: [] + }; + } + } else if (laneId === 'repomix') { + const warm = warmRepomixRepo(group.checkout); + indexElapsedMs = warm.indexElapsedMs; + if (warm.status !== 'completed') { + return { + status: 'setup_failed', + reason: warm.error ?? `Repomix pack failed on ${group.repo.id}`, + taskResults: [] + }; + } + repomixPack = warm.packText; + } + + for (const task of group.tasks) { + let row; + if (laneId === 'codebase-context') { + row = await runCodebaseContextTask(task, group.checkout, projectRoot, indexElapsedMs / group.tasks.length); + } else if (laneId === 'jCodeMunch') { + row = await runJCodeMunchTask(task, group.checkout, jcodemunchRepo, indexElapsedMs / group.tasks.length); + } else if (laneId === 'aider-repo-map') { + row = runAiderTask(task, group.checkout, indexElapsedMs / group.tasks.length); + } else if (laneId === 'repomix') { + row = runRepomixTask(task, repomixPack, indexElapsedMs / group.tasks.length); + } + taskResults.push({ taskId: task.id, ...row }); + } + } + + return aggregateLaneResults( + taskResults.map((row) => ({ + ...row, + status: row.status ?? 'completed' + })) + ); +} + +async function main() { + const { values } = parseArgs({ + options: { + help: { type: 'boolean', default: false }, + 'validate-fixtures': { type: 'boolean', default: false }, + manifest: { type: 'string' }, + protocol: { type: 'string' }, + lanes: { type: 'string' }, + output: { type: 'string' }, + 'skip-build': { type: 'boolean', default: false } + } + }); + + const manifestPath = path.resolve(values.manifest ?? defaultManifest); + const protocolPath = path.resolve(values.protocol ?? defaultProtocol); + const outputPath = path.resolve(values.output ?? defaultOutput); + + if (values.help) { + console.log(`Usage: node scripts/run-niche-benchmark.mjs [options] + + --validate-fixtures Validate frozen manifest/protocol and exit + --lanes Default: ${ALL_LANES.join(',')} + --output Default: ${defaultOutput} + --skip-build Do not auto-build dist before running +`); + process.exit(0); + } + + if (values['validate-fixtures']) { + validateFixtures(manifestPath, protocolPath); + process.exit(0); + } + + const manifest = loadJson(manifestPath); + const protocol = loadJson(protocolPath); + validateFixtures(manifestPath, protocolPath); + + const lanes = (values.lanes ?? ALL_LANES.join(',')) + .split(',') + .map((lane) => lane.trim()) + .filter(Boolean); + for (const lane of lanes) { + if (!ALL_LANES.includes(lane)) { + throw new Error(`Unknown lane ${lane}. Supported: ${ALL_LANES.join(', ')}`); + } + } + + if (!values['skip-build'] && lanes.includes('codebase-context')) { + ensureBuilt(); + } + + console.log(`\nNiche prior-art benchmark (${manifest.tasks.length} tasks, ${lanes.length} lanes)`); + const repoCheckouts = {}; + for (const repo of manifest.repos) { + console.log(`Checking out ${repo.id} @ ${repo.repositoryRef.slice(0, 12)}...`); + repoCheckouts[repo.id] = ensureRepoCheckout(repo, projectRoot); + } + + const laneEvidence = {}; + for (const laneId of lanes) { + console.log(`\n=== Lane: ${laneId} ===`); + try { + laneEvidence[laneId] = await runLane(laneId, manifest, repoCheckouts); + console.log( + ` top-3 hit rate: ${laneEvidence[laneId].top3HitRate === null ? 'n/a' : (laneEvidence[laneId].top3HitRate * 100).toFixed(1) + '%'}` + ); + console.log( + ` avg tokens: ${laneEvidence[laneId].averageEstimatedTokens === null ? 'n/a' : Math.round(laneEvidence[laneId].averageEstimatedTokens)}` + ); + console.log( + ` avg query ms: ${laneEvidence[laneId].averageQueryElapsedMs === null ? 'n/a' : Math.round(laneEvidence[laneId].averageQueryElapsedMs)}` + ); + } catch (error) { + laneEvidence[laneId] = { status: 'setup_failed', reason: error.message, taskResults: [] }; + console.error(` SETUP FAILED: ${error.message}`); + } + } + + const evidence = aggregateEvidence(laneEvidence, manifest, protocol, { + manifestPath, + protocolPath, + manifestSha256: sha256File(manifestPath), + protocolSha256: sha256File(protocolPath), + nodeVersion: process.version, + lanes, + repoCheckouts: Object.fromEntries( + manifest.repos.map((repo) => [repo.id, { ref: repo.repositoryRef, path: repoCheckouts[repo.id] }]) + ) + }); + + mkdirSync(path.dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, `${JSON.stringify(evidence, null, 2)}\n`); + console.log(`\nEvidence written to ${outputPath}`); +} + +main().catch((error) => { + console.error('Fatal:', error); + process.exit(2); +}); diff --git a/tests/fixtures/niche-benchmark-manifest.json b/tests/fixtures/niche-benchmark-manifest.json new file mode 100644 index 0000000..fde60c6 --- /dev/null +++ b/tests/fixtures/niche-benchmark-manifest.json @@ -0,0 +1,204 @@ +{ + "description": "Frozen 18-task niche benchmark across three public repos, two tasks per niche per repo. Expected patterns are path substring checks, not full paths.", + "frozenDate": "2026-10-30", + "cloneRoot": "benchmark-repos", + "repos": [ + { + "id": "angular-spotify", + "repository": "trungk18/angular-spotify", + "repositoryUrl": "https://github.com/trungk18/angular-spotify", + "repositoryRef": "ff9efa765c53cfde78c9a172c62d515ae8ef9fe0", + "language": "typescript", + "sizeClass": "medium", + "sourceFixture": "tests/fixtures/discovery-angular-spotify.json" + }, + { + "id": "excalidraw", + "repository": "excalidraw/excalidraw", + "repositoryUrl": "https://github.com/excalidraw/excalidraw", + "repositoryRef": "e18c1dd213000dde0ae94ef7eb00aab537b39708", + "language": "typescript", + "sizeClass": "large", + "sourceFixture": "tests/fixtures/discovery-excalidraw.json" + }, + { + "id": "fastapi", + "repository": "fastapi/fastapi", + "repositoryUrl": "https://github.com/fastapi/fastapi", + "repositoryRef": "628c34e0cae200564d191c95d7edea78c88c4b5e", + "language": "python", + "sizeClass": "medium", + "sourceFixture": null + } + ], + "tasks": [ + { + "id": "as-team-01", + "repoId": "angular-spotify", + "niche": "team-convention", + "sourceTaskId": "as-find-01", + "query": "What DI pattern dominates here?", + "expectedFilePatterns": ["service", "inject"], + "expectedSignals": ["dependencyInjection"], + "codebaseContext": { "command": "patterns", "args": ["--category", "di", "--json"] } + }, + { + "id": "as-team-02", + "repoId": "angular-spotify", + "niche": "team-convention", + "sourceTaskId": "as-find-02", + "query": "What state-management pattern should an agent imitate?", + "expectedFilePatterns": ["store", "state"], + "expectedSignals": ["stateManagement"], + "codebaseContext": { "command": "patterns", "args": ["--category", "state", "--json"] } + }, + { + "id": "as-ep-01", + "repoId": "angular-spotify", + "niche": "edit-preflight", + "sourceTaskId": "as-ep-01", + "query": "update how authorization token headers are attached to API requests", + "expectedFilePatterns": ["auth", "interceptor"], + "codebaseContext": { "command": "search", "args": ["--intent", "edit", "--limit", "5", "--json"] } + }, + { + "id": "as-ep-02", + "repoId": "angular-spotify", + "niche": "edit-preflight", + "sourceTaskId": "as-ep-02", + "query": "change the logic that skips to the next song", + "expectedFilePatterns": ["player-api", "player/api", "player"], + "codebaseContext": { "command": "search", "args": ["--intent", "edit", "--limit", "5", "--json"] } + }, + { + "id": "as-sym-01", + "repoId": "angular-spotify", + "niche": "symbol-retrieval", + "sourceTaskId": "as-search-01", + "query": "skip to next song", + "expectedFilePatterns": ["player-api", "player/api", "player"], + "codebaseContext": { "command": "search", "args": ["--intent", "explore", "--limit", "5", "--json"] } + }, + { + "id": "as-sym-02", + "repoId": "angular-spotify", + "niche": "symbol-retrieval", + "sourceTaskId": "as-search-03", + "query": "add authorization token to API requests", + "expectedFilePatterns": ["auth", "interceptor"], + "codebaseContext": { "command": "search", "args": ["--intent", "explore", "--limit", "5", "--json"] } + }, + { + "id": "ex-team-01", + "repoId": "excalidraw", + "niche": "team-convention", + "sourceTaskId": "ex-find-01", + "query": "What state-management pattern dominates here?", + "expectedFilePatterns": ["appstate", "state", "store"], + "expectedSignals": ["stateManagement"], + "codebaseContext": { "command": "patterns", "args": ["--category", "state", "--json"] } + }, + { + "id": "ex-team-02", + "repoId": "excalidraw", + "niche": "team-convention", + "sourceTaskId": "ex-find-04", + "query": "What dependency injection or dependency pattern dominates here?", + "expectedFilePatterns": ["component", "app", "src"], + "expectedSignals": ["dependencyInjection"], + "codebaseContext": { "command": "patterns", "args": ["--category", "di", "--json"] } + }, + { + "id": "ex-ep-01", + "repoId": "excalidraw", + "niche": "edit-preflight", + "sourceTaskId": "ex-ep-01", + "query": "change how scene updates are applied", + "expectedFilePatterns": ["scene"], + "codebaseContext": { "command": "search", "args": ["--intent", "edit", "--limit", "5", "--json"] } + }, + { + "id": "ex-ep-02", + "repoId": "excalidraw", + "niche": "edit-preflight", + "sourceTaskId": "ex-ep-03", + "query": "modify scene serialization to json export", + "expectedFilePatterns": ["scene", "json", "data"], + "codebaseContext": { "command": "search", "args": ["--intent", "edit", "--limit", "5", "--json"] } + }, + { + "id": "ex-sym-01", + "repoId": "excalidraw", + "niche": "symbol-retrieval", + "sourceTaskId": "ex-search-01", + "query": "element type definitions", + "expectedFilePatterns": ["element", "type"], + "codebaseContext": { "command": "search", "args": ["--intent", "explore", "--limit", "5", "--json"] } + }, + { + "id": "ex-sym-02", + "repoId": "excalidraw", + "niche": "symbol-retrieval", + "sourceTaskId": "ex-search-02", + "query": "scene serialization export json", + "expectedFilePatterns": ["scene", "json", "data"], + "codebaseContext": { "command": "search", "args": ["--intent", "explore", "--limit", "5", "--json"] } + }, + { + "id": "fa-team-01", + "repoId": "fastapi", + "niche": "team-convention", + "sourceTaskId": null, + "query": "How does FastAPI wire dependency injection for route handlers?", + "expectedFilePatterns": ["param_functions", "depends"], + "expectedSignals": ["Depends", "dependency"], + "codebaseContext": { "command": "patterns", "args": ["--category", "di", "--json"] } + }, + { + "id": "fa-team-02", + "repoId": "fastapi", + "niche": "team-convention", + "sourceTaskId": null, + "query": "Where are API routers defined and included in the application?", + "expectedFilePatterns": ["routing", "applications"], + "expectedSignals": ["APIRouter", "router"], + "codebaseContext": { "command": "patterns", "args": ["--category", "all", "--json"] } + }, + { + "id": "fa-ep-01", + "repoId": "fastapi", + "niche": "edit-preflight", + "sourceTaskId": null, + "query": "change how OAuth2 bearer tokens are extracted from requests", + "expectedFilePatterns": ["oauth2", "security"], + "codebaseContext": { "command": "search", "args": ["--intent", "edit", "--limit", "5", "--json"] } + }, + { + "id": "fa-ep-02", + "repoId": "fastapi", + "niche": "edit-preflight", + "sourceTaskId": null, + "query": "modify OpenAPI schema generation for response models", + "expectedFilePatterns": ["openapi"], + "codebaseContext": { "command": "search", "args": ["--intent", "edit", "--limit", "5", "--json"] } + }, + { + "id": "fa-sym-01", + "repoId": "fastapi", + "niche": "symbol-retrieval", + "sourceTaskId": null, + "query": "validate request body with pydantic field constraints", + "expectedFilePatterns": ["routing", "param"], + "codebaseContext": { "command": "search", "args": ["--intent", "explore", "--limit", "5", "--json"] } + }, + { + "id": "fa-sym-02", + "repoId": "fastapi", + "niche": "symbol-retrieval", + "sourceTaskId": null, + "query": "HTTP exception handler returns JSON error detail", + "expectedFilePatterns": ["exception", "routing"], + "codebaseContext": { "command": "search", "args": ["--intent", "explore", "--limit", "5", "--json"] } + } + ] +} diff --git a/tests/fixtures/niche-benchmark-protocol.json b/tests/fixtures/niche-benchmark-protocol.json new file mode 100644 index 0000000..2490015 --- /dev/null +++ b/tests/fixtures/niche-benchmark-protocol.json @@ -0,0 +1,73 @@ +{ + "name": "niche-prior-art-benchmark", + "frozenDate": "2026-10-30", + "scope": "local-context-retrieval", + "claimAllowed": false, + "purpose": "Reproducible comparison of codebase-context against credited prior art on team-convention lookup, edit-preflight navigation, and symbol retrieval. Not patch quality or agent productivity.", + "niches": ["team-convention", "edit-preflight", "symbol-retrieval"], + "comparators": [ + { + "id": "codebase-context", + "displayName": "Codebase Context", + "kind": "product", + "credit": "This repository (local dist CLI / MCP surfaces).", + "projectUrl": "https://github.com/PatrickSys/codebase-context" + }, + { + "id": "jCodeMunch", + "displayName": "jCodeMunch", + "kind": "mcp-comparator", + "credit": "jCodeMunch MCP symbol search and repo indexing (jcodemunch-mcp on PyPI).", + "projectUrl": "https://github.com/jgravelle/jcodemunch-mcp" + }, + { + "id": "aider-repo-map", + "displayName": "Aider repo map", + "kind": "cli-comparator", + "credit": "Paul Gauthier's Aider RepoMap ranking (aider-chat on PyPI). Query tokens are passed as mentioned identifiers; there is no separate natural-language search API.", + "projectUrl": "https://github.com/Aider-AI/aider" + }, + { + "id": "repomix", + "displayName": "Repomix", + "kind": "cli-comparator", + "credit": "Repomix repository packing (repomix on npm). Repomix does not expose query-specific retrieval; the lane charges pack generation to index time and ranks files by scanning pack sections for query tokens.", + "projectUrl": "https://github.com/yamadashy/repomix" + }, + { + "id": "ripgrep", + "displayName": "ripgrep (lexical baseline)", + "kind": "baseline", + "credit": "BurntSushi ripgrep JSON search over the repo checkout. No index; measures lexical retrieval only.", + "projectUrl": "https://github.com/BurntSushi/ripgrep" + } + ], + "metrics": { + "accuracy": { + "primary": "top3HitRate", + "definition": "Share of tasks where any expectedFilePattern appears in the first three ranked paths returned by the lane." + }, + "tokens": { + "primary": "estimatedTokens", + "definition": "UTF-8 payload bytes divided by four (same heuristic as discovery benchmark)." + }, + "latency": { + "primary": "queryElapsedMs", + "secondary": ["indexElapsedMs", "setupElapsedMs"], + "definition": "Wall time for the lane query step; index and setup are reported separately and amortized per repo where applicable." + } + }, + "fairnessRules": [ + "Task queries and expected patterns are frozen in tests/fixtures/niche-benchmark-manifest.json before benchmark code ships.", + "All lanes receive the same natural-language query string per task.", + "Do not adjust expected patterns after observing lane outputs.", + "Do not add task-specific ranking heuristics to codebase-context for this benchmark.", + "Report setup_failed lanes honestly; do not treat missing competitors as wins.", + "Credit prior art by display name and project URL in public docs." + ], + "blockedClaims": [ + "codebase_context_beats_all_competitors", + "universal_token_savings", + "patch_or_productivity_improvement" + ] +} diff --git a/tests/niche-benchmark.test.ts b/tests/niche-benchmark.test.ts new file mode 100644 index 0000000..0d92dcf --- /dev/null +++ b/tests/niche-benchmark.test.ts @@ -0,0 +1,86 @@ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { aggregateLaneResults, scoreTaskResult } from '../scripts/lib/niche-benchmark/scoring.mjs'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const manifest = JSON.parse( + readFileSync(path.join(__dirname, 'fixtures', 'niche-benchmark-manifest.json'), 'utf8') +); +const protocol = JSON.parse( + readFileSync(path.join(__dirname, 'fixtures', 'niche-benchmark-protocol.json'), 'utf8') +); + +describe('niche benchmark fixtures', () => { + it('freezes 18 tasks across three repos and three niches', () => { + expect(manifest.tasks).toHaveLength(18); + expect(manifest.repos).toHaveLength(3); + const languages = new Set(manifest.repos.map((repo) => repo.language)); + expect(languages.size).toBeGreaterThanOrEqual(2); + for (const niche of protocol.niches) { + expect(manifest.tasks.filter((task) => task.niche === niche)).toHaveLength(6); + } + }); + + it('lists credited prior-art comparators including ripgrep baseline', () => { + const ids = protocol.comparators.map((comparator) => comparator.id); + expect(ids).toEqual([ + 'codebase-context', + 'jCodeMunch', + 'aider-repo-map', + 'repomix', + 'ripgrep' + ]); + for (const comparator of protocol.comparators) { + expect(comparator.projectUrl).toMatch(/^https:\/\//); + expect(comparator.credit.length).toBeGreaterThan(10); + } + expect(protocol.claimAllowed).toBe(false); + }); +}); + +describe('niche benchmark scoring', () => { + it('marks top-3 pattern hits without tuning scores upward', () => { + const task = manifest.tasks.find((entry) => entry.id === 'as-sym-01'); + expect(task).toBeDefined(); + const hit = scoreTaskResult(task, { + payload: 'player-api next track', + topFiles: ['libs/shared/player-api.ts', 'other.ts'] + }); + expect(hit.top3Hit).toBe(true); + const miss = scoreTaskResult(task, { + payload: 'nothing useful', + topFiles: ['README.md'] + }); + expect(miss.top3Hit).toBe(false); + }); + + it('aggregates lane metrics with failures visible', () => { + const aggregated = aggregateLaneResults([ + { + taskId: 't1', + niche: 'symbol-retrieval', + status: 'completed', + top3Hit: true, + usefulnessScore: 1, + estimatedTokens: 100, + queryElapsedMs: 10, + indexElapsedMs: 50 + }, + { + taskId: 't2', + niche: 'edit-preflight', + status: 'tool_error', + top3Hit: false, + usefulnessScore: 0, + estimatedTokens: 0, + queryElapsedMs: 0, + indexElapsedMs: 0 + } + ]); + expect(aggregated.tasksCompleted).toBe(1); + expect(aggregated.tasksFailed).toBe(1); + expect(aggregated.top3HitRate).toBe(1); + }); +});