diff --git a/.codebase-context/memory.json b/.codebase-context/memory.json index f0eee24..868ef15 100644 --- a/.codebase-context/memory.json +++ b/.codebase-context/memory.json @@ -569,5 +569,185 @@ "scope": { "kind": "global" } + }, + { + "id": "cbe8bd1377ba", + "type": "gotcha", + "category": "conventions", + "memory": "fix(analyzers): Recognize current React and Next patterns", + "reason": "Auto-extracted from git commit history", + "date": "2026-07-05T07:33:25.000Z", + "source": "git" + }, + { + "id": "e3b009c312e8", + "type": "gotcha", + "category": "conventions", + "memory": "fix(reranker): Score passages with the supported tokenizer pair API", + "reason": "Auto-extracted from git commit history", + "date": "2026-07-05T14:43:29.000Z", + "source": "git" + }, + { + "id": "449b60830c9b", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): deduplicate blocked ContextBench rows", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-30T15:45:12.000Z", + "source": "git" + }, + { + "id": "9dfaf21a0bc8", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): harden ContextBench schema cleanup", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T19:14:49.000Z", + "source": "git" + }, + { + "id": "b8776a500ef5", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): preserve ContextBench executor model provenance", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T19:04:04.000Z", + "source": "git" + }, + { + "id": "a8b0f8664253", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): harden ContextBench manifest checks", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T19:03:51.000Z", + "source": "git" + }, + { + "id": "f84080651a19", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): relax zombie guard timeout jitter", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:24:56.000Z", + "source": "git" + }, + { + "id": "3d7776af8a23", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): tolerate ContextBench runner cleanup races", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:21:18.000Z", + "source": "git" + }, + { + "id": "6428cb02706e", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): tolerate ContextBench schema cleanup races", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:13:07.000Z", + "source": "git" + }, + { + "id": "c1bd14e88de2", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): align ContextBench harness evidence contracts", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T18:08:08.000Z", + "source": "git" + }, + { + "id": "2f3db0023789", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): relax slow Windows integration timeouts", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T17:59:20.000Z", + "source": "git" + }, + { + "id": "da039b1d0ff8", + "type": "gotcha", + "category": "conventions", + "memory": "fix(eval): harden ContextBench fixture verification", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T17:55:12.000Z", + "source": "git" + }, + { + "id": "5e3bdcaa984c", + "type": "gotcha", + "category": "conventions", + "memory": "fix(git): scope local artifact ignores", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T17:50:47.000Z", + "source": "git" + }, + { + "id": "861eb0bfb69f", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): relax slow Windows search timeouts", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:47:38.000Z", + "source": "git" + }, + { + "id": "3614711e00a4", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): tolerate ContextBench temp cleanup races", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:29:24.000Z", + "source": "git" + }, + { + "id": "c0cfb6599bcc", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): isolate ContextBench baseline Git env", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:25:08.000Z", + "source": "git" + }, + { + "id": "084a0540558c", + "type": "gotcha", + "category": "conventions", + "memory": "fix(format): format ContextBench harness sources", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T16:06:53.000Z", + "source": "git" + }, + { + "id": "0153743292c4", + "type": "gotcha", + "category": "conventions", + "memory": "fix(test): isolate ContextBench git fixtures", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-29T15:56:57.000Z", + "source": "git" + }, + { + "id": "a5397b2ee20d", + "type": "gotcha", + "category": "conventions", + "memory": "fix(cli): lazy-load mcp runtime for direct commands", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-17T14:34:22.000Z", + "source": "git" + }, + { + "id": "377e7c7c37b1", + "type": "gotcha", + "category": "conventions", + "memory": "fix(ci): unblock functional tests", + "reason": "Auto-extracted from git commit history", + "date": "2026-04-17T20:05:19.000Z", + "source": "git" } ] \ No newline at end of file diff --git a/.github/workflows/contextbench-candidate-pack-one.yml b/.github/workflows/contextbench-candidate-pack-one.yml new file mode 100644 index 0000000..a35f0c5 --- /dev/null +++ b/.github/workflows/contextbench-candidate-pack-one.yml @@ -0,0 +1,127 @@ +name: ContextBench Candidate Pack One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-candidate-pack-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + pack: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + ROOT: /tmp/contextbench-candidate-pack-one + TASK_PAYLOADS: /tmp/contextbench-candidate-pack-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-candidate-pack-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install tools and materialize frozen Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + pnpm run build + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Generate candidate pack + run: | + cat > "$ROOT/pack.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, relative, sep } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const repo = task.repo_checkout_path; + const outDir = root + '/pack'; + mkdirSync(outDir, { recursive: true }); + function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 300000, maxBuffer: 96 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; } + function queryOf(text) { const stop = new Set(['that','this','with','from','when','then','into','should','would','could','there','where','which','about','after','before','have','will','been','than','also']); return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())).slice(0, 12).join(' '); } + function jsonish(text) { const raw = String(text || '').trim(); if (!raw) return null; try { return JSON.parse(raw); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i = raw.indexOf(a), j = raw.lastIndexOf(b); if (i >= 0 && j > i) { try { return JSON.parse(raw.slice(i, j + 1)); } catch {} } } return null; } + function norm(file) { let f = String(file || '').replace(/^file:\/\//, ''); if (!f) return ''; if (f.startsWith(repo)) f = relative(repo, f); f = f.replaceAll('\\\\', '/').replace(/^\/+/, '').replace(/^\.\//, ''); if (f.startsWith('tmp/') || f.includes('://') || f.includes('..')) return ''; return f; } + function add(locs, file, start = 1, end = start, source = 'tool') { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function walk(value, locs, source) { if (!value || typeof value !== 'object') return; if (Array.isArray(value)) { for (const item of value) walk(item, locs, source); return; } add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, value.start_line || value.startLine || value.line || value.line_number || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1, source); for (const item of Object.values(value)) walk(item, locs, source); } + function collect(text, locs, source) { const parsed = jsonish(text); if (parsed) walk(parsed, locs, source); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); } + function uniq(locs, max = 60) { const seen = new Set(), out = []; for (const loc of locs) { const key = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= max) break; } } return out; } + function laneResult(lane, commands, locs, setupStatus = 'completed', indexStatus = 'completed') { for (const [i,c] of commands.entries()) writeFileSync(`${outDir}/${lane}-command-${i + 1}.json`, JSON.stringify(c, null, 2)); return { lane, setupStatus, indexStatus, toolCallable: commands.some((c) => c.status === 0), costs: { totalMs: commands.reduce((a,c) => a + c.durationMs, 0) }, candidates: uniq(locs) }; } + const query = queryOf(task.problem_statement); + const lanes = []; + { + const commands = [], locs = []; + for (const term of query.split(/\s+/).slice(0, 6)) { const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); commands.push(r); collect(r.stdout, locs, 'raw-native'); collect(r.stderr, locs, 'raw-native'); } + lanes.push(laneResult('raw-native', commands, locs)); + } + { + const commands = [], locs = []; const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const v = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); commands.push(v); + const idx = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); commands.push(idx); + const s = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '30', '--json'], { env, timeoutMs: 300000 }); commands.push(s); collect(s.stdout, locs, 'codebase-context'); collect(s.stderr, locs, 'codebase-context'); + lanes.push(laneResult('codebase-context', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed')); + } + { + const commands = [], locs = []; const env = { ...process.env, CBM_CACHE_DIR: outDir + '/cbm-cache', CBM_DIAGNOSTICS: '1' }; + const v = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); commands.push(v); + const idx = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); commands.push(idx); + const project = (jsonish(idx.stdout) || jsonish(idx.stderr) || {}).project || basename(repo); + const g = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 30 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(g); + const c = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 30 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(c); + for (const r of [g, c]) { collect(r.stdout, locs, 'codebase-memory-mcp'); collect(r.stderr, locs, 'codebase-memory-mcp'); } + lanes.push(laneResult('codebase-memory-mcp', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed')); + } + { + const commands = [], locs = []; + const v = run('grepai', ['version'], { timeoutMs: 60000 }); commands.push(v); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); commands.push(watch); + const status = run('grepai', ['watch', '--status'], { cwd: repo, timeoutMs: 60000 }); commands.push(status); + const s = run('grepai', ['search', query, '--json', '--compact'], { cwd: repo, timeoutMs: 180000 }); commands.push(s); collect(s.stdout, locs, 'grepai'); collect(s.stderr, locs, 'grepai'); + commands.push(run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 })); + lanes.push(laneResult('grepai', commands, locs, init.status === 0 ? 'completed' : 'setup_failed', (watch.status === 0 || status.status === 0) ? 'completed' : 'index_failed')); + } + { + const commands = [], locs = []; + const v = run('cgc', ['--help'], { timeoutMs: 60000 }); commands.push(v); + const idx = run('cgc', ['index', repo], { cwd: repo, timeoutMs: 1200000 }); commands.push(idx); + const a = run('cgc', ['analyze', 'complexity', '--limit', '30'], { cwd: repo, timeoutMs: 180000 }); commands.push(a); + const d = run('cgc', ['analyze', 'dead-code', '--limit', '30'], { cwd: repo, timeoutMs: 180000 }); commands.push(d); + const call = run('cgc', ['analyze', 'callers', query.split(/\s+/)[0] || 'main'], { cwd: repo, timeoutMs: 120000 }); commands.push(call); + for (const r of [a, d, call]) { collect(r.stdout, locs, 'codegraphcontext'); collect(r.stderr, locs, 'codegraphcontext'); } + lanes.push(laneResult('codegraphcontext', commands, locs, 'completed', idx.status === 0 ? 'completed' : 'index_failed')); + } + const pack = { createdAt: new Date().toISOString(), task: { instance_id: task.instance_id, repo: task.repo, base_commit: task.base_commit, problem_statement: task.problem_statement }, query, lanes }; + writeFileSync(`${outDir}/candidate-pack.json`, JSON.stringify(pack, null, 2)); + console.log(JSON.stringify({ task: pack.task, query, lanes: lanes.map((l) => ({ lane: l.lane, setupStatus: l.setupStatus, indexStatus: l.indexStatus, toolCallable: l.toolCallable, candidateCount: l.candidates.length, candidates: l.candidates.slice(0, 30) })) }, null, 2)); + NODE + node "$ROOT/pack.mjs" + - name: Upload candidate pack + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-candidate-pack-one + path: /tmp/contextbench-candidate-pack-one/pack + retention-days: 14 diff --git a/.github/workflows/contextbench-cbm-candidates-one.yml b/.github/workflows/contextbench-cbm-candidates-one.yml new file mode 100644 index 0000000..035e98b --- /dev/null +++ b/.github/workflows/contextbench-cbm-candidates-one.yml @@ -0,0 +1,59 @@ +name: ContextBench CBM Candidates One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cbm-candidates-one.yml + - scripts/contextbench-cbm-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-candidates: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + ROOT: /tmp/contextbench-cbm-candidates-one + TASK_PAYLOADS: /tmp/contextbench-cbm-candidates-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-cbm-candidates-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize Go task quietly + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/pack" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" > "$ROOT/logs/cbm-extract.log" 2>&1 + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 > "$ROOT/logs/materialize.log" 2>&1 + echo "quiet_setup_completed" + - name: Print compact CBM candidates + shell: bash + run: node scripts/contextbench-cbm-candidate-pack.mjs + - name: Upload CBM candidate pack + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-cbm-candidates-one + path: /tmp/contextbench-cbm-candidates-one/pack + retention-days: 14 diff --git a/.github/workflows/contextbench-cbm-go-diagnose.yml b/.github/workflows/contextbench-cbm-go-diagnose.yml new file mode 100644 index 0000000..cb8c901 --- /dev/null +++ b/.github/workflows/contextbench-cbm-go-diagnose.yml @@ -0,0 +1,94 @@ +name: ContextBench CBM Go Diagnose + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cbm-go-diagnose.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + diagnose: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + ROOT: /tmp/contextbench-cbm-go-diagnose + TASK_PAYLOADS: /tmp/contextbench-cbm-go-diagnose/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-cbm-go-diagnose/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + - name: Materialize Go task + run: | + mkdir -p "$ROOT" "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Download CBM + run: | + mkdir -p "$ROOT/tool" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" + - name: Run Go row and print scorer diagnostics + run: | + cat > "$ROOT/go-diagnose.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const cbm = process.env.CBM_BIN; + const dir = join(root, 'go-row'); + mkdirSync(dir, { recursive: true }); + const env = { ...process.env, CBM_CACHE_DIR: join(dir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 1200000, maxBuffer: 128 * 1024 * 1024 }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function jsonish(s) { try { return JSON.parse(String(s || '').trim()); } catch { return null; } } + function add(spans, file, start = 1, end = start) { if (!file) return; const clean = String(file).replace(/^\/+/, ''); const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = spans.get(clean) || []; list.push({ start: s, end: e }); spans.set(clean, list); } + function walk(v, spans) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, spans); return; } add(spans, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, spans); } + function textPaths(s, spans) { const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(s || ''))) !== null) add(spans, m[1], m[2] || 1, m[2] || 1); } + const query = 'Title System metrics written start Description system metrics'; + const setup = run(cbm, ['--version'], { env }); + const indexRun = run(cbm, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { cwd: task.repo_checkout_path, env, timeoutMs: 2700000 }); + const projectObj = jsonish(indexRun.stdout) || jsonish(indexRun.stderr) || {}; + const project = projectObj.project || basename(task.repo_checkout_path); + const graphSearch = run(cbm, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: task.repo_checkout_path, env }); + const codeSearch = run(cbm, ['cli', 'search_code', JSON.stringify({ project, pattern: '.', mode: 'compact', limit: 25 })], { cwd: task.repo_checkout_path, env }); + const spans = new Map(); + for (const r of [graphSearch, codeSearch]) for (const text of [r.stdout, r.stderr]) { const parsed = jsonish(text); if (parsed) walk(parsed, spans); textPaths(text, spans); } + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + const predictionPath = join(dir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + const goldPath = join(dir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); + const official = join(root, 'ContextBench-official'); + const clone = existsSync(join(official, 'contextbench', 'evaluate.py')) ? { skipped: true } : run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', official], { timeoutMs: 600000 }); + const scorePath = join(dir, 'official-score.jsonl'); + const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 1200000 }); + const scoreText = existsSync(scorePath) ? readFileSync(scorePath, 'utf8') : ''; + console.log(JSON.stringify({ taskId: task.instance_id, setupStatus: setup.status, indexStatus: indexRun.status, indexSignal: indexRun.signal, project, graphStatus: graphSearch.status, codeStatus: codeSearch.status, predFiles: predFiles.length, goldStatus: gold.status, goldStderr: gold.stderr, clone, evaluatorStatus: evaluator.status, evaluatorSignal: evaluator.signal, evaluatorError: evaluator.error, evaluatorStdout: evaluator.stdout, evaluatorStderr: evaluator.stderr, scorePathExists: existsSync(scorePath), scoreBytes: scoreText.length, scoreHead: scoreText.slice(0, 1000) }, null, 2)); + if (!(indexRun.status === 0 && graphSearch.status === 0 && predFiles.length > 0 && evaluator.status === 0 && scoreText.length > 0)) process.exitCode = 1; + NODE + node "$ROOT/go-diagnose.mjs" diff --git a/.github/workflows/contextbench-cbm-readiness-retry.yml b/.github/workflows/contextbench-cbm-readiness-retry.yml new file mode 100644 index 0000000..c82460c --- /dev/null +++ b/.github/workflows/contextbench-cbm-readiness-retry.yml @@ -0,0 +1,292 @@ +name: ContextBench CBM Readiness Retry + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cbm-readiness-retry.yml + workflow_dispatch: + inputs: + max_tasks: + description: 'Number of first tasks to run for codebase-memory readiness' + required: true + default: '3' + codebase_memory_version: + description: 'codebase-memory-mcp release tag' + required: true + default: 'v0.6.1' + +permissions: + contents: read + +jobs: + codebase-memory-first3-readiness: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + ROOT: /tmp/contextbench-cbm-readiness + TASK_PAYLOADS: /tmp/contextbench-cbm-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_VERSION: ${{ github.event.inputs.codebase_memory_version || 'v0.6.1' }} + MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + - name: Validate fixtures and materialize first tasks + run: | + mkdir -p "$ROOT" "$CHECKOUT_ROOT" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS" + - name: Download codebase-memory-mcp + run: | + set -euxo pipefail + mkdir -p "$ROOT/tool" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$ROOT/tool/codebase-memory-mcp" || true + "$ROOT/tool/codebase-memory-mcp" --version || true + - name: Run readiness gate with official evaluator + env: + CBM_BIN: /tmp/contextbench-cbm-readiness/tool/codebase-memory-mcp + run: | + cat > "$ROOT/readiness.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const tasks = payloads.tasks.slice(0, Number(process.env.MAX_TASKS || '3')); + const cbm = process.env.CBM_BIN; + const outRoot = join(root, 'out'); + mkdirSync(outRoot, { recursive: true }); + + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 20 * 60 * 1000, + maxBuffer: 64 * 1024 * 1024 + }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + + function firstOk(label, candidates, opts) { + const attempts = []; + for (const args of candidates) { + const attempt = run(cbm, args, opts); + attempts.push(attempt); + if (attempt.status === 0) return { ...attempt, label, attempts, selectedBecause: 'status0' }; + } + return { ...(attempts.at(-1) || {}), label, attempts, selectedBecause: 'no_status0' }; + } + + function firstUseful(label, candidates, opts) { + const attempts = []; + let firstSuccess = null; + for (const args of candidates) { + const attempt = run(cbm, args, opts); + attempts.push(attempt); + if (attempt.status === 0 && !firstSuccess) firstSuccess = attempt; + if (attempt.status === 0 && hasUsefulLocations(attempt)) return { ...attempt, label, attempts, selectedBecause: 'status0_with_locations' }; + } + if (firstSuccess) return { ...firstSuccess, label, attempts, selectedBecause: 'status0_without_locations' }; + return { ...(attempts.at(-1) || {}), label, attempts, selectedBecause: 'no_status0' }; + } + + function queryOf(text) { + return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 8).join(' '); + } + + function jsonish(s) { + const t = String(s || '').trim(); + if (!t) return null; + try { return JSON.parse(t); } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a), j = t.lastIndexOf(b); + if (i >= 0 && j > i) { try { return JSON.parse(t.slice(i, j + 1)); } catch {} } + } + return null; + } + + function payloadsFrom(result) { + const out = []; + for (const text of [result?.stdout, result?.stderr]) { + const parsed = jsonish(text); + if (!parsed) continue; + out.push(parsed); + const content = parsed.content; + if (Array.isArray(content)) { + for (const item of content) { + const nested = jsonish(item?.text); + if (nested) out.push(nested); + } + } + } + return out; + } + + function projectFrom(...results) { + for (const result of results) { + for (const obj of payloadsFrom(result)) { + if (typeof obj.project === 'string' && obj.project) return obj.project; + if (Array.isArray(obj.projects)) { + for (const entry of obj.projects) { + if (typeof entry === 'string' && entry) return entry; + if (typeof entry?.project === 'string' && entry.project) return entry.project; + if (typeof entry?.name === 'string' && entry.name) return entry.name; + } + } + } + } + return ''; + } + + function add(spans, file, start = 1, end = start) { + if (typeof file !== 'string' || !file) return; + const clean = file.replace(/^\/+/, ''); + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = spans.get(clean) || []; + list.push({ start: s, end: e }); + spans.set(clean, list); + } + + function walk(v, spans) { + if (!v || typeof v !== 'object') return; + if (Array.isArray(v)) { for (const x of v) walk(x, spans); return; } + const file = v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path; + const start = v.start_line || v.startLine || v.line || v.line_number || v.start || 1; + const end = v.end_line || v.endLine || v.end || start; + add(spans, file, start, end); + for (const x of Object.values(v)) walk(x, spans); + } + + function textPaths(s, spans) { + const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = re.exec(String(s || ''))) !== null) add(spans, m[1], m[2] || 1, m[2] || 1); + } + + function addLocationsFrom(result, spans) { + for (const text of [result?.stdout, result?.stderr]) { + const parsed = jsonish(text); + if (parsed) walk(parsed, spans); + textPaths(text, spans); + } + } + + function hasUsefulLocations(result) { + for (const obj of payloadsFrom(result)) { + if (Array.isArray(obj.results) && obj.results.length > 0) return true; + if (Array.isArray(obj.raw_matches) && obj.raw_matches.length > 0) return true; + if (Number(obj.total_results || 0) > 0 || Number(obj.raw_match_count || 0) > 0 || Number(obj.total_grep_matches || 0) > 0) return true; + } + const spans = new Map(); + addLocationsFrom(result, spans); + return spans.size > 0; + } + + function regexLiteral(value) { + return String(value || '').replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + + function allAttempts(...results) { + return results.flatMap((result) => Array.isArray(result?.attempts) && result.attempts.length ? result.attempts : [result]); + } + + function rowReady(report) { + return report.setupStatus === 0 && report.indexStatus === 0 && report.toolCallable && report.nonEmptyPrediction && report.officialEvaluatorScoreable; + } + + function shortFailure(report) { + return { + taskId: report.taskId, + repo: report.repo, + setupStatus: report.setupStatus, + indexStatus: report.indexStatus, + indexSignal: report.indexSignal, + indexError: report.indexError, + toolCallable: report.toolCallable, + nonEmptyPrediction: report.nonEmptyPrediction, + officialEvaluatorScoreable: report.officialEvaluatorScoreable, + indexStderrExcerpt: report.indexStderrExcerpt + }; + } + + const reports = []; + for (const [i, task] of tasks.entries()) { + const dir = join(outRoot, `${i + 1}-${task.instance_id}`); + mkdirSync(dir, { recursive: true }); + const env = { ...process.env, CBM_CACHE_DIR: join(dir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const opts = { cwd: task.repo_checkout_path, env, timeoutMs: 120_000 }; + const query = queryOf(task.problem_statement); + const firstTerm = query.split(/\s+/)[0] || 'import'; + const setup = run(cbm, ['--version'], { env, timeoutMs: 60_000 }); + const indexRun = run(cbm, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { ...opts, timeoutMs: 45 * 60 * 1000 }); + const listProjects = firstOk('list_projects', [['cli', 'list_projects'], ['cli', 'list_projects', '{}']], opts); + const project = projectFrom(indexRun, listProjects) || basename(task.repo_checkout_path); + const graphSchema = firstOk('get_graph_schema', [['cli', 'get_graph_schema', JSON.stringify({ project })]], opts); + const graphSearch = firstUseful('search_graph', [ + ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', query: firstTerm, limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', name_pattern: `.*${regexLiteral(firstTerm)}.*`, limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Function', limit: 25 })], + ['cli', 'search_graph', JSON.stringify({ project, label: 'Class', limit: 25 })] + ], opts); + const codeSearch = firstUseful('search_code', [ + ['cli', 'search_code', JSON.stringify({ project, pattern: query, mode: 'compact', limit: 25 })], + ['cli', 'search_code', JSON.stringify({ project, pattern: firstTerm, mode: 'compact', limit: 25 })], + ['cli', 'search_code', JSON.stringify({ project, pattern: '.', mode: 'compact', limit: 25 })] + ], opts); + + const spans = new Map(); + for (const r of allAttempts(listProjects, graphSchema, graphSearch, codeSearch)) addLocationsFrom(r, spans); + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + const predictionPath = join(dir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + + for (const [name, result] of Object.entries({ setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch })) writeFileSync(join(dir, `${name}.json`), JSON.stringify(result, null, 2)); + const goldPath = join(dir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 10 * 60 * 1000 }); + const official = join(root, 'ContextBench-official'); + if (!existsSync(join(official, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', official], { timeoutMs: 10 * 60 * 1000 }); + const scorePath = join(dir, 'official-score.jsonl'); + const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(dir, 'repo-cache'), '--out', scorePath], { cwd: official, timeoutMs: 20 * 60 * 1000 }); + const scoreText = existsSync(scorePath) ? readFileSync(scorePath, 'utf8').trim() : ''; + const report = { taskId: task.instance_id, repo: task.repo, project, setupStatus: setup.status, indexStatus: indexRun.status, indexSignal: indexRun.signal, indexError: indexRun.error, indexStderrExcerpt: indexRun.stderr.slice(0, 1000), toolCallable: [graphSearch, codeSearch].some((r) => r.status === 0), nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, officialEvaluatorStatus: evaluator.status, officialEvaluatorScoreable: evaluator.status === 0 && scoreText.length > 0, costs: { setupDurationMs: setup.durationMs, indexDurationMs: indexRun.durationMs, queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, evaluatorDurationMs: evaluator.durationMs }, laneIsolation: { allowedTool: 'codebase-memory-mcp', observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], disallowedNativeReadSearchUsedForPrediction: false }, query, predFiles, selectedBecause: { graphSearch: graphSearch.selectedBecause, codeSearch: codeSearch.selectedBecause }, commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } }; + writeFileSync(join(dir, 'readiness-report.json'), JSON.stringify(report, null, 2)); + reports.push(report); + } + const functionalReports = reports.filter(rowReady); + const blockers = reports.filter((r) => !rowReady(r)).map(shortFailure); + const summary = { createdAt: new Date().toISOString(), lane: 'codebase-memory-mcp', ready: functionalReports.length > 0, readinessMeaning: 'lane tool is callable and scoreable on at least one frozen task; per-task setup/index blockers remain separate evidence and are not quality results', attemptedRows: reports.length, functionalRows: functionalReports.length, blockerRows: blockers.length, scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, setupIndexCostReportedSeparately: true, blockers, reports }; + writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2)); + console.log(JSON.stringify({ ready: summary.ready, readinessMeaning: summary.readinessMeaning, attemptedRows: summary.attemptedRows, functionalRows: summary.functionalRows, blockerRows: summary.blockerRows, scoreableRows: summary.scoreableRows, nonEmptyPredictionRows: summary.nonEmptyPredictionRows, blockers: summary.blockers, reports: summary.reports.map((r) => ({ taskId: r.taskId, setupStatus: r.setupStatus, indexStatus: r.indexStatus, indexSignal: r.indexSignal, indexError: r.indexError, toolCallable: r.toolCallable, nonEmptyPrediction: r.nonEmptyPrediction, officialEvaluatorScoreable: r.officialEvaluatorScoreable, predFiles: r.predFiles.length, selectedBecause: r.selectedBecause, costs: r.costs })) }, null, 2)); + if (!summary.ready) process.exitCode = 1; + NODE + node "$ROOT/readiness.mjs" + - name: Upload readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-cbm-readiness-retry + path: /tmp/contextbench-cbm-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-cgc-relevant-readiness.yml b/.github/workflows/contextbench-cgc-relevant-readiness.yml new file mode 100644 index 0000000..233f6ba --- /dev/null +++ b/.github/workflows/contextbench-cgc-relevant-readiness.yml @@ -0,0 +1,74 @@ +name: ContextBench CodeGraphContext Relevant Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-cgc-relevant-readiness.yml + - scripts/contextbench-cgc-relevant-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + cgc-relevant-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-cgc-relevant + TASK_PAYLOADS: /tmp/contextbench-cgc-relevant/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-cgc-relevant-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install CodeGraphContext + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate relevant readiness pack + shell: bash + run: node scripts/contextbench-cgc-relevant-candidate-pack.mjs + - name: Upload relevant CGC readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-cgc-relevant-readiness + path: /tmp/contextbench-cgc-relevant + retention-days: 14 diff --git a/.github/workflows/contextbench-ci-recovery-master.yml b/.github/workflows/contextbench-ci-recovery-master.yml new file mode 100644 index 0000000..6923dcd --- /dev/null +++ b/.github/workflows/contextbench-ci-recovery-master.yml @@ -0,0 +1,281 @@ +name: ContextBench CI Recovery Master + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ci-recovery-master.yml + workflow_dispatch: + inputs: + max_tasks: + description: 'Number of first tasks to run for codebase-memory readiness' + required: true + default: '3' + codebase_memory_version: + description: 'codebase-memory-mcp release tag' + required: true + default: 'v0.6.1' + +permissions: + contents: read + +jobs: + codebase-memory-first3-readiness: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + CI_READINESS_ROOT: /tmp/contextbench-readiness + TASK_PAYLOADS: /tmp/contextbench-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_VERSION: ${{ github.event.inputs.codebase_memory_version || 'v0.6.1' }} + MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }} + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v2 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'pnpm' + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install repo dependencies + run: pnpm install --frozen-lockfile + + - name: Install official evaluator dependencies + run: python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + + - name: Validate frozen ContextBench fixtures + run: node scripts/contextbench-runner.mjs --validate-fixtures + + - name: Materialize first task payloads and checkouts + run: | + mkdir -p "$CI_READINESS_ROOT" "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS" + + - name: Download codebase-memory-mcp + run: | + set -euxo pipefail + mkdir -p "$CI_READINESS_ROOT/tool" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$CI_READINESS_ROOT/tool/cbm.tar.gz" + tar -xzf "$CI_READINESS_ROOT/tool/cbm.tar.gz" -C "$CI_READINESS_ROOT/tool" + chmod +x "$CI_READINESS_ROOT/tool/codebase-memory-mcp" || true + "$CI_READINESS_ROOT/tool/codebase-memory-mcp" --version || true + + - name: Run codebase-memory-mcp first3 readiness with official evaluator + env: + CBM_BIN: /tmp/contextbench-readiness/tool/codebase-memory-mcp + run: | + cat > "$CI_READINESS_ROOT/codebase-memory-first3-readiness.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { join } from 'node:path'; + + const root = process.env.CI_READINESS_ROOT; + const payloadPath = process.env.TASK_PAYLOADS; + const cbmBin = process.env.CBM_BIN; + const maxTasks = Number(process.env.MAX_TASKS || '3'); + const payloads = JSON.parse(readFileSync(payloadPath, 'utf8')); + const tasks = payloads.tasks.slice(0, maxTasks); + const outRoot = join(root, 'codebase-memory-first3'); + mkdirSync(outRoot, { recursive: true }); + + function run(command, args, options = {}) { + const started = Date.now(); + const result = spawnSync(command, args, { + encoding: 'utf8', + timeout: options.timeoutMs ?? 20 * 60 * 1000, + env: options.env ?? process.env, + cwd: options.cwd ?? process.cwd(), + maxBuffer: 64 * 1024 * 1024 + }); + return { + command: [command, ...args].join(' '), + cwd: options.cwd ?? process.cwd(), + status: result.status, + signal: result.signal, + error: result.error?.message ?? null, + durationMs: Date.now() - started, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '' + }; + } + + function runCandidates(label, candidates, options = {}) { + const attempts = []; + for (const args of candidates) { + const attempt = run(cbmBin, args, options); + attempts.push(attempt); + if (attempt.status === 0) return { ...attempt, label, attempts }; + } + return { ...attempts[attempts.length - 1], label, attempts }; + } + + function parseJsonish(text) { + const trimmed = String(text || '').trim(); + if (!trimmed) return null; + try { return JSON.parse(trimmed); } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const first = trimmed.indexOf(open); + const last = trimmed.lastIndexOf(close); + if (first >= 0 && last > first) { + try { return JSON.parse(trimmed.slice(first, last + 1)); } catch {} + } + } + return null; + } + + function addSpan(spans, file, start = 1, end = start) { + if (typeof file !== 'string' || file.length === 0) return; + const clean = file.replace(/^\/+/, ''); + const startLine = Number.isFinite(Number(start)) ? Math.max(1, Number(start)) : 1; + const endLine = Number.isFinite(Number(end)) ? Math.max(startLine, Number(end)) : startLine; + const list = spans.get(clean) || []; + list.push({ start: startLine, end: endLine }); + spans.set(clean, list); + } + + function collectSpans(value, spans = new Map()) { + if (!value || typeof value !== 'object') return spans; + if (Array.isArray(value)) { + for (const item of value) collectSpans(item, spans); + return spans; + } + const file = value.file || value.path || value.file_path || value.rel_path || value.relative_path || value.filename; + const start = value.start_line || value.startLine || value.line || value.line_number || value.line_start || value.start; + const end = value.end_line || value.endLine || value.line_end || value.end || start; + if (typeof file === 'string') addSpan(spans, file, start, end); + for (const item of Object.values(value)) collectSpans(item, spans); + return spans; + } + + function collectTextSpans(text, spans) { + const source = String(text || ''); + const regex = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; + let match; + while ((match = regex.exec(source)) !== null) addSpan(spans, match[1], match[2] || 1, match[2] || 1); + } + + function makeQuery(problem) { + return String(problem || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !/^https?$/.test(w)) + .slice(0, 8) + .join(' '); + } + + const reports = []; + let ready = true; + for (const [index, task] of tasks.entries()) { + const runDir = join(outRoot, `${index + 1}-${task.instance_id}`); + mkdirSync(runDir, { recursive: true }); + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const query = makeQuery(task.problem_statement); + const firstTerm = query.split(/\s+/).find(Boolean) || 'TODO'; + const toolOptions = { env, cwd: task.repo_checkout_path, timeoutMs: 120_000 }; + const setup = run(cbmBin, ['--version'], { env, timeoutMs: 60_000 }); + const indexRun = run(cbmBin, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { env, cwd: task.repo_checkout_path, timeoutMs: 45 * 60 * 1000 }); + const listProjects = runCandidates('list_projects', [['cli', '--raw', 'list_projects'], ['cli', 'list_projects'], ['cli', 'list_projects', '{}']], toolOptions); + const graphSchema = runCandidates('get_graph_schema', [['cli', '--raw', 'get_graph_schema'], ['cli', 'get_graph_schema'], ['cli', 'get_graph_schema', '{}']], toolOptions); + const graphSearch = runCandidates('search_graph', [['cli', '--raw', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })], ['cli', 'search_graph', JSON.stringify({ label: 'Function', limit: 25 })]], toolOptions); + const codeSearch = runCandidates('search_code', [ + ['cli', '--raw', 'search_code', JSON.stringify({ pattern: query, limit: 25 })], + ['cli', 'search_code', JSON.stringify({ pattern: query, limit: 25 })], + ['cli', 'search_code', JSON.stringify({ pattern: firstTerm, limit: 25 })], + ['cli', 'search_code', JSON.stringify({ query, limit: 25 })] + ], toolOptions); + + const spans = new Map(); + for (const result of [codeSearch, graphSearch, graphSchema, listProjects]) { + for (const text of [result.stdout, result.stderr]) { + const parsed = parseJsonish(text); + if (parsed) collectSpans(parsed, spans); + collectTextSpans(text, spans); + } + } + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + const prediction = { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, + model_patch: '' + }; + writeFileSync(join(runDir, 'prediction.json'), JSON.stringify(prediction, null, 2)); + for (const [name, result] of Object.entries({ setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch })) { + writeFileSync(join(runDir, `${name}.stdout.log`), result.stdout || ''); + writeFileSync(join(runDir, `${name}.stderr.log`), result.stderr || ''); + writeFileSync(join(runDir, `${name}.json`), JSON.stringify(result, null, 2)); + } + + const goldPath = join(runDir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', payloadPath], { timeoutMs: 10 * 60 * 1000 }); + const officialRepo = join(root, 'ContextBench-official'); + if (!existsSync(join(officialRepo, 'contextbench', 'evaluate.py'))) run('git', ['clone', '--depth', '1', 'https://github.com/EuniAI/ContextBench.git', officialRepo], { timeoutMs: 10 * 60 * 1000 }); + const scorePath = join(runDir, 'official-score.jsonl'); + const evaluator = run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', join(runDir, 'prediction.json'), '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: officialRepo, timeoutMs: 20 * 60 * 1000 }); + const scoreable = evaluator.status === 0 && existsSync(scorePath); + const nonEmptyPrediction = predFiles.length > 0 && Object.keys(predSpans).length > 0; + const report = { + taskId: task.instance_id, + repo: task.repo, + setupStatus: setup.status, + indexStatus: indexRun.status, + toolCallable: [listProjects, graphSchema, graphSearch, codeSearch].some((r) => r.status === 0), + nonEmptyPrediction, + officialEvaluatorStatus: evaluator.status, + officialEvaluatorScoreable: scoreable, + laneIsolation: { + allowedTool: 'codebase-memory-mcp', + observedCommands: [setup.command, indexRun.command, listProjects.command, graphSchema.command, graphSearch.command, codeSearch.command], + observedCwds: [setup.cwd, indexRun.cwd, listProjects.cwd, graphSchema.cwd, graphSearch.cwd, codeSearch.cwd], + disallowedNativeReadSearchUsedForPrediction: false, + note: 'Prediction spans are derived only from codebase-memory-mcp CLI outputs.' + }, + costs: { + setupDurationMs: setup.durationMs, + indexDurationMs: indexRun.durationMs, + queryDurationMs: listProjects.durationMs + graphSchema.durationMs + graphSearch.durationMs + codeSearch.durationMs, + evaluatorDurationMs: evaluator.durationMs + }, + query, + predFiles, + commands: { setup, indexRun, listProjects, graphSchema, graphSearch, codeSearch, gold, evaluator } + }; + writeFileSync(join(runDir, 'readiness-report.json'), JSON.stringify(report, null, 2)); + reports.push(report); + if (!(setup.status === 0 && indexRun.status === 0 && report.toolCallable && nonEmptyPrediction && scoreable)) ready = false; + } + + const summary = { + createdAt: new Date().toISOString(), + lane: 'codebase-memory-mcp', + ready, + attemptedRows: reports.length, + scoreableRows: reports.filter((r) => r.officialEvaluatorScoreable).length, + nonEmptyPredictionRows: reports.filter((r) => r.nonEmptyPrediction).length, + setupIndexCostReportedSeparately: true, + reports + }; + writeFileSync(join(outRoot, 'lane-readiness-codebase-memory-first3.json'), JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + if (!ready) process.exitCode = 1; + NODE + node "$CI_READINESS_ROOT/codebase-memory-first3-readiness.mjs" + + - name: Upload ContextBench recovery artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-codebase-memory-first3-readiness + path: /tmp/contextbench-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-codegraphcontext-readiness.yml b/.github/workflows/contextbench-codegraphcontext-readiness.yml new file mode 100644 index 0000000..4119a87 --- /dev/null +++ b/.github/workflows/contextbench-codegraphcontext-readiness.yml @@ -0,0 +1,74 @@ +name: ContextBench CodeGraphContext Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-codegraphcontext-readiness.yml + - scripts/contextbench-codegraphcontext-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + codegraphcontext-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-codegraphcontext-readiness + TASK_PAYLOADS: /tmp/contextbench-codegraphcontext-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-codegraphcontext-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install CodeGraphContext lane + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate CodeGraphContext readiness candidate pack + shell: bash + run: node scripts/contextbench-codegraphcontext-ready-pack.mjs + - name: Upload CodeGraphContext readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-codegraphcontext-readiness + path: /tmp/contextbench-codegraphcontext-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-final-lane-readiness.yml b/.github/workflows/contextbench-final-lane-readiness.yml new file mode 100644 index 0000000..bb80542 --- /dev/null +++ b/.github/workflows/contextbench-final-lane-readiness.yml @@ -0,0 +1,94 @@ +name: ContextBench Final Lane Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-final-lane-readiness.yml + - scripts/contextbench-final-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + final-lane-readiness: + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-final-lane/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-final-lane/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-final-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + OLLAMA_HOST: http://127.0.0.1:11434 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Install grepai with local Ollama embeddings + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL https://ollama.com/install.sh | sh > "$ROOT/logs/ollama-install.log" 2>&1 + nohup ollama serve > "$ROOT/logs/ollama-serve.log" 2>&1 & + for i in {1..45}; do + if curl -fsS "$OLLAMA_HOST/api/version" > "$ROOT/logs/ollama-version.log" 2>&1; then + break + fi + sleep 2 + done + curl -fsS "$OLLAMA_HOST/api/version" >> "$ROOT/logs/ollama-version.log" 2>&1 + ollama pull nomic-embed-text > "$ROOT/logs/ollama-pull-nomic-embed-text.log" 2>&1 + - name: Install codegraphcontext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate final lane readiness candidate pack + shell: bash + run: node scripts/contextbench-final-lane-candidate-pack.mjs + - name: Upload final lane readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-final-lane-readiness-${{ matrix.lane }} + path: /tmp/contextbench-final-lane/${{ matrix.lane }} + retention-days: 14 diff --git a/.github/workflows/contextbench-five-lane-candidates.yml b/.github/workflows/contextbench-five-lane-candidates.yml new file mode 100644 index 0000000..416b282 --- /dev/null +++ b/.github/workflows/contextbench-five-lane-candidates.yml @@ -0,0 +1,72 @@ +name: ContextBench Five Lane Candidates + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-five-lane-candidates.yml + - scripts/contextbench-five-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + candidates: + runs-on: ubuntu-latest + timeout-minutes: 100 + env: + ROOT: /tmp/contextbench-five-lane-candidates + TASK_PAYLOADS: /tmp/contextbench-five-lane-candidates/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + CBM_BIN: /tmp/contextbench-five-lane-candidates/tool/codebase-memory-mcp + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install tools and materialize selected Go task quietly + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + pnpm run build > "$ROOT/logs/build.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu > "$ROOT/logs/pip-install.log" 2>&1 + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + echo "five_lane_setup_completed" + - name: Generate five-lane readiness candidate pack + shell: bash + run: node scripts/contextbench-five-lane-candidate-pack.mjs + - name: Upload five-lane candidate artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-five-lane-candidates + path: /tmp/contextbench-five-lane-candidates + retention-days: 14 diff --git a/.github/workflows/contextbench-five-lane-score.yml b/.github/workflows/contextbench-five-lane-score.yml new file mode 100644 index 0000000..79e4513 --- /dev/null +++ b/.github/workflows/contextbench-five-lane-score.yml @@ -0,0 +1,134 @@ +name: ContextBench Five Lane Score + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-five-lane-score.yml + - scripts/contextbench-score-five-lane-selections.mjs + - scripts/contextbench-score-five-lane-artifact-selections.mjs + - scripts/contextbench-build-publishable-report.mjs + - scripts/contextbench-print-publishable-report.mjs + - scripts/contextbench-five-lane-selections.json + - tests/fixtures/contextbench-benchmark-protocol.json + - tests/fixtures/contextbench-lanes.json + - tests/fixtures/contextbench-task-manifest.json + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + five-lane-score: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-five-lane-score + TASK_PAYLOADS: /tmp/contextbench-five-lane-score/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-five-lane-score-checkouts + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-five-lane-score/ContextBench-official + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + SOURCE_SELECTIONS_PATH: scripts/contextbench-five-lane-selections.json + EXTERNAL_READINESS_ROOT: /tmp/contextbench-five-lane-score/external-readiness + REQUIRED_LANES: raw-native,codebase-context,codebase-memory-mcp,grepai,ripgrep-lexical + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" "$EXTERNAL_READINESS_ROOT" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Download GrepAI readiness artifact + uses: actions/download-artifact@v4 + with: + name: contextbench-grepai-readiness + run-id: 25643757046 + github-token: ${{ github.token }} + path: ${{ env.EXTERNAL_READINESS_ROOT }}/grepai + - name: Download ripgrep readiness artifact + uses: actions/download-artifact@v4 + with: + name: contextbench-ripgrep-readiness + run-id: 25644197513 + github-token: ${{ github.token }} + path: ${{ env.EXTERNAL_READINESS_ROOT }}/ripgrep + - name: Score five ready lane selections + shell: bash + run: | + set +e + node scripts/contextbench-score-five-lane-artifact-selections.mjs > "$ROOT/logs/score-five-lane.log" 2>&1 + status=$? + tail -n 120 "$ROOT/logs/score-five-lane.log" + exit "$status" + - name: Build publishable pilot report + shell: bash + run: | + node scripts/contextbench-build-publishable-report.mjs \ + --summary "$ROOT/summary.json" \ + --protocol tests/fixtures/contextbench-benchmark-protocol.json \ + --lanes tests/fixtures/contextbench-lanes.json \ + --task-manifest tests/fixtures/contextbench-task-manifest.json \ + --out "$ROOT/publishable-summary.json" \ + --validation-out "$ROOT/publishable-validation.json" \ + --humanized-out "$ROOT/humanized-summary.md" + - name: Print compact publishable pilot report + shell: bash + run: node scripts/contextbench-print-publishable-report.mjs "$ROOT/publishable-summary.json" + - name: Upload five-lane score artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-five-lane-score + path: | + /tmp/contextbench-five-lane-score/summary.json + /tmp/contextbench-five-lane-score/publishable-summary.json + /tmp/contextbench-five-lane-score/publishable-validation.json + /tmp/contextbench-five-lane-score/humanized-summary.md + /tmp/contextbench-five-lane-score/task-payloads.json + /tmp/contextbench-five-lane-score/logs/** + /tmp/contextbench-five-lane-score/lane-score/summary.json + /tmp/contextbench-five-lane-score/lane-score/gold.json + /tmp/contextbench-five-lane-score/lane-score/gold-command.json + /tmp/contextbench-five-lane-score/lane-score/selections.json + /tmp/contextbench-five-lane-score/lane-score/*/selection.json + /tmp/contextbench-five-lane-score/lane-score/*/prediction.json + /tmp/contextbench-five-lane-score/lane-score/*/official-score.jsonl + /tmp/contextbench-five-lane-score/lane-score/*/evaluator-command.json + /tmp/contextbench-five-lane-score/external-readiness/**/*.json + /tmp/contextbench-five-lane-score/external-readiness/**/*.jsonl + /tmp/contextbench-five-lane-score/external-readiness/**/*.md + retention-days: 14 diff --git a/.github/workflows/contextbench-github-models-probe.yml b/.github/workflows/contextbench-github-models-probe.yml new file mode 100644 index 0000000..b4693a6 --- /dev/null +++ b/.github/workflows/contextbench-github-models-probe.yml @@ -0,0 +1,66 @@ +name: ContextBench GitHub Models Probe + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-github-models-probe.yml + workflow_dispatch: + +permissions: + contents: read + models: read + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + ROOT: /tmp/contextbench-github-models-probe + steps: + - name: Probe model catalog and smoke calls + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p "$ROOT" + api=https://models.github.ai + curl -sS -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2026-03-10" \ + "$api/catalog/models" > "$ROOT/catalog.json" + node - <<'NODE' + const fs = require('fs'); + const raw = fs.readFileSync(process.env.ROOT + '/catalog.json', 'utf8'); + let json; + try { json = JSON.parse(raw); } catch (error) { console.error(raw.slice(0, 2000)); throw error; } + const list = Array.isArray(json) ? json : (json.models || json.data || []); + const ids = list.map((m) => m.id || m.name || m.model).filter(Boolean); + const interesting = ids.filter((id) => /openai|gpt-5|gpt-5\.4|mini/i.test(id)); + fs.writeFileSync(process.env.ROOT + '/interesting-models.json', JSON.stringify(interesting, null, 2)); + console.log(JSON.stringify({ total: ids.length, interesting: interesting.slice(0, 80) }, null, 2)); + NODE + for model in openai/gpt-5.4-mini openai/gpt-5-mini gpt-5.4-mini gpt-5-mini; do + safe=$(echo "$model" | tr '/.' '__') + echo "Trying $model" + code=$(curl -sS -L -o "$ROOT/smoke-$safe.json" -w '%{http_code}' \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "X-GitHub-Api-Version: 2026-03-10" \ + -H "Content-Type: application/json" \ + "$api/inference/chat/completions" \ + -d "{\"model\":\"$model\",\"messages\":[{\"role\":\"user\",\"content\":\"Return JSON: {\\\"ok\\\": true}\"}],\"response_format\":{\"type\":\"json_object\"},\"max_tokens\":32}") || code=000 + echo "$model $code" | tee -a "$ROOT/smoke-status.txt" + done + if grep -E ' 2[0-9][0-9]$' "$ROOT/smoke-status.txt"; then + exit 0 + fi + exit 1 + - name: Upload GitHub Models probe artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-github-models-probe + path: /tmp/contextbench-github-models-probe + retention-days: 14 diff --git a/.github/workflows/contextbench-grepai-readiness.yml b/.github/workflows/contextbench-grepai-readiness.yml new file mode 100644 index 0000000..13f8502 --- /dev/null +++ b/.github/workflows/contextbench-grepai-readiness.yml @@ -0,0 +1,90 @@ +name: ContextBench GrepAI Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-grepai-readiness.yml + - scripts/contextbench-grepai-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + grepai-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-grepai-readiness + TASK_PAYLOADS: /tmp/contextbench-grepai-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-grepai-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + OLLAMA_HOST: http://127.0.0.1:11434 + OLLAMA_NUM_PARALLEL: 1 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install grepai with local Ollama embeddings + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL https://ollama.com/install.sh | sh > "$ROOT/logs/ollama-install.log" 2>&1 + nohup ollama serve > "$ROOT/logs/ollama-serve.log" 2>&1 & + for i in {1..45}; do + if curl -fsS "$OLLAMA_HOST/api/version" > "$ROOT/logs/ollama-version.log" 2>&1; then + break + fi + sleep 2 + done + curl -fsS "$OLLAMA_HOST/api/version" >> "$ROOT/logs/ollama-version.log" 2>&1 + ollama pull nomic-embed-text > "$ROOT/logs/ollama-pull-nomic-embed-text.log" 2>&1 + - name: Generate GrepAI readiness candidate pack + shell: bash + run: node scripts/contextbench-grepai-ready-pack.mjs + - name: Upload GrepAI readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-grepai-readiness + path: /tmp/contextbench-grepai-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-infra-lane-readiness.yml b/.github/workflows/contextbench-infra-lane-readiness.yml new file mode 100644 index 0000000..de20006 --- /dev/null +++ b/.github/workflows/contextbench-infra-lane-readiness.yml @@ -0,0 +1,103 @@ +name: ContextBench Infra Lane Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-infra-lane-readiness.yml + - scripts/contextbench-infra-lane-readiness.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + infra-lane-readiness: + runs-on: ubuntu-latest + timeout-minutes: 55 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-infra-lane/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-infra-lane/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-infra-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + OLLAMA_HOST: http://127.0.0.1:11434 + OLLAMA_NUM_PARALLEL: 1 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Install grepai with local Ollama embeddings + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL https://ollama.com/install.sh | sh > "$ROOT/logs/ollama-install.log" 2>&1 + nohup ollama serve > "$ROOT/logs/ollama-serve.log" 2>&1 & + for i in {1..45}; do + if curl -fsS "$OLLAMA_HOST/api/version" > "$ROOT/logs/ollama-version.log" 2>&1; then + break + fi + sleep 2 + done + curl -fsS "$OLLAMA_HOST/api/version" >> "$ROOT/logs/ollama-version.log" 2>&1 + ollama pull nomic-embed-text > "$ROOT/logs/ollama-pull-nomic-embed-text.log" 2>&1 + - name: Install CodeGraphContext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate infra lane readiness candidate pack + shell: bash + run: node scripts/contextbench-infra-lane-readiness.mjs + - name: Upload infra lane readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-infra-lane-readiness-${{ matrix.lane }} + path: /tmp/contextbench-infra-lane/${{ matrix.lane }} + retention-days: 14 diff --git a/.github/workflows/contextbench-lane-readiness.yml b/.github/workflows/contextbench-lane-readiness.yml new file mode 100644 index 0000000..2b3ee22 --- /dev/null +++ b/.github/workflows/contextbench-lane-readiness.yml @@ -0,0 +1,106 @@ +name: ContextBench Lane Readiness Matrix + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-lane-readiness.yml + - scripts/contextbench-single-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + lane-readiness: + runs-on: ubuntu-latest + timeout-minutes: 50 + strategy: + fail-fast: false + max-parallel: 5 + matrix: + lane: + - raw-native + - codebase-context + - codebase-memory-mcp + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-lane-readiness/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-lane-readiness/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts-${{ matrix.lane }} + CBM_BIN: /tmp/contextbench-lane-readiness/${{ matrix.lane }}/tool/codebase-memory-mcp + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Install raw-native lane + if: matrix.lane == 'raw-native' + shell: bash + run: | + set -euo pipefail + sudo apt-get update > "$ROOT/logs/apt-update.log" 2>&1 + sudo apt-get install -y ripgrep > "$ROOT/logs/apt-ripgrep.log" 2>&1 + - name: Build codebase-context lane + if: matrix.lane == 'codebase-context' + shell: bash + run: pnpm run build > "$ROOT/logs/build.log" 2>&1 + - name: Install codebase-memory-mcp lane + if: matrix.lane == 'codebase-memory-mcp' + shell: bash + run: | + set -euo pipefail + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + - name: Install grepai lane + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + - name: Install codegraphcontext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: python -m pip install uv codegraphcontext kuzu > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 + - name: Generate lane readiness candidate pack + shell: bash + run: node scripts/contextbench-single-lane-candidate-pack.mjs + - name: Upload lane readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-lane-readiness-${{ matrix.lane }} + path: /tmp/contextbench-lane-readiness/${{ matrix.lane }} + retention-days: 14 diff --git a/.github/workflows/contextbench-raw-readiness.yml b/.github/workflows/contextbench-raw-readiness.yml new file mode 100644 index 0000000..38218c9 --- /dev/null +++ b/.github/workflows/contextbench-raw-readiness.yml @@ -0,0 +1,66 @@ +name: ContextBench Raw Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-raw-readiness.yml + - scripts/contextbench-single-lane-candidate-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + raw-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + ROOT: /tmp/contextbench-raw-readiness + TASK_PAYLOADS: /tmp/contextbench-raw-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts-raw + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: raw-native + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected task and install ripgrep + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" "$ROOT/tool" + sudo apt-get update > "$ROOT/logs/apt-update.log" 2>&1 + sudo apt-get install -y ripgrep > "$ROOT/logs/apt-ripgrep.log" 2>&1 + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Generate raw-native readiness candidate pack + shell: bash + run: node scripts/contextbench-single-lane-candidate-pack.mjs + - name: Upload raw readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-raw-readiness + path: /tmp/contextbench-raw-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-ready-lane-score.yml b/.github/workflows/contextbench-ready-lane-score.yml new file mode 100644 index 0000000..6d2b804 --- /dev/null +++ b/.github/workflows/contextbench-ready-lane-score.yml @@ -0,0 +1,76 @@ +name: ContextBench Ready Lane Score + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ready-lane-score.yml + - scripts/contextbench-score-five-lane-selections.mjs + - scripts/contextbench-ready-lane-selections.json + workflow_dispatch: + +permissions: + contents: read + +jobs: + ready-lane-score: + runs-on: ubuntu-latest + timeout-minutes: 35 + env: + ROOT: /tmp/contextbench-ready-lane-score + TASK_PAYLOADS: /tmp/contextbench-ready-lane-score/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ready-lane-score-checkouts + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-ready-lane-score/ContextBench-official + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + SELECTIONS_PATH: scripts/contextbench-ready-lane-selections.json + REQUIRED_LANES: raw-native,codebase-context,codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Score ready lane selections + shell: bash + run: node scripts/contextbench-score-five-lane-selections.mjs + - name: Upload ready lane score artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ready-lane-score + path: /tmp/contextbench-ready-lane-score + retention-days: 14 diff --git a/.github/workflows/contextbench-real-cbm-go-one.yml b/.github/workflows/contextbench-real-cbm-go-one.yml new file mode 100644 index 0000000..ccfc1a6 --- /dev/null +++ b/.github/workflows/contextbench-real-cbm-go-one.yml @@ -0,0 +1,108 @@ +name: ContextBench Real CBM Go One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-cbm-go-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-go-one: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-real-cbm-go-one + TASK_PAYLOADS: /tmp/contextbench-real-cbm-go-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-cbm-go-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Run one bounded scoreable CBM row + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-cbm-go-one/ContextBench-official + run: | + cat > "$ROOT/one.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const runDir = join(root, 'row-codebase-memory-mcp-go'); + mkdirSync(runDir, { recursive: true }); + function run(cmd, args, opts = {}) { const started = Date.now(); const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 128 * 1024 * 1024 }); return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; } + function queryOf(s) { return String(s || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; } + function add(locs, file, start = 1, end = start, source = 'cbm') { if (!file || typeof file !== 'string' || file.includes('://')) return; const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); if (!clean || clean.includes('..')) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function walk(v, locs) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1); for (const x of Object.values(v)) walk(x, locs); } + function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; } + function addSpan(map, file, start = 1, end = start) { const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = map.get(file) || []; list.push({ start: s, end: e }); map.set(file, list); } + async function ask(candidates, query) { + const started = Date.now(); + if (!process.env.OPENAI_API_KEY) return { ok: false, status: 'model_unavailable', error: 'missing_OPENAI_API_KEY_secret', durationMs: 0 }; + const candidateFiles = new Set(candidates.map((c) => c.file)); + const body = { model: process.env.OPENAI_MODEL, reasoning: { effort: process.env.OPENAI_REASONING_EFFORT }, max_output_tokens: 1200, instructions: 'Select likely ContextBench edit locations using only the provided codebase-memory-mcp candidate locations. Return JSON only.', input: JSON.stringify({ taskId: task.instance_id, repo: task.repo, lane: 'codebase-memory-mcp', query, problemStatement: task.problem_statement, candidateLocations: candidates.slice(0, 60) }), text: { format: { type: 'json_schema', name: 'contextbench_selection', strict: true, schema: { type: 'object', additionalProperties: false, required: ['files','spans','notes'], properties: { files: { type: 'array', maxItems: 20, items: { type: 'string' } }, spans: { type: 'array', maxItems: 40, items: { type: 'object', additionalProperties: false, required: ['file','start','end'], properties: { file: { type: 'string' }, start: { type: 'integer', minimum: 1 }, end: { type: 'integer', minimum: 1 } } } }, notes: { type: 'string' } } } } } }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...body, input: JSON.parse(body.input) }, null, 2)); + const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 300000); + try { const res = await fetch('https://api.openai.com/v1/responses', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, body: JSON.stringify(body), signal: controller.signal }); const text = await res.text(); clearTimeout(timer); writeFileSync(join(runDir, 'openai-response.json'), text); if (!res.ok) return { ok: false, status: 'model_error', httpStatus: res.status, error: text.slice(0, 2000), durationMs: Date.now() - started }; const json = JSON.parse(text); const out = json.output_text || (json.output || []).flatMap((i) => i.content || []).filter((i) => i.type === 'output_text').map((i) => i.text).join('\n'); const parsed = JSON.parse(out); const files = [...new Set((parsed.files || []).filter((f) => candidateFiles.has(f)))].slice(0, 20); const spans = (parsed.spans || []).filter((s) => candidateFiles.has(s.file)).slice(0, 40); if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', parsed, durationMs: Date.now() - started }; return { ok: true, status: 'completed', parsed: { files, spans, notes: parsed.notes || '' }, usage: json.usage || null, durationMs: Date.now() - started }; } catch (error) { clearTimeout(timer); return { ok: false, status: 'model_timeout_or_error', error: String(error?.message || error), durationMs: Date.now() - started }; } + } + const query = queryOf(task.problem_statement); + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { cwd: task.repo_checkout_path, env, timeoutMs: 2700000 }); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(task.repo_checkout_path); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(runDir, `${name}.json`), JSON.stringify(value, null, 2)); + const locs = []; for (const r of [graph, code]) { collect(r.stdout, locs); collect(r.stderr, locs); } + const candidates = uniq(locs); writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(candidates, null, 2)); + const model = await ask(candidates, query); writeFileSync(join(runDir, 'model-result.json'), JSON.stringify(model, null, 2)); + const spanMap = new Map(); if (model.ok) { for (const s of model.parsed.spans || []) addSpan(spanMap, s.file, s.start, s.end); for (const f of model.parsed.files || []) addSpan(spanMap, f, 1, 1); } + const predFiles = [...spanMap.keys()].slice(0, 20); const predSpans = Object.fromEntries([...spanMap.entries()].slice(0, 20)); + const predPath = join(runDir, 'prediction.json'); writeFileSync(predPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + const goldPath = join(runDir, 'gold.json'); const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); + const scorePath = join(runDir, 'official-score.jsonl'); const evaluator = model.ok && predFiles.length ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.ROOT + '/ContextBench-official', timeoutMs: 1200000 }) : { status: null, error: 'skipped_no_model_prediction', durationMs: 0 }; writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + let score = null; if (existsSync(scorePath)) { const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); if (lines.length) score = JSON.parse(lines.at(-1)); } + const row = { lane_id: 'codebase-memory-mcp', task_id: task.instance_id, model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, status: evaluator.status === 0 && score ? 'completed' : (model.status || 'judge_failed'), setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, nonEmptyPrediction: predFiles.length > 0, predFiles: predFiles.length, officialEvaluatorScoreable: evaluator.status === 0 && Boolean(score), setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, modelStatus: model.status, modelUsage: model.usage || null, score }; + const summary = { createdAt: new Date().toISOString(), attemptedRows: 1, scoreableRows: row.officialEvaluatorScoreable ? 1 : 0, setupIndexCostReportedSeparately: true, resultsTable: row.officialEvaluatorScoreable ? [{ lane: row.lane_id, fileCoverage: score.final.file.coverage, filePrecision: score.final.file.precision, symbolCoverage: score.final.symbol.coverage, spanCoverage: score.final.span.coverage, lineCoverage: score.final.line.coverage, editlocRecall: score.editloc?.recall ?? null }] : [], rows: [row] }; + writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); console.log(JSON.stringify(summary, null, 2)); if (!row.officialEvaluatorScoreable) process.exitCode = 1; + NODE + node "$ROOT/one.mjs" + - name: Upload one-row artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-cbm-go-one + path: /tmp/contextbench-real-cbm-go-one + retention-days: 14 diff --git a/.github/workflows/contextbench-real-gpt54mini-auth-one.yml b/.github/workflows/contextbench-real-gpt54mini-auth-one.yml new file mode 100644 index 0000000..0ac57cc --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini-auth-one.yml @@ -0,0 +1,174 @@ +name: ContextBench Real GPT54 Mini Auth One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini-auth-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-go-auth-one: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-real-gpt54mini-auth-one + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini-auth-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || secrets.CODEX_OPENAI_API_KEY || secrets.OPENAI_KEY || secrets.OPENAI_API_KEY_GPT54 || secrets.CONTEXTBENCH_OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-gpt54mini-auth-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check model auth + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "::error::Missing model auth. Set one Actions secret: OPENAI_API_KEY, CODEX_OPENAI_API_KEY, OPENAI_KEY, OPENAI_API_KEY_GPT54, or CONTEXTBENCH_OPENAI_API_KEY." + exit 78 + fi + + - name: Install and materialize frozen Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + + - name: Run authenticated scoreable CBM row + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini-auth-one/ContextBench-official + run: | + cat > "$ROOT/run.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const runDir = join(root, 'row-codebase-memory-mcp-go'); + mkdirSync(runDir, { recursive: true }); + + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024 + }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function queryOf(text) { return String(text || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function jsonish(text) { const raw = String(text || '').trim(); if (!raw) return null; try { return JSON.parse(raw); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i = raw.indexOf(a), j = raw.lastIndexOf(b); if (i >= 0 && j > i) { try { return JSON.parse(raw.slice(i, j + 1)); } catch {} } } return null; } + function norm(file) { return String(file || '').replace(/^\/+/, '').replace(/^\.\//, ''); } + function add(locs, file, start = 1, end = start, source = 'cbm') { const clean = norm(file); if (!clean || clean.includes('..') || clean.includes('://')) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function walk(value, locs) { if (!value || typeof value !== 'object') return; if (Array.isArray(value)) { for (const item of value) walk(item, locs); return; } add(locs, value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path, value.start_line || value.line || 1, value.end_line || value.line || 1); for (const item of Object.values(value)) walk(item, locs); } + function collect(text, locs) { const parsed = jsonish(text); if (parsed) walk(parsed, locs); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const key = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(key)) { seen.add(key); out.push(loc); if (out.length >= 80) break; } } return out; } + function addSpan(map, file, start = 1, end = start) { const clean = norm(file); if (!clean) return; const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = map.get(clean) || []; list.push({ start: s, end: e }); map.set(clean, list); } + + async function askModel(candidates, query) { + const started = Date.now(); + const candidateFiles = new Set(candidates.map((c) => c.file)); + const body = { + model: process.env.OPENAI_MODEL, + reasoning: { effort: process.env.OPENAI_REASONING_EFFORT }, + max_output_tokens: 1200, + instructions: 'Select likely ContextBench edit locations using only the provided codebase-memory-mcp candidate locations. Return JSON only.', + input: JSON.stringify({ taskId: task.instance_id, repo: task.repo, lane: 'codebase-memory-mcp', query, problemStatement: task.problem_statement, candidateLocations: candidates.slice(0, 60) }), + text: { format: { type: 'json_schema', name: 'contextbench_selection', strict: true, schema: { type: 'object', additionalProperties: false, required: ['files','spans','notes'], properties: { files: { type: 'array', maxItems: 20, items: { type: 'string' } }, spans: { type: 'array', maxItems: 40, items: { type: 'object', additionalProperties: false, required: ['file','start','end'], properties: { file: { type: 'string' }, start: { type: 'integer', minimum: 1 }, end: { type: 'integer', minimum: 1 } } } }, notes: { type: 'string' } } } } } + }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...body, input: JSON.parse(body.input) }, null, 2)); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 300000); + try { + const res = await fetch('https://api.openai.com/v1/responses', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, body: JSON.stringify(body), signal: controller.signal }); + const text = await res.text(); + clearTimeout(timer); + writeFileSync(join(runDir, 'openai-response.json'), text); + if (!res.ok) return { ok: false, status: 'model_error', httpStatus: res.status, error: text.slice(0, 2000), durationMs: Date.now() - started }; + const json = JSON.parse(text); + const output = json.output_text || (json.output || []).flatMap((item) => item.content || []).filter((item) => item.type === 'output_text').map((item) => item.text).join('\n'); + const parsed = JSON.parse(output); + const files = [...new Set((parsed.files || []).map(norm).filter((file) => candidateFiles.has(file)))].slice(0, 20); + const spans = (parsed.spans || []).map((span) => ({ file: norm(span.file), start: span.start, end: span.end })).filter((span) => candidateFiles.has(span.file)).slice(0, 40); + if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', parsed, durationMs: Date.now() - started }; + return { ok: true, status: 'completed', parsed: { files, spans, notes: parsed.notes || '' }, usage: json.usage || null, durationMs: Date.now() - started }; + } catch (error) { + clearTimeout(timer); + return { ok: false, status: 'model_timeout_or_error', error: String(error?.message || error), durationMs: Date.now() - started }; + } + } + + const query = queryOf(task.problem_statement); + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: task.repo_checkout_path })], { cwd: task.repo_checkout_path, env, timeoutMs: 2700000 }); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(task.repo_checkout_path); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 25 })], { cwd: task.repo_checkout_path, env, timeoutMs: 120000 }); + for (const [name, value] of Object.entries({ setup, index, graph, code })) writeFileSync(join(runDir, `${name}.json`), JSON.stringify(value, null, 2)); + + const locs = []; + for (const result of [graph, code]) { collect(result.stdout, locs); collect(result.stderr, locs); } + const candidates = uniq(locs); + writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(candidates, null, 2)); + + const model = await askModel(candidates, query); + writeFileSync(join(runDir, 'model-result.json'), JSON.stringify(model, null, 2)); + const spanMap = new Map(); + if (model.ok) { for (const span of model.parsed.spans || []) addSpan(spanMap, span.file, span.start, span.end); for (const file of model.parsed.files || []) addSpan(spanMap, file, 1, 1); } + const predFiles = [...spanMap.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spanMap.entries()].slice(0, 20)); + const predictionPath = join(runDir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify({ instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, model_patch: '' }, null, 2)); + + const goldPath = join(runDir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); + writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); + const scorePath = join(runDir, 'official-score.jsonl'); + const evaluator = model.ok && predFiles.length ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.OFFICIAL_CONTEXTBENCH, timeoutMs: 1200000 }) : { status: null, error: 'skipped_no_model_prediction', durationMs: 0 }; + writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + let score = null; + if (existsSync(scorePath)) { const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); if (lines.length) score = JSON.parse(lines.at(-1)); } + + const row = { lane_id: 'codebase-memory-mcp', task_id: task.instance_id, model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, status: evaluator.status === 0 && score ? 'completed' : (model.status || 'judge_failed'), setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', indexStatus: index.status === 0 ? 'completed' : 'index_failed', toolCallable: graph.status === 0 || code.status === 0, candidateCount: candidates.length, nonEmptyPrediction: predFiles.length > 0, predFiles: predFiles.length, officialEvaluatorScoreable: evaluator.status === 0 && Boolean(score), setupIndex: { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: graph.durationMs + code.durationMs }, modelStatus: model.status, modelUsage: model.usage || null, score }; + const summary = { createdAt: new Date().toISOString(), attemptedRows: 1, scoreableRows: row.officialEvaluatorScoreable ? 1 : 0, setupIndexCostReportedSeparately: true, resultsTable: row.officialEvaluatorScoreable ? [{ lane: row.lane_id, fileCoverage: score.final.file.coverage, filePrecision: score.final.file.precision, symbolCoverage: score.final.symbol.coverage, spanCoverage: score.final.span.coverage, lineCoverage: score.final.line.coverage, editlocRecall: score.editloc?.recall ?? null }] : [], rows: [row] }; + writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); + writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); + console.log(JSON.stringify(summary, null, 2)); + if (!row.officialEvaluatorScoreable) process.exitCode = 1; + NODE + node "$ROOT/run.mjs" + + - name: Upload authenticated one-row artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini-auth-one + path: /tmp/contextbench-real-gpt54mini-auth-one + retention-days: 14 diff --git a/.github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml b/.github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml new file mode 100644 index 0000000..cb6f14d --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml @@ -0,0 +1,82 @@ +name: ContextBench Real GPT54 Mini Auth Vars One + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini-auth-vars-one.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + cbm-go-auth-vars-one: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + ROOT: /tmp/contextbench-real-gpt54mini-auth-vars-one + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini-auth-vars-one/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || vars.OPENAI_API_KEY || secrets.CODEX_OPENAI_API_KEY || vars.CODEX_OPENAI_API_KEY || secrets.OPENAI_KEY || vars.OPENAI_KEY || secrets.OPENAI_API_KEY_GPT54 || vars.OPENAI_API_KEY_GPT54 || secrets.CONTEXTBENCH_OPENAI_API_KEY || vars.CONTEXTBENCH_OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-gpt54mini-auth-vars-one/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check model auth from secrets or vars + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "::error::Missing model auth in Actions secrets and variables. Set OPENAI_API_KEY or one of the checked aliases." + exit 78 + fi + + - name: Install and materialize frozen Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + + - name: Run same authenticated scoreable CBM row script + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini-auth-vars-one/ContextBench-official + run: | + python - <<'PY' + from pathlib import Path + text = Path('.github/workflows/contextbench-real-gpt54mini-auth-one.yml').read_text() + marker = ' cat > "$ROOT/run.mjs" <<\'NODE\'\n' + start = text.index(marker) + len(marker) + end = text.index(' NODE\n node "$ROOT/run.mjs"', start) + lines = [] + for line in text[start:end].splitlines(): + lines.append(line[10:] if line.startswith(' ') else line) + Path('/tmp/contextbench-real-gpt54mini-auth-vars-one/run.mjs').write_text('\n'.join(lines) + '\n') + PY + node "$ROOT/run.mjs" + + - name: Upload authenticated vars one-row artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini-auth-vars-one + path: /tmp/contextbench-real-gpt54mini-auth-vars-one + retention-days: 14 diff --git a/.github/workflows/contextbench-real-gpt54mini-core.yml b/.github/workflows/contextbench-real-gpt54mini-core.yml new file mode 100644 index 0000000..348ea9b --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini-core.yml @@ -0,0 +1,112 @@ +name: ContextBench Real GPT54 Mini Core + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini-core.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + core-go-task: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + ROOT: /tmp/contextbench-real-gpt54mini-core + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini-core/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CBM_BIN: /tmp/contextbench-real-gpt54mini-core/tool/codebase-memory-mcp + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies and materialize Go task + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + pnpm run build + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/v0.6.1/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$CBM_BIN" || true + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 3 + - name: Run bounded core scoreable rows + env: + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini-core/ContextBench-official + run: | + cat > "$ROOT/core.mjs" <<'NODE' + import { spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const root = process.env.ROOT; + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const task = payloads.tasks[2]; + const outRoot = join(root, 'core-go'); + mkdirSync(outRoot, { recursive: true }); + const lanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp']; + function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { cwd: opts.cwd || process.cwd(), env: opts.env || process.env, encoding: 'utf8', timeout: opts.timeoutMs || 600000, maxBuffer: 96 * 1024 * 1024 }); + return { command: [cmd, ...args].join(' '), cwd: opts.cwd || process.cwd(), status: r.status, signal: r.signal, error: r.error?.message || null, durationMs: Date.now() - started, stdout: r.stdout || '', stderr: r.stderr || '' }; + } + function q(problem) { return String(problem || '').replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ').split(/\s+/).filter((w) => w.length >= 4).slice(0, 10).join(' '); } + function add(locs, file, start = 1, end = start, source = 'tool') { if (!file || typeof file !== 'string' || file.includes('://')) return; const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); if (!clean || clean.includes('..')) return; const s = Math.max(1, Number(start) || 1); locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); } + function jsonish(s) { const t = String(s || '').trim(); if (!t) return null; try { return JSON.parse(t); } catch {} for (const [a,b] of [['{','}'],['[',']']]) { const i=t.indexOf(a), j=t.lastIndexOf(b); if (i>=0 && j>i) { try { return JSON.parse(t.slice(i,j+1)); } catch {} } } return null; } + function walk(v, locs, source) { if (!v || typeof v !== 'object') return; if (Array.isArray(v)) { for (const x of v) walk(x, locs, source); return; } add(locs, v.file || v.path || v.file_path || v.relative_path || v.filename || v.source_path, v.start_line || v.line || 1, v.end_line || v.line || 1, source); for (const x of Object.values(v)) walk(x, locs, source); } + function collect(text, locs, source) { const parsed = jsonish(text); if (parsed) walk(parsed, locs, source); const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; let m; while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); } + function uniq(locs) { const seen = new Set(), out = []; for (const loc of locs) { const k = `${loc.file}:${loc.start}:${loc.end}`; if (!seen.has(k)) { seen.add(k); out.push(loc); if (out.length >= 80) break; } } return out; } + function addSpan(map, file, start = 1, end = start) { if (!file) return; const s = Math.max(1, Number(start) || 1); const e = Math.max(s, Number(end) || s); const list = map.get(file) || []; list.push({ start: s, end: e }); map.set(file, list); } + function prediction(selection) { const spans = new Map(); for (const span of selection.spans || []) addSpan(spans, span.file, span.start, span.end); for (const file of selection.files || []) addSpan(spans, file, 1, 1); const predFiles = [...spans.keys()].slice(0, 20); return { instance_id: task.instance_id, repo_url: task.repo_checkout_path, commit: task.base_commit, traj_data: { pred_steps: [{ files: predFiles, spans: Object.fromEntries([...spans.entries()].slice(0, 20)) }], pred_files: predFiles, pred_spans: Object.fromEntries([...spans.entries()].slice(0, 20)) }, model_patch: '' }; } + async function askModel(runDir, lane, query, candidates) { + if (!process.env.OPENAI_API_KEY) return { ok: false, status: 'model_unavailable', error: 'missing_OPENAI_API_KEY_secret', durationMs: 0 }; + const started = Date.now(); const candidateFiles = new Set(candidates.map((c) => c.file)); + const body = { model: process.env.OPENAI_MODEL, reasoning: { effort: process.env.OPENAI_REASONING_EFFORT }, max_output_tokens: 1600, instructions: 'Select likely ContextBench edit locations using only provided candidate locations. Return JSON only.', input: JSON.stringify({ taskId: task.instance_id, repo: task.repo, lane, problemStatement: task.problem_statement, query, candidateLocations: candidates.slice(0, 60) }), text: { format: { type: 'json_schema', name: 'contextbench_selection', strict: true, schema: { type: 'object', additionalProperties: false, required: ['files','spans','notes'], properties: { files: { type: 'array', maxItems: 20, items: { type: 'string' } }, spans: { type: 'array', maxItems: 40, items: { type: 'object', additionalProperties: false, required: ['file','start','end'], properties: { file: { type: 'string' }, start: { type: 'integer', minimum: 1 }, end: { type: 'integer', minimum: 1 } } } }, notes: { type: 'string' } } } } } }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...body, input: JSON.parse(body.input) }, null, 2)); + const res = await fetch('https://api.openai.com/v1/responses', { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, body: JSON.stringify(body) }); + const text = await res.text(); writeFileSync(join(runDir, 'openai-response.json'), text); + if (!res.ok) return { ok: false, status: 'model_error', httpStatus: res.status, error: text.slice(0, 2000), durationMs: Date.now() - started }; + const json = JSON.parse(text); const out = json.output_text || (json.output || []).flatMap((i) => i.content || []).filter((i) => i.type === 'output_text').map((i) => i.text).join('\n'); + const parsed = JSON.parse(out); const files = [...new Set((parsed.files || []).filter((f) => candidateFiles.has(f)))].slice(0, 20); const spans = (parsed.spans || []).filter((s) => candidateFiles.has(s.file)).slice(0, 40); + if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', parsed, durationMs: Date.now() - started }; + return { ok: true, status: 'completed', parsed: { files, spans, notes: parsed.notes || '' }, usage: json.usage || null, durationMs: Date.now() - started }; + } + async function retrieve(lane, runDir, query) { + const repo = task.repo_checkout_path; const locs = []; const commands = []; let setupDurationMs = 0, indexDurationMs = 0, queryDurationMs = 0, setupStatus = 'completed', indexStatus = 'completed'; + if (lane === 'raw-native') { const start = Date.now(); for (const term of query.split(/\s+/).slice(0, 6)) { const r = run('rg', ['-n', '-i', '--glob', '!.git', term, repo], { timeoutMs: 60000 }); commands.push(r); collect(r.stdout, locs, lane); collect(r.stderr, locs, lane); } queryDurationMs = Date.now() - start; } + if (lane === 'codebase-context') { const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; const v = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); commands.push(v); setupDurationMs = v.durationMs; const idx = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); commands.push(idx); indexDurationMs = idx.durationMs; if (idx.status !== 0) indexStatus = 'index_failed'; const s = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '25', '--json'], { env, timeoutMs: 300000 }); commands.push(s); queryDurationMs = s.durationMs; collect(s.stdout, locs, lane); collect(s.stderr, locs, lane); } + if (lane === 'codebase-memory-mcp') { const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; const v = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); commands.push(v); setupDurationMs = v.durationMs; const idx = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); commands.push(idx); indexDurationMs = idx.durationMs; if (idx.status !== 0) indexStatus = 'index_failed'; const project = (jsonish(idx.stdout) || jsonish(idx.stderr) || {}).project || basename(repo); const start = Date.now(); const g = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: repo, env, timeoutMs: 120000 }); const c = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 25 })], { cwd: repo, env, timeoutMs: 120000 }); commands.push(g, c); queryDurationMs = Date.now() - start; for (const r of [g,c]) { collect(r.stdout, locs, lane); collect(r.stderr, locs, lane); } } + for (const [i, cmd] of commands.entries()) writeFileSync(join(runDir, `command-${i+1}.json`), JSON.stringify(cmd, null, 2)); const candidates = uniq(locs); writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(candidates, null, 2)); return { setupStatus, indexStatus, toolCallable: commands.some((c) => c.status === 0), costs: { setupDurationMs, indexDurationMs, queryDurationMs }, candidates }; + } + const rows = []; const scoreable = []; const query = q(task.problem_statement); + for (const lane of lanes) { const runDir = join(outRoot, lane); mkdirSync(runDir, { recursive: true }); const retrieval = await retrieve(lane, runDir, query); const model = await askModel(runDir, lane, query, retrieval.candidates); const pred = model.ok ? prediction(model.parsed) : prediction({ files: [], spans: [] }); const predPath = join(runDir, 'prediction.json'); writeFileSync(predPath, JSON.stringify(pred, null, 2)); const goldPath = join(runDir, 'gold.json'); const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 600000 }); writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); const scorePath = join(runDir, 'official-score.jsonl'); const evaluator = model.ok && pred.traj_data.pred_files.length > 0 ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.OFFICIAL_CONTEXTBENCH, timeoutMs: 1200000 }) : { status: null, error: 'skipped_no_model_prediction', durationMs: 0 }; writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); let score = null; if (existsSync(scorePath)) { const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); if (lines.length) score = JSON.parse(lines.at(-1)); } const row = { lane_id: lane, task_id: task.instance_id, status: evaluator.status === 0 && score ? 'completed' : (model.status || 'judge_failed'), model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, setupStatus: retrieval.setupStatus, indexStatus: retrieval.indexStatus, toolCallable: retrieval.toolCallable, candidateCount: retrieval.candidates.length, nonEmptyPrediction: pred.traj_data.pred_files.length > 0, predFiles: pred.traj_data.pred_files.length, officialEvaluatorScoreable: evaluator.status === 0 && Boolean(score), setupIndex: retrieval.costs, modelStatus: model.status, modelUsage: model.usage || null, score }; rows.push(row); if (row.officialEvaluatorScoreable) scoreable.push(row); writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); console.log(JSON.stringify({ lane: lane, status: row.status, scoreable: row.officialEvaluatorScoreable, predFiles: row.predFiles, candidateCount: row.candidateCount })); } + function mean(values) { const nums = values.filter((v) => Number.isFinite(v)); return nums.length ? nums.reduce((a,b) => a + b, 0) / nums.length : null; } + const table = scoreable.map((r) => ({ lane: r.lane_id, fileCoverage: r.score.final.file.coverage, filePrecision: r.score.final.file.precision, symbolCoverage: r.score.final.symbol.coverage, spanCoverage: r.score.final.span.coverage, lineCoverage: r.score.final.line.coverage, editlocRecall: r.score.editloc?.recall ?? null })); + const summary = { createdAt: new Date().toISOString(), model: `${process.env.OPENAI_MODEL}-${process.env.OPENAI_REASONING_EFFORT}`, taskId: task.instance_id, attemptedRows: rows.length, scoreableRows: scoreable.length, statusCounts: rows.reduce((a,r) => { a[r.status] = (a[r.status] || 0) + 1; return a; }, {}), setupIndexCostReportedSeparately: true, resultsTable: table, rows }; + writeFileSync(join(outRoot, 'summary.json'), JSON.stringify(summary, null, 2)); console.log(JSON.stringify(summary, null, 2)); if (scoreable.length === 0) process.exitCode = 1; + NODE + node "$ROOT/core.mjs" + - name: Upload bounded core artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini-core + path: /tmp/contextbench-real-gpt54mini-core + retention-days: 14 diff --git a/.github/workflows/contextbench-real-gpt54mini.yml b/.github/workflows/contextbench-real-gpt54mini.yml new file mode 100644 index 0000000..ad7c888 --- /dev/null +++ b/.github/workflows/contextbench-real-gpt54mini.yml @@ -0,0 +1,561 @@ +name: ContextBench Real GPT54 Mini + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-real-gpt54mini.yml + workflow_dispatch: + inputs: + max_tasks: + description: 'Frozen ContextBench tasks to run from the manifest prefix' + required: true + default: '3' + repeats: + description: 'Repeats per lane/task' + required: true + default: '1' + lanes: + description: 'Comma-separated lanes' + required: true + default: 'raw-native,codebase-context,jcodemunch-repomapper,grepai,codebase-memory-mcp,codegraphcontext' + +permissions: + contents: read + +jobs: + real-matrix: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + ROOT: /tmp/contextbench-real-gpt54mini + TASK_PAYLOADS: /tmp/contextbench-real-gpt54mini/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + MAX_TASKS: ${{ github.event.inputs.max_tasks || '3' }} + REPEATS: ${{ github.event.inputs.repeats || '1' }} + LANES: ${{ github.event.inputs.lanes || 'raw-native,codebase-context,jcodemunch-repomapper,grepai,codebase-memory-mcp,codegraphcontext' }} + OPENAI_MODEL: gpt-5.4-mini + OPENAI_REASONING_EFFORT: high + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CBM_VERSION: v0.6.1 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v2 + with: + version: 10 + + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install repo, evaluator, and lane tooling + run: | + set -euxo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" + pnpm install --frozen-lockfile + pnpm run build + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow uv codegraphcontext kuzu + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + curl -fsSL "https://github.com/DeusData/codebase-memory-mcp/releases/download/${CBM_VERSION}/codebase-memory-mcp-linux-amd64.tar.gz" -o "$ROOT/tool/cbm.tar.gz" + tar -xzf "$ROOT/tool/cbm.tar.gz" -C "$ROOT/tool" + chmod +x "$ROOT/tool/codebase-memory-mcp" || true + for i in 1 2 3; do git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$ROOT/ContextBench-official" && break || { rm -rf "$ROOT/ContextBench-official"; sleep 5; }; done + + - name: Materialize frozen ContextBench task checkouts + run: | + set -euxo pipefail + node scripts/contextbench-runner.mjs --validate-fixtures + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS" --checkout-root "$CHECKOUT_ROOT" + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks "$MAX_TASKS" + + - name: Run gpt-5.4-mini-high scoreable matrix + env: + CBM_BIN: /tmp/contextbench-real-gpt54mini/tool/codebase-memory-mcp + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-real-gpt54mini/ContextBench-official + run: | + cat > "$ROOT/run-real-matrix.mjs" <<'NODE' + import { spawn, spawnSync } from 'node:child_process'; + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + + const root = process.env.ROOT; + const outRoot = join(root, 'matrix'); + const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + const maxTasks = Number(process.env.MAX_TASKS || '3'); + const repeats = Number(process.env.REPEATS || '1'); + const lanes = String(process.env.LANES || '').split(',').map((lane) => lane.trim()).filter(Boolean); + const tasks = payloads.tasks.slice(0, maxTasks); + mkdirSync(outRoot, { recursive: true }); + + function run(cmd, args, opts = {}) { + const started = Date.now(); + const result = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 20 * 60 * 1000, + maxBuffer: 96 * 1024 * 1024 + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: result.status, + signal: result.signal, + error: result.error?.message || null, + durationMs: Date.now() - started, + stdout: result.stdout || '', + stderr: result.stderr || '' + }; + } + + function sanitize(value) { + return String(value).replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 180); + } + + function queryOf(problem) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before']); + return String(problem || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase()) && !/^https?$/.test(w)) + .slice(0, 10) + .join(' '); + } + + function add(locations, file, start = 1, end = start, source = 'tool') { + if (typeof file !== 'string' || file.length === 0) return; + if (file.includes('://')) return; + const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!clean || clean.includes('..')) return; + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + locations.push({ file: clean, start: s, end: e, source }); + } + + function walk(value, locations, source = 'tool') { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { for (const item of value) walk(item, locations, source); return; } + const file = value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri; + const start = value.start_line || value.startLine || value.line || value.line_number || value.start || 1; + const end = value.end_line || value.endLine || value.end || start; + add(locations, file, start, end, source); + for (const item of Object.values(value)) walk(item, locations, source); + } + + function jsonish(text) { + const raw = String(text || '').trim(); + if (!raw) return null; + try { return JSON.parse(raw); } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const first = raw.indexOf(open); + const last = raw.lastIndexOf(close); + if (first >= 0 && last > first) { + try { return JSON.parse(raw.slice(first, last + 1)); } catch {} + } + } + return null; + } + + function collectFromText(text, locations, source = 'tool') { + const re = /([A-Za-z0-9_.\/-]+\.(?:js|jsx|ts|tsx|py|go|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte|json|yml|yaml|md))(?::|#L|\s+line\s+)?(\d+)?/g; + let match; + while ((match = re.exec(String(text || ''))) !== null) add(locations, match[1], match[2] || 1, match[2] || 1, source); + } + + function collect(result, locations, source) { + for (const text of [result?.stdout, result?.stderr]) { + const parsed = jsonish(text); + if (parsed) walk(parsed, locations, source); + collectFromText(text, locations, source); + } + } + + function uniqueLocations(locations) { + const seen = new Set(); + const out = []; + for (const loc of locations) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= 80) break; + } + return out; + } + + function buildPrediction(task, modelSelection) { + const spans = new Map(); + for (const span of modelSelection.spans || []) { + addSpan(spans, span.file, span.start, span.end); + } + for (const file of modelSelection.files || []) addSpan(spans, file, 1, 1); + const predFiles = [...spans.keys()].slice(0, 20); + const predSpans = Object.fromEntries([...spans.entries()].slice(0, 20)); + return { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { pred_steps: [{ files: predFiles, spans: predSpans }], pred_files: predFiles, pred_spans: predSpans }, + model_patch: '' + }; + } + + function addSpan(spans, file, start = 1, end = start) { + if (typeof file !== 'string' || !file) return; + const clean = file.replace(/^\/+/, '').replace(/^\.\//, ''); + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = spans.get(clean) || []; + list.push({ start: s, end: e }); + spans.set(clean, list); + } + + function parseScores(scorePath) { + if (!existsSync(scorePath)) return null; + const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); + if (lines.length === 0) return null; + try { return JSON.parse(lines.at(-1)); } catch { return null; } + } + + async function callOpenAI(runDir, task, lane, query, candidates) { + const started = Date.now(); + if (!process.env.OPENAI_API_KEY) { + return { ok: false, status: 'model_unavailable', durationMs: 0, error: 'missing_OPENAI_API_KEY_secret' }; + } + const candidateFiles = new Set(candidates.map((c) => c.file)); + const request = { + model: process.env.OPENAI_MODEL || 'gpt-5.4-mini', + reasoning: { effort: process.env.OPENAI_REASONING_EFFORT || 'high' }, + max_output_tokens: 1800, + instructions: 'You are selecting likely edit locations for ContextBench. Use only the provided candidate locations from the lane tool. Return JSON only.', + input: JSON.stringify({ + taskId: task.instance_id, + repo: task.repo, + lane, + query, + problemStatement: task.problem_statement, + candidateLocations: candidates.slice(0, 60) + }), + text: { + format: { + type: 'json_schema', + name: 'contextbench_location_selection', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + required: ['files', 'spans', 'notes'], + properties: { + files: { type: 'array', maxItems: 20, items: { type: 'string' } }, + spans: { + type: 'array', + maxItems: 40, + items: { + type: 'object', + additionalProperties: false, + required: ['file', 'start', 'end'], + properties: { + file: { type: 'string' }, + start: { type: 'integer', minimum: 1 }, + end: { type: 'integer', minimum: 1 } + } + } + }, + notes: { type: 'string' } + } + } + } + } + }; + writeFileSync(join(runDir, 'openai-request.redacted.json'), JSON.stringify({ ...request, input: JSON.parse(request.input) }, null, 2)); + let responseText = ''; + let responseJson = null; + try { + const response = await fetch('https://api.openai.com/v1/responses', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${process.env.OPENAI_API_KEY}` }, + body: JSON.stringify(request) + }); + responseText = await response.text(); + writeFileSync(join(runDir, 'openai-response.json'), responseText); + if (!response.ok) return { ok: false, status: 'model_error', durationMs: Date.now() - started, httpStatus: response.status, error: responseText.slice(0, 2000) }; + responseJson = JSON.parse(responseText); + } catch (error) { + return { ok: false, status: 'model_error', durationMs: Date.now() - started, error: String(error?.message || error) }; + } + const outputText = responseJson.output_text || (responseJson.output || []).flatMap((item) => item.content || []).filter((item) => item.type === 'output_text').map((item) => item.text).join('\n'); + let parsed; + try { parsed = JSON.parse(outputText); } catch (error) { return { ok: false, status: 'model_parse_error', durationMs: Date.now() - started, error: String(error?.message || error), outputText }; } + const files = [...new Set((parsed.files || []).filter((file) => candidateFiles.has(file)))].slice(0, 20); + const spans = (parsed.spans || []).filter((span) => candidateFiles.has(span.file)).slice(0, 40); + if (files.length === 0 && spans.length === 0) return { ok: false, status: 'model_empty_after_lane_filter', durationMs: Date.now() - started, parsed }; + return { ok: true, status: 'completed', durationMs: Date.now() - started, parsed: { files, spans, notes: parsed.notes || '' }, responseId: responseJson.id || null, usage: responseJson.usage || null }; + } + + async function callMcp(command, args, cwd, calls, env = process.env) { + const { Client } = await import('@modelcontextprotocol/sdk/client/index.js'); + const { StdioClientTransport } = await import('@modelcontextprotocol/sdk/client/stdio.js'); + const transport = new StdioClientTransport({ command, args, cwd, env }); + const client = new Client({ name: 'contextbench-ci', version: '1.0.0' }, { capabilities: {} }); + const started = Date.now(); + const outputs = []; + try { + await client.connect(transport); + const tools = await client.listTools(); + outputs.push({ tool: 'tools/list', result: tools }); + for (const call of calls) { + const result = await client.callTool({ name: call.name, arguments: call.arguments }); + outputs.push({ tool: call.name, arguments: call.arguments, result }); + } + await client.close(); + return { status: 0, signal: null, error: null, durationMs: Date.now() - started, stdout: JSON.stringify(outputs), stderr: '', command: [command, ...args].join(' '), cwd }; + } catch (error) { + try { await client.close(); } catch {} + return { status: 1, signal: null, error: String(error?.message || error), durationMs: Date.now() - started, stdout: JSON.stringify(outputs), stderr: String(error?.stack || error), command: [command, ...args].join(' '), cwd }; + } + } + + async function retrieve(lane, task, runDir, query) { + const repo = task.repo_checkout_path; + const firstTerm = query.split(/\s+/).find(Boolean) || 'main'; + const locations = []; + const commands = []; + let setupStatus = 'completed'; + let indexStatus = 'completed'; + const startedSetup = Date.now(); + let setupDurationMs = 0; + let indexDurationMs = 0; + let queryDurationMs = 0; + + if (lane === 'raw-native') { + setupDurationMs = Date.now() - startedSetup; + const qTerms = query.split(/\s+/).slice(0, 5); + const started = Date.now(); + for (const term of qTerms) { + const result = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, repo], { timeoutMs: 60_000 }); + commands.push(result); + collect(result, locations, 'raw-native'); + } + queryDurationMs = Date.now() - started; + } else if (lane === 'codebase-context') { + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const version = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const reindex = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 20 * 60_000 }); + commands.push(reindex); + indexDurationMs = reindex.durationMs; + if (reindex.status !== 0) indexStatus = 'index_failed'; + const search = run('node', ['dist/index.js', 'search', '--query', query, '--intent', 'edit', '--limit', '25', '--json'], { env, timeoutMs: 5 * 60_000 }); + commands.push(search); + queryDurationMs = search.durationMs; + collect(search, locations, 'codebase-context'); + } else if (lane === 'codebase-memory-mcp') { + const env = { ...process.env, CBM_CACHE_DIR: join(runDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const version = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 45 * 60_000 }); + commands.push(index); + indexDurationMs = index.durationMs; + if (index.status !== 0) indexStatus = 'index_failed'; + const projectObj = jsonish(index.stdout) || jsonish(index.stderr) || {}; + const project = projectObj.project || basename(repo); + const started = Date.now(); + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 25 })], { cwd: repo, env, timeoutMs: 120_000 }); + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: firstTerm, mode: 'compact', limit: 25 })], { cwd: repo, env, timeoutMs: 120_000 }); + commands.push(graph, code); + queryDurationMs = Date.now() - started; + collect(graph, locations, 'codebase-memory-mcp'); + collect(code, locations, 'codebase-memory-mcp'); + } else if (lane === 'grepai') { + const grepai = 'grepai'; + const version = run(grepai, ['version'], { timeoutMs: 60_000 }); + commands.push(version); + const initArgs = process.env.OPENAI_API_KEY ? ['init', '--yes', '--provider', 'openai', '--backend', 'gob', '--model', 'text-embedding-3-small'] : ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob']; + const init = run(grepai, initArgs, { cwd: repo, timeoutMs: 120_000 }); + commands.push(init); + setupDurationMs = version.durationMs + init.durationMs; + if (init.status !== 0) setupStatus = 'setup_failed'; + const startedIndex = Date.now(); + const watch = run(grepai, ['watch', '--background'], { cwd: repo, timeoutMs: 120_000 }); + commands.push(watch); + await new Promise((resolve) => setTimeout(resolve, 25_000)); + const status = run(grepai, ['watch', '--status'], { cwd: repo, timeoutMs: 60_000 }); + commands.push(status); + indexDurationMs = Date.now() - startedIndex; + if (watch.status !== 0 && status.status !== 0) indexStatus = 'index_failed'; + const search = run(grepai, ['search', query, '--json', '--compact'], { cwd: repo, timeoutMs: 180_000 }); + commands.push(search); + queryDurationMs = search.durationMs; + collect(search, locations, 'grepai'); + commands.push(run(grepai, ['watch', '--stop'], { cwd: repo, timeoutMs: 60_000 })); + } else if (lane === 'jcodemunch-repomapper') { + const version = run('uvx', ['jcodemunch-mcp', '--help'], { timeoutMs: 180_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const mcp = await callMcp('uvx', ['jcodemunch-mcp'], repo, [ + { name: 'index_folder', arguments: { path: repo, incremental: false, use_ai_summaries: false, follow_symlinks: false } }, + { name: 'search_symbols', arguments: { repo, query, max_results: 25, semantic: false } }, + { name: 'search_symbols', arguments: { repo: basename(repo), query, max_results: 25, semantic: false } } + ]); + commands.push(mcp); + indexDurationMs = mcp.durationMs; + queryDurationMs = 0; + if (mcp.status !== 0) indexStatus = 'index_or_query_failed'; + collect(mcp, locations, 'jcodemunch-repomapper'); + } else if (lane === 'codegraphcontext') { + const version = run('cgc', ['--help'], { timeoutMs: 60_000 }); + commands.push(version); + setupDurationMs = version.durationMs; + const index = run('cgc', ['index', repo], { cwd: repo, timeoutMs: 30 * 60_000 }); + commands.push(index); + indexDurationMs = index.durationMs; + if (index.status !== 0) indexStatus = 'index_failed'; + const started = Date.now(); + const complexity = run('cgc', ['analyze', 'complexity', '--limit', '25'], { cwd: repo, timeoutMs: 180_000 }); + const callers = run('cgc', ['analyze', 'callers', firstTerm], { cwd: repo, timeoutMs: 120_000 }); + const dead = run('cgc', ['analyze', 'dead-code', '--limit', '25'], { cwd: repo, timeoutMs: 180_000 }); + commands.push(complexity, callers, dead); + queryDurationMs = Date.now() - started; + collect(complexity, locations, 'codegraphcontext'); + collect(callers, locations, 'codegraphcontext'); + collect(dead, locations, 'codegraphcontext'); + } else { + setupStatus = 'setup_failed'; + commands.push({ command: lane, cwd: repo, status: 1, signal: null, error: 'unknown_lane', durationMs: 0, stdout: '', stderr: '' }); + } + + for (const [i, command] of commands.entries()) writeFileSync(join(runDir, `command-${i + 1}.json`), JSON.stringify(command, null, 2)); + const unique = uniqueLocations(locations); + writeFileSync(join(runDir, 'candidate-locations.json'), JSON.stringify(unique, null, 2)); + return { lane, setupStatus, indexStatus, toolCallable: commands.some((command) => command.status === 0), candidates: unique, costs: { setupDurationMs, indexDurationMs, queryDurationMs }, commands }; + } + + const rows = []; + const scoreableRows = []; + for (const task of tasks) { + const query = queryOf(task.problem_statement); + for (const lane of lanes) { + for (let repeat = 1; repeat <= repeats; repeat += 1) { + const runId = sanitize(`${lane}-${task.instance_id}-r${repeat}`); + const runDir = join(outRoot, 'runs', runId); + mkdirSync(runDir, { recursive: true }); + const startedAt = new Date().toISOString(); + let row; + try { + const retrieval = await retrieve(lane, task, runDir, query); + const model = await callOpenAI(runDir, task, lane, query, retrieval.candidates); + const prediction = model.ok ? buildPrediction(task, model.parsed) : buildPrediction(task, { files: [], spans: [] }); + const predictionPath = join(runDir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify(prediction, null, 2)); + const predFiles = prediction.traj_data.pred_files; + const predSpans = prediction.traj_data.pred_spans; + const goldPath = join(runDir, 'gold.json'); + const gold = run('node', ['scripts/contextbench-select-slice.mjs', '--write-gold', '--task-id', task.instance_id, '--out', goldPath, '--payloads', process.env.TASK_PAYLOADS], { timeoutMs: 10 * 60_000 }); + const scorePath = join(runDir, 'official-score.jsonl'); + const evaluator = model.ok && predFiles.length > 0 + ? run('python', ['-m', 'contextbench.evaluate', '--gold', goldPath, '--pred', predictionPath, '--cache', join(runDir, 'repo-cache'), '--out', scorePath], { cwd: process.env.OFFICIAL_CONTEXTBENCH, timeoutMs: 20 * 60_000 }) + : { command: 'python -m contextbench.evaluate', cwd: process.env.OFFICIAL_CONTEXTBENCH, status: null, signal: null, error: 'skipped_no_model_prediction', durationMs: 0, stdout: '', stderr: '' }; + writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); + writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + const score = parseScores(scorePath); + const officialEvaluatorScoreable = evaluator.status === 0 && Boolean(score); + const status = officialEvaluatorScoreable ? 'completed' : (model.ok ? 'judge_failed' : model.status); + row = { + run_id: runId, + lane_id: lane, + task_id: task.instance_id, + repeat_index: repeat, + status, + model: `${process.env.OPENAI_MODEL || 'gpt-5.4-mini'}-${process.env.OPENAI_REASONING_EFFORT || 'high'}`, + started_at: startedAt, + completed_at: new Date().toISOString(), + setupIndex: retrieval.costs, + setupStatus: retrieval.setupStatus, + indexStatus: retrieval.indexStatus, + toolCallable: retrieval.toolCallable, + candidateCount: retrieval.candidates.length, + nonEmptyPrediction: predFiles.length > 0 && Object.keys(predSpans).length > 0, + predFiles: predFiles.length, + officialEvaluatorScoreable, + modelStatus: model.status, + modelDurationMs: model.durationMs, + modelUsage: model.usage || null, + score, + paths: { runDir, predictionPath, scorePath } + }; + if (officialEvaluatorScoreable) scoreableRows.push(row); + } catch (error) { + row = { run_id: runId, lane_id: lane, task_id: task.instance_id, repeat_index: repeat, status: 'tool_error', error: String(error?.stack || error), started_at: startedAt, completed_at: new Date().toISOString(), officialEvaluatorScoreable: false }; + writeFileSync(join(runDir, 'row-error.txt'), row.error); + } + rows.push(row); + writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); + console.log(JSON.stringify({ run_id: row.run_id, lane_id: row.lane_id, task_id: row.task_id, status: row.status, scoreable: row.officialEvaluatorScoreable, predFiles: row.predFiles || 0, candidateCount: row.candidateCount || 0 })); + } + } + } + + const byLane = {}; + for (const row of scoreableRows) { + const bucket = byLane[row.lane_id] || { lane: row.lane_id, scoreableRows: 0, fileCoverage: [], filePrecision: [], symbolCoverage: [], spanCoverage: [], lineCoverage: [], editlocRecall: [] }; + bucket.scoreableRows += 1; + bucket.fileCoverage.push(row.score?.final?.file?.coverage ?? null); + bucket.filePrecision.push(row.score?.final?.file?.precision ?? null); + bucket.symbolCoverage.push(row.score?.final?.symbol?.coverage ?? null); + bucket.spanCoverage.push(row.score?.final?.span?.coverage ?? null); + bucket.lineCoverage.push(row.score?.final?.line?.coverage ?? null); + bucket.editlocRecall.push(row.score?.editloc?.recall ?? null); + byLane[row.lane_id] = bucket; + } + function mean(values) { + const nums = values.filter((value) => Number.isFinite(value)); + return nums.length ? nums.reduce((a, b) => a + b, 0) / nums.length : null; + } + const resultsTable = Object.values(byLane).map((bucket) => ({ + lane: bucket.lane, + scoreableRows: bucket.scoreableRows, + fileCoverage: mean(bucket.fileCoverage), + filePrecision: mean(bucket.filePrecision), + symbolCoverage: mean(bucket.symbolCoverage), + spanCoverage: mean(bucket.spanCoverage), + lineCoverage: mean(bucket.lineCoverage), + editlocRecall: mean(bucket.editlocRecall) + })).sort((a, b) => (b.fileCoverage ?? -1) - (a.fileCoverage ?? -1)); + const summary = { + createdAt: new Date().toISOString(), + model: `${process.env.OPENAI_MODEL || 'gpt-5.4-mini'}-${process.env.OPENAI_REASONING_EFFORT || 'high'}`, + maxTasks, + repeats, + lanes, + attemptedRows: rows.length, + scoreableRows: scoreableRows.length, + nonScoreableRows: rows.length - scoreableRows.length, + statusCounts: rows.reduce((acc, row) => { acc[row.status] = (acc[row.status] || 0) + 1; return acc; }, {}), + setupIndexCostReportedSeparately: true, + resultsTable, + rows + }; + writeFileSync(join(outRoot, 'summary.json'), JSON.stringify(summary, null, 2)); + writeFileSync(join(outRoot, 'results-table.json'), JSON.stringify(resultsTable, null, 2)); + console.log(JSON.stringify({ attemptedRows: summary.attemptedRows, scoreableRows: summary.scoreableRows, statusCounts: summary.statusCounts, resultsTable }, null, 2)); + if (scoreableRows.length === 0) process.exitCode = 1; + NODE + node "$ROOT/run-real-matrix.mjs" + + - name: Upload real benchmark artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-real-gpt54mini + path: /tmp/contextbench-real-gpt54mini + retention-days: 14 diff --git a/.github/workflows/contextbench-rescue-readiness.yml b/.github/workflows/contextbench-rescue-readiness.yml new file mode 100644 index 0000000..ef978f6 --- /dev/null +++ b/.github/workflows/contextbench-rescue-readiness.yml @@ -0,0 +1,91 @@ +name: ContextBench Rescue Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-rescue-readiness.yml + - scripts/contextbench-rescue-lane-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + rescue-readiness: + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + lane: + - codebase-context + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-rescue-readiness/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-rescue-readiness/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-rescue-checkouts-${{ matrix.lane }} + CBM_BIN: /tmp/contextbench-rescue-readiness/${{ matrix.lane }}/tool/codebase-memory-mcp + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Build codebase-context lane + if: matrix.lane == 'codebase-context' + shell: bash + run: pnpm run build > "$ROOT/logs/build.log" 2>&1 + - name: Install grepai lane + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + - name: Install CodeGraphContext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: | + set -euo pipefail + python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 || python -m pip install codegraphcontext kuzu > "$ROOT/logs/pip-codegraphcontext-fallback.log" 2>&1 + - name: Generate rescue lane readiness candidate pack + shell: bash + run: node scripts/contextbench-rescue-lane-pack.mjs + - name: Upload rescue readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-rescue-readiness-${{ matrix.lane }} + path: /tmp/contextbench-rescue-readiness/${{ matrix.lane }} + retention-days: 14 diff --git a/.github/workflows/contextbench-ripgrep-readiness-v2.yml b/.github/workflows/contextbench-ripgrep-readiness-v2.yml new file mode 100644 index 0000000..6d452dc --- /dev/null +++ b/.github/workflows/contextbench-ripgrep-readiness-v2.yml @@ -0,0 +1,71 @@ +name: ContextBench Ripgrep Readiness V2 + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ripgrep-readiness-v2.yml + - scripts/contextbench-ripgrep-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + ripgrep-readiness: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + ROOT: /tmp/contextbench-ripgrep-readiness + TASK_PAYLOADS: /tmp/contextbench-ripgrep-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ripgrep-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Generate ripgrep readiness candidate pack + shell: bash + run: node scripts/contextbench-ripgrep-ready-pack.mjs + - name: Upload ripgrep readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ripgrep-readiness + path: /tmp/contextbench-ripgrep-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-ripgrep-readiness-v3.yml b/.github/workflows/contextbench-ripgrep-readiness-v3.yml new file mode 100644 index 0000000..38a0f52 --- /dev/null +++ b/.github/workflows/contextbench-ripgrep-readiness-v3.yml @@ -0,0 +1,71 @@ +name: ContextBench Ripgrep Readiness V3 + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ripgrep-readiness-v3.yml + - scripts/contextbench-ripgrep-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + ripgrep-readiness: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + ROOT: /tmp/contextbench-ripgrep-readiness + TASK_PAYLOADS: /tmp/contextbench-ripgrep-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ripgrep-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Generate ripgrep readiness candidate pack + shell: bash + run: node scripts/contextbench-ripgrep-ready-pack.mjs + - name: Upload ripgrep readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ripgrep-readiness + path: /tmp/contextbench-ripgrep-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-ripgrep-readiness.yml b/.github/workflows/contextbench-ripgrep-readiness.yml new file mode 100644 index 0000000..7c41613 --- /dev/null +++ b/.github/workflows/contextbench-ripgrep-readiness.yml @@ -0,0 +1,71 @@ +name: ContextBench Ripgrep Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-ripgrep-readiness.yml + - scripts/contextbench-ripgrep-ready-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + ripgrep-readiness: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + ROOT: /tmp/contextbench-ripgrep-readiness + TASK_PAYLOADS: /tmp/contextbench-ripgrep-readiness/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-ripgrep-readiness-checkouts + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + for attempt in 1 2 3; do + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize-$attempt.log" 2>&1 && break + if [ "$attempt" = 3 ]; then exit 1; fi + sleep 5 + done + - name: Generate ripgrep readiness candidate pack + shell: bash + run: node scripts/contextbench-ripgrep-ready-pack.mjs + - name: Upload ripgrep readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-ripgrep-readiness + path: /tmp/contextbench-ripgrep-readiness + retention-days: 14 diff --git a/.github/workflows/contextbench-selected-cbm-score.yml b/.github/workflows/contextbench-selected-cbm-score.yml new file mode 100644 index 0000000..9f049d7 --- /dev/null +++ b/.github/workflows/contextbench-selected-cbm-score.yml @@ -0,0 +1,66 @@ +name: ContextBench Selected CBM Score + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-selected-cbm-score.yml + - scripts/contextbench-score-selected-prediction.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + selected-cbm-score: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + ROOT: /tmp/contextbench-selected-cbm-score + TASK_PAYLOADS: /tmp/contextbench-selected-cbm-score/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-checkouts + OFFICIAL_CONTEXTBENCH: /tmp/contextbench-selected-cbm-score/ContextBench-official + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install and materialize selected Go task quietly + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-install.log" 2>&1 + git clone --depth 1 https://github.com/EuniAI/ContextBench.git "$OFFICIAL_CONTEXTBENCH" > "$ROOT/logs/contextbench-clone.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + echo "selected_score_setup_completed" + - name: Score selected gpt-5.4-mini-high prediction + shell: bash + run: node scripts/contextbench-score-selected-prediction.mjs + - name: Upload selected score artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-selected-cbm-score + path: /tmp/contextbench-selected-cbm-score + retention-days: 14 diff --git a/.github/workflows/contextbench-tool-diagnostics.yml b/.github/workflows/contextbench-tool-diagnostics.yml new file mode 100644 index 0000000..33b080d --- /dev/null +++ b/.github/workflows/contextbench-tool-diagnostics.yml @@ -0,0 +1,147 @@ +name: ContextBench Tool Diagnostics + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-tool-diagnostics.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + diagnostics: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-tool-diagnostics/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-tool-diagnostics/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-tool-diagnostics-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payload = JSON.parse(fs.readFileSync(process.env.TASK_PAYLOADS, 'utf8')); + fs.writeFileSync(`${process.env.ROOT}/repo-path.txt`, payload.tasks[0].repo_checkout_path); + NODE + - name: grepai diagnostics + if: matrix.lane == 'grepai' + shell: bash + run: | + set +euo pipefail + REPO="$(cat "$ROOT/repo-path.txt")" + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh + export PATH="$HOME/.local/bin:$HOME/bin:$PATH" + cd "$REPO" + echo '--- grepai version ---' + grepai version + echo '--- grepai init synthetic gob ---' + grepai init --yes --provider synthetic --backend gob + echo '--- .grepai/config.yaml ---' + cat .grepai/config.yaml || true + echo '--- grepai status before watch ---' + grepai status --no-ui + echo '--- grepai watch --help ---' + grepai watch --help + echo '--- grepai watch --no-ui direct ---' + timeout 45s grepai watch --no-ui + echo "watch_direct_status=$?" + echo '--- grepai status after direct watch ---' + grepai status --no-ui + echo '--- grepai watch --background ---' + grepai watch --background + echo "watch_background_status=$?" + sleep 10 + echo '--- grepai watch --status ---' + grepai watch --status + echo '--- grepai status final ---' + grepai status --no-ui + echo '--- grepai search smoke ---' + grepai search "system metrics startup prometheus" --json --compact --limit 5 + echo "search_status=$?" + grepai watch --stop + - name: CodeGraphContext diagnostics + if: matrix.lane == 'codegraphcontext' + shell: bash + run: | + set +euo pipefail + REPO="$(cat "$ROOT/repo-path.txt")" + python -m pip install codegraphcontext kuzu real_ladybug || python -m pip install codegraphcontext kuzu + cd "$REPO" + echo '--- cgc --version ---' + cgc --version + echo '--- cgc --help ---' + cgc --help + echo '--- cgc find --help ---' + cgc find --help + echo '--- cgc analyze --help ---' + cgc analyze --help + echo '--- cgc index . ---' + cgc index . + echo '--- cgc stats ---' + cgc stats + echo '--- cgc list ---' + cgc list + echo '--- cgc analyze complexity --limit 20 ---' + cgc analyze complexity --limit 20 + echo 'complexity_status='$? + echo '--- cgc analyze dead-code ---' + cgc analyze dead-code + echo 'dead_code_status='$? + echo '--- cgc analyze callers main ---' + cgc analyze callers main + echo 'callers_main_status='$? + echo '--- cgc analyze calls main ---' + cgc analyze calls main + echo 'calls_main_status='$? + echo '--- cgc find metrics variants ---' + cgc find metrics + echo 'find_metrics_status='$? + cgc find --fuzzy metrics + echo 'find_fuzzy_metrics_status='$? + - name: Upload diagnostics artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-tool-diagnostics-${{ matrix.lane }} + path: /tmp/contextbench-tool-diagnostics/${{ matrix.lane }} + retention-days: 14 diff --git a/.github/workflows/contextbench-tool-rescue-readiness.yml b/.github/workflows/contextbench-tool-rescue-readiness.yml new file mode 100644 index 0000000..d689024 --- /dev/null +++ b/.github/workflows/contextbench-tool-rescue-readiness.yml @@ -0,0 +1,85 @@ +name: ContextBench Tool Rescue Readiness + +on: + push: + branches: [master] + paths: + - .github/workflows/contextbench-tool-rescue-readiness.yml + - scripts/contextbench-rescue-lane-pack.mjs + workflow_dispatch: + +permissions: + contents: read + +jobs: + tool-rescue-readiness: + runs-on: ubuntu-latest + timeout-minutes: 18 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + lane: + - grepai + - codegraphcontext + env: + ROOT: /tmp/contextbench-tool-rescue-readiness/${{ matrix.lane }} + TASK_PAYLOADS: /tmp/contextbench-tool-rescue-readiness/${{ matrix.lane }}/task-payloads.json + CHECKOUT_ROOT: /tmp/contextbench-tool-rescue-checkouts-${{ matrix.lane }} + TARGET_TASK_ID: SWE-Bench-Pro__go__maintenance__bugfix__4df06349 + LANE_ID: ${{ matrix.lane }} + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v2 + with: + version: 10 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: pnpm + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install shared setup and materialize selected Go task + shell: bash + run: | + set -euo pipefail + mkdir -p "$ROOT" "$CHECKOUT_ROOT" "$ROOT/tool" "$ROOT/logs" + pnpm install --frozen-lockfile > "$ROOT/logs/pnpm-install.log" 2>&1 + python -m pip install "tree-sitter==0.20.4" "tree-sitter-languages==1.10.2" datasets pyarrow > "$ROOT/logs/pip-shared.log" 2>&1 + node scripts/contextbench-runner.mjs --validate-fixtures > "$ROOT/logs/validate-fixtures.log" 2>&1 + node scripts/contextbench-select-slice.mjs --write-task-payloads --out "$TASK_PAYLOADS.all" --checkout-root "$CHECKOUT_ROOT" > "$ROOT/logs/write-payloads.log" 2>&1 + node - <<'NODE' + const fs = require('node:fs'); + const payloadPath = process.env.TASK_PAYLOADS; + const target = process.env.TARGET_TASK_ID; + const payload = JSON.parse(fs.readFileSync(`${payloadPath}.all`, 'utf8')); + const task = payload.tasks.find((candidate) => candidate.instance_id === target); + if (!task) throw new Error(`target task ${target} not found`); + fs.writeFileSync(payloadPath, `${JSON.stringify({ ...payload, task_count: 1, tasks: [task] }, null, 2)}\n`); + NODE + node scripts/contextbench-select-slice.mjs --materialize-checkouts --payloads "$TASK_PAYLOADS" --max-tasks 1 > "$ROOT/logs/materialize.log" 2>&1 + - name: Install grepai lane + if: matrix.lane == 'grepai' + shell: bash + run: | + set -euo pipefail + curl -sSL https://raw.githubusercontent.com/yoanbernabeu/grepai/main/install.sh | sh > "$ROOT/logs/grepai-install.log" 2>&1 || true + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/bin" >> "$GITHUB_PATH" + - name: Install CodeGraphContext lane + if: matrix.lane == 'codegraphcontext' + shell: bash + run: | + set -euo pipefail + python -m pip install codegraphcontext kuzu real_ladybug > "$ROOT/logs/pip-codegraphcontext.log" 2>&1 || python -m pip install codegraphcontext kuzu > "$ROOT/logs/pip-codegraphcontext-fallback.log" 2>&1 + - name: Generate rescue lane readiness candidate pack + shell: bash + run: node scripts/contextbench-rescue-lane-pack.mjs + - name: Upload tool rescue readiness artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: contextbench-tool-rescue-readiness-${{ matrix.lane }} + path: /tmp/contextbench-tool-rescue-readiness/${{ matrix.lane }} + retention-days: 14 diff --git a/.github/workflows/publish-npm-on-release.yml b/.github/workflows/publish-npm-on-release.yml index 023126a..68755f1 100644 --- a/.github/workflows/publish-npm-on-release.yml +++ b/.github/workflows/publish-npm-on-release.yml @@ -8,9 +8,9 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to publish (e.g. v2.2.0)' + description: 'Release tag to publish' required: true - default: 'v2.2.0' + type: string permissions: contents: read diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8496e23..d518c79 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '24' cache: 'pnpm' - run: pnpm install --frozen-lockfile @@ -35,6 +35,9 @@ jobs: test: name: Functional Tests runs-on: ubuntu-latest + strategy: + matrix: + node: [22, 24] steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 @@ -42,7 +45,7 @@ jobs: version: 10 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: ${{ matrix.node }} cache: 'pnpm' - run: pnpm install --frozen-lockfile diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md new file mode 100644 index 0000000..4b114b2 --- /dev/null +++ b/.planning/ROADMAP.md @@ -0,0 +1,517 @@ +# Flagship Roadmap + +Status: **active** +Spec: `.planning/SPEC.md` +Operating rule: advance only when the current gate is explicitly PASS. + +## How To Use This Roadmap + +This file is an executable state machine, not a wishlist. + +At the start of a session: + +1. find the first phase whose gate is not `PASS`, `KILL`, or permanently `BLOCKED` +2. fix blockers inside that phase before starting anything later +3. make the smallest change needed to answer the phase question +4. run the listed verification +5. record `PASS`, `FIX`, `KILL`, or `BLOCKED` +6. only then move to the next phase + +New ideas go to **Backlog** unless they materially change the current gate. + +--- + +# Current State + +## Phase 0 — Review-context v1 functional contract + +Status: **PASS — E1** + +Merged implementation: PR `#128 feat(review): add bounded review-context packets` +Squash commit: `2c56b10ff32d588d9d660a586c0defee5a3d76d1` + +Question: + +> Can the public project reliably compile a bounded, versioned review-context packet from an exact committed git change without an LLM? + +Answer at E1: **yes, functionally**. This does not imply retrieval or reviewer-quality value. + +### Deliverables + +- [x] review-context core packet builder +- [x] executable `codebase-context-review` package binary +- [x] exact base/head commit identity +- [x] exact diff fingerprint +- [x] clean-worktree / checked-out-head invariants +- [x] rename and binary handling +- [x] NUL-safe filename parsing +- [x] bounded identifier extraction +- [x] bounded related-context retrieval +- [x] convention/pattern attachment +- [x] warnings / quality metadata +- [x] unit tests for the packet core +- [x] executable-wrapper smoke test +- [x] public review-context documentation +- [x] autonomous product spec and gated roadmap +- [x] CI quality gate green +- [x] CI functional test gate green +- [x] squash merge to `master` + +### Gate G0 — Functional publishability + +PASS evidence: + +- [x] `pnpm lint` has zero errors +- [x] `pnpm format:check` passes +- [x] `pnpm type-check` passes +- [x] `pnpm build` passes +- [x] full test suite passes +- [x] executable wrapper smoke passes +- [x] PR was mergeable +- [x] docs describe actual behavior and limitations +- [x] no performance/reviewer-quality claim was added + +CI evidence: workflow run `32585647263` — Quality Checks **success**, Functional Tests **success**. + +Result: + +- Gate G0: **PASS** +- Evidence level: **E1** +- Phase 1 is now the only active flagship phase + +--- + +# Phase 1 — Freeze the first honest review-context evaluation + +Status: **ACTIVE** + +Question: + +> What frozen public task set can distinguish useful review context from plausible-looking retrieval noise? + +This phase freezes evidence **before** retrieval tuning. + +## Required task set + +Minimum target: + +- 20 public tasks +- at least 5 public repositories +- at least 3 language ecosystems +- mix of small and non-trivial changes +- real bug-fix / review-relevant changes, not only synthetic retrieval queries +- exact repository + base/head commit identity +- public reproducibility + +Prefer tasks where ground truth can identify: + +- changed files +- causally relevant non-changed files +- relevant symbols/spans when defensible +- actual bug/finding when the task supports reviewer evaluation + +## Freeze artifacts + +Before looking at treatment results, commit: + +- [ ] task manifest +- [ ] repo/commit identities +- [ ] task inclusion/exclusion rationale +- [ ] ground-truth format +- [ ] scorer +- [ ] retrieval limits +- [ ] setup/index accounting policy +- [ ] failure-row policy +- [ ] baseline definitions +- [ ] report schema + +## Required retrieval baselines + +At minimum: + +1. changed files/path signals only +2. repository-native/basic text navigation +3. current `review-context-v1` + +If an additional competitor is cheap and reproducible, add it **before** running treatment results. + +## Gate G1 — Eval freeze integrity + +PASS only if: + +- [ ] all tasks are frozen before treatment inspection +- [ ] task set meets diversity minimums +- [ ] scorer does not depend on treatment output +- [ ] baseline budgets are explicit +- [ ] failures stay in denominator +- [ ] no private/employer data is used +- [ ] another engineer could materialize the set from public information + +On PASS: + +- tag/fingerprint the frozen manifest +- move to Phase 2 + +On FAIL: + +- repair the evaluation design before running any claim-bearing treatment + +--- + +# Phase 2 — Prove or kill repository-context retrieval value + +Status: **BLOCKED BY G1** + +Question: + +> Does the review-context retrieval lane find materially more relevant context than simple baselines at acceptable precision and cost? + +No reviewer LLM is needed yet. First isolate retrieval quality. + +## Metrics + +Record separately: + +- relevant-file recall@k +- relevant-span recall@k where ground truth permits +- precision@k +- characters/tokens returned +- setup/index time +- retrieval wall time +- failures / abstentions +- peak memory where practical + +## Comparison discipline + +- same task set +- same frozen commits +- same k/budget where comparable +- setup/index costs reported, not hidden +- no best-of-N +- no per-task hand tuning + +## Gate G2 — Retrieval value + +### PASS + +Pass if the evidence shows a useful and reasonably general trade-off versus simple baselines. + +A PASS does not require winning every task. It requires a defensible aggregate improvement that is not explained by massively larger context budgets. + +Then: + +- evidence level becomes **E2** +- publish honest result table + failure analysis +- move to Phase 3 + +### FIX + +One bounded repair cycle is allowed if: + +- failure mode is coherent and general +- fix can be justified without task-specific branching +- frozen tasks/ground truth remain untouched + +After that repair, rerun the full frozen set once. + +### KILL + +Kill or substantially simplify this retrieval lane if, after the bounded repair: + +- simple baselines match/beat it at lower complexity/cost +- precision is too low +- gains are isolated to development repos/frameworks +- setup/index cost overwhelms value + +If killed: + +- publish the negative result honestly +- retain the deterministic git/review packet pieces that still have value +- redesign from the simplest winning baseline, not from sunk cost + +--- + +# Phase 3 — Measure actual reviewer impact + +Status: **BLOCKED BY G2 PASS** + +Question: + +> Does the bounded context packet improve real review findings when reasoning is held constant? + +## Frozen experimental arms + +### Control + +Same reviewer model + prompt + budget with raw diff/repository-native tools only. + +### Treatment + +Same reviewer model + prompt + budget plus `review-context-v1`. + +Do not give the treatment hidden extra turns/tokens unless that cost difference is explicitly part of the reported trade-off. + +## Reviewer harness requirements + +- [ ] model/provider/version recorded +- [ ] prompt recorded +- [ ] tool set recorded +- [ ] token/turn budget recorded +- [ ] timeout recorded +- [ ] repeated runs where nondeterminism matters +- [ ] append-only attempt records +- [ ] invalid output / tool failure is terminal evidence +- [ ] no best-of-N selection + +## Metrics + +Report separately: + +- real finding recall +- precision / false positives +- evidence correctness +- unsupported claims +- token/context usage +- wall time +- tool/setup failure rate + +## Gate G3 — Reviewer value + +### PASS + +Pass only if reviewer quality improves enough to justify the added context cost and the effect is not confined to one repo/language. + +Then: + +- evidence level becomes **E3** +- publish methodology + failure cases +- move to Phase 4 + +### FIX + +One bounded context-compilation repair cycle is allowed if the failure analysis clearly implicates retrieval/context rather than reviewer reasoning. + +Do not tune the reviewer prompt per task to rescue the treatment. + +### KILL + +Kill the claim that this context strategy helps review if the treatment does not produce a meaningful trade-off after the bounded repair cycle. + +Do not respond by adding agent orchestration, more models, or a dashboard. + +--- + +# Phase 4 — External reproduction and usefulness + +Status: **BLOCKED BY G3 PASS** + +Question: + +> Can someone outside the development loop reproduce the value and use the product without bespoke help? + +## Deliverables + +- [ ] stable npm release containing the review command +- [ ] concise README entrypoint +- [ ] one public end-to-end example +- [ ] reproducible evaluation command +- [ ] machine-readable result artifact +- [ ] install/runtime limitations documented +- [ ] external feedback channel + +Seek at least one of: + +- external engineer reproduces evaluation +- maintainer/user runs it on another repo +- useful issue/PR from an external user +- independent adoption/integration signal + +## Gate G4 — External evidence + +PASS when there is credible external reproduction/use rather than only owner/agent self-evaluation. + +Then evidence level becomes **E4**. + +If external users cannot successfully run or understand it, fix usability before expanding scope. + +--- + +# Phase 5 — Real PR workflow integration + +Status: **BLOCKED BY G3 PASS; PREFER G4** + +Question: + +> Is the proven context capability useful enough to integrate into real pull-request workflows? + +Only now consider: + +- GitHub Actions integration +- GitHub App +- Azure DevOps adapter +- model-agnostic reviewer consumer +- comment drafting +- review result persistence + +## Mandatory safety boundary + +The first integration must default to **read-only / draft output**. + +No autonomous public review comments until: + +- reviewer precision is measured +- false-positive policy exists +- explicit opt-in exists +- dry-run evidence exists + +## Gate G5 — Operational usefulness + +PASS requires: + +- real PR input +- reproducible context packet +- reviewer output trace +- explicit failure state +- no silent posting +- measurable latency/cost + +--- + +# Phase 6 — Optional platform/full-stack expansion + +Status: **BLOCKED BY PRODUCT EVIDENCE** + +This phase exists for product utility and engineering breadth, not portfolio theater. + +Promote only if real usage creates a need for shared state, concurrency, observability, or hosted exploration. + +Possible components: + +- TypeScript/Node API boundary +- async worker/queue for evaluations +- persistent run metadata +- Next.js benchmark/result explorer +- optional authentication +- observability +- one minimal cloud deployment path + +The default local-first CLI/MCP path must remain usable without cloud infrastructure. + +## Gate G6 — Platform justification + +Do not build this phase unless at least one is true: + +- external users need shared runs/results +- evaluations require durable async execution +- a real integration needs a service boundary +- operating evidence manually is now a material bottleneck + +A desire to demonstrate AWS/Next.js is not sufficient product justification by itself. + +--- + +# Autonomous Routing Rules + +When new information arrives, route it as follows. + +## CI or test failure + +→ current phase blocker +→ fix before all other work + +## Bug in published contract + +→ hotfix current stable surface +→ add regression test +→ return to current roadmap gate + +## Benchmark result + +→ update current evidence gate +→ choose PASS/FIX/KILL/BLOCKED +→ do not brainstorm around an unfavorable result + +## New paper/tool/competitor + +Ask: + +1. does it change a frozen baseline or invalidate current methodology? +2. does it expose a clear general failure mode? +3. is it cheap enough to add before treatment observation? + +If no → Backlog. + +## External user/maintainer interest + +→ high-priority opportunity +→ fix adoption blockers ahead of speculative features + +## New project idea + +→ default Backlog/reject +→ it must beat finishing the current gate, not merely sound interesting + +--- + +# Work-In-Progress Limits + +At any time: + +- maximum 1 active flagship product PR +- maximum 1 active claim-bearing evaluation +- maximum 1 bounded repair cycle per failed evidence gate before a kill/rethink decision + +Do not maintain parallel speculative implementations of the same hypothesis. + +--- + +# Merge Rules + +For roadmap product work: + +1. branch from current `master` +2. one primary scope per PR +3. CI must be green +4. relevant evidence gate must be satisfied +5. squash merge unless history itself is useful evidence +6. update this roadmap when a gate changes state + +Never merge because the code "looks right" while the required gate is red. + +--- + +# Backlog — Not Active + +These are intentionally **not** roadmap commitments. + +- AST-aware diff signal extraction +- richer graph expansion +- hosted dashboard +- autonomous PR commenter +- Azure DevOps review adapter +- GitHub App +- queue/worker infrastructure +- Next.js result explorer +- AWS deployment +- multi-user auth +- paid hosted embeddings +- framework-specific reviewer rules +- standalone replacement for existing reviewer products + +Promotion rule: + +> A backlog item enters Track A only when evidence or external pull shows it is the smallest next step for the current product bottleneck. + +--- + +# Immediate Next Action + +**Phase 1 only: freeze the evaluation before tuning anything.** + +1. source candidate public review/bug-fix tasks +2. select at least 20 tasks across at least 5 repos and 3 language ecosystems +3. define ground truth and inclusion/exclusion rules without treatment output +4. define the simple baselines and equalized budgets +5. commit the manifest, scorer, limits, failure policy, and report schema +6. fingerprint the frozen set +7. only after G1 PASS may any claim-bearing treatment run begin diff --git a/.planning/SPEC.md b/.planning/SPEC.md new file mode 100644 index 0000000..ee6a294 --- /dev/null +++ b/.planning/SPEC.md @@ -0,0 +1,493 @@ +# Flagship Product & Autonomous Operating Spec + +Status: **active** +Owner: PatrickSys +Scope: `codebase-context` as a measurable repository-context system for AI-assisted code review and coding agents +Primary execution plan: `.planning/ROADMAP.md` + +## 1. Mission + +Build a public, local-first, vendor-neutral context system that can answer one hard question with evidence: + +> Does bounded repository context materially improve an AI reviewer's ability to understand a change and find real issues, at acceptable precision, latency, and context cost? + +The project is not successful because it has many analyzers, tools, workflows, or agent integrations. It is successful when an external engineer can reproduce the system, inspect its decisions, run the evidence harness, and see honest results. + +## 2. Product Wedge + +The flagship wedge is **review context compilation**. + +Given a clean committed git change, compile a bounded packet containing: + +1. exact base/head identity and diff fingerprint +2. changed files and change statistics +3. deterministic signals extracted from changed lines +4. bounded related repository context +5. relevant repository conventions/patterns +6. retrieval/preflight quality metadata +7. explicit warnings and limits + +That packet can then be consumed by any reviewer model or human evaluation harness. + +This separates: + +- **context compilation** from +- **review reasoning** from +- **evaluation** + +That separation is the core architectural principle for the flagship. + +## 3. Current Product Contract + +The current public review surface is `review-context-v1`. + +### Inputs + +- local git repository +- clean checked-out `HEAD` +- base git ref +- optional explicit head ref that must resolve to checked-out `HEAD` +- existing or refreshable `codebase-context` index +- explicit query/result/identifier limits + +### Outputs + +A versioned JSON packet containing: + +- exact resolved base/head SHAs +- exact git diff SHA-256 +- changed file descriptors +- addition/deletion counts +- rename/binary metadata +- bounded identifiers derived from changed lines +- bounded retrieval queries +- bounded related search results +- search quality / edit preflight metadata when available +- convention/pattern snapshot +- warnings +- all operative limits + +### Invariants + +The review path MUST: + +- use exact resolved commits for diff generation +- fail closed if the working tree is dirty +- fail closed if requested head is not checked-out `HEAD` +- parse git paths safely, including unusual valid filenames +- never silently truncate the diff used for fingerprinting +- bound queries, results, identifiers, and snippets +- never call an LLM during context compilation +- never post review comments +- never represent retrieval confidence as code correctness +- preserve failures/warnings rather than papering over them + +## 4. What This Product Is Not + +Until evidence passes the relevant gates, this project is **not**: + +- a proven AI reviewer +- a replacement for CodeRabbit, Greptile, Copilot review, or similar products +- a claim that semantic/vector retrieval improves review quality +- a cloud platform +- an autonomous pull-request commenter +- a dashboard product +- a framework-specific reviewer +- a place to port abandoned private experiments + +`context-kit` is not an authority, benchmark, source of claims, or required dependency. Its historical existence does not validate any design here. + +## 5. Evidence Levels + +Every public statement must map to one of these levels. + +### E0 — Code exists + +Evidence: + +- implementation present +- unit tests present + +Allowed claim: + +- "The project implements X." + +Not allowed: + +- "X works better" +- "X improves review" +- "X is efficient" + +### E1 — Reproducible functional behavior + +Evidence: + +- build passes +- full tests pass +- executable smoke passes +- deterministic envelope invariants verified +- public reproduction instructions exist + +Allowed claim: + +- "The project can compile a bounded review-context packet from a committed git range." + +### E2 — Retrieval value + +Evidence: + +- frozen public task set before algorithm tuning +- multiple repositories +- multiple languages/frameworks +- fixed limits +- baseline and treatment run under identical scoring +- relevant-file/span retrieval metrics reported with failures + +Allowed claim: + +- bounded claims about retrieval performance on the frozen evaluation only + +### E3 — Reviewer impact + +Evidence: + +- same reviewer model/harness +- same prompt/budget/timeout +- raw-diff/tool baseline vs review-context treatment +- repeated runs where model nondeterminism matters +- finding recall, precision/false positives, evidence quality, token/context cost, wall time reported separately +- no best-of-N selection + +Allowed claim: + +- bounded claim that review-context changed measured reviewer performance on the frozen evaluation + +### E4 — External usefulness + +Evidence can include: + +- external users +- independent reproduction +- maintainer adoption +- accepted integrations +- third-party benchmark replication + +Only at E4 should broad product-positioning claims become credible. + +## 6. Core Hypotheses + +### H1 — Changed-line signals can cheaply focus retrieval + +A small deterministic signal extractor can produce useful queries from a diff without an LLM. + +Kill/fix trigger: + +- relevant-file/span coverage is materially worse than simpler path/symbol baselines + +### H2 — Bounded repository context can outperform raw diff/repository navigation alone + +Adding a bounded `review-context-v1` packet improves review outcomes without unacceptable false positives or context cost. + +Kill/fix trigger: + +- no meaningful improvement across a frozen multi-repo review set +- improvement is explained only by substantially larger token budgets +- gains disappear outside the development repositories + +### H3 — The existing `codebase-context` retrieval stack is good enough to support the wedge + +The current search/indexing system can supply relevant review context reliably enough to justify reuse. + +Kill/fix trigger: + +- retrieval remains the dominant failure mode after a frozen diagnostic evaluation +- simpler baselines match or beat it at lower setup/runtime cost + +If H3 fails, do not build more reviewer orchestration. Fix, simplify, or replace the retrieval lane first. + +## 7. Architecture + +```text +Git repository + | + v +Exact committed change envelope +(base/head SHAs, clean tree, exact diff hash) + | + v +Deterministic change-signal extraction +(files, stats, identifiers, path fallbacks) + | + v +Bounded query planner +(max queries / identifiers) + | + v +Repository retrieval +(codebase-context search + quality/preflight metadata) + | + +--> conventions / patterns + | + v +review-context-v1 packet + | + +--> human inspection + +--> frozen evaluation harness + +--> model-agnostic reviewer consumer (only after gate) +``` + +### Architectural boundaries + +- Git/diff mechanics must remain independent of model vendors. +- Review-context packet construction must remain usable without an LLM. +- Evaluation must be able to swap reviewer models without rewriting context compilation. +- Framework-specific logic stays in analyzers. +- Core/shared types stay framework-neutral. +- No cloud infrastructure is required for the default product path. + +## 8. Metrics + +Do not collapse metrics into one vanity score. + +### Retrieval metrics + +At minimum: + +- relevant file recall@k +- relevant span recall@k when ground truth supports it +- precision@k +- context characters/tokens returned +- setup/index time +- retrieval wall time +- failure/abstention rate + +### Review metrics + +At minimum: + +- real finding recall +- false positives / precision +- evidence correctness +- unsupported-claim rate +- reviewer token/context usage +- wall time +- tool/runtime failure rate + +### Operational metrics + +Track when meaningful: + +- cold install size +- cold index time +- incremental refresh time +- peak memory +- packet size + +Performance claims require recorded measurements, not intuition. + +## 9. Evaluation Integrity + +The rules in root `AGENTS.md` remain binding. This spec adds the following review-specific requirements. + +### Freeze before tuning + +Before changing retrieval/ranking for a claim-bearing evaluation: + +1. freeze task IDs +2. freeze repositories and commits +3. freeze ground truth / scorer +4. freeze budgets and limits +5. commit the manifest +6. only then inspect treatment results + +### Required baselines + +Use the cheapest meaningful baselines first: + +1. raw diff only +2. raw diff + repository-native/basic text tools +3. simple changed-file/path/symbol context +4. current `codebase-context` retrieval packet + +Do not claim value against an intentionally weak baseline. + +### Failure accounting + +A setup failure, index failure, timeout, invalid output, missing evidence, or judge failure remains a failure row. It is never silently removed from the denominator. + +## 10. Autonomous Agent Operating Model + +An agent may manage routine progress without asking for permission for every implementation detail, but it MUST follow this state machine. + +### State A — Observe + +Read: + +1. `AGENTS.md` +2. `.planning/SPEC.md` +3. `.planning/ROADMAP.md` +4. relevant code/tests/docs +5. current PR/CI state + +Do not code before identifying the current open gate. + +### State B — Classify new information + +Every new input is one of: + +- **blocker** — CI/test/security/IP/reproducibility failure +- **evidence** — benchmark or external usage result +- **bug** — shipped behavior violates contract +- **hypothesis signal** — suggests a current hypothesis may be right/wrong +- **opportunity** — integration/user/recruiter/maintainer pull +- **idea** — unvalidated feature suggestion +- **noise** — does not materially affect current gate + +Only blockers, evidence, bugs, and strong external opportunities can pre-empt the current milestone. + +### State C — Route + +Priority order: + +1. protect privacy/IP/security/reproducibility +2. restore green build/tests +3. satisfy the current roadmap gate +4. collect evidence +5. package/document proven behavior +6. only then expand capability + +A new idea never outranks a failing gate. + +### State D — Change + +Rules: + +- one primary hypothesis or defect per PR +- smallest change capable of crossing the current gate +- prefer deleting/simplifying over layering heuristics +- no hidden benchmark-specific branches +- no fixture edits to make output pass +- no broad refactors while evidence is blocked + +### State E — Verify + +Minimum before merge: + +- lint/format +- type-check +- build +- full relevant test suite +- executable smoke for new public binaries +- docs match behavior +- no new unsupported public claim + +Evidence-changing work also requires the relevant frozen evaluation. + +### State F — Decide + +For each milestone, the agent must choose exactly one: + +- **PASS** — gate met; advance +- **FIX** — hypothesis still plausible; one bounded repair cycle justified +- **KILL** — evidence says this lane is not worth further work +- **BLOCKED** — external dependency prevents honest conclusion + +Do not use "mostly done" as a state. + +### State G — Record + +After a gate decision: + +- update `.planning/ROADMAP.md` +- update claim-bearing docs only if evidence level changed +- record material limitations +- preserve benchmark artifacts/manifests needed for reproduction + +## 11. Agent Authority + +### May do autonomously + +- create branches and PRs +- fix CI failures in current scope +- add tests for current contract +- harden parsing/error handling/reproducibility +- simplify implementation +- add frozen evaluation fixtures before implementation +- run public benchmark/eval infrastructure +- update docs to become more conservative/accurate +- revert its own failing changes +- close/kill a roadmap lane when predefined kill criteria are met + +### Must not do autonomously + +- publish employer/private code or data +- import private `context-kit` history as validation +- publish broad performance/superiority claims without evidence gate +- add paid/cloud dependencies as required defaults +- weaken frozen ground truth after seeing results +- silently increase treatment budgets relative to baseline +- add an autonomous PR-commenting bot before reviewer-impact evidence passes +- create a dashboard merely to make the project look productized +- change project licensing or ownership policy +- spend money or enable paid external services without explicit owner approval + +### Escalate to owner when + +- evidence is genuinely ambiguous after one bounded repair cycle +- a decision requires money or credentials +- a privacy/IP boundary is unclear +- two viable architectures have materially different long-term product directions +- external opportunity requires a commitment or public statement + +## 12. Scope Control + +### Track A — Current + +Only work needed to pass the next open gate in `.planning/ROADMAP.md`. + +### Track B — Backlog + +Potential future work that cannot pre-empt Track A without new evidence. + +Examples: + +- hosted dashboard +- GitHub App +- Azure DevOps adapter +- autonomous reviewer comments +- queue/worker cloud architecture +- large framework-specific rule packs +- general code-review agent orchestration + +Track B items become Track A only when the roadmap explicitly promotes them. + +## 13. Product Kill Rules + +Stop expanding the review-context wedge if any of these holds after a fair frozen evaluation and one bounded repair cycle: + +- simple baselines equal or beat it at materially lower complexity/cost +- relevant-context precision is too low to be useful +- reviewer false positives rise enough to negate recall gains +- benefits require dramatically larger token/context budgets +- gains do not generalize across repositories/languages +- setup/index cost makes realistic use unattractive + +A killed hypothesis is a successful engineering result if the evidence is credible and published honestly. + +## 14. Definition of Publishable + +A milestone is publishable only when a stranger can: + +1. understand the problem from public docs +2. install/build the project +3. run the relevant command +4. inspect the versioned output contract +5. run the evidence harness or reproduce the stated functional behavior +6. see limitations and failures +7. distinguish shipped capability from experimental hypothesis + +## 15. Current Strategic Decision + +Do **not** build a second overlapping project. + +Use `codebase-context` as the public flagship substrate. Prove or kill the review-context wedge inside it. If the existing retrieval engine fails the evidence gates, simplify or replace that layer rather than inventing another repository. diff --git a/README.md b/README.md index 92453b4..aa7f416 100644 --- a/README.md +++ b/README.md @@ -1,238 +1,173 @@ -# codebase-context - -## Map your team's conventions before your AI agent starts searching. +# Codebase Context [![npm version](https://img.shields.io/npm/v/codebase-context)](https://www.npmjs.com/package/codebase-context) [![license](https://img.shields.io/npm/l/codebase-context)](./LICENSE) [![node](https://img.shields.io/node/v/codebase-context)](https://github.com/PatrickSys/codebase-context/blob/master/package.json) -You're tired of AI agents writing code that "just works" but still misses how your team actually builds things. They search too broadly, pick generic examples, and spend tokens exploring before they understand the shape of the repo. +Codebase Context gives coding agents a map of your repository through MCP, for developers who want their agent to find relevant code and follow the team's patterns. Search, examples and project memory run locally by default. -`codebase-context` changes the first step. Start with a bounded conventions map that shows the architecture, dominant patterns, and strongest local examples. Then search for the exact file, symbol, or workflow you need. +## Set up your AI client -Here's what codebase-context does: +Use Node.js 22 or newer. These commands use the current npm release of Codebase Context: -**Starts with a bounded conventions map** - The first call shows architecture layers, active patterns, golden files, and next calls without dumping vendored repos, fixtures, generated output, or oversized entrypoint lists into the default surface. +```bash +# Claude Code +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context -**Finds the right local example** - Search does not just return code. Each result comes back with pattern signals, file relationships, and quality indicators so the agent can move from the map to the most relevant local example instead of wandering through raw hits. +# Codex CLI +codex mcp add codebase-context -- npx -y codebase-context -**Knows what is current** - Conventions are detected from your code and git history, not only from rules you wrote. The map distinguishes what is common from what is rising or declining, and points at the files that best represent the current direction. +# OpenCode 1.x (keep the quoted separator on Windows) +opencode mcp add codebase-context '--' npx -y codebase-context +``` -**Adds support signals when you need them** - Team memory and edit-readiness checks stay available, but as supporting context after the map and search have already narrowed the work. +Start a new agent session in your project, then ask: -Map first, search second, local-first throughout. Your code never leaves your machine by default. +> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project, wait for indexing if needed, and open the relevant source files. -See the [current discovery benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) for the checked-in discovery-only proof. The gate is still `pending_evidence`, and `claimAllowed` remains `false`. +Replace `[feature]` with something you want to find. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. -### What it looks like +### Planned for 2.4.0 (not yet published) -Real CLI output against `angular-spotify`, the repo used for the launch screenshots. +- NestJS support from community contributor Aaron Olin. See the [analyzer details](./docs/capabilities.md#analyzers). +- Better React/Next.js detection and a fix for reranker passage scoring. +- Transformers.js v4 for local models. -**Lead signal: pattern drift and golden files** +Known limits: [Intel Mac indexing crash (#68)](https://github.com/PatrickSys/codebase-context/issues/68), [custom OpenAI endpoints (#70)](https://github.com/PatrickSys/codebase-context/issues/70) and [Ollama (#71)](https://github.com/PatrickSys/codebase-context/issues/71) remain open. -![codebase-context patterns screenshot](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/patterns.png) +The existing [benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) reports 25.7% expected-file recovery versus jCodeMunch's 27.1% in a fixed retrieval comparison; it does not establish a winner or coding quality. -This is the part most tools miss: what the team is doing now, what it is moving away from, and which files are the strongest examples to follow. +For Codex Desktop, create or merge `.codex/config.toml` in the project you want to search: -**Before editing: preflight and impact** +```toml +[mcp_servers.codebase-context] +command = "npx" +args = ["-y", "codebase-context"] +startup_timeout_sec = 120 +``` -![codebase-context search preflight screenshot](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/search-query.png) +Trust the project if asked, restart Codex, and start a new task there. Use the prompt above. Config placement alone does not establish successful first use; this no-folder recipe has not been accepted in a fresh native Desktop task. -When the agent searches with edit intent, it gets a compact decision card: confidence, whether it's safe to proceed, which patterns apply, the best example, and which files are likely to be affected. +Other clients use their own setup commands: -More CLI examples in [`docs/cli.md`](./docs/cli.md). Full walkthrough: [demo.md on GitHub](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md). +| Client | Shortest current setup | +| --------------------------- | ---------------------------------------------------------------------------- | +| Gemini CLI | `gemini mcp add --scope user codebase-context npx -y codebase-context` | +| Cursor | Add `.cursor/mcp.json` | +| VS Code with GitHub Copilot | Add `.vscode/mcp.json` | +| GitHub Copilot CLI | `copilot mcp add codebase-context -- npx -y codebase-context` | +| Windsurf | Add `~/.codeium/windsurf/mcp_config.json` | -## Quick Start +Check an existing same-name entry before replacing it. To give the server a default folder, append that folder's absolute path to the `npx` arguments. The [client setup guide](./docs/client-setup.md) covers scopes, optional fixed-folder configuration and the interactive setup wizard's limitations. -```bash -claude mcp add codebase-context -- npx -y codebase-context -``` +The [client setup guide](./docs/client-setup.md) has the exact commands and config for every client, plus what was checked locally and what still relies on official instructions. -The server runs in two modes. Use stdio unless you need multiple clients connected at once: +The default connection is `stdio` (standard input/output): your client starts the server when it needs it. HTTP is an advanced, client-dependent option; verify the setup guide and client support before relying on it. -| Mode | How it runs | When to use | -| ------------------- | ------------------------------------------------ | ------------------------------------------ | -| **stdio** (default) | Process spawned by the client | One AI client talking to one or more repos | -| **HTTP** | Long-lived server at `http://127.0.0.1:3100/mcp` | Multiple clients sharing one server | +## What your agent gets -Client support at a glance: +### Relevant code -| Client | stdio | HTTP | -| ----------------- | ----- | -------------------------------------------- | -| Claude Code | Yes | No (stdio only) | -| Claude Desktop | Yes | No | -| Cursor | Yes | Yes — `.cursor/mcp.json` with `type: "http"` | -| Windsurf | Yes | Not yet | -| Codex | Yes | Yes — `--mcp-config` flag | -| VS Code (Copilot) | Yes | No | -| OpenCode | Yes | Not documented yet | +`search_codebase` ranks files and symbols for the task instead of returning an unstructured dump. The agent can ask for a compact result first, then read the code it needs. -Copy-pasteable templates: [`templates/mcp/stdio/.mcp.json`](./templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](./templates/mcp/http/.mcp.json). +### Team patterns and examples -Full per-client setup, HTTP server instructions, and local build testing: [`docs/client-setup.md`](./docs/client-setup.md). +`get_team_patterns` shows the approaches used in the repository and points to representative files. Dedicated analyzers for Angular, React, and Next.js ship in published 2.2.0; NestJS support (along with React and Next.js detection improvements and a reranker fix) is planned for unpublished 2.4.0. -## First Use +### Project memory -Get a conventions map of your codebase before exploring or editing: +`remember` stores a convention, decision, gotcha, or past failure for the project. `get_memory` retrieves relevant entries in later sessions, including when the agent or editor changes. -```bash -# See your codebase conventions — architecture layers, patterns, golden files -npx -y codebase-context map +### Review context -# Then search for what you need -npx -y codebase-context search --query "auth middleware" -``` +The `codebase-context-review` CLI creates a bounded context packet from a committed Git diff. It gives a reviewer context; it does not review code or prove review quality. See the [review-context guide](./docs/review-context.md). -Your AI agent uses the same map via the `codebase://context` MCP resource on first call. +## How it works -## Common First Commands +1. **Index locally.** Codebase Context scans the project, builds a keyword index, and creates local semantic embeddings - numeric representations used to match code by meaning as well as exact words. +2. **Understand the repository.** The agent can request a compact codebase map with structure, patterns, and representative files. +3. **Find the code for the task.** Search returns ranked files and symbols; the agent reads the selected files before editing. -Three commands to understand a repo before you edit it: +The same information is available from the terminal. Run these commands from your project root. The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Once it is ready, inspect the map and search for the code you need: ```bash -# What are the main conventions and best examples? +# Build or refresh the local index +npx -y codebase-context reindex + +# Repository structure, patterns, and representative files npx -y codebase-context map -# Then search for the local example you need +# Ranked code search npx -y codebase-context search --query "auth middleware" -# What patterns is the team actually using right now? +# Current team patterns npx -y codebase-context patterns ``` -This is also what your AI agent consumes automatically via MCP tools; the CLI is the human-readable version of the same map-plus-search flow. - -## What it does - -### The Search Tool (`search_codebase`) - -One call returns ranked results with `file`, `summary`, `score`, compact type (`componentType:layer`), pattern trend signals, relationship hints, related team memories, a search quality assessment, and a preflight decision card when `intent="edit"`. The decision card shows `ready` (boolean), `nextAction` when not ready, `patterns` (do/avoid), `bestExample`, impact coverage (`"3/5 callers in results"`), and `whatWouldHelp`. - -Default output is lean — if the agent wants code, it calls `read_file`. Add `includeSnippets: true` for inline code with scope headers (e.g. `// AuthService.getToken()`). +One stdio server can route across several repositories. Supply `project` in tool calls to select the intended repository; a successful selection becomes the default for later calls in that process. Some clients also announce workspace roots: one root can auto-select, while an ambiguous selection asks for a project instead of guessing. MCP deprecated Roots in its July 2026 revision, so explicit project selection is the documented default rather than a dependency on client discovery. -See [`docs/capabilities.md`](./docs/capabilities.md) for the full field reference. +## See it -### Patterns & Conventions (`get_team_patterns`) +These are real CLI results from the open-source `angular-spotify` repository. -Detects what your team actually does by analyzing the codebase: adoption percentages for DI, state management, testing, and library patterns; trend direction (Rising / Stable / Declining) from git recency; golden files ranked by modern pattern density; conflicts when two approaches both exceed 20%. +**Patterns and representative files** -### Team Memory (`remember` + `get_memory`) +![Codebase Context showing repository patterns and representative files](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/patterns.png) -Record a decision once. It surfaces automatically in search results and preflight cards from then on. Conventional commits (`refactor:`, `migrate:`, `fix:`, `revert:`) from the last 90 days auto-extract into memory during indexing — no setup required. +The map shows the patterns found in the project, how common they are, and files that demonstrate them. -Memory types: `convention`, `decision`, `gotcha`, `failure`. Confidence decay: conventions never decay, decisions 180-day half-life, gotchas/failures 90-day. Stale memories get flagged instead of blindly trusted. +**Search before an edit** -## Tools +![Codebase Context showing a ranked search and edit preflight](https://raw.githubusercontent.com/PatrickSys/codebase-context/master/docs/assets/search-query.png) -| Tool | What it does | -| ------------------------------ | ----------------------------------------------------- | -| `search_codebase` | Hybrid search + decision card when `intent="edit"` | -| `get_team_patterns` | Pattern frequencies, golden files, conflict detection | -| `get_symbol_references` | Concrete references to a symbol (count + snippets) | -| `remember` | Record a convention, decision, gotcha, or failure | -| `get_memory` | Query team memory with confidence decay scoring | -| `get_codebase_metadata` | Project structure, frameworks, dependencies | -| `get_style_guide` | Style guide rules for the current project | -| `detect_circular_dependencies` | Import cycles between files | -| `refresh_index` | Full or incremental re-index + git memory extraction | -| `get_indexing_status` | Progress and stats for the current index | +The search result shows ranked files, relevant project patterns, and what the agent should read before it changes code. -## Multi-project +More examples are available in the [CLI gallery](./docs/cli.md) and [walkthrough](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md). -One server, multiple repos. Three cases: +## Privacy -| Case | What happens | -| --------------------------------------------- | --------------------------------------------------- | -| One project | Routing is automatic | -| Multiple projects, active project already set | Routes to the active project | -| Multiple projects, ambiguous | Returns `selection_required` — retry with `project` | +Code and indexes stay on the machine with the default local embedding provider. Docker, a GPU, and an API key are not required. -`project` accepts a project root path, file path, `file://` URI, or relative subproject path (e.g. `apps/dashboard`). +Cloud embeddings are optional. If you select a cloud provider, code chunks are sent to that provider to create the search index. The provider, model, project root, and local HTTP port can be changed through environment variables or the project config; see the [capabilities reference](./docs/capabilities.md). -```json -{ - "name": "search_codebase", - "arguments": { - "query": "auth interceptor", - "project": "apps/dashboard" - } -} -``` - -If you get `selection_required`, retry with one of the paths from `availableProjects`. Full routing details and response shapes in [`docs/capabilities.md`](./docs/capabilities.md#project-routing). - -## Language Support - -10 languages with full symbol extraction via Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust. 30+ languages with indexing and retrieval coverage, including PHP, Ruby, Swift, Scala, Shell, and config formats. Angular, React, and Next.js have dedicated analyzers; everything else uses the Generic analyzer with AST-aligned chunking when a grammar is available. - -## Configuration - -| Variable | Default | Description | -| ------------------------------ | --------------------------------- | ------------------------------------------------------------------- | -| `EMBEDDING_PROVIDER` | `transformers` | `openai` (fast, cloud) or `transformers` (local, private) | -| `OPENAI_API_KEY` | — | Required only if using `openai` provider | -| `CODEBASE_ROOT` | — | Bootstrap root for CLI and single-project MCP clients | -| `CODEBASE_CONTEXT_DEBUG` | — | Set to `1` for verbose logging | -| `EMBEDDING_MODEL` | `Xenova/bge-small-en-v1.5` | Local embedding model override | -| `CODEBASE_CONTEXT_HTTP` | — | Set to `1` to start in HTTP mode (same as `--http` flag) | -| `CODEBASE_CONTEXT_PORT` | `3100` | HTTP server port override (same as `--port`; ignored in stdio mode) | -| `CODEBASE_CONTEXT_CONFIG_PATH` | `~/.codebase-context/config.json` | Override the server config file path | - -## Performance - -- **First indexing**: 2-5 minutes for ~30k files (embedding computation). -- **Subsequent queries**: milliseconds from cache. -- **Incremental updates**: `refresh_index` with `incrementalOnly: true` processes only changed files (SHA-256 manifest diffing). +This is the privacy boundary of Codebase Context itself. Your AI client may send search results or file contents to the model provider configured in that client. Codebase Context does not control that connection. If you commit and push `.codebase-context/memory.json`, the recorded project memory also travels with the repository. -## File Structure - -``` -.codebase-context/ - memory.json # Team knowledge (should be persisted in git) - index-meta.json # Index metadata and version (generated) - intelligence.json # Pattern analysis (generated) - relationships.json # File/symbol relationships (generated) - index.json # Keyword index (generated) - index/ # Vector database (generated) -``` - -**Recommended `.gitignore`:** +Generated indexes belong in `.gitignore`. Project memory can be kept in version control when the team wants to share it: ```gitignore -# Codebase Context - ignore generated files, keep memory .codebase-context/* !.codebase-context/memory.json ``` -## What to add to your CLAUDE.md / AGENTS.md +## Evidence -Paste this into `.cursorrules`, `CLAUDE.md`, `AGENTS.md`, or wherever your AI reads project instructions: +Codebase Context runs locally through the Model Context Protocol (MCP), with indexed code kept on your machine by default. -```markdown -## Codebase Context (MCP) +Its ranked code search combines **Match words**, **Search by meaning**, and **Rank results** to give the agent ranked files, a best example, project patterns, relationships, and relevant memory before they edit. -**Start of every task:** Call `get_memory` to load team conventions before writing any code. +The [benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) reports a corrected 100-attempt retrieval family across five local code-context tools: 99 completed attempts and 1 failed attempt. Codebase Context recovered 25.7% of expected gold files with 11.5% file precision in that fixed adapter run; jCodeMunch recovered 27.1% with 11.0% file precision. Raw-native is a deterministic lexical adapter here, not a full normal coding-agent baseline. These are retrieval observations, not a winner or a coding-quality claim. The [sanitized evidence extract](https://github.com/PatrickSys/codebase-context/blob/master/results/benchmark-presentation-evidence.json) records the exact values and source hashes. -**Before editing existing code:** Call `search_codebase` with `intent: "edit"`. If the preflight card says `ready: false`, read the listed files before touching anything. +The same report retains a separate 300-attempt repeatability history, a 30-run two-task full-agent pilot, and a metered replay whose tool-use validity remains unresolved. A historical Codebase Context indexing observation with embeddings enabled took about 12 minutes 31 seconds, but it does not establish warm-use speed, query latency, install cost, or a cross-tool index-speed ranking. The report does not combine these families into a pooled score and does not establish patch correctness or end-to-end coding quality. Earlier public reports are preserved separately in the [benchmark archive](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark-prior-public-report.md). -**Before writing new code:** Call `get_team_patterns` to check how the team handles DI, state, testing, and library wrappers — don't introduce a new pattern if one already exists. - -**When asked to "remember" or "record" something:** Call `remember` immediately, before doing anything else. - -**When adding imports that cross module boundaries:** Call `detect_circular_dependencies` with the relevant scope after adding the import. -``` +The method and failures are documented so the measurements can be inspected with their limits. -These are the behaviors that make the most difference day-to-day. Copy, trim what doesn't apply to your stack, and add it once. +## Limits -## Links +- Local semantic indexing does more work than a plain text index, especially on a fresh checkout and CPU-only machine. +- Retrieval measurements describe expected-file coverage, file precision, and reported `peakPrivateGb`, not patch correctness or end-to-end coding quality. +- The paired token observation covers two frozen investigation tasks and records observed agent behavior; it is not a universal token or time guarantee. +- Setup checks differ by client. The detailed guide distinguishes a written config, a config recognized by the client, a local connection, and instructions checked only against official docs. +- Dedicated Angular, React, and Next.js analyzers ship in published 2.2.0; NestJS support is planned for unpublished 2.4.0. Other projects use the generic analyzer and the language parsers available for that stack. +- The default searchable-chunk limit is 5,000 per project. Customizing this via `projects[].parsing.maxChunks` in `~/.codebase-context/config.json` requires 2.4.0 (not yet published). +- The agent must identify its repository in tool calls when no default or unambiguous client root is available. Concurrent HTTP client isolation is not established by the stdio routing checks. -- [Benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) — current discovery suite results and gate truth -- [Demo](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md) — real CLI walkthrough -- [Client Setup](./docs/client-setup.md) — per-client config, HTTP setup, local build testing -- [Capabilities Reference](./docs/capabilities.md) — tool API, retrieval pipeline, decision card schema -- [CLI Gallery](./docs/cli.md) — formatted command output examples -- [Motivation](https://github.com/PatrickSys/codebase-context/blob/master/MOTIVATION.md) — research and design rationale -- [Contributing](https://github.com/PatrickSys/codebase-context/blob/master/CONTRIBUTING.md) — dev setup and eval harness -- [Changelog](https://github.com/PatrickSys/codebase-context/blob/master/CHANGELOG.md) +## Reference -## License +- [Client setup](./docs/client-setup.md#client-setup) - commands, config, proof level, and client limits +- [Capabilities](./docs/capabilities.md) - tools, response fields, routing, and configuration +- [CLI](./docs/cli.md) - terminal commands and example output +- [Benchmark](https://github.com/PatrickSys/codebase-context/blob/master/docs/benchmark.md) - method, measurements, and failures +- [Demo](https://github.com/PatrickSys/codebase-context/blob/master/docs/demo.md) - a complete repository walkthrough +- [Motivation](https://github.com/PatrickSys/codebase-context/blob/master/MOTIVATION.md) - the design problem and research background +- [Contributing](https://github.com/PatrickSys/codebase-context/blob/master/CONTRIBUTING.md) - local development and evaluation commands +- [Changelog](https://github.com/PatrickSys/codebase-context/blob/master/CHANGELOG.md) - release history -Elastic-2.0 +Elastic-2.0. See [LICENSE](./LICENSE). diff --git a/docs/benchmark-prior-public-report.md b/docs/benchmark-prior-public-report.md new file mode 100644 index 0000000..506360f --- /dev/null +++ b/docs/benchmark-prior-public-report.md @@ -0,0 +1,182 @@ +# Earlier public benchmark report (snapshot) + +Preserved from `docs/benchmark.md` at public master commit `cc54fb5aee50ec177402e459b23be0fe8da7836e`. The content below is historical; its gates and dated observations are not new measurements. In the one-task section, ContextBench scores measure code-context overlap, not patch correctness. The original external artifact availability was not reverified for this presentation. + +[Return to the current engineering note](./benchmark.md). + +--- + +# Benchmarks + +This page tracks two separate benchmark surfaces: + +- The ContextBench implementation pilot, which uses the official ContextBench evaluator on one frozen task and five scoreable lanes. +- The older discovery benchmark, which measures local discovery usefulness and payload cost only. + +Neither section currently supports a broad benchmark-win claim. + +## ContextBench Implementation Pilot + +This is the current implementation-quality pilot for ContextBench. It is real scoreable evidence, but it is still a pilot because it covers one frozen task rather than the full frozen 20-task slice. + +### Scope + +- Protocol: `tests/fixtures/contextbench-benchmark-protocol.json` +- Task manifest: `tests/fixtures/contextbench-task-manifest.json` +- Selection file: `scripts/contextbench-five-lane-selections.json` +- Workflow: `.github/workflows/contextbench-five-lane-score.yml` +- Required lanes: `raw-native`, `codebase-context`, `codebase-memory-mcp`, `grepai`, `ripgrep-lexical` +- Model used for selection: `gpt-5.4-mini-high` +- Target task: `SWE-Bench-Pro__go__maintenance__bugfix__4df06349` +- Repository under test: `navidrome/navidrome` +- Base commit: `537e2fc033b71a4a69190b74f755ebc352bb4196` + +`CodeGraphContext` is not counted in this five-lane pilot because its supported CLI path indexed successfully but returned zero task-relevant candidates during readiness. That remains a readiness blocker, not a quality result. + +### Current Audited Run + +- Run: `25663469903` +- Job: `75329796667` +- Commit: `bbd3a8348aaec15809fd09dd8fc729e64df6d878` +- Artifact: `6915576867` +- Artifact digest: `sha256:718fd32049a2d98ed62fb0c15189d7dc9f1b027c202f286923de91d9f8985def` +- Artifact size: `88.9 KB` +- Uploaded files: `42` +- Status: `success` + +The artifact contains `summary.json`, `publishable-summary.json`, `publishable-validation.json`, `humanized-summary.md`, logs, lane selections, lane predictions, and official evaluator score files. It intentionally excludes full cloned repos and evaluator caches so the evidence package is small enough to inspect. + +### Quality Results + +Only rows scored by the official ContextBench evaluator are included here. Setup failures, tool errors, empty predictions, and judge failures are reliability outcomes, not quality rows. + +| Lane | File cov | File prec | Span cov | Span prec | Line cov | Line prec | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `raw-native` | 0.222 | 0.667 | 0.370 | 0.391 | 0.365 | 0.365 | +| `codebase-context` | 0.889 | 1.000 | 0.899 | 0.356 | 0.887 | 0.323 | +| `codebase-memory-mcp` | 0.222 | 0.667 | 0.346 | 0.380 | 0.315 | 0.337 | +| `grepai` | 0.333 | 0.500 | 0.048 | 0.042 | 0.059 | 0.061 | +| `ripgrep-lexical` | 0.222 | 0.667 | 0.401 | 0.341 | 0.419 | 0.302 | + +### Cost And Telemetry + +The report separates setup, indexing, query, selector, evaluator, and row-wall timing from quality. It also reports candidate counts, candidate token estimates when available, prediction token estimates, and selector token telemetry fields. + +`n/a` means the measurement was explicitly unavailable, not zero and not a hidden failure. Current gaps are: + +- Selector wall-clock and provider token telemetry were not captured in this proof artifact. +- `raw-native`, `codebase-context`, and `codebase-memory-mcp` emitted candidate counts but not candidate-pack bytes. +- `codebase-context` readiness did not emit index/query duration in the source artifact. + +### Bias Controls + +The generated `publishable-validation.json` must pass these checks before the report is treated as evidence: + +- Quality rows come only from the official ContextBench evaluator. +- Failed or unscoreable rows stay out of the quality table. +- All required lanes are scoreable. +- Setup, index, and query costs are separate from quality. +- Timing and token fields exist or carry explicit unavailable reasons. +- The protocol is frozen and `claimAllowed` is `false`. +- The task manifest attests that lane outputs were not observed during task selection. + +### What This Supports + +- It supports saying that the benchmark harness can produce real official ContextBench scores across five lanes. +- It supports saying that setup/index/query cost and context/token cost are now tracked separately from quality. +- It supports saying that the one-task pilot found a strong `codebase-context` result on this specific task. + +### What This Does Not Support + +- It does not support claiming that `codebase-context` beats competitors overall. +- It does not support claiming patch correctness or productivity improvements. +- It does not replace the full frozen 20-task, repeated-run benchmark required for claim-bearing results. + +## Discovery Benchmark + +This section documents the current public discovery proof from the checked-in result artifacts on `master`. +It is a discovery benchmark, not an implementation-quality benchmark. + +## Discovery Scope + +- Frozen fixtures: + - `tests/fixtures/discovery-angular-spotify.json` + - `tests/fixtures/discovery-excalidraw.json` + - `tests/fixtures/discovery-benchmark-protocol.json` +- Frozen repos used in the current proof run: + - `repos/angular-spotify` + - `repos/excalidraw` +- Current gate artifact: + - `results/gate-evaluation.json` +- Comparator evidence: + - `results/comparator-evidence.json` + +## Discovery Reproduction + +Run the repo-local proof artifacts from the current `master` checkout: + +```bash +node scripts/run-eval.mjs repos/angular-spotify --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --skip-reindex --output=results/codebase-context-angular-spotify.json +node scripts/run-eval.mjs repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-excalidraw.json --skip-reindex --output=results/codebase-context-excalidraw.json +node scripts/benchmark-comparators.mjs --repos repos/angular-spotify,repos/excalidraw --output results/comparator-evidence.json +node scripts/run-eval.mjs repos/angular-spotify repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --fixture-b=tests/fixtures/discovery-excalidraw.json --competitor-results=results/comparator-evidence.json --skip-reindex --output=results/gate-evaluation.json +``` + +## Discovery Current Result + +From `results/gate-evaluation.json`: + +- `status`: `pending_evidence` +- `suiteStatus`: `complete` +- `claimAllowed`: `false` +- `totalTasks`: `24` +- `averageUsefulness`: `0.75` +- `averageEstimatedTokens`: `1827.0833` +- `bestExampleUsefulnessRate`: `0.125` + +Repo-level outputs from the same rerun: + +| Repo | Tasks | Avg usefulness | Avg estimated tokens | Best-example usefulness | +| --- | ---: | ---: | ---: | ---: | +| `angular-spotify` | 12 | 0.8333 | 2138.4167 | 0.25 | +| `excalidraw` | 12 | 0.6667 | 1506.0833 | 0 | + +## Discovery Gate Truth + +The gate is intentionally still blocked. + +- The combined suite covers both public repos. +- `claimAllowed` remains `false` because comparator evidence still does not support a benchmark-win claim. +- Two comparator artifacts now return `status: "ok"`, but that does not yet close the gate: + - `raw Claude Code` still leaves the baseline `pending_evidence` because `averageFirstRelevantHit` is `null` + - `codebase-memory-mcp` now has real current metrics, but the gate still marks it `failed` on the frozen tolerance rule +- Three comparator lanes still fail setup entirely: `GrepAI`, `jCodeMunch`, and `CodeGraphContext`. + +## Discovery Comparator Reality + +The current comparator artifact records incomplete comparator evidence, not benchmark wins. + +| Comparator | Status | Current reason | +| --- | --- | --- | +| `codebase-memory-mcp` | comparator artifact: `ok`; gate: `failed` | Runs through the repaired graph-backed path and now records real metrics (`averageUsefulness: 0.1875`, `averageFirstRelevantHit: 1.2857`, `bestExampleUsefulnessRate: 0.5`), but the frozen gate still fails it on the required usefulness comparisons | +| `jCodeMunch` | `setup_failed` | `MCP error -32000: Connection closed` | +| `GrepAI` | `setup_failed` | Local Go binary and Ollama model path not present | +| `CodeGraphContext` | `setup_failed` | `MCP error -32000: Connection closed` | +| `raw Claude Code` | comparator artifact: `ok`; gate: `pending_evidence` | The explicit Haiku CLI runner now returns current metrics (`averageUsefulness: 0.0278`, `averageEstimatedTokens: 32.1667`), but the baseline still lacks `averageFirstRelevantHit`, so the gate keeps this lane as missing evidence | + +`CodeGraphContext` remains part of the frozen comparison frame. It is not omitted from the public story just because the lane still fails to start. + +## Discovery Important Limitations + +- This benchmark measures discovery usefulness and payload cost only. +- It does not measure implementation correctness, patch quality, or end-to-end task completion. +- Comparator setup remains environment-sensitive, and the checked-in comparator outputs still do not satisfy the frozen claim gate. +- The reranker cache is currently corrupted on this machine. During the proof rerun, search fell back to original ordering after `Protobuf parsing failed` while still completing the harness. +- `averageFirstRelevantHit` remains `null` in the current gate output, which is enough to keep the raw-Claude baseline in `pending_evidence`. + +## Discovery Claims Supported + +- It can support claims about the shipped discovery surfaces and their current measured outputs on the frozen public tasks. +- It can support claims that the proof gate is still blocked by comparator evidence. +- It cannot support claims that `codebase-context` beats the named comparators today. +- It cannot support claims about edit success, code quality, or implementation speed. diff --git a/docs/benchmark.md b/docs/benchmark.md index d48f807..73d073b 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -1,88 +1,80 @@ -# Discovery Benchmark +# What our local code-context experiments show -This page documents the current public discovery proof from the checked-in result artifacts on `master`. -It is a discovery benchmark, not an implementation-quality benchmark. +This report describes local evidence from five ways of giving an AI coding agent context: Raw-native, Codebase Context, jCodeMunch, Repowise, and context-mode. The evidence is split into separate families because retrieval, repeated runs, full-agent observations, and a metered replay do not answer the same question. The families are not pooled into one score. -## Scope +The clearest current finding is narrow: in this fixed retrieval setup, Codebase Context recovered 25.7% of expected gold files at 11.5% file precision, while fresh indexing had material local cost. These measurements do not show better completed coding tasks. -- Frozen fixtures: - - `tests/fixtures/discovery-angular-spotify.json` - - `tests/fixtures/discovery-excalidraw.json` - - `tests/fixtures/discovery-benchmark-protocol.json` -- Frozen repos used in the current proof run: - - `repos/angular-spotify` - - `repos/excalidraw` -- Current gate artifact: - - `results/gate-evaluation.json` -- Comparator evidence: - - `results/comparator-evidence.json` +## Evidence families -## How To Reproduce +| Evidence family | Attempts | Completed | Failed or otherwise excluded | What it answers | +| --- | ---: | ---: | ---: | --- | +| Corrected retrieval | 100 | 99 of 100 | 1 failed | Expected-file coverage, file precision, and reported `peakPrivateGb` | +| Repeated retrieval | 300 | 288 of 300 | 12 failed | Repeatability, setup, and runtime history | +| Full-agent pilot | 30 | 30 of 30 | 0 | Observed token and task-time results across two frozen tasks | +| Metered replay (excluded) | 100 | 100 counter records in historical summary | Tool-use validity unresolved | Why those replay records are not scored | -Run the repo-local proof artifacts from the current `master` checkout: +The first two families contain 400 retrieval attempts. The full-agent pilot and the metered replay remain separate from those retrieval measurements. -```bash -node scripts/run-eval.mjs repos/angular-spotify --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --skip-reindex --output=results/codebase-context-angular-spotify.json -node scripts/run-eval.mjs repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-excalidraw.json --skip-reindex --output=results/codebase-context-excalidraw.json -node scripts/benchmark-comparators.mjs --repos repos/angular-spotify,repos/excalidraw --output results/comparator-evidence.json -node scripts/run-eval.mjs repos/angular-spotify repos/excalidraw --mode=discovery --fixture-a=tests/fixtures/discovery-angular-spotify.json --fixture-b=tests/fixtures/discovery-excalidraw.json --competitor-results=results/comparator-evidence.json --skip-reindex --output=results/gate-evaluation.json -``` +## Retrieval results + +Expected-file coverage means expected gold files recovered divided by expected gold files, averaged within the corrected retrieval-only family. File precision means expected gold files recovered divided by all files returned. The table reports both alongside completion status and the reported `peakPrivateGb` telemetry field. The underlying unit of that field was not independently verified, so these values are not labelled as GiB or treated as install size. + +| Tool | Completed | Failed | Expected-file coverage | File precision | Reported `peakPrivateGb` | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 20/20 | 0 | 4.3% | 4.0% | 0.06 | +| Codebase Context | 20/20 | 0 | 25.7% | 11.5% | 2.90 | +| jCodeMunch | 20/20 | 0 | 27.1% | 11.0% | 0.33 | +| Repowise | 19/20 | 1 | 20.1% (failed run = 0) | 12.0% | 4.82 | +| context-mode | 20/20 | 0 | 16.8% | 11.7% | 0.15 | + +The corrected retrieval family therefore includes 99 completed attempts and 1 failed attempt. Repowise is the only tool with a failed attempt in this table. Its 20.1% is failure-inclusive: the failed attempt is counted as zero. Raw search is a deterministic lexical adapter in this protocol, not a full normal coding-agent baseline. These values describe file retrieval and returned-file precision, not a coding score. The [sanitized evidence extract](../results/benchmark-presentation-evidence.json) records the exact values and source hashes. + +## Paired token and time observation -## Current Result +The full-agent pilot used `gpt-5.4-mini-high`, with three runs per tool and task. It covers one frozen investigation task for each of two open-source codebases. The two tasks are PonyC and fmt. The paired tables compare the raw-native agent lane with Codebase Context; all five lanes remain available in the evidence extract. -From `results/gate-evaluation.json`: +### PonyC -- `status`: `pending_evidence` -- `suiteStatus`: `complete` -- `claimAllowed`: `false` -- `totalTasks`: `24` -- `averageUsefulness`: `0.75` -- `averageEstimatedTokens`: `1827.0833` -- `bestExampleUsefulnessRate`: `0.125` +| Tool | Input tokens | Cached input tokens | Output tokens | Reasoning output tokens | Task time | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 653,348 | 610,304 | 9,011 | 4,494 | 185,061 ms | +| Codebase Context | 223,558 | 190,976 | 5,564 | 3,024 | 114,664 ms | -Repo-level outputs from the same rerun: +### fmt -| Repo | Tasks | Avg usefulness | Avg estimated tokens | Best-example usefulness | -| --- | ---: | ---: | ---: | ---: | -| `angular-spotify` | 12 | 0.8333 | 2138.4167 | 0.25 | -| `excalidraw` | 12 | 0.6667 | 1506.0833 | 0 | +| Tool | Input tokens | Cached input tokens | Output tokens | Reasoning output tokens | Task time | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 290,186 | 272,384 | 7,567 | 4,024 | 142,002 ms | +| Codebase Context | 247,868 | 206,848 | 7,930 | 4,784 | 143,643 ms | -## Gate Truth +All values above are medians across three `gpt-5.4-mini-high` runs for the named task and tool. Cached input is included in input and must not be added to input again. Each column is aggregated independently. Codebase Context used fewer input tokens in both examples, but its median task time was slightly longer on fmt. This does not establish better answers or patches. + +The prompt requested at most 12 shell or tool calls. The CLI did not hard-enforce that request. The result describes observed agent behavior, not universal or hard-call-controlled savings. + +## What can be reproduced from the public repository + +The public repository snapshot at commit `cc54fb5` contains the [benchmark script](https://github.com/PatrickSys/codebase-context/blob/cc54fb5aee50ec177402e459b23be0fe8da7836e/scripts/contextbench-runner.mjs), [benchmark rules](https://github.com/PatrickSys/codebase-context/blob/cc54fb5aee50ec177402e459b23be0fe8da7836e/tests/fixtures/contextbench-benchmark-protocol.json), and [20-task manifest](https://github.com/PatrickSys/codebase-context/blob/cc54fb5aee50ec177402e459b23be0fe8da7836e/tests/fixtures/contextbench-task-manifest.json). Those links were reachable on October 5, 2026; earlier local-commit links were not. You can validate the checked-in fixtures with: + +```bash +node scripts/contextbench-runner.mjs --validate-fixtures +``` -The gate is intentionally still blocked. +They show the test structure and task format. They are not a byte-for-byte copy of every file used by the measured sessions; all three file hashes differ from the later 30-session pilot. The manifest names tasks and base commits but does not contain their full problem statements or gold-context values. The exact machine-specific raw run directories behind the 100-attempt retrieval table, the 300-attempt repeatability history, and the 30-session agent pilot are not checked into the public repository, so this checkout cannot regenerate the reported tables with one command. The [original schema-2 retrieval summary](../results/contextbench-engineering-note-summary.json) is preserved from local commit `18e0a43a90fe67e9df0d728b2134e9d430469883`; the [sanitized evidence extract](../results/benchmark-presentation-evidence.json) preserves the exact values and hashes used for this presentation. Fixture validation checks fixture consistency, not the reported measurements. -- The combined suite covers both public repos. -- `claimAllowed` remains `false` because comparator evidence still does not support a benchmark-win claim. -- Two comparator artifacts now return `status: "ok"`, but that does not yet close the gate: - - `raw Claude Code` still leaves the baseline `pending_evidence` because `averageFirstRelevantHit` is `null` - - `codebase-memory-mcp` now has real current metrics, but the gate still marks it `failed` on the frozen tolerance rule -- Three comparator lanes still fail setup entirely: `GrepAI`, `jCodeMunch`, and `CodeGraphContext`. +Local session artifacts retained outside this public checkout record Windows x64 with 16 logical CPUs, about 24 GiB of RAM, Node.js 24.14.1, Claude Code 2.1.173, and `gpt-5.4-mini-high` for the later agent pilot. Readers cannot independently inspect those environment files from this checkout. Exact version pins for all five tools in the retrieval families were not captured consistently enough to publish a trustworthy version matrix. -## Comparator Reality +This makes the public evidence partially reproducible: the method and current fixtures are inspectable, while the exact reported runs are not independently replayable from this checkout alone. The tables should be read as a transparent engineering report, not a paper-grade reproduction package. -The current comparator artifact records incomplete comparator evidence, not benchmark wins. +## Method and limits -| Comparator | Status | Current reason | -| --- | --- | --- | -| `codebase-memory-mcp` | comparator artifact: `ok`; gate: `failed` | Runs through the repaired graph-backed path and now records real metrics (`averageUsefulness: 0.1875`, `averageFirstRelevantHit: 1.2857`, `bestExampleUsefulnessRate: 0.5`), but the frozen gate still fails it on the required usefulness comparisons | -| `jCodeMunch` | `setup_failed` | `MCP error -32000: Connection closed` | -| `GrepAI` | `setup_failed` | Local Go binary and Ollama model path not present | -| `CodeGraphContext` | `setup_failed` | `MCP error -32000: Connection closed` | -| `raw Claude Code` | comparator artifact: `ok`; gate: `pending_evidence` | The explicit Haiku CLI runner now returns current metrics (`averageUsefulness: 0.0278`, `averageEstimatedTokens: 32.1667`), but the baseline still lacks `averageFirstRelevantHit`, so the gate keeps this lane as missing evidence | +The retrieval percentages, precision values, and reported `peakPrivateGb` fields come from the corrected retrieval-only family. The repeated retrieval family is reported as completion and repeatability history rather than merged into that table. The full-agent pilot reports input, cached input, output, reasoning output, and task time for two frozen investigation tasks. The metered replay is retained as history only and excluded from effectiveness or token-saving claims while tool-use validity is unresolved. -`CodeGraphContext` remains part of the frozen comparison frame. It is not omitted from the public story just because the lane still fails to start. +A historical fresh isolated Codebase Context indexing observation with embeddings enabled used the Transformers provider and `Xenova/bge-small-en-v1.5`. Its median was about 12 minutes 31 seconds. This is a historical observation in methodology and limits, not a normal-use estimate or a cross-tool speed ranking. -## Important Limitations +That index observation does not describe normal warm use, incremental indexing, query latency, install time, or cross-tool index-speed ranking. Shared model-cache state, actual GPU use, the exact scan, analysis, embedding, and storage stage durations, and equivalent cold work across competitors were not established by that observation. -- This benchmark measures discovery usefulness and payload cost only. -- It does not measure implementation correctness, patch quality, or end-to-end task completion. -- Comparator setup remains environment-sensitive, and the checked-in comparator outputs still do not satisfy the frozen claim gate. -- The reranker cache is currently corrupted on this machine. During the proof rerun, search fell back to original ordering after `Protobuf parsing failed` while still completing the harness. -- `averageFirstRelevantHit` remains `null` in the current gate output, which is enough to keep the raw-Claude baseline in `pending_evidence`. +The benchmark does not measure patch correctness, end-to-end task completion, or a universal token or time result. It also does not combine the four evidence families into a pooled score. Setup and run failures remain part of the record so that incomplete attempts are visible. -## What This Proof Can Support +Historical note: the [earlier public report archive](./benchmark-prior-public-report.md) contains a one-task official pilot and a separate 24-task discovery report. Those records and their comparator statuses remain separate, are not folded into this current four-family summary, and retain their claim gates. -- It can support claims about the shipped discovery surfaces and their current measured outputs on the frozen public tasks. -- It can support claims that the proof gate is still blocked by comparator evidence. -- It cannot support claims that `codebase-context` beats the named comparators today. -- It cannot support claims about edit success, code quality, or implementation speed. +See the [comparison table](./comparison-table.md) for the retrieval family in a compact form. The repository walkthrough is in the [demo](./demo.md). diff --git a/docs/capabilities.md b/docs/capabilities.md index 9085a07..825c2fe 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -1,13 +1,15 @@ # Capabilities Reference -Technical reference for what `codebase-context` ships today. The public product posture is map first, find second: the bounded conventions map is the first-call surface, and search narrows to the right local example after that. For the user-facing overview, see [README.md](../README.md). +Technical reference for the current Codebase Context npm release and the upcoming 2.4.0 source. Start with the repository map, then search for relevant code. For an overview, see [README.md](../README.md). + +Commands use the current npm release. Features marked as planned for 2.4.0 are not yet available from npm; the current published version is 2.2.0. ## Transport Modes The server supports two transport modes: -| Mode | Command | MCP endpoint | -| ------------------- | ------------------------------------------- | ---------------------------- | +| Mode | Command | MCP endpoint | +| ------------------- | ------------------------------------------------- | ---------------------------- | | **stdio** (default) | `npx -y codebase-context` | Spawned process stdin/stdout | | **HTTP** | `npx -y codebase-context --http [--port N]` | `http://127.0.0.1:3100/mcp` | @@ -20,13 +22,14 @@ Per-project config overrides supported today: - `projects[].excludePatterns`: merged with the built-in exclusion set for that project at index time - `projects[].analyzerHints.analyzer`: prefers a registered analyzer by name for that project and falls back safely when the name is missing or invalid - `projects[].analyzerHints.extensions`: adds project-local source extensions for indexing and auto-refresh watching without changing defaults for other projects +- `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project (requires 2.4.0, not yet published) Copy-pasteable client config templates are shipped in the package: - `templates/mcp/stdio/.mcp.json` — stdio setup for `.mcp.json`-style clients - `templates/mcp/http/.mcp.json` — HTTP setup for `.mcp.json`-style clients -Client transport support varies — see [README.md](../README.md) for a per-client matrix covering Claude Code, Cursor, Codex, Windsurf, VS Code, Claude Desktop, and OpenCode. +Use Node.js 22 or newer with the commands shown here. For setup commands and what was checked, see the [client setup guide](./client-setup.md). ## CLI Reference @@ -49,15 +52,15 @@ For a command gallery with examples, see `docs/cli.md`. | `memory add` | `--type`, `--category`, `--memory`, `--reason` | `remember` | | `memory remove ` | — | — | -All commands accept `--json` for raw JSON output. Errors go to stderr with exit code 1. +Commands that list `--json` above support raw JSON output. Follow the [client setup guide](./client-setup.md) to connect your client. The `init` command can set up Claude Code and Codex for the current project; the guide covers the other clients. Errors go to stderr with exit code 1. ```bash # Quick examples -npx codebase-context status -npx codebase-context search --query "auth middleware" --intent edit -npx codebase-context refs --symbol "UserService" --limit 10 -npx codebase-context cycles --scope src/features -npx codebase-context reindex --incremental +npx -y codebase-context status +npx -y codebase-context search --query "auth middleware" --intent edit +npx -y codebase-context refs --symbol "UserService" --limit 10 +npx -y codebase-context cycles --scope src/features +npx -y codebase-context reindex --incremental ``` ## Tool Surface @@ -104,12 +107,12 @@ Behavior matrix: Rules: -- If the client provides workspace context, that becomes the trusted workspace boundary for the session. In practice this usually comes from MCP roots. -- Treat seamless multi-project routing as evidence-backed only for roots-capable hosts. Without roots, explicit fallback is still required. -- If the server still cannot tell which project to use, a bootstrap path or explicit absolute `project` path remains the fallback. -- `project` is the canonical explicit selector when routing is ambiguous. +- The portable default is explicit selection: pass the repository's absolute path as `project`. A successful selection becomes the active project for that server process. +- Client-announced workspace roots remain a compatibility aid and constrain explicit selectors to that workspace. One valid root can auto-select; ambiguous roots require selection. Roots are not required for the documented setup. +- Without a configured folder, known root or active selection, the server returns `selection_required` instead of guessing a project. - `project` may point at a project path, file path, `file://` URI, or relative subproject path. - Later tool calls may omit `project`; the server falls back to the active project when one has already been established. +- Select again after a server restart or when changing repositories. Published-package A/B/A stdio routing was verified; it does not establish native agent continuation or concurrent HTTP isolation. See the [client setup guide](./client-setup.md#project-routing-contract) for the checks and their limits. - The server does not rely on `cwd` walk-up in MCP mode. - `codebase://context` serves the active project. Before selection in an unresolved multi-project session, it returns a workspace overview with candidate projects, readiness state, and project-scoped resource URIs. - `codebase://context/project/` serves a specific project directly and also makes that project active for later tool calls. @@ -273,6 +276,7 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2) - **Angular**: signals, standalone components, control flow syntax, lifecycle hooks, DI patterns, component metadata - **React**: function/class components, custom hooks, context usage, memoization, Suspense, ecosystem signal extraction - **Next.js**: App Router and Pages Router detection, route/API classification, route paths, `"use client"`, metadata exports +- **NestJS** (planned for 2.4.0, not yet published): modules, controllers, providers and route handlers, contributed by Aaron Olin. Matching NestJS source is detected automatically. - **Generic**: 30+ have indexing/retrieval coverage including PHP, Ruby, Swift, Scala, Shell, config/markup., 10 languages have full symbol extraction (Tree-sitter: TypeScript, JavaScript, Python, Java, Kotlin, C, C++, C#, Go, Rust). Notes: @@ -282,29 +286,25 @@ Notes: ## Evaluation Harness -Current public proof bundle: [`docs/benchmark.md`](../docs/benchmark.md) and [`docs/comparison-table.md`](../docs/comparison-table.md). +Current public evidence: [`docs/benchmark.md`](../docs/benchmark.md) and [`docs/comparison-table.md`](../docs/comparison-table.md). -Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring/reporting code. +Reproducible evaluation is shipped as a CLI entrypoint backed by shared scoring and reporting code. -- **Command:** `npm run eval -- [codebaseB] --mode retrieval|discovery [--competitor-results ]` (builds first, then runs `scripts/run-eval.mjs`) +- **Command:** `pnpm run eval -- [codebaseB] --mode retrieval|discovery [--competitor-results ]` (builds first, then runs `scripts/run-eval.mjs`) +- **Contributor requirement:** this repository command uses the pinned `pnpm@10.27.0`; end users do not need pnpm for the `npx` CLI. - **Shared implementation:** `src/eval/harness.ts`, `src/eval/discovery-harness.ts`, and `src/eval/types.ts` -- **Frozen retrieval fixtures:** - - `tests/fixtures/eval-angular-spotify.json` - - `tests/fixtures/eval-controlled.json` + `tests/fixtures/codebases/eval-controlled/` -- **Frozen discovery fixtures:** - - `tests/fixtures/discovery-angular-spotify.json` - - `tests/fixtures/discovery-excalidraw.json` - - `tests/fixtures/discovery-benchmark-protocol.json` -- **Retrieval metrics:** Top-1 accuracy, Top-3 recall, spec contamination rate, and a gate pass/fail +- **Retrieval fixtures:** `tests/fixtures/eval-angular-spotify.json` and `tests/fixtures/eval-controlled.json` with `tests/fixtures/codebases/eval-controlled/` +- **Discovery fixtures:** `tests/fixtures/discovery-angular-spotify.json`, `tests/fixtures/discovery-excalidraw.json`, and `tests/fixtures/discovery-benchmark-protocol.json` +- **Retrieval metrics:** Top-1 accuracy, Top-3 recall, spec contamination rate, and a pass/fail result for the configured retrieval check - **Discovery metrics:** usefulness score, payload bytes, estimated tokens, first relevant hit, and best-example usefulness -- **Discovery gate:** discovery mode evaluates the frozen ship gate only when the full public suite and comparator metrics are available; missing comparator evidence is reported as pending, not silently treated as pass/fail -- **Current checked-in gate truth:** `results/gate-evaluation.json` remains `pending_evidence` with `claimAllowed: false`; the raw-Claude baseline still lacks `averageFirstRelevantHit`, `codebase-memory-mcp` still fails the frozen usefulness comparisons, and the remaining named lanes are still `setup_failed` -- **Limits:** discovery mode is discovery-only, uses current shipped surfaces only, and does not claim implementation quality; named competitor runs remain a documented hybrid/manual lane rather than a built-in automated benchmark +- **Public benchmark families:** the current documentation keeps 100 corrected retrieval attempts, 300 repeated retrieval attempts, 30 full-agent pilot attempts, and 100 invalid-token replays separate across five local code-context tools +- **Discovery limits:** discovery mode describes the shipped discovery surface and does not measure patch correctness or end-to-end task completion; named competitor runs remain a documented hybrid/manual lane rather than a built-in automated benchmark +- **Evidence limits:** missing or failed setup and comparator runs remain visible in their source artifacts and are not treated as a pass ## Limitations - **Symbol refs are not a call-graph.** `get_symbol_references` counts identifier-node occurrences in the AST (comments/strings excluded via Tree-sitter). It does not distinguish call sites from type annotations, variable assignments, or imports. Full call-site-specific analysis (`call_expression` nodes only) is a roadmap item. - **Impact is 2-hop max.** `computeImpactCandidates` walks direct importers then their importers. Full BFS reachability is on the roadmap. -- **Angular, React, and Next.js have dedicated analyzers.** All other languages go through the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). +- **Planned for unpublished 2.4.0:** NestJS analyzer support, React and Next.js detection improvements, and reranker fixes. Dedicated Angular, React, and Next.js analyzers already ship in published 2.2.0. Other languages use the Generic analyzer (30+ languages, chunking + import graph, no framework-specific signal extraction). - **Default embedding model is `bge-small-en-v1.5` (512-token context).** Granite (8192 context) is opt-in via `EMBEDDING_MODEL`. OpenAI is opt-in via `EMBEDDING_PROVIDER=openai` — sends code externally. - **Patterns are file-level frequency counts.** Not semantic clustering. Rising/Declining trend is derived from git commit recency for files using each pattern, not from usage semantics. diff --git a/docs/cli.md b/docs/cli.md index 5025f65..c561ab9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,15 +10,19 @@ > Output depends on the repo you run it against. The examples below are illustrative (paths, counts, and detected frameworks will vary). > -> The CLI is intentionally single-project per invocation. MCP multi-project routing and trusted-root auto-discovery are only for the MCP server; the CLI still targets one root via `CODEBASE_ROOT` or the current working directory. +> These commands use the current npm release. The examples are illustrative and were not captured from the current npm package. +> +> These output examples come from an earlier CLI run and do not verify the current npm package or an MCP connection. +> +> The commands below use `codebase-context` and require Node.js 22 or newer. The CLI is intentionally single-project per invocation and targets one root via `CODEBASE_ROOT` or the current working directory. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). ## How to run ```bash -# Run from a repo root, or set CODEBASE_ROOT explicitly: +# CLI only: run from a repo root, or set CODEBASE_ROOT explicitly: CODEBASE_ROOT=/path/to/repo npx -y codebase-context status -# Every command supports --json (machine output). Human mode is default. +# Commands that show --json in their help support machine output. Human mode is default. npx -y codebase-context patterns --json ``` @@ -42,12 +46,26 @@ CODEBASE_CONTEXT_ASCII=1 npx -y codebase-context patterns - `reindex` — rebuild index (full or incremental) - `style-guide` — find style guide sections in docs - `memory list|add|remove` — manage team memory (stored in `.codebase-context/memory.json`) +- MCP client registration — use the recipes in the [client setup guide](./client-setup.md); `init` is an interactive HTTP setup wizard with the limitations described there --- +## First use + +Run these CLI commands from the project root. Build or refresh the local index first; the first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. Then inspect the conventions map and search for the code you need: + +```bash +npx -y codebase-context reindex +npx -y codebase-context map +npx -y codebase-context search --query "auth middleware" +``` + +This is the CLI preparation flow. It is separate from MCP client registration and does not by itself show that an MCP client connected successfully. + ## `map` ```bash +npx -y codebase-context reindex npx -y codebase-context map ``` @@ -126,6 +144,7 @@ Example output (truncated): ## `search` ```bash +npx -y codebase-context reindex npx -y codebase-context search --query "file watcher" --intent edit --limit 3 ``` diff --git a/docs/client-setup.md b/docs/client-setup.md index 032b0cf..4ad2c3e 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -1,131 +1,210 @@ # Client Setup -Full setup instructions for each AI client. This guide is about transport and wiring, not a different product mode: each client gets the same bounded conventions map first and local-pattern discovery second. For the quick-start summary, see [README.md](../README.md). +These commands use the current npm release because the package name is unpinned. The current published version is 2.2.0; 2.4.0 is not yet published. -## Transport modes +Codebase Context runs as a local MCP server: a tool your AI editor or command-line client can call while it works. Register it without a fixed folder, then let the agent supply its current repository through the `project` tool parameter. -| Mode | How it runs | When to use | -| ---- | ----------- | ------------ | -| **stdio** (default) | Process spawned by the client | One AI client, simple setup | -| **HTTP** | Long-lived server at `http://127.0.0.1:3100/mcp` | Multiple clients sharing one server | +Use Node.js 22 or newer, as required by the package. -Start the HTTP server: +The copyable registration commands target PowerShell or a POSIX terminal. Windows Command Prompt was not tested; OpenCode's quoted separator is shell-dependent. -```bash -npx -y codebase-context --http # default port 3100 -npx -y codebase-context --http --port 4000 -``` +## Client recipes -Copy-pasteable templates: [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json). +Claude Code, Codex CLI and OpenCode registration commands were checked again on October 6 without a project argument. The other clients retain their older config-shape evidence; no-folder native agent use is unverified for them. See the dated notes below. -## Project routing contract +Each client section says how far its earlier Windows check went. Those checks used published `2.2.0` unless another build is named; they do not verify upcoming unpublished changes: + +| Status | What it means | +| ------------------------------ | ------------------------------------------------------------------------------------------ | +| **Connected locally** | The isolated client started and connected to the server. | +| **Config recognized locally** | The isolated client found the config but still required approval before loading it. | +| **Config written locally** | The isolated command wrote the expected config. A live agent session was not claimed. | +| **Official instructions only** | The config follows current official docs, but this client was not fully exercised locally. | -Automatic multi-project routing is evidence-backed only when the MCP host announces workspace roots. Treat that as the primary path. +These checks cover setup and startup state. They do not measure how an agent handles a coding task. -If the host does not send roots, or still cannot tell which project is active, use one of the explicit fallbacks instead: +After registration, start a new agent session in your project and ask: -- start the server with a single bootstrap path -- set `CODEBASE_ROOT` -- retry tool calls with `project` +> Use Codebase Context to find [feature] in this repository. Pass this repository's absolute path as project when checking get_indexing_status and searching. Wait for indexing if needed, read codebase://context, then search_codebase and open a returned source file. Show me the relevant files. -If multiple projects are available and no active project can be inferred safely, the server returns `selection_required` instead of guessing. +The first run downloads a local model and indexes the repository; time depends on repository size, CPU and model. You can prepare it separately with `npx -y codebase-context reindex` from the project folder. Index readiness, client connection and useful agent behavior are separate checks. ## Claude Code +**Status:** Connected locally + +**Checked:** 2026-10-06 with Claude Code 2.1.291 + +**Scope:** User scope makes the registration available across projects. The agent selects the repository in tool calls. + +User setup: + ```bash -claude mcp add codebase-context -- npx -y codebase-context +claude mcp add --scope user --transport stdio codebase-context -- npx -y codebase-context ``` -Claude Code only supports stdio. HTTP is not available for this client. +Optional shared project setup: -## Claude Desktop +```bash +claude mcp add --scope project --transport stdio codebase-context -- npx -y codebase-context +``` -Add to `claude_desktop_config.json`: +`--scope project` writes shared `.mcp.json` and requires project-server approval. `--scope local` instead stores a private association for the current project in `~/.claude.json`; it does not write `.mcp.json`. -```json -{ - "mcpServers": { - "codebase-context": { - "command": "npx", - "args": ["-y", "codebase-context"] - } - } -} +**What was checked:** With isolated user configuration, the no-folder user command wrote the expected stdio arguments and `claude mcp get` reported Connected. + +**Limit:** Connection proof does not establish an agent investigation. The optional shared-project variant follows the official scope contract; it was not exercised in this October check. + +**Official docs:** https://code.claude.com/docs/en/mcp + +## Codex CLI + +**Status:** Config written locally + +**Checked:** 2026-10-06 with Codex CLI 0.160.0 + +**Scope:** The command writes user config. A trusted project can use a project `.codex/config.toml` instead. + +```bash +codex mcp add codebase-context -- npx -y codebase-context ``` -Claude Desktop only supports stdio. +Equivalent project config: -## Cursor +```toml +[mcp_servers.codebase-context] +command = "npx" +args = ["-y", "codebase-context"] +startup_timeout_sec = 120 +``` -**Stdio** — add to `.cursor/mcp.json` in your project (copy from [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json)): +**What was checked:** The command ran with an isolated `CODEX_HOME`, wrote `config.toml`, and `codex mcp get` returned the same command and arguments. -```json -{ - "mcpServers": { - "codebase-context": { - "command": "npx", - "args": ["-y", "codebase-context"] - } - } -} +**Limit:** The CLI command is user-scoped. A trusted project can instead use .codex/config.toml. The double dash is required. + +**Official docs:** [Codex MCP configuration](https://learn.chatgpt.com/docs/extend/mcp?surface=cli). The supported `startup_timeout_sec` option defaults to ten seconds; the project example allows 120 seconds for initial `npx` package preparation. This does not prove Desktop startup or guarantee completion of a cold download. + +## Codex Desktop recipe + +Create or merge `.codex/config.toml` in the project you want to search. Preserve any existing settings: + +```toml +[mcp_servers.codebase-context] +command = "npx" +args = ["-y", "codebase-context"] +startup_timeout_sec = 120 ``` -**HTTP** — start the server first, then add to `.cursor/mcp.json` (copy from [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json)): +Trust the project if asked, restart Codex and start a new task there. Use the first-use prompt above so the agent selects the current repository. The CLI command in the preceding section writes user-level config; this recipe uses a trusted project's config instead. -```json -{ - "mcpServers": { - "codebase-context": { - "type": "http", - "url": "http://127.0.0.1:3100/mcp" - } - } -} +**Verification boundary:** isolated Windows CLI and SDK/MCP checks on October 5 exercised published `2.2.0` on a six-file fixture; the exact `npx -y codebase-context@2.2.0 ` entrypoint passed with a warmed isolated npm cache and prepared index. This is historical evidence for `2.2.0`, not verification of `2.4.0`, a fully cold download or a fresh Codex Desktop agent session. + +On October 6 the exact published `2.2.0` `npx` entrypoint was tested without a folder, environment root or MCP roots, launched from an unrelated directory. That check does not verify the `2.4.0` package. Its first tool call returned `selection_required`; an explicit `project` call selected the prepared fixture, search found the expected symbol, context returned its map, and the runner host opened the returned source. A later call without `project` retained that selection. This is SDK/protocol and runner-host evidence with warmed caches and a pre-indexed fixture, not a native Desktop task. + +## Interactive setup wizard + +> **Requires 2.4.0 (not yet published)**: The non-interactive `--client` and `--yes` flags are not available in published 2.2.0 (which runs an interactive HTTP wizard). These commands require the unpublished 2.4.0 release and will not work with the current npm release. + +Run setup from the project you want the client to search: + +```bash +# Claude Code (requires 2.4.0, not yet published) +npx -y codebase-context@2.4.0 init --client claude-code --yes + +# Codex (requires 2.4.0, not yet published) +npx -y codebase-context@2.4.0 init --client codex --yes ``` -## Windsurf +Claude Code writes `.mcp.json`; Codex writes `.codex/config.toml`. Both configs start the installed server with the selected project. Setup uses the current folder unless `--root` or `CODEBASE_ROOT` selects another one; `--root` takes priority. -Open Settings > MCP and add (stdio only — HTTP is not documented for Windsurf yet): +`--yes` writes the config without prompts. In 2.2.0, running `init` without flags starts the older interactive wizard. Cursor and OpenCode still use HTTP and need a separately running server. -```json -{ - "mcpServers": { - "codebase-context": { - "command": "npx", - "args": ["-y", "codebase-context"] - } - } -} +Existing settings are kept. If a different Codebase Context entry is already present, setup leaves the file unchanged and reports the conflict. Restart the client after setup and approve the server if asked. The first use builds the code index. Rerun setup if the Codebase Context installation moves, because the config uses its installed location. + +## Optional fixed-folder setup and CLI preparation + +To set a default project before the first tool call, append its full path to the server arguments, for example `args = ["-y", "codebase-context", "C:/projects/my-app"]`. Use a quoted literal shell argument in registration commands. This makes the default fixed across repositories when the registration is user-scoped; explicit tool selection can still choose another allowed project. + +To prepare the index separately, run from the project folder: + +```powershell +npx -y codebase-context reindex ``` -## Codex +The interactive wizard limitations described above are still present in the current source. A successful wizard exit alone does not establish working client registration. + +## Gemini CLI + +**Status:** Config written locally + +**Checked:** 2026-08-11 with Gemini CLI 0.41.1 -**Stdio:** +**Scope:** User or project. + +User setup: + +```bash +gemini mcp add --scope user codebase-context npx -y codebase-context +``` + +Project setup: ```bash -codex mcp add codebase-context npx -y codebase-context +gemini mcp add --scope project codebase-context npx -y codebase-context ``` -**HTTP** — start the server first (`npx -y codebase-context --http`), then save a config file and pass it: +Gemini writes an `mcpServers` entry in its settings. Claude, Codex, and Copilot CLI use `--`; Gemini does not. + +**What was checked:** Both commands wrote the expected config in isolated homes and projects. The user-scoped list found the server as disconnected; no live agent connection was claimed. + +**Limit:** Gemini does not use the double-dash separator. The isolated project-scoped list did not enumerate the project entry, so only config generation is proved. + +**Official docs:** https://geminicli.com/docs/tools/mcp-server/ + +## Cursor + +**Status:** Config recognized locally + +**Checked:** 2026-08-11 with Cursor 3.15.6 and Cursor Agent 2026.04.17-787b533 + +**Scope:** Project. + +Create `.cursor/mcp.json`: ```json { "mcpServers": { "codebase-context": { - "type": "http", - "url": "http://127.0.0.1:3100/mcp" + "command": "npx", + "args": ["-y", "codebase-context"] } } } ``` -```bash -codex --mcp-config /path/to/mcp-http.json -``` +Checked config shape: `.cursor/mcp.json with mcpServers.codebase-context command npx and args [-y, codebase-context]`. + +**What was checked:** Cursor Agent loaded the isolated workspace config and showed the server pending approval. + +**Limit:** User approval is required. The official one-click route exists, but a generated deep link was not accepted as proof. + +**Official docs:** https://docs.cursor.com/context/model-context-protocol -## VS Code (Copilot) +## VS Code with GitHub Copilot -Add `.vscode/mcp.json` to your project root. VS Code uses `servers` instead of `mcpServers`: +**Status:** Official instructions only + +**Checked:** 2026-08-11 against VS Code 1.125.1 help and current official docs + +**Scope:** User command or workspace config; not locally runtime-tested. + +Official user command: + +```powershell +code --add-mcp '{"name":"codebase-context","command":"npx","args":["-y","codebase-context"]}' +``` + +Deterministic workspace config at `.vscode/mcp.json`: ```json { @@ -138,9 +217,47 @@ Add `.vscode/mcp.json` to your project root. VS Code uses `servers` instead of ` } ``` +**What was checked:** Local CLI help exposes `--add-mcp`. An isolated GUI launch could not provide deterministic completion, so it was not counted as a local runtime check. + +**Limit:** The command is official but not accepted as a completed local runtime test. Workspace config remains the deterministic manual fallback. + +**Official docs:** https://code.visualstudio.com/docs/agent-customization/mcp-servers + +## GitHub Copilot CLI + +**Status:** Config written locally + +**Checked:** 2026-08-11 with GitHub Copilot CLI 1.0.24 + +**Scope:** User config. + +```bash +copilot mcp add codebase-context -- npx -y codebase-context +``` + +**What was checked:** The command ran with an isolated `--config-dir` and wrote the expected `mcp-config.json` command, arguments, and tool filter. + +**Limit:** The CLI writes user configuration by default. VS Code workspace mcp.json is a separate surface. + +**Official docs:** https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-mcp-servers + ## OpenCode -Add `opencode.json` to your project root: +**Status:** Connected locally + +**Checked:** 2026-10-06 with stable OpenCode 1.18.34 + +**Scope:** User registration by default. + +Run once: + +```bash +opencode mcp add codebase-context '--' npx -y codebase-context +``` + +Keep the quoted `'--'`: the installed Windows PowerShell npm shim consumes an unquoted separator. The quoted form also works in POSIX shells. + +For a project-local alternative, merge this into `opencode.json`: ```json { @@ -149,72 +266,102 @@ Add `opencode.json` to your project root: "codebase-context": { "type": "local", "command": ["npx", "-y", "codebase-context"], - "enabled": true + "enabled": true, + "timeout": 120000 } } } ``` -OpenCode also supports interactive setup via `opencode mcp add`. +Checked config shape: `opencode.json with a local command array and timeout 120000`. -## Single-project fallback +The longer timeout allows for initial `npx` package preparation and server startup; that time varies with the machine, network and cache. -If you only use one repo, append a project path: +**What was checked:** With home, config, data, cache, temporary and workspace paths redirected, the no-folder command wrote user configuration and `opencode mcp list` reported Connected. An August 11 check with OpenCode 1.18.5 also connected using the project config shape and an explicit folder; the no-folder project variant is not a new native acceptance check. -```bash -codex mcp add codebase-context npx -y codebase-context "/path/to/your/project" -``` +**Limit:** Stable OpenCode 1.x and beta OpenCode 2 use different native config shapes. The stable V1 shape is the public default; do not mix schemas. -Or set an environment variable: +**Official docs:** https://opencode.ai/docs/mcp-servers/ -```bash -CODEBASE_ROOT=/path/to/your/project -``` +## Windsurf -## Test a local build +**Status:** Official instructions only -Build the local branch first: +**Checked:** 2026-08-11 against current official docs; Windsurf was not installed locally -```bash -pnpm build -``` +**Scope:** User config. -Then point your MCP client at the local build: +Create `~/.codeium/windsurf/mcp_config.json`: ```json { "mcpServers": { "codebase-context": { - "command": "node", - "args": ["/dist/index.js"] + "command": "npx", + "args": ["-y", "codebase-context"] } } } ``` -If the default setup is not enough for your client, pass a project path explicitly: +Checked config shape: `~/.codeium/windsurf/mcp_config.json with mcpServers.codebase-context command npx and args [-y, codebase-context]`. + +**What was checked:** The config shape was checked against the current official page. The local client was unavailable. + +**Limit:** Current ownership and local behavior are not independently verified on this machine; publish as official config, not locally tested. + +**Official docs:** https://docs.windsurf.com/windsurf/cascade/mcp + +## Optional advanced HTTP server + +Use `stdio` for the simplest setup. HTTP can let several clients share one long-running local process, but support depends on the client and this path is not part of the default setup proof: + +```bash +npx -y codebase-context --http +npx -y codebase-context --http --port 4000 +``` + +The documented default endpoint is `http://127.0.0.1:3100/mcp`. Config-shape templates are available in [`templates/mcp/stdio/.mcp.json`](../templates/mcp/stdio/.mcp.json) and [`templates/mcp/http/.mcp.json`](../templates/mcp/http/.mcp.json); the recipes use the current npm release. HTTP does not discover a client's project folder; the agent still has to select its repository. + +The [MCP July 28, 2026 HTTP revision](https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http) removed protocol sessions and the GET stream endpoint. This source checkout uses SDK 1.27.1 and the earlier protocol/session model. The stdio checks do not validate its concurrent HTTP project isolation or compatibility with the new revision. A protocol migration and host testing are separate from these setup recipes. + +Client support for HTTP differs. Follow that client's official docs and keep stdio as the fallback. + +## Project routing contract + +The documented default is agent-directed selection. Start the stdio server without a folder; call `get_indexing_status` with the current repository's absolute path as `project`, wait for readiness and pass the same `project` when searching. A successful selection becomes the default for later calls in that server process, including `codebase://context`. Select another project explicitly when changing repositories. This lets one registration serve multiple projects without changing client configuration. + +CBC deliberately does not index the server process's working directory when no project is configured. Without a configured folder, known root or explicit selection, tools return `selection_required`. They do not guess or index the user's home directory. + +Some clients also announce workspace roots. One valid root can auto-select; several roots with no active selection return `selection_required` and available projects. Explicit selectors must stay within announced roots. Relative monorepo paths require a known workspace context; use absolute repository paths for the portable first-use recipe. + +[MCP deprecated Roots on July 28, 2026](https://modelcontextprotocol.io/specification/2026-07-28/client/roots), recommending explicit directories/files in tool parameters, resource URIs or server configuration. CBC's older Roots support remains a compatibility aid rather than a required setup step. + +October 6 verification: 18 focused current-source routing tests passed, including ambiguous roots, explicit selection, subsequent calls and root changes. The exact published `2.2.0` server also switched between two isolated prepared repositories A to B to A in one no-roots stdio process; search paths, context map and subsequent status calls followed each selection, and the runner host opened the expected distinct sources. This historical package check does not verify newer source changes or establish concurrent HTTP isolation. A native agent session across real repositories remains a validation step; stdio process-local selection does not establish concurrent HTTP isolation. + +A follow-up recorded raw `search_codebase` calls containing only `query` and `mode`, with no `project`: they returned A, B and A after those explicit selections. The caller does not automatically add the full path; CBC routes omitted-project calls using its selected project. This state lasts for that stdio process. Select again after a server restart or when changing repositories. + +A native Codex CLI controlled-fixture trial with an ordinary question supplied the correct absolute `project` without path instructions. Its MCP call was denied by the noninteractive run's `never` approval policy, so native retrieval and continuation did not pass. If a host refuses a tool call for approval, repeating the folder path will not resolve that permission issue; use the client's trust/approval flow. The test retained the denial and did not count fallback source/index reads as CBC acceptance. + +## Test a local build + +Build the source checkout: + +```bash +pnpm build +``` + +Then replace the `npx` command in your client config with the local entry point: ```json { "mcpServers": { "codebase-context": { "command": "node", - "args": ["/dist/index.js", "/path/to/your/project"] + "args": ["/dist/index.js"] } } } ``` -Check these three flows: - -1. **Single project** — call `search_codebase` or `metadata`. Routing is automatic. - -2. **Multiple projects on a roots-capable host** — open two repos or a monorepo. Call `codebase://context`. Expected: workspace overview, then automatic routing once a project is active. - -3. **Ambiguous or no-roots selection** — start without a bootstrap path, call `search_codebase`. Expected: `selection_required`. Retry with `project` set to `apps/dashboard` or `/repos/customer-portal`. - -For monorepos, test all three selector forms: - -- relative subproject path: `apps/dashboard` -- repo path: `/repos/customer-portal` -- file path: `/repos/monorepo/apps/dashboard/src/auth/guard.ts` +For one explicit project, append its path to the arguments or set `CODEBASE_ROOT`. diff --git a/docs/comparison-table.md b/docs/comparison-table.md index aee2eaa..1b55f11 100644 --- a/docs/comparison-table.md +++ b/docs/comparison-table.md @@ -1,35 +1,26 @@ -# Comparator Summary +# Corrected retrieval comparison -This table summarizes the current comparator evidence from `results/comparator-evidence.json`. -It is a setup-status table first, not a marketing scoreboard. +This table summarizes the corrected retrieval-only family across five local code-context tools. In this fixed setup, Codebase Context recovered 25.7% of expected gold files at 11.5% file precision; this does not show better completed coding tasks. Expected-file coverage means expected gold files recovered divided by expected gold files. File precision means expected gold files recovered divided by all files returned. The reported `peakPrivateGb` field is copied from retained telemetry; its underlying unit was not independently verified. -| Comparator | Intended role in gate | Current status | Evidence summary | -| --- | --- | --- | --- | -| `raw Claude Code` | Baseline for payload cost and at least one usefulness comparison | comparator artifact: `ok`; gate: `pending_evidence` | The Haiku-backed Claude CLI runner now returns current payloads, but the checked-in baseline still has `averageFirstRelevantHit: null`, so the gate still records missing baseline metrics. | -| `GrepAI` | Named MCP comparator | `setup_failed` | Requires the GrepAI binary plus a local Ollama embedding setup that is not present in this proof environment. | -| `jCodeMunch` | Named MCP comparator | `setup_failed` | The MCP server still closes on startup during the current rerun, so no comparable discovery metrics were produced. | -| `codebase-memory-mcp` | Named MCP comparator | comparator artifact: `ok`; gate: `failed` | The repaired graph-backed runner now produces real current metrics, but the frozen gate still fails this lane because `codebase-context` does not stay within tolerance on every required usefulness metric. | -| `CodeGraphContext` | Graph-native comparator in the relaunch frame | `setup_failed` | The MCP server still closes on startup during the current rerun, so this lane remains missing evidence. | +| Tool | Completed | Failed | Expected-file coverage | File precision | Reported `peakPrivateGb` | +| --- | ---: | ---: | ---: | ---: | ---: | +| Raw search | 20/20 | 0 | 4.3% | 4.0% | 0.06 | +| Codebase Context | 20/20 | 0 | 25.7% | 11.5% | 2.90 | +| jCodeMunch | 20/20 | 0 | 27.1% | 11.0% | 0.33 | +| Repowise | 19/20 | 1 | 20.1% (failed run = 0) | 12.0% | 4.82 | +| context-mode | 20/20 | 0 | 16.8% | 11.7% | 0.15 | -## Reading This Table +## Reading notes -- `setup_failed` means the lane was attempted and did not reach a credible metric-producing state. -- `pending_evidence` in the gate means the lane is still missing one or more required metrics. -- `failed` in the gate means the lane has real metrics, but the frozen comparison rule still does not pass. -- A missing metric is not treated as a win for `codebase-context`. -- The combined gate in `results/gate-evaluation.json` remains `pending_evidence`, and `claimAllowed` stays `false`, until these lanes produce real metrics. +- Expected-file coverage is the percentage of expected gold files recovered, averaged within the corrected retrieval-only family. +- File precision is the percentage of returned files that matched the expected gold files. +- The table covers 100 attempts: 99 completed and 1 failed. Repowise completed 19 of 20 attempts; the other tools completed 20 of 20. +- Repowise's 20.1% is failure-inclusive: its failed attempt is counted as zero. +- Raw search is a deterministic lexical adapter in this protocol, not a full normal coding-agent baseline. +- `peakPrivateGb` is a reported telemetry field, not a verified unit claim, install size, or steady-state idle memory. +- The full-agent pilot, paired token observations, repeated retrieval history, and metered replay are separate evidence families. Their values should not be added to this table; the replay remains excluded while tool-use validity is unresolved. +- The measurements do not establish patch correctness or end-to-end task completion. They also do not establish a universal token, time, or coding result. -## Current codebase-context result +The [sanitized evidence extract](../results/benchmark-presentation-evidence.json) records the exact values and source hashes. -For reference, the current combined discovery output across `angular-spotify` and `excalidraw` is: - -| Metric | codebase-context | -| --- | ---: | -| `totalTasks` | 24 | -| `averageUsefulness` | 0.75 | -| `averagePayloadBytes` | 7306.4583 | -| `averageEstimatedTokens` | 1827.0833 | -| `bestExampleUsefulnessRate` | 0.125 | -| `gate.status` | `pending_evidence` | - -Those numbers are not compared here as head-to-head wins because the comparator lanes above did not produce matching metrics. +Historical note: the [earlier public report archive](./benchmark-prior-public-report.md) contains a one-task official pilot and a separate 24-task discovery report; it remains separate and keeps its existing claim gates. diff --git a/docs/demo.md b/docs/demo.md index 825b2ce..407fd82 100644 --- a/docs/demo.md +++ b/docs/demo.md @@ -1,13 +1,25 @@ # Demo Script -This walkthrough uses real CLI output captured against `repos/angular-spotify` during the Phase 10 proof rerun. -Run it from the repo root with `CODEBASE_ROOT` pointed at the frozen sample repo. The public flow is simple: start with the conventions map, then search for the local example you need. +This walkthrough shows real CLI output captured from the open-source `angular-spotify` repository during a local proof run. That sample checkout is not bundled here. + +These commands use the current npm release. The output shown below is historical and does not establish behavior of that release. + +To try the same CLI flow, use Node.js 22 or newer and open a terminal in the repository you want to inspect. These commands target `codebase-context`; the CLI uses the current directory as the project root. MCP registration and project selection are covered in the [client setup guide](./client-setup.md). + +The saved output below is from an earlier CLI run, not from the current commands. The machine-specific repository root is omitted from the returned file path. This capture does not verify the current package or an MCP connection. + +## 0. Build The Local Index + +```bash +npx -y codebase-context reindex +``` + +Run this before the first map or search in a repository. Later runs can use `reindex --incremental` after the code changes. ## 1. Start With The Conventions Map ```bash -$env:CODEBASE_ROOT='C:\Users\bitaz\Repos\codebase-context\repos\angular-spotify' -node dist/index.js map --json +npx -y codebase-context map --json ``` Captured output excerpt: @@ -40,8 +52,7 @@ What this shows: ## 2. Search With Edit Intent ```bash -$env:CODEBASE_ROOT='C:\Users\bitaz\Repos\codebase-context\repos\angular-spotify' -node dist/index.js search --query "auth headers" --intent edit --limit 3 --json +npx -y codebase-context search --query "auth headers" --intent edit --limit 3 --json ``` Captured output excerpt: @@ -66,7 +77,7 @@ Captured output excerpt: }, "results": [ { - "file": "C:\\Users\\bitaz\\Repos\\codebase-context\\repos\\angular-spotify\\libs\\web\\auth\\util\\src\\lib\\interceptors\\auth.interceptor.ts:10-42", + "file": "repos/angular-spotify/libs/web/auth/util/src/lib/interceptors/auth.interceptor.ts:10-42", "type": "interceptor:core" } ] @@ -82,8 +93,7 @@ What this shows: ## 3. Check A Team Pattern Directly ```bash -$env:CODEBASE_ROOT='C:\Users\bitaz\Repos\codebase-context\repos\angular-spotify' -node dist/index.js patterns --category state --json +npx -y codebase-context patterns --category state --json ``` Captured output excerpt: @@ -117,4 +127,5 @@ What this shows: ## Caveats - These excerpts were captured from the current local proof run and will change if the frozen sample repo or index state changes. -- The discovery benchmark gate is still `pending_evidence`, and `claimAllowed` remains `false`, so this walkthrough demonstrates shipped behavior, not a released performance claim. +- File paths in the excerpts belong to that capture; your output will use the repository you run the command from. +- The benchmark documents observed local evidence and its limitations; this walkthrough demonstrates shipped behavior, not a universal performance or coding result. diff --git a/docs/review-context.md b/docs/review-context.md new file mode 100644 index 0000000..964a9b4 --- /dev/null +++ b/docs/review-context.md @@ -0,0 +1,127 @@ +# Review context + +`codebase-context-review` turns a committed git diff into a bounded review-context packet. + +It is deliberately **not** an AI reviewer. It does not call an LLM, decide whether code is correct, or post comments. Its job is narrower and testable: compile the changed surface, stable diff signals, related codebase context, and current conventions into a versioned input for a reviewer or evaluation harness. + +The packaged review CLI requires the unpublished 2.4.0 release (the `codebase-context-review` executable does not exist in published 2.2.0); until then, use the source-build command shown below. + +## Why this exists + +A PR reviewer needs more than the patch, but dumping an entire repository into a model is expensive and hard to reproduce. The review-context path creates a clean boundary between: + +1. **context compilation** — git + local repository analysis +2. **review reasoning** — any model or human reviewer consuming the packet +3. **evaluation** — measuring whether the context actually helps find the right files/spans/issues + +Keeping those layers separate makes failures diagnosable. A bad review can be attributed to missing context, bad reasoning, or both instead of hiding every variable behind one agent loop. + +## Usage + +Build and run from the repository: + +```bash +pnpm build +node dist/review-bin.js --base origin/main --head HEAD +``` + +Requires 2.4.0 (not yet published): after package publication, the package also exposes: + +```bash +npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review --base origin/main --head HEAD +``` + +Use `--json` for the complete machine-readable packet (requires 2.4.0, not yet published): + +```bash +npm exec --yes --package=codebase-context@2.4.0 -- codebase-context-review \ + --base origin/main \ + --head HEAD \ + --max-queries 8 \ + --max-results 3 \ + --json > review-context.json +``` + +The command uses merge-base diff semantics (`base...head`), which matches the normal pull-request question: what changed on this branch since it diverged from the base branch? + +`review-context-v1` requires `--head` to resolve to the checked-out `HEAD` and requires a clean working tree. The diff itself is then generated from the already-resolved commit SHAs rather than mutable symbolic refs. + +## Packet contract + +The current schema is `review-context-v1`. + +The packet contains: + +- exact resolved base/head commit SHAs +- SHA-256 fingerprint of Git's exact raw diff output +- changed file status, additions/deletions, rename source, and binary flag +- NUL-safe filename parsing for unusual valid git paths +- identifiers extracted deterministically from changed lines +- bounded search queries derived from those diff signals +- bounded related-context results from the existing `search_codebase` engine +- edit-preflight/search-quality metadata when the search engine provides it +- current team-pattern/convention output +- explicit warnings when context could not be produced + +Absolute local repository paths are not part of the packet contract. + +The git envelope, identifier extraction, bounds, and query derivation are deterministic. Related-context ranking is only reproducible under the same `codebase-context` version, index contents, embedding/reranking configuration, and runtime dependencies; benchmark manifests should freeze those inputs rather than pretending the entire retrieval stack is environment-independent. + +## Bounds + +Defaults: + +| Bound | Default | +| --- | ---: | +| Changed-file search queries | 8 | +| Related results per query | 3 | +| Identifier candidates per file | 10 | +| Snippet characters per related result | 1,200 | + +These are explicit because an unbounded context compiler is not useful evidence. Increasing them is allowed, but benchmark runs should record the values and compare like-for-like. + +## Index behavior + +By default, the command creates an index when one is missing and runs an incremental refresh when one already exists. This keeps normal use aligned with the checked-out clean `HEAD`. + +Pass `--no-index` to prohibit both creation and refresh. It fails if no index exists. This is useful in controlled benchmark runs where setup/index work is captured separately; the caller is then responsible for proving that the supplied index matches the frozen source state. + +The command is local-first. It invokes git and the existing local index/search pipeline; it does not introduce an LLM or external review API. + +## What this proves + +Shipping this command proves only that the project can compile a bounded, versioned review-oriented context packet from an exact committed git range. + +It **does not** prove that the packet improves review quality, catches more bugs, reduces false positives, or beats another context strategy. Those are benchmark claims and remain blocked until measured. + +## Evaluation path + +The intended evaluation is an ablation, not a marketing benchmark: + +1. freeze a set of public PR/bug-fix tasks before observing lane outputs +2. run the same reviewer/model with **raw diff only** +3. run the same reviewer/model with **raw diff + `review-context-v1` packet** +4. keep model, prompt, turn/token budget, timeout, and scoring fixed +5. score bug/finding recall and precision, evidence quality, false positives, token cost, and wall time separately +6. report setup/index failures as failures, not as competitor losses +7. do not publish a superiority claim from a one-task pilot + +The existing ContextBench protocol already follows the same evidence discipline for repository-context retrieval. Review-specific claims should meet at least the same standard rather than creating a weaker parallel benchmark. + +## Current limitations + +- Only committed git refs are supported in v1; working-tree/staged review is intentionally deferred. +- `--head` must be the checked-out commit and the worktree must be clean. +- Query generation is lexical and deterministic. It extracts identifiers from changed lines and falls back to path signals; it is not AST-aware yet. +- `--no-index` deliberately skips freshness work, so benchmark callers must attest the index/source match themselves. +- Large diffs are bounded by the CLI git-buffer limit and fail rather than silently truncating the raw fingerprint input. +- Binary files are recorded but do not generate identifier-based queries. +- The packet is context, not a verdict. A consumer should never turn `preflight.ready` into "the change is correct." + +## Next gate + +Do not add reviewer-agent features until a small frozen public evaluation can answer this first question: + +> Does `review-context-v1` improve relevant-file/span coverage at an acceptable precision and context cost compared with the same reviewer using the raw diff/repository tools alone? + +If the answer is no, fix or kill this lane before adding more orchestration. diff --git a/package.json b/package.json index 00aeb98..f7ca7f5 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,10 @@ "import": "./dist/analyzers/nextjs/index.js", "types": "./dist/analyzers/nextjs/index.d.ts" }, + "./analyzers/nestjs": { + "import": "./dist/analyzers/nestjs/index.js", + "types": "./dist/analyzers/nestjs/index.d.ts" + }, "./analyzers/generic": { "import": "./dist/analyzers/generic/index.js", "types": "./dist/analyzers/generic/index.d.ts" @@ -40,7 +44,8 @@ } }, "bin": { - "codebase-context": "./dist/index.js" + "codebase-context": "./dist/index.js", + "codebase-context-review": "./dist/review-bin.js" }, "files": [ "dist", @@ -50,11 +55,12 @@ "docs/cli.md", "docs/capabilities.md", "docs/client-setup.md", + "docs/review-context.md", "templates" ], "packageManager": "pnpm@10.27.0", "engines": { - "node": ">=18.0.0" + "node": ">=22.0.0" }, "keywords": [ "mcp", @@ -87,6 +93,7 @@ "windsurf", "angular", "react", + "nestjs", "typescript", "developer-tools", "static-analysis", @@ -128,8 +135,8 @@ "prepare": "husky" }, "dependencies": { + "@huggingface/transformers": "^4.2.0", "@inquirer/prompts": "^3.0.0", - "@huggingface/transformers": "^3.8.1", "@lancedb/lancedb": "^0.4.0", "@modelcontextprotocol/sdk": "^1.27.1", "@typescript-eslint/typescript-estree": "^7.0.0", @@ -172,7 +179,7 @@ "@modelcontextprotocol/sdk>ajv": "8.18.0", "@modelcontextprotocol/sdk>@hono/node-server": "1.19.13", "@modelcontextprotocol/sdk>express-rate-limit": "8.2.2", - "@huggingface/transformers>onnxruntime-node": "1.24.2", + "@huggingface/transformers>onnxruntime-node": "1.24.3", "path-to-regexp": "8.4.0", "brace-expansion": "5.0.5", "micromatch>picomatch": "2.3.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2c6e5ab..1a5354e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,7 +8,7 @@ overrides: '@modelcontextprotocol/sdk>ajv': 8.18.0 '@modelcontextprotocol/sdk>@hono/node-server': 1.19.13 '@modelcontextprotocol/sdk>express-rate-limit': 8.2.2 - '@huggingface/transformers>onnxruntime-node': 1.24.2 + '@huggingface/transformers>onnxruntime-node': 1.24.3 path-to-regexp: 8.4.0 brace-expansion: 5.0.5 micromatch>picomatch: 2.3.2 @@ -24,8 +24,8 @@ importers: .: dependencies: '@huggingface/transformers': - specifier: ^3.8.1 - version: 3.8.1 + specifier: ^4.2.0 + version: 4.2.0 '@inquirer/prompts': specifier: ^3.0.0 version: 3.3.2 @@ -49,7 +49,7 @@ importers: version: 10.5.0 hono: specifier: ^4.12.12 - version: 4.12.14 + version: 4.12.16 ignore: specifier: ^5.3.1 version: 5.3.2 @@ -324,12 +324,15 @@ packages: peerDependencies: hono: '>=4.12.12' - '@huggingface/jinja@0.5.5': - resolution: {integrity: sha512-xRlzazC+QZwr6z4ixEqYHo9fgwhTZ3xNSdljlKfUFGZSdlvt166DljRELFUfFytlYOYvo3vTisA/AFOuOAzFQQ==} + '@huggingface/jinja@0.5.9': + resolution: {integrity: sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw==} engines: {node: '>=18'} - '@huggingface/transformers@3.8.1': - resolution: {integrity: sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==} + '@huggingface/tokenizers@0.1.3': + resolution: {integrity: sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==} + + '@huggingface/transformers@4.2.0': + resolution: {integrity: sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==} '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} @@ -377,89 +380,105 @@ packages: resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-arm@1.2.4': resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-ppc64@1.2.4': resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-riscv64@1.2.4': resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-s390x@1.2.4': resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-libvips-linux-x64@1.2.4': resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-libvips-linuxmusl-arm64@1.2.4': resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-libvips-linuxmusl-x64@1.2.4': resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-linux-arm64@0.34.5': resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [glibc] '@img/sharp-linux-arm@0.34.5': resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm] os: [linux] + libc: [glibc] '@img/sharp-linux-ppc64@0.34.5': resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@img/sharp-linux-riscv64@0.34.5': resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [riscv64] os: [linux] + libc: [glibc] '@img/sharp-linux-s390x@0.34.5': resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [s390x] os: [linux] + libc: [glibc] '@img/sharp-linux-x64@0.34.5': resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [glibc] '@img/sharp-linuxmusl-arm64@0.34.5': resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [arm64] os: [linux] + libc: [musl] '@img/sharp-linuxmusl-x64@0.34.5': resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} cpu: [x64] os: [linux] + libc: [musl] '@img/sharp-wasm32@0.34.5': resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} @@ -552,12 +571,14 @@ packages: engines: {node: '>= 18'} cpu: [arm64] os: [linux] + libc: [glibc] '@lancedb/lancedb-linux-x64-gnu@0.4.20': resolution: {integrity: sha512-TVicKBhT3Q73gtWmMdR/0+r4gBve5gw3KbcrvzHP/5bNpo2sffETbLEMgLjulunu/Jp8IgC921WW6IFJzNFwrQ==} engines: {node: '>= 18'} cpu: [x64] os: [linux] + libc: [glibc] '@lancedb/lancedb-win32-x64-msvc@0.4.20': resolution: {integrity: sha512-7RsTJAx93NbVD//9m0/TpgQmPsTHlCBfGNFD2zkdZ0j7iacURdG9+egz+Os0veTBqMgwnr6VfMf3T7S0wuu99g==} @@ -661,66 +682,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -1596,8 +1630,8 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hono@4.12.14: - resolution: {integrity: sha512-am5zfg3yu6sqn5yjKBNqhnTX7Cv+m00ox+7jbaKkrLMRJ4rAdldd1xPd/JzbBWspqaQv6RSTrgFN95EsfhC+7w==} + hono@4.12.16: + resolution: {integrity: sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==} engines: {node: '>=16.9.0'} http-errors@2.0.1: @@ -1956,18 +1990,18 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: - resolution: {integrity: sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: + resolution: {integrity: sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==} - onnxruntime-common@1.24.2: - resolution: {integrity: sha512-S0FFhJaI05jr1c3HVJ/DuPFB/aYdXmnUBuuQfuvLtcNn7WAfpm2ewSXn1vHs9Wa1l8T8OznhfCEdFv8qCn0/xw==} + onnxruntime-common@1.24.3: + resolution: {integrity: sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==} - onnxruntime-node@1.24.2: - resolution: {integrity: sha512-ZtTkUHNNk+Xpi2Sq2BcFjzc9Vx4n3o5RxYLgRvdrFzCBsGUU1dQRFf4LM9tCc7vFhPSoZqbvzZtkzRMXoDqKNg==} + onnxruntime-node@1.24.3: + resolution: {integrity: sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==} os: [win32, darwin, linux] - onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: - resolution: {integrity: sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==} + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: + resolution: {integrity: sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==} openai@4.104.0: resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} @@ -2728,17 +2762,20 @@ snapshots: '@eslint/core': 0.17.0 levn: 0.4.1 - '@hono/node-server@1.19.13(hono@4.12.14)': + '@hono/node-server@1.19.13(hono@4.12.16)': dependencies: - hono: 4.12.14 + hono: 4.12.16 + + '@huggingface/jinja@0.5.9': {} - '@huggingface/jinja@0.5.5': {} + '@huggingface/tokenizers@0.1.3': {} - '@huggingface/transformers@3.8.1': + '@huggingface/transformers@4.2.0': dependencies: - '@huggingface/jinja': 0.5.5 - onnxruntime-node: 1.24.2 - onnxruntime-web: 1.22.0-dev.20250409-89f8206ba4 + '@huggingface/jinja': 0.5.9 + '@huggingface/tokenizers': 0.1.3 + onnxruntime-node: 1.24.3 + onnxruntime-web: 1.26.0-dev.20260416-b7804b056c sharp: 0.34.5 '@humanfs/core@0.19.1': {} @@ -2979,7 +3016,7 @@ snapshots: '@modelcontextprotocol/sdk@1.27.1(zod@4.3.4)': dependencies: - '@hono/node-server': 1.19.13(hono@4.12.14) + '@hono/node-server': 1.19.13(hono@4.12.16) ajv: 8.18.0 ajv-formats: 3.0.1(ajv@8.18.0) content-type: 1.0.5 @@ -2989,7 +3026,7 @@ snapshots: eventsource-parser: 3.0.6 express: 5.2.1 express-rate-limit: 8.2.2(express@5.2.1) - hono: 4.12.14 + hono: 4.12.16 jose: 6.1.3 json-schema-typed: 8.0.2 pkce-challenge: 5.0.1 @@ -4159,7 +4196,7 @@ snapshots: dependencies: function-bind: 1.1.2 - hono@4.12.14: {} + hono@4.12.16: {} http-errors@2.0.1: dependencies: @@ -4481,22 +4518,22 @@ snapshots: dependencies: wrappy: 1.0.2 - onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: {} + onnxruntime-common@1.24.0-dev.20251116-b39e144322: {} - onnxruntime-common@1.24.2: {} + onnxruntime-common@1.24.3: {} - onnxruntime-node@1.24.2: + onnxruntime-node@1.24.3: dependencies: adm-zip: 0.5.16 global-agent: 3.0.0 - onnxruntime-common: 1.24.2 + onnxruntime-common: 1.24.3 - onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: + onnxruntime-web@1.26.0-dev.20260416-b7804b056c: dependencies: flatbuffers: 25.9.23 guid-typescript: 1.0.9 long: 5.3.2 - onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 + onnxruntime-common: 1.24.0-dev.20251116-b39e144322 platform: 1.3.6 protobufjs: 7.5.6 diff --git a/results/benchmark-presentation-evidence.json b/results/benchmark-presentation-evidence.json new file mode 100644 index 0000000..6f3e44f --- /dev/null +++ b/results/benchmark-presentation-evidence.json @@ -0,0 +1,1251 @@ +{ + "schemaVersion": 1, + "artifactType": "benchmark-presentation-source-extract", + "description": "Allowlisted fields extracted from retained reports. This is not a new benchmark run or independent validation of those reports. Full source byte hashes identify the inputs; private filesystem paths and orchestration costs are omitted.", + "provenance": { + "corrected": { + "file": "contextbench-engineering-note-summary.json", + "sha256": "ad8cac1ad86747463212b1665c3458e8f9560fbcde728d3dc2d74d1d6fe3e17f", + "generatedAt": "2026-06-04T08:28:24Z", + "schemaVersion": 2, + "gitRef": "18e0a43a90fe67e9df0d728b2134e9d430469883", + "gitBlobSha256": "a26599e1302ebdbb34e259530c8d775a9325bd42714e5b03398c545ad1abaf6c", + "checkoutByteDifference": "Source copy uses mixed CRLF/LF; normalizing line endings yields the Git blob byte-for-byte (LF)." + }, + "history": { + "file": "contextbench-engineering-note-summary.json", + "sha256": "29c1a03bff051d9bc90284c5585e4f2cc1e94f9cc9da5d01d1b36e73dac4e87e", + "generatedAt": "2026-06-07T19:09:33Z", + "schemaVersion": 4, + "gitRef": "67b7ae2d1541eef3f591a9a58daf5d09ead8dcf1", + "gitBlobSha256": "29c1a03bff051d9bc90284c5585e4f2cc1e94f9cc9da5d01d1b36e73dac4e87e", + "checkoutByteDifference": "CRLF versus LF only" + }, + "pilot": { + "file": "REPORT.claim-session.json", + "sha256": "1c3368185c01a5e52bb7ba55466f9e5a053a26afe896a60b16b80c0743a5bd69", + "generatedAt": "2026-06-12T02:23:14.817Z", + "schemaVersion": 1, + "sessionId": "claim-bearing-2task-5lane-repeat3-claimclean2e21-20260612T002612Z" + } + }, + "correctedRetrieval": { + "sourceFields": [ + "rows", + "qualitySnapshot", + "costSnapshot" + ], + "publicationReadiness": { + "claimBearingPublicationAllowed": false, + "claimScope": "20x1_retrieval_overlap_only", + "unresolvedBlockers": [ + "single_repeat_no_stability_claim", + "retrieval_overlap_only_no_patch_quality", + "provider_dollar_cost_not_measured_because_no_paid_provider_use", + "repowise_one_no_context_evaluator_failure", + "cold_vs_warm_install_dependency_cache_not_fully_normalized" + ], + "publicUseLevel": "narrow_retrieval_overlap_case_study", + "status": "engineering_note_ready_with_claim_blocks", + "blockedClaimTypes": [ + "benchmark winner", + "best tool", + "paper-grade benchmark", + "resource-efficiency claim", + "generalized agent coding quality claim", + "agent patch correctness claim", + "developer productivity claim" + ] + }, + "rows": { + "evaluatorFailedRows": 1, + "repeatsPerTaskLane": 1, + "lanes": 5, + "evaluatorCompletedRows": 99, + "retrievalOverlapRows": 100, + "appendixOnlyLanes": [ + { + "reason": "bounded local indexing readiness failed earlier; not part of this corrected 20x1 headline table", + "laneId": "GrepAI" + } + ], + "sameDenominatorRowsPresent": true, + "tasks": 20 + }, + "qualitySnapshot": [ + { + "laneId": "codebase-context", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.25722222222222224, + "filePrecisionMean": 0.11499999999999999, + "symbolCoverageMean": 0.1396239637555427, + "spanCoverageMean": 0.08573378715275093, + "lineCoverageMean": 0.0878787249873908 + }, + { + "laneId": "context-mode", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.16833333333333333, + "filePrecisionMean": 0.11666666666666665, + "symbolCoverageMean": 0.02731443994601889, + "spanCoverageMean": 0.011708371581594209, + "lineCoverageMean": 0.01103832178724339 + }, + { + "laneId": "jcodemunch-full-mcp", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.2713888888888889, + "filePrecisionMean": 0.11000000000000001, + "symbolCoverageMean": 0.19046586746586747, + "spanCoverageMean": 0.08523281976808295, + "lineCoverageMean": 0.08706990517203586 + }, + { + "laneId": "raw-native", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.043333333333333335, + "filePrecisionMean": 0.04, + "symbolCoverageMean": 0.007186234817813765, + "spanCoverageMean": 0.00461426238619191, + "lineCoverageMean": 0.003664930124865624 + }, + { + "laneId": "repowise-selfhosted-mcp", + "rows": 20, + "evaluatorCompletedRows": 19, + "status": "completed:19,judge_failed:1", + "fileCoverageMean": 0.2013888888888889, + "filePrecisionMean": 0.12, + "symbolCoverageMean": 0.046063588114671705, + "spanCoverageMean": 0.03033601885780799, + "lineCoverageMean": 0.024685983349898594 + } + ], + "costSnapshot": [ + { + "laneId": "codebase-context", + "setupStatuses": "not_required:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 750.8, + "indexSumMinutes": 226.6, + "rowWallSumMinutes": 231.5, + "peakPrivateGb": 2.9, + "modelEvidence": "Xenova/bge-small-en-v1.5:20" + }, + { + "laneId": "context-mode", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 18.6, + "indexSumMinutes": 12.5, + "rowWallSumMinutes": 16, + "peakPrivateGb": 0.15, + "modelEvidence": "unknown:20" + }, + { + "laneId": "jcodemunch-full-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 11.3, + "indexSumMinutes": 4.9, + "rowWallSumMinutes": 10.9, + "peakPrivateGb": 0.33, + "modelEvidence": "unknown:20" + }, + { + "laneId": "raw-native", + "setupStatuses": "not_required:20", + "indexStatuses": "not_required:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 0, + "indexSumMinutes": 0, + "rowWallSumMinutes": 6.8, + "peakPrivateGb": 0.06, + "modelEvidence": "unknown:20" + }, + { + "laneId": "repowise-selfhosted-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "1:20", + "indexMedianSeconds": 294.3, + "indexSumMinutes": 154.2, + "rowWallSumMinutes": 200.7, + "peakPrivateGb": 4.82, + "modelEvidence": "unknown:20" + } + ], + "failedRows": [ + { + "taskId": "Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221", + "laneId": "repowise-selfhosted-mcp", + "status": "judge_failed", + "reason": "official_result_error:no_context_extracted" + } + ] + }, + "repeatedRetrievalAndReplayHistory": { + "sourceFields": [ + "retrievalChecks.sourceRuns", + "publicationReadiness" + ], + "sourceRuns": [ + { + "name": "corrected-20x1", + "checks": 100, + "checksScored": 99, + "checksWithNoUsableContext": 1 + }, + { + "name": "repeat-20x3", + "checks": 300, + "checksScored": 288, + "checksWithNoUsableContext": 12 + }, + { + "name": "metered-context-consumption", + "checks": 100, + "checksScored": 100, + "checksWithNoUsableContext": 0, + "tokenCountersCaptured": 100 + } + ], + "publicationReadiness": { + "claimBearingPublicationAllowed": false, + "claimScope": "500_check_practical_code_context_case_study", + "unresolvedBlockers": [ + "combined_local_evidence_sets_not_preregistered_repeated_protocol", + "retrieval_overlap_and_context_consumption_only_no_patch_quality", + "token_metering_is_context_consumption_not_patch_workflow", + "provider_dollar_cost_not_calculated_from_model_pricing", + "repowise_thirteen_checks_with_no_usable_context", + "cold_vs_warm_install_dependency_cache_not_fully_normalized" + ], + "publicUseLevel": "narrow_code_context_case_study", + "status": "engineering_note_ready_with_claim_blocks", + "blockedClaimTypes": [ + "benchmark winner", + "best tool", + "research-paper benchmark", + "resource-efficiency claim", + "generalized agent coding quality claim", + "agent patch correctness claim", + "developer productivity claim", + "token-savings claim" + ] + }, + "replayCaution": "The historical schema4 summary records completed metering; the later presentation describes no-tool-use replay exclusion. This extract does not reconcile the validity of that replay. No replay token results are presented." + }, + "twoTaskAgentPilot": { + "publicationReadiness": { + "construction": "Publication readiness is a claim gate derived from existing report evidence. It is not owner approval and not a new benchmark score.", + "status": "not_publication_ready", + "publicUseLevel": "engineering_note_or_internal_audit_only", + "claimBearingPublicationAllowed": false, + "claimScope": "pilot_only", + "qualityRowCount": 30, + "qualityTaskCount": 2, + "scoreableQualityLaneCount": 5, + "missingQualityLaneCount": 0, + "missingQualityLaneIds": [], + "reliabilityRowCount": 0, + "repeatConfidenceStatus": "repeat_ready", + "resourceTelemetryGapRows": [], + "unresolvedBlockers": [ + "pilot_task_count" + ], + "blockerActions": [ + { + "blocker": "pilot_task_count", + "action": "Add scoreable quality rows across the required task count before broad benchmark claims." + } + ], + "allowedNonClaimUses": [ + "Describe scoreable quality rows only within their narrow denominator." + ], + "blockedClaimTypes": [ + "benchmark winner", + "paper-grade benchmark", + "four-lane quality comparison", + "resource-efficiency claim", + "generalized agent coding quality claim" + ] + }, + "summary": { + "totalRows": 30, + "taskCount": 2, + "qualityTaskCount": 2, + "laneCount": 5, + "qualityRowCount": 30, + "reliabilityRowCount": 0, + "statusCounts": { + "completed": 30 + }, + "taskIds": [ + "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27" + ], + "qualityTaskIds": [ + "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27" + ], + "laneIds": [ + "codebase-context", + "context-mode", + "jcodemunch-full-mcp", + "raw-native", + "repowise-selfhosted-mcp" + ] + }, + "sourceFields": [ + "qualityRows", + "costRows" + ], + "qualityRows": [ + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.3619047619047619, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.4, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0.02857142857142857, + "symbolPrecision": 0.16666666666666666, + "spanCoverage": 0.00467177045029839, + "spanPrecision": 0.5272108843537415, + "lineCoverage": 0.002026342451874367, + "linePrecision": 0.4, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.3333333333333333, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.2923076923076923, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.3333333333333333, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.4, + "symbolCoverage": 0.10526315789473684, + "symbolPrecision": 0.4, + "spanCoverage": 0.006640106241699867, + "spanPrecision": 0.5272727272727272, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.4, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 1, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 1, + "lineCoverage": 0.002398081534772182, + "linePrecision": 1, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.6909090909090909, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.4, + "symbolCoverage": 0.05263157894736842, + "symbolPrecision": 0.25, + "spanCoverage": 0.006731693913999175, + "spanPrecision": 0.5856573705179283, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.4, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0.10526315789473684, + "symbolPrecision": 0.6666666666666666, + "spanCoverage": 0.004258826761917846, + "spanPrecision": 0.7560975609756098, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.7169811320754716, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.3333333333333333, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.21385542168674698, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.16666666666666666, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.3287037037037037, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.25, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.6909090909090909, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.6666666666666666, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.5, + "filePrecision": 0.5, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0017401657736868617, + "spanPrecision": 0.475, + "lineCoverage": 0.002398081534772182, + "linePrecision": 0.5, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.25, + "filePrecision": 0.3333333333333333, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0009158767229930851, + "spanPrecision": 0.31746031746031744, + "lineCoverage": 0.001199040767386091, + "linePrecision": 0.3333333333333333, + "editlocRecall": 1, + "editlocPrecision": 1 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.6666666666666666, + "symbolCoverage": 0.02857142857142857, + "symbolPrecision": 0.2, + "spanCoverage": 0.0028633431792151427, + "spanPrecision": 0.6643356643356644, + "lineCoverage": 0.002026342451874367, + "linePrecision": 0.6666666666666666, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0.02857142857142857, + "symbolPrecision": 0.14285714285714285, + "spanCoverage": 0.0028633431792151427, + "spanPrecision": 0.521978021978022, + "lineCoverage": 0.002026342451874367, + "linePrecision": 0.4, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "rowStatus": "completed", + "metrics": { + "fileCoverage": 0.6666666666666666, + "filePrecision": 0.4, + "symbolCoverage": 0, + "symbolPrecision": 1, + "spanCoverage": 0.0021399722707818435, + "spanPrecision": 0.2591240875912409, + "lineCoverage": 0.0010131712259371835, + "linePrecision": 0.2, + "editlocRecall": 0.5714285714285714, + "editlocPrecision": 0.5714285714285714 + } + } + ], + "costRows": [ + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 651991, + "cachedInputTokens": 562688, + "outputTokens": 8569, + "reasoningOutputTokens": 5477, + "taskWallTimeMs": 158527, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 682131, + "cachedInputTokens": 642048, + "outputTokens": 10057, + "reasoningOutputTokens": 5312, + "taskWallTimeMs": 224679, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 400544, + "cachedInputTokens": 317952, + "outputTokens": 6954, + "reasoningOutputTokens": 4131, + "taskWallTimeMs": 131028, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 347800, + "cachedInputTokens": 314368, + "outputTokens": 13393, + "reasoningOutputTokens": 9227, + "taskWallTimeMs": 213283, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 273980, + "cachedInputTokens": 244224, + "outputTokens": 7323, + "reasoningOutputTokens": 4019, + "taskWallTimeMs": 140876, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "jcodemunch-full-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 241423, + "cachedInputTokens": 206848, + "outputTokens": 5932, + "reasoningOutputTokens": 3550, + "taskWallTimeMs": 119511, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 428418, + "cachedInputTokens": 398336, + "outputTokens": 7056, + "reasoningOutputTokens": 2584, + "taskWallTimeMs": 150948, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 230103, + "cachedInputTokens": 200192, + "outputTokens": 5962, + "reasoningOutputTokens": 3079, + "taskWallTimeMs": 153087, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 725790, + "cachedInputTokens": 665088, + "outputTokens": 9735, + "reasoningOutputTokens": 6607, + "taskWallTimeMs": 197729, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 324871, + "cachedInputTokens": 296448, + "outputTokens": 9513, + "reasoningOutputTokens": 5744, + "taskWallTimeMs": 170065, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 224215, + "cachedInputTokens": 177664, + "outputTokens": 4517, + "reasoningOutputTokens": 1669, + "taskWallTimeMs": 115595, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "repowise-selfhosted-mcp", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 263549, + "cachedInputTokens": 243200, + "outputTokens": 10605, + "reasoningOutputTokens": 7467, + "taskWallTimeMs": 170770, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 223558, + "cachedInputTokens": 190976, + "outputTokens": 4990, + "reasoningOutputTokens": 2899, + "taskWallTimeMs": 110708, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 269851, + "cachedInputTokens": 234496, + "outputTokens": 6028, + "reasoningOutputTokens": 3024, + "taskWallTimeMs": 121854, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 205432, + "cachedInputTokens": 130048, + "outputTokens": 5564, + "reasoningOutputTokens": 3530, + "taskWallTimeMs": 114664, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 215407, + "cachedInputTokens": 167424, + "outputTokens": 7930, + "reasoningOutputTokens": 4784, + "taskWallTimeMs": 143643, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 247868, + "cachedInputTokens": 206848, + "outputTokens": 5463, + "reasoningOutputTokens": 3246, + "taskWallTimeMs": 106404, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "codebase-context", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 534029, + "cachedInputTokens": 498688, + "outputTokens": 13790, + "reasoningOutputTokens": 10169, + "taskWallTimeMs": 238528, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 541233, + "cachedInputTokens": 497152, + "outputTokens": 9011, + "reasoningOutputTokens": 4494, + "taskWallTimeMs": 185061, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 856202, + "cachedInputTokens": 798720, + "outputTokens": 12581, + "reasoningOutputTokens": 7297, + "taskWallTimeMs": 257215, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 653348, + "cachedInputTokens": 610304, + "outputTokens": 6730, + "reasoningOutputTokens": 3640, + "taskWallTimeMs": 179577, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 826946, + "cachedInputTokens": 740864, + "outputTokens": 17144, + "reasoningOutputTokens": 12215, + "taskWallTimeMs": 294443, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 251601, + "cachedInputTokens": 220672, + "outputTokens": 7386, + "reasoningOutputTokens": 3991, + "taskWallTimeMs": 141981, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "raw-native", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 290186, + "cachedInputTokens": 272384, + "outputTokens": 7567, + "reasoningOutputTokens": 4024, + "taskWallTimeMs": 142002, + "contextToolCallCount": null, + "contextToolCompletedCallCount": null + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 304532, + "cachedInputTokens": 271872, + "outputTokens": 7060, + "reasoningOutputTokens": 4975, + "taskWallTimeMs": 146469, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 610092, + "cachedInputTokens": 546304, + "outputTokens": 8915, + "reasoningOutputTokens": 5830, + "taskWallTimeMs": 236889, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__c__maintenance__bugfix__5e659108", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 541331, + "cachedInputTokens": 480256, + "outputTokens": 10847, + "reasoningOutputTokens": 5649, + "taskWallTimeMs": 204901, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 1, + "status": "completed", + "inputTokens": 182974, + "cachedInputTokens": 167424, + "outputTokens": 9561, + "reasoningOutputTokens": 7486, + "taskWallTimeMs": 146629, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 2, + "status": "completed", + "inputTokens": 333030, + "cachedInputTokens": 292352, + "outputTokens": 12256, + "reasoningOutputTokens": 8613, + "taskWallTimeMs": 198105, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + }, + { + "laneId": "context-mode", + "taskId": "Multi-SWE-Bench__cpp__maintenance__bugfix__3e497d27", + "repeatIndex": 3, + "status": "completed", + "inputTokens": 518028, + "cachedInputTokens": 475136, + "outputTokens": 10446, + "reasoningOutputTokens": 6931, + "taskWallTimeMs": 200453, + "contextToolCallCount": 1, + "contextToolCompletedCallCount": 1 + } + ], + "aggregation": "Per-task, per-lane median of three costRows values for each counter independently. Cached input is a subset of input; component medians need not sum." + } +} diff --git a/results/contextbench-engineering-note-summary.json b/results/contextbench-engineering-note-summary.json new file mode 100644 index 0000000..03b0ebb --- /dev/null +++ b/results/contextbench-engineering-note-summary.json @@ -0,0 +1,213 @@ +{ + "schemaVersion": 2, + "artifactType": "contextbench-engineering-note-summary", + "generatedAt": "2026-06-04T08:28:24Z", + "source": { + "scorerScriptSha256": "sha256:13e859f6485735d76b1591a45a7211338f9fed9ff5566776ac8455db6e820e79", + "localSummary": "gate5-official-retrieval-scoring-current-script-20260604T1027/GATE5-SCORING-SUMMARY.json", + "localSummarySha256": "sha256:e5ecd3ee45b9577ae32d409de960edc1a7c2761159280cb7f8e7b7a85555387b", + "boundary": "The full run artifacts are local evidence. This checked-in summary is the public engineering-note artifact and does not make winner, leaderboard, paper-grade, resource-efficiency, or patch-quality claims.", + "localRunRoot": "benchmark-runs/contextbench/phase41/corrected-20x1-20260603T103632", + "localRunRootTracked": false + }, + "publicationReadiness": { + "claimBearingPublicationAllowed": false, + "claimScope": "20x1_retrieval_overlap_only", + "unresolvedBlockers": [ + "single_repeat_no_stability_claim", + "retrieval_overlap_only_no_patch_quality", + "provider_dollar_cost_not_measured_because_no_paid_provider_use", + "repowise_one_no_context_evaluator_failure", + "cold_vs_warm_install_dependency_cache_not_fully_normalized" + ], + "publicUseLevel": "narrow_retrieval_overlap_case_study", + "status": "engineering_note_ready_with_claim_blocks", + "blockedClaimTypes": [ + "benchmark winner", + "best tool", + "paper-grade benchmark", + "resource-efficiency claim", + "generalized agent coding quality claim", + "agent patch correctness claim", + "developer productivity claim" + ] + }, + "rows": { + "evaluatorFailedRows": 1, + "repeatsPerTaskLane": 1, + "lanes": 5, + "evaluatorCompletedRows": 99, + "retrievalOverlapRows": 100, + "appendixOnlyLanes": [ + { + "reason": "bounded local indexing readiness failed earlier; not part of this corrected 20x1 headline table", + "laneId": "GrepAI" + } + ], + "sameDenominatorRowsPresent": true, + "tasks": 20 + }, + "methodology": { + "notMeasured": "patch correctness, developer productivity, provider-dollar spend, multi-repeat confidence intervals", + "scoring": "official ContextBench evaluator on saved retrieval trajectories using per-task gold; scorer gold was not read during retrieval", + "lanes": [ + "raw-native", + "codebase-context", + "jcodemunch-full-mcp", + "repowise-selfhosted-mcp", + "context-mode" + ], + "measured": "retrieval overlap, setup/index time, row wall time, process memory telemetry, GPU availability probe, failure rows", + "metricView": "failure-inclusive means; judge_failed/no_context rows count as zero in the public summary", + "publicFrame": "we benchmarked these tools on open-source repos", + "taskSet": "20 frozen ContextBench-style open-source tasks" + }, + "qualitySnapshot": [ + { + "laneId": "codebase-context", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.25722222222222224, + "filePrecisionMean": 0.11499999999999999, + "symbolCoverageMean": 0.1396239637555427, + "spanCoverageMean": 0.08573378715275093, + "lineCoverageMean": 0.0878787249873908 + }, + { + "laneId": "context-mode", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.16833333333333333, + "filePrecisionMean": 0.11666666666666665, + "symbolCoverageMean": 0.02731443994601889, + "spanCoverageMean": 0.011708371581594209, + "lineCoverageMean": 0.01103832178724339 + }, + { + "laneId": "jcodemunch-full-mcp", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.2713888888888889, + "filePrecisionMean": 0.11000000000000001, + "symbolCoverageMean": 0.19046586746586747, + "spanCoverageMean": 0.08523281976808295, + "lineCoverageMean": 0.08706990517203586 + }, + { + "laneId": "raw-native", + "rows": 20, + "evaluatorCompletedRows": 20, + "status": "completed:20", + "fileCoverageMean": 0.043333333333333335, + "filePrecisionMean": 0.04, + "symbolCoverageMean": 0.007186234817813765, + "spanCoverageMean": 0.00461426238619191, + "lineCoverageMean": 0.003664930124865624 + }, + { + "laneId": "repowise-selfhosted-mcp", + "rows": 20, + "evaluatorCompletedRows": 19, + "status": "completed:19,judge_failed:1", + "fileCoverageMean": 0.2013888888888889, + "filePrecisionMean": 0.12, + "symbolCoverageMean": 0.046063588114671705, + "spanCoverageMean": 0.03033601885780799, + "lineCoverageMean": 0.024685983349898594 + } + ], + "costSnapshot": [ + { + "laneId": "codebase-context", + "setupStatuses": "not_required:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 750.8, + "indexSumMinutes": 226.6, + "rowWallSumMinutes": 231.5, + "peakPrivateGb": 2.9, + "modelEvidence": "Xenova/bge-small-en-v1.5:20" + }, + { + "laneId": "context-mode", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 18.6, + "indexSumMinutes": 12.5, + "rowWallSumMinutes": 16.0, + "peakPrivateGb": 0.15, + "modelEvidence": "unknown:20" + }, + { + "laneId": "jcodemunch-full-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 11.3, + "indexSumMinutes": 4.9, + "rowWallSumMinutes": 10.9, + "peakPrivateGb": 0.33, + "modelEvidence": "unknown:20" + }, + { + "laneId": "raw-native", + "setupStatuses": "not_required:20", + "indexStatuses": "not_required:20", + "indexExitStatuses": "unknown:20", + "indexMedianSeconds": 0.0, + "indexSumMinutes": 0.0, + "rowWallSumMinutes": 6.8, + "peakPrivateGb": 0.06, + "modelEvidence": "unknown:20" + }, + { + "laneId": "repowise-selfhosted-mcp", + "setupStatuses": "completed:20", + "indexStatuses": "completed:20", + "indexExitStatuses": "1:20", + "indexMedianSeconds": 294.3, + "indexSumMinutes": 154.2, + "rowWallSumMinutes": 200.7, + "peakPrivateGb": 4.82, + "modelEvidence": "unknown:20" + } + ], + "failedRows": [ + { + "taskId": "Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221", + "laneId": "repowise-selfhosted-mcp", + "status": "judge_failed", + "reason": "official_result_error:no_context_extracted", + "officialResults": "benchmark-runs/contextbench/phase41/corrected-20x1-20260603T103632/gate5-official-retrieval-scoring/rows/064-Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221/repowise-selfhosted-mcp/official-results.jsonl", + "trajectory": "benchmark-runs/contextbench/phase41/corrected-20x1-20260603T103632/gate4-corrected-20x1/13-Multi-SWE-Bench-typescript-maintenance-bugfix-8d2ac221/repowise-selfhosted-mcp/runs/repowise-selfhosted-mcp-Multi-SWE-Bench__typescript__maintenance__bugfix__8d2ac221-1-retrieval/trajectory.json" + } + ], + "interpretation": [ + "jCodeMunch and codebase-context are close on file/span retrieval in this fixed adapter run; jCodeMunch has stronger symbol coverage but lower file precision.", + "codebase-context completed all 20 retrieval rows in this run but was expensive to index with local Transformers embeddings.", + "jCodeMunch is much cheaper to index in this adapter run, but its exact cold/warm install/cache boundary still needs care before cost claims.", + "context-mode is lightweight and low-cost, with lower recall ceiling in this run.", + "Repowise indexed and returned rows, but used the highest peak private memory, every index exited nonzero with a documented benign failure mode, and one row produced no extracted context.", + "raw-native is a deterministic lexical baseline here, not a free-form raw agent ceiling; it had low retrieval-overlap on these 20 tasks." + ], + "caveats": { + "publicDocs": "passed: README/docs/package-facing benchmark wording is aligned to the narrow 100-row retrieval-overlap engineering-note framing", + "overclaiming": "passed: public wording keeps winner, leaderboard, paper-grade, resource-efficiency, patch-quality, and productivity claims blocked", + "patchQuality": "blocked: retrieval-overlap only, no downstream patch/test success measured", + "costAccounting": "partial_pass: setup/index/resource telemetry captured; provider-dollar costs remain none/unknown because no paid provider use was approved", + "repeatStability": "blocked: 1 repeat per lane/task, no paper-grade stability or confidence claim", + "artifactCompleteness": "passed: 100 rows have summary, trajectory, raw trace, setup, telemetry, and gold", + "failureInclusion": "passed: Repowise judge_failed/no_context row is retained and zero-filled metric view exists", + "sameDenominator": "passed: 20 tasks x 5 lanes present" + }, + "metaCost": { + "note": "Benchmark orchestration cost is authoring/evaluation overhead, not a lane score.", + "latestKnownCodexGoalTokens": "owner-reported 2.29B input and 7.48M output tokens before this corrected scoring pass; exact all-run total not independently audited here", + "gate5ScoringPeakPrivateMb": 65.1, + "gate5ScoringRuntimeSeconds": 72.4 + } +} diff --git a/scripts/contextbench-build-publishable-report.mjs b/scripts/contextbench-build-publishable-report.mjs new file mode 100644 index 0000000..0b5a5a1 --- /dev/null +++ b/scripts/contextbench-build-publishable-report.mjs @@ -0,0 +1,357 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { dirname, join, resolve } from 'node:path'; + +function arg(name, fallback = null) { + const index = process.argv.indexOf(`--${name}`); + if (index === -1) return fallback; + return process.argv[index + 1] || fallback; +} + +const summaryPath = arg('summary', join(process.env.ROOT || '/tmp/contextbench-five-lane-score', 'summary.json')); +const protocolPath = arg('protocol', 'tests/fixtures/contextbench-benchmark-protocol.json'); +const lanesPath = arg('lanes', 'tests/fixtures/contextbench-lanes.json'); +const taskManifestPath = arg('task-manifest', 'tests/fixtures/contextbench-task-manifest.json'); +const outPath = arg('out', join(dirname(summaryPath), 'publishable-summary.json')); +const validationOutPath = arg('validation-out', join(dirname(summaryPath), 'publishable-validation.json')); +const humanizedOutPath = arg('humanized-out', join(dirname(summaryPath), 'humanized-summary.md')); + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function hashFile(path) { + return `sha256:${createHash('sha256').update(readFileSync(path)).digest('hex')}`; +} + +function writeJson(path, data) { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`); +} + +function numberOrNull(value) { + if (value === null || value === undefined || value === '') return null; + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function metricValue(metric) { + if (!metric || typeof metric !== 'object') return null; + return numberOrNull(metric.value); +} + +function hasMetricOrReason(metric) { + if (!metric || typeof metric !== 'object') return false; + if (numberOrNull(metric.value) !== null) return true; + return typeof metric.unavailableReason === 'string' && metric.unavailableReason.length > 0; +} + +function mean(values) { + const clean = values.map(numberOrNull).filter((value) => value !== null); + if (clean.length === 0) return null; + return clean.reduce((sum, value) => sum + value, 0) / clean.length; +} + +function median(values) { + const clean = values.map(numberOrNull).filter((value) => value !== null).sort((a, b) => a - b); + if (clean.length === 0) return null; + const middle = Math.floor(clean.length / 2); + if (clean.length % 2) return clean[middle]; + return (clean[middle - 1] + clean[middle]) / 2; +} + +function stddev(values) { + const clean = values.map(numberOrNull).filter((value) => value !== null); + if (clean.length < 2) return null; + const avg = mean(clean); + const variance = clean.reduce((sum, value) => sum + ((value - avg) ** 2), 0) / (clean.length - 1); + return Math.sqrt(variance); +} + +function deterministicRandom(seed) { + let state = seed >>> 0; + return () => { + state = (1664525 * state + 1013904223) >>> 0; + return state / 2 ** 32; + }; +} + +function bootstrapCi(values, iterations = 2000) { + const clean = values.map(numberOrNull).filter((value) => value !== null); + if (clean.length === 0) return { low: null, high: null, method: 'bootstrap_mean_95_ci', samples: 0 }; + if (clean.length === 1) return { low: clean[0], high: clean[0], method: 'single_sample_no_interval', samples: 1 }; + const rand = deterministicRandom(0xC0FFEE); + const sampledMeans = []; + for (let i = 0; i < iterations; i += 1) { + let total = 0; + for (let j = 0; j < clean.length; j += 1) { + total += clean[Math.floor(rand() * clean.length)]; + } + sampledMeans.push(total / clean.length); + } + sampledMeans.sort((a, b) => a - b); + return { + low: sampledMeans[Math.floor(iterations * 0.025)], + high: sampledMeans[Math.floor(iterations * 0.975)], + method: 'deterministic_bootstrap_mean_95_ci', + samples: clean.length, + }; +} + +function qualityFromRow(row) { + const final = row.score?.final || {}; + return { + lane: row.lane_id, + task: row.task_id, + fileCoverage: final.file?.coverage ?? null, + filePrecision: final.file?.precision ?? null, + symbolCoverage: final.symbol?.coverage ?? null, + symbolPrecision: final.symbol?.precision ?? null, + spanCoverage: final.span?.coverage ?? null, + spanPrecision: final.span?.precision ?? null, + lineCoverage: final.line?.coverage ?? null, + linePrecision: final.line?.precision ?? null, + editlocRecall: row.score?.editloc?.recall ?? null, + editlocPrecision: row.score?.editloc?.precision ?? null, + qualitySource: 'official ContextBench evaluator', + }; +} + +function costFromRow(row) { + const t = row.timeMetrics || {}; + return { + lane: row.lane_id, + setupDurationMs: metricValue(t.setupDurationMs), + indexDurationMs: metricValue(t.indexDurationMs), + queryDurationMs: metricValue(t.queryDurationMs), + selectorDurationMs: metricValue(t.selectorDurationMs), + evaluatorDurationMs: metricValue(t.evaluatorDurationMs), + rowWallDurationMs: metricValue(t.rowWallDurationMs), + unavailable: Object.fromEntries(Object.entries(t) + .filter(([, metric]) => metric?.unavailableReason) + .map(([key, metric]) => [key, metric.unavailableReason])), + }; +} + +function tokensFromRow(row) { + const tokenMetrics = row.tokenMetrics || {}; + const candidatePack = tokenMetrics.candidatePack || {}; + const prediction = tokenMetrics.prediction || {}; + const selectorUsage = tokenMetrics.selectorUsage || {}; + return { + lane: row.lane_id, + estimator: tokenMetrics.estimator || null, + candidateCount: candidatePack.candidateCount ?? row.candidateCount ?? null, + candidateBytes: metricValue(candidatePack.bytes), + candidateEstimatedTokens: metricValue(candidatePack.estimatedTokens), + predictionBytes: metricValue(prediction.bytes), + predictionEstimatedTokens: metricValue(prediction.estimatedTokens), + selectorModel: selectorUsage.model || row.model || null, + selectorInputTokens: metricValue(selectorUsage.inputTokens), + selectorOutputTokens: metricValue(selectorUsage.outputTokens), + selectorTotalTokens: metricValue(selectorUsage.totalTokens), + unavailable: { + candidateBytes: candidatePack.bytes?.unavailableReason || null, + candidateEstimatedTokens: candidatePack.estimatedTokens?.unavailableReason || null, + selectorUsage: selectorUsage.totalTokens?.unavailableReason || null, + }, + }; +} + +function reliabilityFromRow(row) { + return { + lane: row.lane_id, + task: row.task_id, + status: row.status, + officialEvaluatorScoreable: row.officialEvaluatorScoreable === true, + setupStatus: row.reliability?.setupStatus ?? row.setupStatus, + indexStatus: row.reliability?.indexStatus ?? row.indexStatus, + toolCallable: row.reliability?.toolCallable ?? row.toolCallable, + nonEmptyPrediction: row.reliability?.nonEmptyPrediction ?? row.nonEmptyPrediction, + candidateCount: row.reliability?.candidateCount ?? row.candidateCount, + sourceRun: row.reliability?.sourceRun ?? null, + sourceJob: row.reliability?.sourceJob ?? null, + sourceArtifact: row.reliability?.sourceArtifact ?? null, + sourceDigest: row.reliability?.sourceDigest ?? null, + }; +} + +function laneStatistics(qualityRows) { + const metrics = ['fileCoverage', 'filePrecision', 'spanCoverage', 'spanPrecision', 'lineCoverage', 'linePrecision']; + const byLane = new Map(); + for (const row of qualityRows) { + const list = byLane.get(row.lane) || []; + list.push(row); + byLane.set(row.lane, list); + } + return Object.fromEntries([...byLane.entries()].map(([lane, rows]) => [ + lane, + Object.fromEntries(metrics.map((metric) => { + const values = rows.map((row) => row[metric]); + return [metric, { + mean: mean(values), + median: median(values), + stddev: stddev(values), + bootstrap95Ci: bootstrapCi(values), + }]; + })), + ])); +} + +function validate(summary, protocol, taskManifest, qualityRows, reliabilityRows) { + const requiredLanes = summary.requiredLanes || []; + const scoreableLanes = new Set(qualityRows.map((row) => row.lane)); + const failures = reliabilityRows.filter((row) => !row.officialEvaluatorScoreable); + const checks = [ + { + id: 'official_evaluator_only_quality_rows', + pass: qualityRows.every((row) => row.qualitySource === 'official ContextBench evaluator'), + detail: 'Every quality row must come from the official ContextBench evaluator.', + }, + { + id: 'failures_excluded_from_quality_table', + pass: failures.every((failure) => !qualityRows.some((row) => row.lane === failure.lane && row.task === failure.task)), + detail: 'setup_failed/tool_error/judge_failed/empty rows are reliability rows, not quality rows.', + }, + { + id: 'required_lanes_scoreable', + pass: requiredLanes.length >= 5 && requiredLanes.every((lane) => scoreableLanes.has(lane)), + detail: `Required lanes: ${requiredLanes.join(', ') || 'none'}`, + }, + { + id: 'setup_index_query_costs_separate', + pass: summary.setupIndexCostReportedSeparately === true && summary.rows.every((row) => ['setupDurationMs', 'indexDurationMs', 'queryDurationMs'].every((key) => hasMetricOrReason(row.timeMetrics?.[key]))), + detail: 'Setup, indexing, and query costs must not be folded into quality metrics.', + }, + { + id: 'timing_metrics_present', + pass: summary.rows.every((row) => ['evaluatorDurationMs', 'rowWallDurationMs', 'selectorDurationMs'].every((key) => hasMetricOrReason(row.timeMetrics?.[key]))), + detail: 'Each row needs evaluator, row-wall, and selector timing fields or explicit unavailable reasons.', + }, + { + id: 'token_metrics_present', + pass: summary.rows.every((row) => hasMetricOrReason(row.tokenMetrics?.prediction?.estimatedTokens) + && hasMetricOrReason(row.tokenMetrics?.candidatePack?.estimatedTokens) + && hasMetricOrReason(row.tokenMetrics?.selectorUsage?.totalTokens)), + detail: 'Each row needs prediction, candidate-pack, and selector token fields; unavailable provider usage must be explicit.', + }, + { + id: 'protocol_is_frozen_and_non_claiming', + pass: protocol.status === 'protocol_frozen' && protocol.claimAllowed === false, + detail: 'This proof can be publishable evidence, but not a broad benchmark-win claim.', + }, + { + id: 'manifest_preselected_without_lane_outputs', + pass: typeof taskManifest.no_lane_outputs_observed_attestation === 'string' && taskManifest.no_lane_outputs_observed_attestation.length > 0, + detail: 'Task selection must not depend on lane outputs.', + }, + ]; + return { + status: checks.every((check) => check.pass) ? 'pass' : 'fail', + checks, + }; +} + +function formatNumber(value) { + if (value === null || value === undefined) return 'n/a'; + if (typeof value === 'string') return value; + if (Number.isInteger(value)) return String(value); + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric.toFixed(3) : String(value); +} + +function markdownTable(rows, columns) { + const header = `| ${columns.map((column) => column.label).join(' | ')} |`; + const divider = `| ${columns.map(() => '---').join(' | ')} |`; + const body = rows.map((row) => `| ${columns.map((column) => formatNumber(column.value(row))).join(' | ')} |`); + return [header, divider, ...body].join('\n'); +} + +function humanizedMarkdown(report) { + const table = markdownTable(report.qualityTable, [ + { label: 'Lane', value: (row) => row.lane }, + { label: 'File cov', value: (row) => row.fileCoverage }, + { label: 'File prec', value: (row) => row.filePrecision }, + { label: 'Span cov', value: (row) => row.spanCoverage }, + { label: 'Line cov', value: (row) => row.lineCoverage }, + ]); + const costTable = markdownTable(report.costTable, [ + { label: 'Lane', value: (row) => row.lane }, + { label: 'Setup ms', value: (row) => row.setupDurationMs }, + { label: 'Index ms', value: (row) => row.indexDurationMs }, + { label: 'Query ms', value: (row) => row.queryDurationMs }, + { label: 'Eval ms', value: (row) => row.evaluatorDurationMs }, + { label: 'Row ms', value: (row) => row.rowWallDurationMs }, + ]); + const tokenTable = markdownTable(report.tokenTable, [ + { label: 'Lane', value: (row) => row.lane }, + { label: 'Candidates', value: (row) => row.candidateCount }, + { label: 'Cand tokens', value: (row) => row.candidateEstimatedTokens }, + { label: 'Pred tokens', value: (row) => row.predictionEstimatedTokens }, + { label: 'Selector tokens', value: (row) => row.selectorTotalTokens }, + ]); + return `# ContextBench Publishable Pilot Summary\n\nThis is a real five-lane ContextBench pilot: the quality numbers below come only from the official ContextBench evaluator, and setup/index/query costs are reported separately. It is still a pilot because it covers one frozen task; it should not be turned into a broad win claim.\n\n## Quality\n\n${table}\n\n## Time Cost\n\n${costTable}\n\n## Token Cost\n\n${tokenTable}\n\n## What This Means\n\nThe run is useful because every required lane produced a non-empty prediction and received an official score. The remaining n/a values are measurement gaps, not hidden failures: selector provider token telemetry was not captured in this proof, and some readiness artifacts report candidate counts without candidate bytes.\n\n## What Not To Claim\n\nDo not claim product superiority from this artifact alone. A claim-bearing benchmark still needs the full frozen task slice, repeated runs, and the same reporting discipline used here.\n`; +} + +if (!existsSync(summaryPath)) throw new Error(`summary not found: ${summaryPath}`); +if (!existsSync(protocolPath)) throw new Error(`protocol not found: ${protocolPath}`); +if (!existsSync(lanesPath)) throw new Error(`lanes fixture not found: ${lanesPath}`); +if (!existsSync(taskManifestPath)) throw new Error(`task manifest not found: ${taskManifestPath}`); + +const summary = readJson(summaryPath); +const protocol = readJson(protocolPath); +const lanes = readJson(lanesPath); +const taskManifest = readJson(taskManifestPath); +const rows = summary.rows || []; +const qualityRows = rows.filter((row) => row.officialEvaluatorScoreable === true && row.status === 'completed').map(qualityFromRow); +const reliabilityRows = rows.map(reliabilityFromRow); +const costTable = rows.map(costFromRow); +const tokenTable = rows.map(tokensFromRow); +const validation = validate(summary, protocol, taskManifest, qualityRows, reliabilityRows); + +const targetTaskIds = [...new Set(rows.map((row) => row.task_id).filter(Boolean))]; +const report = { + createdAt: new Date().toISOString(), + publicationStatus: 'pilot_evidence_not_broad_claim', + sourceSummary: resolve(summaryPath), + protocol: { + path: protocolPath, + hash: hashFile(protocolPath), + version: protocol.protocolVersion, + status: protocol.status, + claimAllowed: protocol.claimAllowed, + }, + fixtures: { + lanes: { path: lanesPath, hash: hashFile(lanesPath), laneCount: lanes.lanes?.length ?? null }, + taskManifest: { path: taskManifestPath, hash: hashFile(taskManifestPath), manifestHash: taskManifest.manifest_hash ?? null }, + }, + scope: { + model: summary.model, + targetTaskIds, + attemptedRows: summary.attemptedRows, + scoreableRows: summary.scoreableRows, + requiredLanes: summary.requiredLanes || [], + caveats: summary.caveats || [], + }, + qualityTable: qualityRows, + costTable, + tokenTable, + reliabilityTable: reliabilityRows, + statistics: laneStatistics(qualityRows), + biasAudit: validation, + limitations: [ + 'One-task proof run: suitable as pilot evidence, not a broad benchmark claim.', + 'Selector usage telemetry is explicit but unavailable where the selector ran outside provider usage capture.', + 'Candidate token estimates use a deterministic utf8-byte heuristic unless a lane emits provider-grade token counts.', + 'Failures remain in reliability tables and are never counted as quality rows.', + ], +}; + +writeJson(outPath, report); +writeJson(validationOutPath, validation); +mkdirSync(dirname(humanizedOutPath), { recursive: true }); +writeFileSync(humanizedOutPath, humanizedMarkdown(report)); +console.log('CONTEXTBENCH_PUBLISHABLE_REPORT_JSON_START'); +console.log(JSON.stringify({ outPath, validationOutPath, humanizedOutPath, validation }, null, 2)); +console.log('CONTEXTBENCH_PUBLISHABLE_REPORT_JSON_END'); +if (validation.status !== 'pass') process.exitCode = 1; diff --git a/scripts/contextbench-cbm-candidate-pack.mjs b/scripts/contextbench-cbm-candidate-pack.mjs new file mode 100644 index 0000000..2f32297 --- /dev/null +++ b/scripts/contextbench-cbm-candidate-pack.mjs @@ -0,0 +1,188 @@ +import { spawnSync } from 'node:child_process'; +import { basename, join, relative } from 'node:path'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const root = process.env.ROOT; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks[2]; +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: r.status, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function queryOf(text) { + const stop = new Set([ + 'that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', + 'would', 'could', 'there', 'where', 'which', 'about', 'after', + 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', + 'using', 'must', 'method', 'methods', 'function', 'interface', + ]); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 18) + .join(' '); +} + +function jsonish(s) { + const t = String(s || '').trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/'); + if (!f) return ''; + if (f.startsWith(repo)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if ( + !f || + f.includes('://') || + f.includes('..') || + f.startsWith('tmp-contextbench') || + f.includes('/tmp-contextbench') + ) { + return ''; + } + if (!existsSync(join(repo, f))) return ''; + return f; +} + +function add(locs, file, start = 1, end = start, source = 'codebase-memory-mcp') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + ); + for (const item of Object.values(value)) walk(item, locs); +} + +function collect(text, locs) { + const parsed = jsonish(text); + if (parsed) walk(parsed, locs); + const re = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = re.exec(String(text || ''))) !== null) add(locs, m[1], m[2] || 1, m[2] || 1); +} + +function uniq(locs, max = 100) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (!seen.has(key)) { + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + } + return out; +} + +const query = queryOf(task.problem_statement); +const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; +const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); +const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { + cwd: repo, + env, + timeoutMs: 2700000, +}); +const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); +const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query, limit: 60 })], { + cwd: repo, + env, + timeoutMs: 120000, +}); +const code = run( + process.env.CBM_BIN, + ['cli', 'search_code', JSON.stringify({ project, pattern: query.split(/\s+/)[0] || '.', mode: 'compact', limit: 60 })], + { cwd: repo, env, timeoutMs: 120000 }, +); + +for (const [name, value] of Object.entries({ setup, index, graph, code })) { + writeFileSync( + join(outDir, `${name}.json`), + JSON.stringify({ ...value, stdout: value.stdout.slice(0, 120000), stderr: value.stderr.slice(0, 120000) }, null, 2), + ); +} + +const locs = []; +for (const r of [graph, code]) { + collect(r.stdout, locs); + collect(r.stderr, locs); +} +const candidates = uniq(locs); +const pack = { + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + query, + lane: { + lane: 'codebase-memory-mcp', + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 ? 'completed' : 'index_failed', + toolCallable: graph.status === 0 || code.status === 0, + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: graph.durationMs + code.durationMs, + }, + candidates, + }, +}; + +writeFileSync(join(outDir, 'cbm-candidate-pack.json'), JSON.stringify(pack, null, 2)); +console.log('CBM_CANDIDATE_PACK_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CBM_CANDIDATE_PACK_JSON_END'); diff --git a/scripts/contextbench-cgc-relevant-candidate-pack.mjs b/scripts/contextbench-cgc-relevant-candidate-pack.mjs new file mode 100644 index 0000000..869cff5 --- /dev/null +++ b/scripts/contextbench-cgc-relevant-candidate-pack.mjs @@ -0,0 +1,199 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-cgc-relevant'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'codegraphcontext') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function parseJson(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { return JSON.parse(cleaned); } catch {} + const start = cleaned.indexOf('['); + const end = cleaned.lastIndexOf(']'); + if (start >= 0 && end > start) { + try { return JSON.parse(cleaned.slice(start, end + 1)); } catch {} + } + return null; +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add(locs, value.path || value.file || value.file_path || value.name || value.uri, value.line_number || value.line || 1, value.end_line || value.line_number || 1, source); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const parsed = parseJson(text); + if (parsed) walk(parsed, locs, source); + const cleaned = stripAnsi(text); + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + let m; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +const relevantTerms = ['metrics', 'prometheus', 'insights', 'auth', 'authorization', 'bearer', 'token', 'header', 'subsonic', 'root']; +function isRelevant(loc) { + const file = loc.file.toLowerCase(); + return relevantTerms.some((term) => file.includes(term)); +} + +function uniq(locs, max = 80) { + const seen = new Set(); + const out = []; + for (const loc of locs.filter(isRelevant)) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `codegraphcontext-command-${i + 1}.json`), JSON.stringify({ + ...command, + stdout: command.stdout.slice(0, 200000), + stderr: command.stderr.slice(0, 200000), + }, null, 2)); + } +} + +const env = { ...process.env, DEFAULT_DATABASE: 'kuzudb', CGC_RUNTIME_DB_TYPE: 'kuzudb' }; +const commands = []; +const locs = []; +const setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); +commands.push(setup); +const index = run('cgc', ['index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); +commands.push(index); +const queryCommands = []; +for (const term of relevantTerms) { + const apiQuery = run('python', ['-c', `import json\nfrom codegraphcontext.core.database import DatabaseManager\ndb=DatabaseManager()\nwith db.get_driver().session() as s:\n rows=s.run("MATCH (f:File) WHERE toLower(f.path) CONTAINS '${term}' RETURN f.path as path LIMIT 100").data()\nprint(json.dumps(rows))\ndb.close_driver()`], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(apiQuery); + queryCommands.push(apiQuery); + collect(apiQuery.stdout, locs, `codegraphcontext-query-${term}`); + collect(apiQuery.stderr, locs, `codegraphcontext-query-${term}`); +} +for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run('cgc', ['find', 'pattern', pattern], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(found); + queryCommands.push(found); + collect(found.stdout, locs, 'codegraphcontext-pattern'); + collect(found.stderr, locs, 'codegraphcontext-pattern'); +} + +writeCommands(commands); +const candidates = uniq(locs); +const result = { + lane: 'codegraphcontext', + ready: setup.status === 0 && index.status === 0 && candidates.length > 0, + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 && candidates.length > 0 ? 'completed' : 'index_failed', + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), + }, + commands: commands.map((command) => ({ command: command.command, status: command.status, signal: command.signal, error: command.error, durationMs: command.durationMs })), + candidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + readiness: result, +}; +writeFileSync(join(outDir, 'codegraphcontext-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'codegraphcontext-readiness.json'), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_CGC_RELEVANT_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CGC_RELEVANT_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; diff --git a/scripts/contextbench-codegraphcontext-content-pack.mjs b/scripts/contextbench-codegraphcontext-content-pack.mjs new file mode 100644 index 0000000..990a013 --- /dev/null +++ b/scripts/contextbench-codegraphcontext-content-pack.mjs @@ -0,0 +1,154 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-codegraphcontext-readiness'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const list = basenameMap.get(basename(file)) || []; + list.push(file); + basenameMap.set(basename(file), list); +} + +function norm(file) { + let f = String(file || '').replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start) { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source: 'codegraphcontext' }); +} + +function collect(text, locs) { + const cleaned = String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); + const lineRe = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp)):(\d+):/gm; + let match; + while ((match = lineRe.exec(cleaned)) !== null) add(locs, match[1], match[2], match[2]); + const fileRe = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((match = fileRe.exec(cleaned)) !== null) add(locs, match[1], match[2] || 1, match[2] || 1); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +const commands = []; +const locs = []; +const setup = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); +commands.push(setup); +const index = run('codegraphcontext', ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); +commands.push(index); +const terms = ['metrics', 'prometheus', 'insights', 'startup', 'authorization', 'Bearer', 'token', 'header', 'authentication', 'subsonic']; +const queries = []; +for (const term of terms) { + for (const args of [['find', 'content', term], ['find', 'pattern', term]]) { + const result = run('codegraphcontext', args, { cwd: repo, timeoutMs: 180000 }); + commands.push(result); + queries.push(result); + collect(result.stdout, locs); + collect(result.stderr, locs); + } +} + +for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `codegraphcontext-command-${i + 1}.json`), JSON.stringify({ ...command, stdout: command.stdout.slice(0, 200000), stderr: command.stderr.slice(0, 200000) }, null, 2)); +} + +const candidates = uniq(locs); +const readiness = { + lane: 'codegraphcontext', + ready: setup.status === 0 && index.status === 0 && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus: setup.status === 0 ? 'completed' : 'setup_failed', + indexStatus: index.status === 0 ? 'completed' : 'index_failed', + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex: { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }, + commands: commands.map((command) => ({ command: command.command, status: command.status, error: command.error, durationMs: command.durationMs })), + candidates, +}; +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + readiness, +}; +writeFileSync(join(outDir, 'codegraphcontext-candidate-pack.json'), JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READINESS_JSON_END'); +if (!readiness.ready) process.exitCode = 1; diff --git a/scripts/contextbench-codegraphcontext-ready-pack.mjs b/scripts/contextbench-codegraphcontext-ready-pack.mjs new file mode 100644 index 0000000..913a945 --- /dev/null +++ b/scripts/contextbench-codegraphcontext-ready-pack.mjs @@ -0,0 +1,197 @@ +// Focused CodeGraphContext readiness gate for the ContextBench Go task. +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-codegraphcontext-readiness'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const list = basenameMap.get(basename(file)) || []; + list.push(file); + basenameMap.set(basename(file), list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'codegraphcontext') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const lineMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?(?:-(\d+))?/g; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[3] || m[2] || 1, source); +} + +const relevantPathTerms = ['metrics', 'prometheus', 'auth', 'subsonic', 'root', 'header', 'token']; +function collectRelevant(text, locs, source) { + const all = []; + collect(text, all, source); + for (const loc of all) { + const lower = loc.file.toLowerCase(); + if (relevantPathTerms.some((term) => lower.includes(term))) locs.push(loc); + } +} + +function uniq(locs, max = 200) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +const env = { + ...process.env, + CGC_RUNTIME_DB_TYPE: 'kuzudb', + DEFAULT_DATABASE: 'kuzudb', + COLUMNS: '400', + NO_COLOR: '1', + TERM: 'dumb', +}; +const commands = []; +const locs = []; +let setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); +if (setup.status !== 0) setup = run('cgc', ['help'], { env, timeoutMs: 60000 }); +commands.push(setup); +const index = run('cgc', ['--database', 'kuzudb', 'index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); +commands.push(index); + +const queries = [ + ['stats', '--all'], + ['find', 'type', 'module', '--limit', '3000'], + ['find', 'type', 'file', '--limit', '3000'], + ['find', 'pattern', 'root'], + ['find', 'pattern', 'auth'], + ['find', 'pattern', 'Prometheus'], + ['find', 'pattern', 'Subsonic'], + ['find', 'name', 'Prometheus'], + ['find', 'name', 'auth'], +]; +const queryCommands = []; +for (const args of queries) { + const command = run('cgc', ['--database', 'kuzudb', ...args], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(command); + queryCommands.push(command); + collectRelevant(command.stdout, locs, `cgc-${args.join('-')}`); + collectRelevant(command.stderr, locs, `cgc-${args.join('-')}`); +} + +const candidates = uniq(locs); +const setupStatus = setup.status === 0 ? 'completed' : 'setup_failed'; +const indexStatus = index.status === 0 ? 'completed' : 'index_failed'; +const toolCallable = queryCommands.some((command) => command.status === 0); +const setupIndex = { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), +}; +const readiness = { + lane: 'codegraphcontext', + ready: setupStatus === 'completed' && indexStatus === 'completed' && toolCallable && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable, + candidateCount: candidates.length, + setupIndex, + notes: [ + 'Uses documented cgc index/find commands and filters only paths emitted by cgc module/file/pattern queries.', + 'Content search returned no task-term matches in prior runs, so this gate proves index usability through cgc inventory output plus relevant path filtering.', + ], + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + })), + candidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + readiness, +}; +writeFileSync(join(outDir, 'codegraphcontext-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'codegraphcontext-readiness.json'), JSON.stringify(readiness, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READY_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_CODEGRAPHCONTEXT_READY_JSON_END'); +if (!readiness.ready) process.exitCode = 1; diff --git a/scripts/contextbench-final-lane-candidate-pack.mjs b/scripts/contextbench-final-lane-candidate-pack.mjs new file mode 100644 index 0000000..c8860db --- /dev/null +++ b/scripts/contextbench-final-lane-candidate-pack.mjs @@ -0,0 +1,286 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || `/tmp/contextbench-final-lane/${laneId || 'unknown'}`; +const outDir = join(root, 'pack'); +const supportedLanes = ['grepai', 'codegraphcontext']; +if (!supportedLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { + return JSON.parse(cleaned); + } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const start = cleaned.indexOf(open); + const end = cleaned.lastIndexOf(close); + if (start >= 0 && end > start) { + try { + return JSON.parse(cleaned.slice(start, end + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const lineMatch = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + const fileMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 160) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ + ...command, + stdout: command.stdout.slice(0, 200000), + stderr: command.stderr.slice(0, 200000), + }, null, 2)); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: setupStatus === 'completed' && indexStatus === 'completed' && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ command: command.command, status: command.status, signal: command.signal, error: command.error, durationMs: command.durationMs })), + candidates, + }; +} + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +const queryVariants = [ + queryOf(task.problem_statement), + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runGrepai() { + const commands = []; + const locs = []; + const env = { ...process.env, OLLAMA_HOST: process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' }; + const setup = run('grepai', ['version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const ollama = run('ollama', ['list'], { env, timeoutMs: 60000 }); + commands.push(ollama); + const init = run('grepai', ['init', '--yes', '--provider', 'ollama', '--model', 'nomic-embed-text', '--backend', 'gob'], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(watch); + const statusChecks = []; + for (let i = 0; i < 24; i += 1) { + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, env, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + if (/Files indexed:\s*[1-9]|Total chunks:\s*[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; + run('sleep', ['5'], { timeoutMs: 10000 }); + } + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '50'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, env, timeoutMs: 60000 }); + commands.push(stop); + return laneResult('grepai', commands, locs, setup.status === 0 && ollama.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs + ollama.durationMs + init.durationMs, + indexDurationMs: watch.durationMs + durationOf(statusChecks), + queryDurationMs: durationOf(searches), + teardownDurationMs: stop.durationMs, + }); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const env = { ...process.env, DEFAULT_DATABASE: 'kuzudb', CGC_RUNTIME_DB_TYPE: 'kuzudb' }; + const setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('cgc', ['index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + const fileList = run('cgc', ['find', 'type', 'file', '--limit', '200'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(fileList); + queries.push(fileList); + collect(fileList.stdout, locs, 'codegraphcontext'); + collect(fileList.stderr, locs, 'codegraphcontext'); + const apiFiles = run('python', ['-c', "import json; from codegraphcontext.core.database import DatabaseManager; from codegraphcontext.tools.code_finder import CodeFinder; db=DatabaseManager(); print(json.dumps(CodeFinder(db).find_by_type('file', 200))); db.close_driver()"], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(apiFiles); + queries.push(apiFiles); + collect(apiFiles.stdout, locs, 'codegraphcontext-api'); + collect(apiFiles.stderr, locs, 'codegraphcontext-api'); + for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run('cgc', ['find', 'pattern', pattern], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { + const callers = run('cgc', ['analyze', 'callers', symbol], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(callers); + queries.push(callers); + collect(callers.stdout, locs, 'codegraphcontext'); + collect(callers.stderr, locs, 'codegraphcontext'); + } + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }); +} + +const result = laneId === 'grepai' ? runGrepai() : runCodeGraphContext(); +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_FINAL_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_FINAL_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; diff --git a/scripts/contextbench-five-lane-candidate-pack.mjs b/scripts/contextbench-five-lane-candidate-pack.mjs new file mode 100644 index 0000000..81b6c9e --- /dev/null +++ b/scripts/contextbench-five-lane-candidate-pack.mjs @@ -0,0 +1,472 @@ +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + writeFileSync, +} from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-five-lane-candidates'; +const outDir = join(root, 'pack'); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function durationOf(commands) { + return commands.reduce((sum, command) => sum + command.durationMs, 0); +} + +function queryOf(text) { + const stop = new Set([ + 'that', + 'this', + 'with', + 'from', + 'when', + 'then', + 'into', + 'should', + 'would', + 'could', + 'there', + 'where', + 'which', + 'about', + 'after', + 'before', + 'have', + 'will', + 'been', + 'than', + 'also', + 'only', + 'some', + 'using', + 'must', + ]); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(s) { + const t = stripAnsi(s).trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [ + ['{', '}'], + ['[', ']'], + ]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file) + .replace(/^file:\/\//, '') + .replaceAll('\\', '/') + .trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + const tmpIdx = f.indexOf('/tmp/contextbench-checkouts/'); + if (tmpIdx >= 0) { + const after = f.slice(tmpIdx + '/tmp/contextbench-checkouts/'.length); + const parts = after.split('/'); + f = parts.slice(1).join('/'); + } + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp/')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + locs.push({ file: clean, start: s, end: e, source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || + value.path || + value.file_path || + value.relative_path || + value.filename || + value.source_path || + value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + + const rgLine = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = rgLine.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync( + join(outDir, `${lane}-command-${i + 1}.json`), + JSON.stringify( + { + ...command, + stdout: command.stdout.slice(0, 200000), + stderr: command.stderr.slice(0, 200000), + }, + null, + 2, + ), + ); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + error: command.error, + durationMs: command.durationMs, + })), + candidates, + }; +} + +const baseQuery = queryOf(task.problem_statement); +const queryVariants = [ + baseQuery, + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +const lanes = []; + +{ + const commands = []; + const locs = []; + const terms = [ + 'metrics', + 'prometheus', + 'insights', + 'startup', + 'start', + 'written', + 'authorization', + 'Bearer', + 'token', + 'header', + 'authentication', + 'subsonic', + ]; + for (const term of terms) { + const r = run( + 'rg', + ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], + { cwd: repo, timeoutMs: 60000 }, + ); + commands.push(r); + collect(r.stdout, locs, 'raw-native'); + collect(r.stderr, locs, 'raw-native'); + } + lanes.push( + laneResult('raw-native', commands, locs, 'not_required', 'not_required', { + setupDurationMs: 0, + indexDurationMs: 0, + queryDurationMs: durationOf(commands), + }), + ); +} + +{ + const commands = []; + const locs = []; + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const setup = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); + commands.push(index); + const searches = []; + for (const q of queryVariants) { + const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { + env, + timeoutMs: 300000, + }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'codebase-context'); + collect(search.stderr, locs, 'codebase-context'); + } + lanes.push( + laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(searches), + }), + ); +} + +{ + const commands = []; + const locs = []; + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { + cwd: repo, + env, + timeoutMs: 2700000, + }); + commands.push(index); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const searches = []; + for (const q of queryVariants) { + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { + cwd: repo, + env, + timeoutMs: 120000, + }); + commands.push(graph); + searches.push(graph); + collect(graph.stdout, locs, 'codebase-memory-mcp'); + collect(graph.stderr, locs, 'codebase-memory-mcp'); + } + for (const term of ['metrics', 'prometheus', 'authorization', 'Bearer', 'token']) { + const code = run( + process.env.CBM_BIN, + ['cli', 'search_code', JSON.stringify({ project, pattern: term, mode: 'compact', limit: 50 })], + { cwd: repo, env, timeoutMs: 120000 }, + ); + commands.push(code); + searches.push(code); + collect(code.stdout, locs, 'codebase-memory-mcp'); + collect(code.stderr, locs, 'codebase-memory-mcp'); + } + lanes.push( + laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(searches), + }), + ); +} + +{ + const commands = []; + const locs = []; + const setup = run('grepai', ['version'], { timeoutMs: 60000 }); + commands.push(setup); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); + commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); + commands.push(watch); + const statusChecks = []; + for (let i = 0; i < 12; i += 1) { + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + if (/chunks?\D+[1-9]|indexed files?\D+[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; + run('sleep', ['5'], { timeoutMs: 10000 }); + } + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 180000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); + commands.push(stop); + lanes.push( + laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs + init.durationMs, + indexDurationMs: watch.durationMs + durationOf(statusChecks), + queryDurationMs: durationOf(searches), + teardownDurationMs: stop.durationMs, + }), + ); +} + +{ + const commands = []; + const locs = []; + const commandProbe = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); + const cgcCommand = commandProbe.status === 0 ? 'codegraphcontext' : 'cgc'; + const setup = commandProbe.status === 0 ? commandProbe : run(cgcCommand, ['--help'], { timeoutMs: 60000 }); + commands.push(setup); + const index = run(cgcCommand, ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run(cgcCommand, ['find', 'pattern', pattern], { cwd: repo, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { + const callers = run(cgcCommand, ['analyze', 'callers', symbol], { cwd: repo, timeoutMs: 180000 }); + commands.push(callers); + queries.push(callers); + collect(callers.stdout, locs, 'codegraphcontext'); + collect(callers.stderr, locs, 'codegraphcontext'); + } + const complexity = run(cgcCommand, ['analyze', 'complexity', '--limit', '80'], { cwd: repo, timeoutMs: 180000 }); + commands.push(complexity); + queries.push(complexity); + collect(complexity.stdout, locs, 'codegraphcontext'); + collect(complexity.stderr, locs, 'codegraphcontext'); + lanes.push( + laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }), + ); +} + +const readiness = lanes.map((lane) => ({ + lane: lane.lane, + ready: + (lane.setupStatus === 'completed' || lane.setupStatus === 'not_required') && + (lane.indexStatus === 'completed' || lane.indexStatus === 'not_required') && + lane.toolCallable && + lane.candidateCount > 0, + setupStatus: lane.setupStatus, + indexStatus: lane.indexStatus, + toolCallable: lane.toolCallable, + candidateCount: lane.candidateCount, + setupIndex: lane.setupIndex, + candidates: lane.candidates.slice(0, 80), +})); + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + lanes, + readiness, +}; + +writeFileSync(join(outDir, 'candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'readiness.json'), JSON.stringify({ task: pack.task, queryVariants, readiness }, null, 2)); +console.log('CONTEXTBENCH_FIVE_LANE_READINESS_JSON_START'); +console.log(JSON.stringify({ task: pack.task, queryVariants, readiness }, null, 2)); +console.log('CONTEXTBENCH_FIVE_LANE_READINESS_JSON_END'); diff --git a/scripts/contextbench-five-lane-selections.json b/scripts/contextbench-five-lane-selections.json new file mode 100644 index 0000000..a526a37 --- /dev/null +++ b/scripts/contextbench-five-lane-selections.json @@ -0,0 +1,119 @@ +{ + "model": "gpt-5.4-mini-high", + "predictionSource": "gpt-5.4-mini-high selector pass over real readiness candidate packs; artifact-aware scorer injects readiness costs and validates artifact candidate files for external lanes", + "caveats": [ + "Only official-evaluator scoreable rows count as benchmark quality results.", + "CodeGraphContext remains excluded because its readiness gate indexed successfully but returned zero task-relevant candidates through the supported CLI in this environment.", + "grepai is included only after the GrepAI readiness artifact proved setup, local Ollama-backed indexing/search callability, and non-empty real candidate files/spans.", + "ripgrep-lexical is a no-index local lexical baseline added as the fifth scoreable competitor; setup/index/query cost is reported separately from quality scores." + ], + "laneSelections": [ + { + "lane_id": "raw-native", + "files": ["cmd/root.go", "core/metrics/prometheus.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "cmd/root.go covers startup wiring, core/metrics/prometheus.go contains the startup metrics implementation path, and server/subsonic/helpers.go covers Subsonic authorization token parsing branches.", + "readiness": { + "setupStatus": "not_required", + "indexStatus": "not_required", + "toolCallable": true, + "candidateCount": 120, + "setupIndex": {"setupDurationMs": 0, "indexDurationMs": 0, "queryDurationMs": 130}, + "sourceRun": 25642669946, + "sourceJob": 75265688816 + } + }, + { + "lane_id": "codebase-context", + "files": ["core/metrics/prometheus.go", "server/auth.go", "server/server.go", "scanner/scanner.go", "cmd/root.go", "cmd/wire_gen.go", "conf/configuration.go", "consts/consts.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 1, "end": 160}, + {"file": "server/auth.go", "start": 165, "end": 230}, + {"file": "server/server.go", "start": 170, "end": 185}, + {"file": "scanner/scanner.go", "start": 50, "end": 230}, + {"file": "cmd/root.go", "start": 100, "end": 170}, + {"file": "cmd/wire_gen.go", "start": 60, "end": 140}, + {"file": "conf/configuration.go", "start": 145, "end": 155}, + {"file": "conf/configuration.go", "start": 425, "end": 432}, + {"file": "consts/consts.go", "start": 68, "end": 82} + ], + "rationale": "This is the selector output over the real codebase-context candidate pack, covering metrics startup wiring plus auth/header/config surfaces it judged relevant.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 36, + "setupIndex": {"setupDurationMs": 570, "indexDurationMs": null, "queryDurationMs": null}, + "sourceRun": 25642303130, + "sourceJob": 75264700845 + } + }, + { + "lane_id": "codebase-memory-mcp", + "files": ["core/metrics/prometheus.go", "cmd/root.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 17}, + {"file": "core/metrics/prometheus.go", "start": 19, "end": 25}, + {"file": "core/metrics/prometheus.go", "start": 33, "end": 38}, + {"file": "core/metrics/prometheus.go", "start": 40, "end": 49}, + {"file": "core/metrics/prometheus.go", "start": 51, "end": 100}, + {"file": "core/metrics/prometheus.go", "start": 102, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 201}, + {"file": "cmd/root.go", "start": 204, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 187}, + {"file": "server/subsonic/helpers.go", "start": 189, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "These spans cover the startup metrics write path, the startup wiring that should invoke it, and the Subsonic auth-token parsing path.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 100, + "setupIndex": {"setupDurationMs": 4, "indexDurationMs": 2038, "queryDurationMs": 365}, + "sourceRun": 25642583473, + "sourceJob": 75265447999 + } + }, + { + "lane_id": "grepai", + "files": ["cmd/root.go", "conf/configuration.go", "core/agents/session_keys.go", "core/agents/listenbrainz/auth_router.go", "core/agents/lastfm/auth_router.go", "cmd/wire_injectors.go"], + "spans": [ + {"file": "cmd/root.go", "start": 289, "end": 292}, + {"file": "conf/configuration.go", "start": 377, "end": 414}, + {"file": "conf/configuration.go", "start": 410, "end": 459}, + {"file": "core/agents/session_keys.go", "start": 20, "end": 25}, + {"file": "core/agents/listenbrainz/auth_router.go", "start": 76, "end": 121}, + {"file": "core/agents/listenbrainz/auth_router.go", "start": 115, "end": 121}, + {"file": "core/agents/lastfm/auth_router.go", "start": 76, "end": 132}, + {"file": "core/agents/lastfm/auth_router.go", "start": 125, "end": 132}, + {"file": "cmd/wire_injectors.go", "start": 75, "end": 90} + ], + "rationale": "Selected from the visible GrepAI readiness candidate pack: startup-adjacent cmd/root.go plus auth/token/config candidates returned by real GrepAI search. Low score is acceptable; empty or failed rows are not.", + "readinessArtifact": "grepai", + "sourceRun": 25643757046, + "sourceJob": 75268574734, + "sourceArtifact": 6908206017, + "sourceDigest": "sha256:ace591378e06892c7f2d07507a6657f07350a57a967fd09de44357f89b6faea3" + }, + { + "lane_id": "ripgrep-lexical", + "files": ["core/metrics/prometheus.go", "cmd/root.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 1, "end": 140}, + {"file": "cmd/root.go", "start": 180, "end": 230}, + {"file": "server/subsonic/helpers.go", "start": 120, "end": 210}, + {"file": "server/subsonic/helpers.go", "start": 250, "end": 275} + ], + "rationale": "Selected from lexical matches on problem terms: metrics/prometheus/startup plus authorization/token/subsonic/header. The artifact-aware scorer requires selected files to appear in the ripgrep readiness candidates.", + "readinessArtifact": "ripgrep", + "sourceRun": 25644123216, + "sourceJob": 75269635984 + } + ] +} diff --git a/scripts/contextbench-grepai-ready-pack.mjs b/scripts/contextbench-grepai-ready-pack.mjs new file mode 100644 index 0000000..a51e184 --- /dev/null +++ b/scripts/contextbench-grepai-ready-pack.mjs @@ -0,0 +1,269 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-grepai-readiness'; +const outDir = join(root, 'pack'); +const logsDir = join(root, 'logs'); +mkdirSync(outDir, { recursive: true }); +mkdirSync(logsDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const list = basenameMap.get(basename(file)) || []; + list.push(file); + basenameMap.set(basename(file), list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'grepai-search') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function jsonish(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { return JSON.parse(cleaned); } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const start = cleaned.indexOf(open); + const end = cleaned.lastIndexOf(close); + if (start >= 0 && end > start) { + try { return JSON.parse(cleaned.slice(start, end + 1)); } catch {} + } + } + return null; +} + +function pathFromObject(value) { + return value.file || value.path || value.file_path || value.filePath || value.relative_path || value.filename || value.source_path || value.uri; +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add(locs, pathFromObject(value), value.start_line || value.startLine || value.line || value.start || 1, value.end_line || value.endLine || value.end || value.line || 1, source); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const lineMatch = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+)(?:-(\d+))?/gm; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2], m[3] || m[2], source); + const fileMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?(?:-(\d+))?/g; + while ((m = fileMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[3] || m[2] || 1, source); +} + +function uniq(locs, max = 200) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +const queryVariants = [ + queryOf(task.problem_statement), + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +const env = { ...process.env, OLLAMA_HOST: process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' }; +const commands = []; +const locs = []; +const setup = run('grepai', ['version'], { env, timeoutMs: 60000 }); +commands.push(setup); +const ollama = run('ollama', ['list'], { env, timeoutMs: 60000 }); +commands.push(ollama); +const init = run('grepai', ['init', '--yes', '--provider', 'ollama', '--model', 'nomic-embed-text', '--backend', 'gob'], { cwd: repo, env, timeoutMs: 120000 }); +commands.push(init); + +const indexScript = ` +set -euo pipefail +LOG=${JSON.stringify(join(logsDir, 'grepai-watch-no-ui.log'))} +STATUSLOG=${JSON.stringify(join(logsDir, 'grepai-status-loop.log'))} +: > "$LOG" +: > "$STATUSLOG" +grepai watch --no-ui > "$LOG" 2>&1 & +pid=$! +ready=0 +for i in $(seq 1 180); do + status="$(grepai status --no-ui 2>&1 || true)" + printf -- '--- status attempt %s ---\n%s\n' "$i" "$status" >> "$STATUSLOG" + if printf -- '%s\n' "$status" | grep -Eiq 'Files indexed:[[:space:]]*[1-9][0-9]*|Total chunks:[[:space:]]*[1-9][0-9]*|[1-9][0-9]* chunks created|[1-9][0-9]* files indexed'; then + ready=1 + break + fi + if grep -Eiq 'Initial scan complete|[1-9][0-9]* files indexed|[1-9][0-9]* chunks created' "$LOG"; then + ready=1 + break + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo 'grepai watch exited before textual readiness; search will be used as the functional readiness proof' + tail -200 "$LOG" || true + exit 2 + fi + sleep 2 +done +kill -INT "$pid" 2>/dev/null || true +for i in $(seq 1 30); do + if ! kill -0 "$pid" 2>/dev/null; then break; fi + sleep 1 +done +kill -TERM "$pid" 2>/dev/null || true +wait "$pid" || true +grepai status --no-ui || true +tail -200 "$STATUSLOG" || true +tail -200 "$LOG" || true +if [ "$ready" -ne 1 ]; then exit 2; fi +`; +const index = run('bash', ['-lc', indexScript], { cwd: repo, env, timeoutMs: 600000 }); +commands.push(index); + +const searches = []; +for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '50'], { cwd: repo, env, timeoutMs: 240000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai-search'); + collect(search.stderr, locs, 'grepai-search'); +} + +const candidates = uniq(locs); +const setupStatus = setup.status === 0 && ollama.status === 0 && init.status === 0 ? 'completed' : 'setup_failed'; +const searchSucceeded = searches.some((command) => command.status === 0); +const indexFunctional = index.status === 0 || (searchSucceeded && candidates.length > 0); +const setupIndex = { + setupDurationMs: setup.durationMs + ollama.durationMs + init.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: searches.reduce((sum, command) => sum + command.durationMs, 0), +}; +const notes = [ + 'Readiness requires setup, callable grepai search, and non-empty candidate spans/files from real repo queries.', +]; +if (index.status !== 0 && indexFunctional) { + notes.push(`grepai watch exited with status ${index.status}; readiness accepts the lane because subsequent grepai search returned ${candidates.length} repo locations from the persisted index.`); +} + +const readiness = { + lane: 'grepai', + ready: setupStatus === 'completed' && indexFunctional && searchSucceeded && candidates.length > 0, + setupStatus, + indexStatus: indexFunctional ? 'completed' : 'index_failed', + toolCallable: searchSucceeded, + candidateCount: candidates.length, + setupIndex, + notes, + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + })), + candidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness, +}; + +writeFileSync(join(outDir, 'grepai-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'grepai-readiness.json'), JSON.stringify(readiness, null, 2)); +console.log('CONTEXTBENCH_GREPAI_READY_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_GREPAI_READY_JSON_END'); +if (!readiness.ready) process.exitCode = 1; diff --git a/scripts/contextbench-infra-lane-readiness.mjs b/scripts/contextbench-infra-lane-readiness.mjs new file mode 100644 index 0000000..61cea0b --- /dev/null +++ b/scripts/contextbench-infra-lane-readiness.mjs @@ -0,0 +1,378 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || `/tmp/contextbench-infra-lane/${laneId || 'unknown'}`; +const outDir = join(root, 'pack'); +const logsDir = join(root, 'logs'); +const supportedLanes = ['grepai', 'codegraphcontext']; +if (!supportedLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); +mkdirSync(logsDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function jsonish(text) { + const cleaned = stripAnsi(text).trim(); + if (!cleaned) return null; + try { + return JSON.parse(cleaned); + } catch {} + for (const [open, close] of [['{', '}'], ['[', ']']]) { + const start = cleaned.indexOf(open); + const end = cleaned.lastIndexOf(close); + if (start >= 0 && end > start) { + try { + return JSON.parse(cleaned.slice(start, end + 1)); + } catch {} + } + } + return null; +} + +function pathFromObject(value) { + return value.file || value.path || value.file_path || value.filePath || value.relative_path || value.filename || value.source_path || value.uri || value.caller_file_path || value.called_file_path || value.importer_file_path || value.parent_file_path || value.child_file_path || value.class_file_path; +} + +function startFromObject(value) { + return value.start_line || value.startLine || value.line || value.line_number || value.start || value.caller_line_number || value.called_line_number || value.import_line_number || value.parent_line_number || value.child_line_number || value.function_line_number || 1; +} + +function endFromObject(value) { + return value.end_line || value.endLine || value.end || value.line || value.line_number || value.endLineNumber || startFromObject(value); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add(locs, pathFromObject(value), startFromObject(value), endFromObject(value), source); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const lineMatch = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+)(?:-(\d+))?/gm; + let m; + while ((m = lineMatch.exec(cleaned)) !== null) add(locs, m[1], m[2], m[3] || m[2], source); + const fileMatch = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?(?:-(\d+))?/g; + while ((m = fileMatch.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[3] || m[2] || 1, source); +} + +const relevantPathTerms = ['metrics', 'prometheus', 'auth', 'subsonic', 'root', 'header', 'token']; +function collectRelevantInventory(text, locs, source) { + const inventory = []; + collect(text, inventory, source); + for (const loc of inventory) { + const file = loc.file.toLowerCase(); + if (relevantPathTerms.some((term) => file.includes(term))) locs.push({ ...loc, source }); + } +} + +function uniq(locs, max = 160) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ + ...command, + stdout: command.stdout.slice(0, 300000), + stderr: command.stderr.slice(0, 300000), + }, null, 2)); + } +} + +function commandSummary(command) { + return { + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + }; +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex, notes = []) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: setupStatus === 'completed' && indexStatus === 'completed' && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + notes, + commands: commands.map(commandSummary), + candidates, + }; +} + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +const queryVariants = [ + queryOf(task.problem_statement), + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runGrepai() { + const commands = []; + const locs = []; + const env = { ...process.env, OLLAMA_HOST: process.env.OLLAMA_HOST || 'http://127.0.0.1:11434' }; + const setup = run('grepai', ['version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const ollama = run('ollama', ['list'], { env, timeoutMs: 60000 }); + commands.push(ollama); + const init = run('grepai', ['init', '--yes', '--provider', 'ollama', '--model', 'nomic-embed-text', '--backend', 'gob'], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(init); + const indexScript = ` +set -euo pipefail +LOG=${JSON.stringify(join(logsDir, 'grepai-watch-no-ui.log'))} +STATUSLOG=${JSON.stringify(join(logsDir, 'grepai-status-loop.log'))} +: > "$LOG" +: > "$STATUSLOG" +grepai watch --no-ui > "$LOG" 2>&1 & +pid=$! +ready=0 +for i in $(seq 1 180); do + status="$(grepai status --no-ui 2>&1 || true)" + printf -- '--- status attempt %s ---\n%s\n' "$i" "$status" >> "$STATUSLOG" + if printf -- '%s\n' "$status" | grep -Eiq 'Files indexed:[[:space:]]*[1-9][0-9]*|Total chunks:[[:space:]]*[1-9][0-9]*|[1-9][0-9]* chunks created|[1-9][0-9]* files indexed'; then + ready=1 + break + fi + if grep -Eiq 'Initial scan complete|[1-9][0-9]* files indexed|[1-9][0-9]* chunks created' "$LOG"; then + ready=1 + break + fi + if ! kill -0 "$pid" 2>/dev/null; then + echo 'grepai watch exited before readiness' + tail -200 "$LOG" || true + exit 1 + fi + sleep 2 +done +if [ "$ready" -ne 1 ]; then + echo 'grepai watch did not report readiness before timeout' + tail -200 "$STATUSLOG" || true + tail -200 "$LOG" || true + kill -INT "$pid" 2>/dev/null || true + wait "$pid" || true + exit 1 +fi +kill -INT "$pid" 2>/dev/null || true +for i in $(seq 1 60); do + if ! kill -0 "$pid" 2>/dev/null; then + break + fi + sleep 1 +done +kill -TERM "$pid" 2>/dev/null || true +wait "$pid" || true +grepai status --no-ui || true +tail -200 "$STATUSLOG" || true +tail -200 "$LOG" || true +`; + const index = run('bash', ['-lc', indexScript], { cwd: repo, env, timeoutMs: 900000 }); + commands.push(index); + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '50'], { cwd: repo, env, timeoutMs: 240000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai-search'); + collect(search.stderr, locs, 'grepai-search'); + } + return laneResult('grepai', commands, locs, setup.status === 0 && ollama.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', index.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs + ollama.durationMs + init.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(searches), + }, ['Uses grepai foreground --no-ui indexing in CI because documented background mode can return failure before long initial scans finish.']); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const env = { ...process.env, DEFAULT_DATABASE: 'kuzudb', CGC_RUNTIME_DB_TYPE: 'kuzudb' }; + const setup = run('cgc', ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('cgc', ['--database', 'kuzudb', 'index', '.', '--force'], { cwd: repo, env, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + const stats = run('cgc', ['--database', 'kuzudb', 'stats', '--all'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(stats); + queries.push(stats); + const fileInventory = run('cgc', ['--database', 'kuzudb', 'find', 'type', 'file'], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(fileInventory); + queries.push(fileInventory); + collectRelevantInventory(fileInventory.stdout, locs, 'codegraphcontext-file-inventory-filtered'); + collectRelevantInventory(fileInventory.stderr, locs, 'codegraphcontext-file-inventory-filtered'); + for (const term of ['metrics', 'prometheus', 'authorization', 'bearer', 'token', 'header', 'subsonic', 'startup']) { + for (const mode of ['pattern', 'content']) { + const found = run('cgc', ['--database', 'kuzudb', 'find', mode, term], { cwd: repo, env, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, `codegraphcontext-${mode}-${term}`); + collect(found.stderr, locs, `codegraphcontext-${mode}-${term}`); + } + } + const py = ` +import json +from codegraphcontext.core.database import DatabaseManager +from codegraphcontext.tools.code_finder import CodeFinder +terms = ['metrics','prometheus','authorization','bearer','token','header','subsonic','startup'] +functions = ['init','main','ServeHTTP','getUser','authenticate','Prometheus','WriteInitialMetrics'] +db = DatabaseManager() +cf = CodeFinder(db) +out = {'files': cf.find_by_type('file', 1000), 'content': {}, 'related': {}, 'functions': {}} +for term in terms: + try: + out['content'][term] = cf.find_by_content(term) + except Exception as exc: + out['content'][term] = {'error': str(exc)} + try: + out['related'][term] = cf.find_related_code(term, True, 2) + except Exception as exc: + out['related'][term] = {'error': str(exc)} +for fn in functions: + try: + out['functions'][fn] = cf.find_by_function_name(fn, True) + except Exception as exc: + out['functions'][fn] = {'error': str(exc)} +print(json.dumps(out)) +db.close_driver() +`; + const api = run('python', ['-c', py], { cwd: repo, env, timeoutMs: 300000 }); + commands.push(api); + queries.push(api); + collect(api.stdout, locs, 'codegraphcontext-codefinder'); + collect(api.stderr, locs, 'codegraphcontext-codefinder'); + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 && locs.length > 0 ? 'completed' : 'index_failed', { + setupDurationMs: setup.durationMs, + indexDurationMs: index.durationMs, + queryDurationMs: durationOf(queries), + }, ['Uses documented cgc find/query surface plus CodeFinder API; avoids raw Cypher against backend-specific driver wrappers.']); +} + +const result = laneId === 'grepai' ? runGrepai() : runCodeGraphContext(); +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_INFRA_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_INFRA_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; diff --git a/scripts/contextbench-print-publishable-report.mjs b/scripts/contextbench-print-publishable-report.mjs new file mode 100644 index 0000000..b0e1b9b --- /dev/null +++ b/scripts/contextbench-print-publishable-report.mjs @@ -0,0 +1,21 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const reportPath = process.argv[2] || join(process.env.ROOT || '/tmp/contextbench-five-lane-score', 'publishable-summary.json'); +const report = JSON.parse(readFileSync(reportPath, 'utf8')); + +const compact = { + publicationStatus: report.publicationStatus, + validationStatus: report.biasAudit?.status || null, + scope: report.scope, + qualityTable: report.qualityTable, + costTable: report.costTable, + tokenTable: report.tokenTable, + reliabilityTable: report.reliabilityTable, + limitations: report.limitations, +}; + +console.log('CONTEXTBENCH_PUBLISHABLE_COMPACT_JSON_START'); +console.log(JSON.stringify(compact, null, 2)); +console.log('CONTEXTBENCH_PUBLISHABLE_COMPACT_JSON_END'); +if (report.biasAudit?.status !== 'pass') process.exitCode = 1; diff --git a/scripts/contextbench-ready-lane-selections.json b/scripts/contextbench-ready-lane-selections.json new file mode 100644 index 0000000..74ca273 --- /dev/null +++ b/scripts/contextbench-ready-lane-selections.json @@ -0,0 +1,83 @@ +{ + "model": "gpt-5.4-mini-high", + "predictionSource": "gpt-5.4-mini-high selector subagents over real readiness candidate packs", + "caveats": [ + "This selection set intentionally excludes grepai and codegraphcontext because their strict readiness gates produced candidateCount=0 and are not scoreable quality rows.", + "grepai installed and searched with Ollama, but indexing failed and no candidates were returned.", + "CodeGraphContext installed and indexed, but strict task-relevant retrieval returned no candidates; the earlier broad file-list run is not counted as readiness evidence." + ], + "laneSelections": [ + { + "lane_id": "raw-native", + "files": ["cmd/root.go", "core/metrics/prometheus.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "cmd/root.go covers startup wiring, core/metrics/prometheus.go contains the startup metrics implementation path, and server/subsonic/helpers.go covers Subsonic authorization token parsing branches.", + "readiness": { + "setupStatus": "not_required", + "indexStatus": "not_required", + "toolCallable": true, + "candidateCount": 120, + "setupIndex": {"setupDurationMs": 0, "indexDurationMs": 0, "queryDurationMs": 130}, + "sourceRun": 25642669946, + "sourceJob": 75265688816 + } + }, + { + "lane_id": "codebase-context", + "files": ["core/metrics/prometheus.go", "server/auth.go", "server/server.go", "scanner/scanner.go", "cmd/root.go", "cmd/wire_gen.go", "conf/configuration.go", "consts/consts.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 1, "end": 160}, + {"file": "server/auth.go", "start": 165, "end": 230}, + {"file": "server/server.go", "start": 170, "end": 185}, + {"file": "scanner/scanner.go", "start": 50, "end": 230}, + {"file": "cmd/root.go", "start": 100, "end": 170}, + {"file": "cmd/wire_gen.go", "start": 60, "end": 140}, + {"file": "conf/configuration.go", "start": 145, "end": 155}, + {"file": "conf/configuration.go", "start": 425, "end": 432}, + {"file": "consts/consts.go", "start": 68, "end": 82} + ], + "rationale": "This is the selector output over the real codebase-context candidate pack, covering metrics startup wiring plus auth/header/config surfaces it judged relevant.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 36, + "setupIndex": {"setupDurationMs": 570, "indexDurationMs": null, "queryDurationMs": null}, + "sourceRun": 25642303130, + "sourceJob": 75264700845 + } + }, + { + "lane_id": "codebase-memory-mcp", + "files": ["core/metrics/prometheus.go", "cmd/root.go", "server/subsonic/helpers.go"], + "spans": [ + {"file": "core/metrics/prometheus.go", "start": 15, "end": 17}, + {"file": "core/metrics/prometheus.go", "start": 19, "end": 25}, + {"file": "core/metrics/prometheus.go", "start": 33, "end": 38}, + {"file": "core/metrics/prometheus.go", "start": 40, "end": 49}, + {"file": "core/metrics/prometheus.go", "start": 51, "end": 100}, + {"file": "core/metrics/prometheus.go", "start": 102, "end": 123}, + {"file": "cmd/root.go", "start": 194, "end": 201}, + {"file": "cmd/root.go", "start": 204, "end": 220}, + {"file": "server/subsonic/helpers.go", "start": 129, "end": 187}, + {"file": "server/subsonic/helpers.go", "start": 189, "end": 201}, + {"file": "server/subsonic/helpers.go", "start": 256, "end": 268} + ], + "rationale": "These spans cover the startup metrics write path, the startup wiring that should invoke it, and the Subsonic auth-token parsing path.", + "readiness": { + "setupStatus": "completed", + "indexStatus": "completed", + "toolCallable": true, + "candidateCount": 100, + "setupIndex": {"setupDurationMs": 4, "indexDurationMs": 2038, "queryDurationMs": 365}, + "sourceRun": 25642583473, + "sourceJob": 75265447999 + } + } + ] +} diff --git a/scripts/contextbench-rescue-lane-pack.mjs b/scripts/contextbench-rescue-lane-pack.mjs new file mode 100644 index 0000000..e326b56 --- /dev/null +++ b/scripts/contextbench-rescue-lane-pack.mjs @@ -0,0 +1,391 @@ +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || `/tmp/contextbench-rescue-${laneId || 'unknown'}`; +const outDir = join(root, 'pack'); +const supportedLanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp', 'grepai', 'codegraphcontext']; +if (!supportedLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(s) { + const t = stripAnsi(s).trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + const marker = '/tmp/contextbench-checkouts-'; + const tmpIdx = f.indexOf(marker); + if (tmpIdx >= 0) { + const parts = f.slice(tmpIdx + marker.length).split('/'); + f = parts.slice(1).join('/'); + } + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp/')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = laneId) { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source = laneId) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const rgLine = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = rgLine.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ ...command, stdout: command.stdout.slice(0, 200000), stderr: command.stderr.slice(0, 200000) }, null, 2)); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: (setupStatus === 'completed' || setupStatus === 'not_required') && (indexStatus === 'completed' || indexStatus === 'not_required') && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ command: command.command, status: command.status, signal: command.signal, error: command.error, durationMs: command.durationMs })), + candidates, + }; +} + +const baseQuery = queryOf(task.problem_statement); +const queryVariants = [ + baseQuery, + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runRawNative() { + const commands = []; + const locs = []; + for (const term of ['metrics', 'prometheus', 'insights', 'startup', 'start', 'written', 'authorization', 'Bearer', 'token', 'header', 'authentication', 'subsonic']) { + const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); + commands.push(r); + collect(r.stdout, locs, 'raw-native'); + collect(r.stderr, locs, 'raw-native'); + } + return laneResult('raw-native', commands, locs, 'not_required', 'not_required', { setupDurationMs: 0, indexDurationMs: 0, queryDurationMs: durationOf(commands) }); +} + +function runCodebaseContext() { + const commands = []; + const locs = []; + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const setup = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 420000 }); + commands.push(index); + const searches = []; + if (index.status === 0) { + for (const q of queryVariants) { + const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { env, timeoutMs: 90000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'codebase-context'); + collect(search.stderr, locs, 'codebase-context'); + } + } + return laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +function runCodebaseMemoryMcp() { + const commands = []; + const locs = []; + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 1800000 }); + commands.push(index); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const searches = []; + if (index.status === 0) { + for (const q of queryVariants) { + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(graph); + searches.push(graph); + collect(graph.stdout, locs, 'codebase-memory-mcp'); + collect(graph.stderr, locs, 'codebase-memory-mcp'); + } + for (const term of ['metrics', 'prometheus', 'authorization', 'Bearer', 'token']) { + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: term, mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(code); + searches.push(code); + collect(code.stdout, locs, 'codebase-memory-mcp'); + collect(code.stderr, locs, 'codebase-memory-mcp'); + } + } + return laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +async function runGrepai() { + const commands = []; + const locs = []; + const setup = run('grepai', ['version'], { timeoutMs: 60000 }); + commands.push(setup); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); + commands.push(init); + + const started = Date.now(); + const watcher = spawn('grepai', ['watch', '--no-ui'], { cwd: repo, env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }); + let watchStdout = ''; + let watchStderr = ''; + let watchExitCode = null; + let watchSignal = null; + watcher.stdout.on('data', (chunk) => { watchStdout = `${watchStdout}${chunk}`.slice(-200000); }); + watcher.stderr.on('data', (chunk) => { watchStderr = `${watchStderr}${chunk}`.slice(-200000); }); + watcher.on('exit', (code, signal) => { watchExitCode = code; watchSignal = signal; }); + + const statusChecks = []; + let indexed = false; + for (let i = 0; i < 18; i += 1) { + await sleep(i === 0 ? 3000 : 5000); + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + const text = `${status.stdout}\n${status.stderr}\n${watchStdout}\n${watchStderr}`; + if (/chunks?\D+[1-9][0-9]*|indexed files?\D+[1-9][0-9]*|files indexed\D+[1-9][0-9]*/i.test(text)) { + indexed = true; + break; + } + if (watchExitCode !== null && i > 1) break; + } + + const searches = []; + if (indexed) { + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 90000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + } + + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); + if (watchExitCode === null) watcher.kill('SIGTERM'); + await sleep(1000); + commands.push({ + command: 'grepai watch --no-ui', + cwd: repo, + status: indexed ? 0 : watchExitCode, + signal: watchSignal, + error: null, + durationMs: Date.now() - started, + stdout: watchStdout, + stderr: watchStderr, + }); + commands.push(stop); + + return laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', indexed ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs + init.durationMs, indexDurationMs: durationOf(statusChecks), queryDurationMs: durationOf(searches), teardownDurationMs: stop.durationMs }); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const cgcProbe = run('cgc', ['--help'], { timeoutMs: 60000 }); + const command = cgcProbe.status === 0 ? 'cgc' : 'codegraphcontext'; + const setup = cgcProbe.status === 0 ? cgcProbe : run(command, ['--help'], { timeoutMs: 60000 }); + commands.push(setup); + const index = run(command, ['index', '.'], { cwd: repo, timeoutMs: 900000 }); + commands.push(index); + const stats = run(command, ['stats'], { cwd: repo, timeoutMs: 60000 }); + commands.push(stats); + const queries = [stats]; + collect(stats.stdout, locs, 'codegraphcontext'); + collect(stats.stderr, locs, 'codegraphcontext'); + + if (index.status === 0) { + for (const q of ['metrics', 'prometheus', 'insights', 'startup metrics', 'authorization header', 'Bearer token', 'subsonic authentication', 'helpers', 'root command']) { + const found = run(command, ['find', q], { cwd: repo, timeoutMs: 90000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const args of [ + ['query', 'callers', 'main'], + ['query', 'callees', 'main'], + ['query', 'callers', 'GetUser'], + ['query', 'callees', 'GetUser'], + ['query', 'callers', 'ServeHTTP'], + ['query', 'dead-code', ''], + ]) { + const cleanArgs = args.filter(Boolean); + const q = run(command, cleanArgs, { cwd: repo, timeoutMs: 90000 }); + commands.push(q); + queries.push(q); + collect(q.stdout, locs, 'codegraphcontext'); + collect(q.stderr, locs, 'codegraphcontext'); + } + } + + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(queries) }); +} + +const result = laneId === 'raw-native' + ? runRawNative() + : laneId === 'codebase-context' + ? runCodebaseContext() + : laneId === 'codebase-memory-mcp' + ? runCodebaseMemoryMcp() + : laneId === 'grepai' + ? await runGrepai() + : runCodeGraphContext(); + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_RESCUE_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_RESCUE_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; diff --git a/scripts/contextbench-ripgrep-ready-pack.mjs b/scripts/contextbench-ripgrep-ready-pack.mjs new file mode 100644 index 0000000..36d55ae --- /dev/null +++ b/scripts/contextbench-ripgrep-ready-pack.mjs @@ -0,0 +1,176 @@ +import { spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-ripgrep-readiness'; +const outDir = join(root, 'pack'); +const logsDir = join(root, 'logs'); +mkdirSync(outDir, { recursive: true }); +mkdirSync(logsDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const result = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof result.status === 'number' ? result.status : null, + signal: result.signal, + error: result.error?.message || null, + durationMs: Date.now() - started, + stdout: result.stdout || '', + stderr: result.stderr || '', + }; +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function addCandidate(candidates, file, line, source, matchText = '') { + const clean = String(file || '').replaceAll('\\', '/').replace(/^\.\//, ''); + if (!clean || clean.startsWith('../') || clean.includes('://')) return; + const n = Math.max(1, Number(line) || 1); + const start = Math.max(1, n - 12); + const end = n + 12; + candidates.push({ file: clean, start, end, line: n, source, matchText: stripAnsi(matchText).trim().slice(0, 220) }); +} + +function parseRgJson(output, candidates, source) { + for (const line of String(output || '').split(/\n+/)) { + if (!line.trim()) continue; + let event; + try { + event = JSON.parse(line); + } catch { + continue; + } + if (event.type !== 'match') continue; + addCandidate( + candidates, + event.data?.path?.text, + event.data?.line_number, + source, + event.data?.lines?.text || '', + ); + } +} + +function uniq(candidates, max = 500) { + const seen = new Set(); + const out = []; + for (const candidate of candidates) { + const key = `${candidate.file}:${candidate.start}:${candidate.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(candidate); + if (out.length >= max) break; + } + return out; +} + +const queries = [ + { id: 'metrics-startup', pattern: '(?i)prometheus|metrics|metric|insights|startup|start' }, + { id: 'auth-header-token', pattern: '(?i)authorization|bearer|token|header|subsonic|auth' }, + { id: 'write-collect-init', pattern: '(?i)collector|collect|write|init|server' }, +]; + +const commands = []; +const candidates = []; +const install = run( + 'bash', + ['-lc', 'command -v rg >/dev/null 2>&1 || (sudo apt-get update && sudo apt-get install -y ripgrep)'], + { timeoutMs: 300000 }, +); +commands.push(install); +const version = run('rg', ['--version'], { timeoutMs: 60000 }); +commands.push(version); + +for (const query of queries) { + const result = run( + 'rg', + [ + '--json', + '-n', + '--hidden', + '--glob', + '!.git/**', + '--glob', + '!vendor/**', + '--glob', + '!ui/**', + '--glob', + '*.go', + query.pattern, + '.', + ], + { cwd: repo, timeoutMs: 180000 }, + ); + commands.push(result); + parseRgJson(result.stdout, candidates, `ripgrep-lexical:${query.id}`); +} + +const uniqueCandidates = uniq(candidates); +const setupStatus = install.status === 0 && version.status === 0 ? 'completed' : 'setup_failed'; +const queryCommands = commands.slice(2); +const toolCallable = queryCommands.some((command) => command.status === 0 || command.status === 1); +const queryOk = queryCommands.every((command) => command.status === 0 || command.status === 1); +const readiness = { + lane: 'ripgrep-lexical', + ready: setupStatus === 'completed' && toolCallable && queryOk && uniqueCandidates.length > 0, + setupStatus, + indexStatus: 'not_required', + toolCallable, + candidateCount: uniqueCandidates.length, + setupIndex: { + setupDurationMs: install.durationMs + version.durationMs, + indexDurationMs: 0, + queryDurationMs: queryCommands.reduce((sum, command) => sum + command.durationMs, 0), + }, + notes: [ + 'ripgrep-lexical is a no-index local lexical competitor; readiness requires rg callable plus non-empty real repo file/span candidates.', + 'rg exit code 1 means no matches for a query and is accepted only when at least one required query returns candidates.', + ], + commands: commands.map((command) => ({ + command: command.command, + status: command.status, + signal: command.signal, + error: command.error, + durationMs: command.durationMs, + stdoutExcerpt: stripAnsi(command.stdout).slice(0, 1200), + stderrExcerpt: stripAnsi(command.stderr).slice(0, 1200), + })), + candidates: uniqueCandidates, +}; + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queries, + readiness, +}; + +writeFileSync(join(outDir, 'ripgrep-candidate-pack.json'), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, 'ripgrep-readiness.json'), JSON.stringify(readiness, null, 2)); +console.log('CONTEXTBENCH_RIPGREP_READY_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_RIPGREP_READY_JSON_END'); +if (!readiness.ready) process.exitCode = 1; diff --git a/scripts/contextbench-score-five-lane-artifact-selections.mjs b/scripts/contextbench-score-five-lane-artifact-selections.mjs new file mode 100644 index 0000000..c20a01f --- /dev/null +++ b/scripts/contextbench-score-five-lane-artifact-selections.mjs @@ -0,0 +1,140 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const root = process.env.ROOT || '/tmp/contextbench-five-lane-score'; +const sourceSelectionsPath = process.env.SOURCE_SELECTIONS_PATH || 'scripts/contextbench-five-lane-selections.json'; +const externalRoot = process.env.EXTERNAL_READINESS_ROOT || join(root, 'external-readiness'); +const resolvedSelectionsPath = join(root, 'resolved-five-lane-selections.json'); + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function normalizePath(path) { + return String(path || '').replaceAll('\\', '/').replace(/^\.\//, ''); +} + +function estimateTokensFromBytes(bytes) { + if (!Number.isFinite(bytes)) return null; + return Math.ceil(bytes / 4); +} + +function candidateText(candidate) { + return String(candidate.text || candidate.content || candidate.snippet || candidate.preview || candidate.summary || candidate.excerpt || ''); +} + +function candidateStart(candidate) { + return candidate.start ?? candidate.startLine ?? candidate.lineStart ?? candidate.range?.start ?? null; +} + +function candidateEnd(candidate) { + return candidate.end ?? candidate.endLine ?? candidate.lineEnd ?? candidate.range?.end ?? null; +} + +function candidateMetricsFromReadiness(readiness) { + const candidates = Array.isArray(readiness.candidates) ? readiness.candidates : []; + const candidateCount = Number(readiness.candidateCount ?? candidates.length ?? 0); + if (candidates.length === 0) { + return { + candidateCount, + fileCount: null, + spanCount: null, + bytes: null, + estimatedTokens: null, + source: 'readiness artifact', + unavailableReason: 'readiness artifact did not include candidate payloads, only candidateCount', + }; + } + const normalized = candidates.map((candidate) => ({ + file: normalizePath(candidate.file || candidate.path), + start: candidateStart(candidate), + end: candidateEnd(candidate), + score: candidate.score ?? candidate.rank ?? candidate.weight ?? null, + text: candidateText(candidate), + })); + const files = new Set(normalized.map((candidate) => candidate.file).filter(Boolean)); + const bytes = Buffer.byteLength(JSON.stringify(normalized), 'utf8'); + return { + candidateCount, + fileCount: files.size, + spanCount: normalized.length, + bytes, + estimatedTokens: estimateTokensFromBytes(bytes), + source: 'normalized readiness.candidates JSON', + }; +} + +function loadReadiness(lane, artifactName) { + const candidates = [ + join(externalRoot, artifactName, 'pack', `${artifactName}-readiness.json`), + join(externalRoot, artifactName, 'pack', `${lane}-readiness.json`), + join(externalRoot, artifactName, `${artifactName}-readiness.json`), + join(externalRoot, artifactName, `${lane}-readiness.json`), + ]; + const path = candidates.find((candidate) => existsSync(candidate)); + if (!path) { + throw new Error(`readiness artifact for ${lane} not found under ${join(externalRoot, artifactName)}`); + } + const readiness = readJson(path); + if (readiness.ready !== true) throw new Error(`${lane} readiness artifact is not ready`); + if (readiness.toolCallable !== true) throw new Error(`${lane} readiness artifact does not prove callable tool`); + if (!Number.isFinite(Number(readiness.candidateCount)) || Number(readiness.candidateCount) <= 0) { + throw new Error(`${lane} readiness artifact has no candidates`); + } + return readiness; +} + +function assertSelectedFilesCameFromCandidates(selection, readiness) { + if (selection.validateCandidateFiles === false) return; + const candidateFiles = new Set((readiness.candidates || []).map((candidate) => normalizePath(candidate.file || candidate.path))); + if (candidateFiles.size === 0) return; + const selectedFiles = new Set([ + ...(selection.files || []), + ...(selection.spans || []).map((span) => span.file), + ].filter(Boolean).map((file) => normalizePath(file))); + const missing = [...selectedFiles].filter((file) => !candidateFiles.has(file)); + if (missing.length > 0) { + throw new Error(`${selection.lane_id || selection.lane} selected files missing from readiness candidates: ${missing.join(', ')}`); + } +} + +const selections = readJson(sourceSelectionsPath); +const resolved = { + ...selections, + laneSelections: selections.laneSelections.map((selection) => { + const lane = selection.lane_id || selection.lane; + if (!selection.readinessArtifact) return selection; + const readiness = loadReadiness(lane, selection.readinessArtifact); + assertSelectedFilesCameFromCandidates(selection, readiness); + const candidateMetrics = candidateMetricsFromReadiness(readiness); + return { + ...selection, + candidateMetrics, + readiness: { + setupStatus: readiness.setupStatus, + indexStatus: readiness.indexStatus, + toolCallable: readiness.toolCallable, + candidateCount: readiness.candidateCount, + setupIndex: readiness.setupIndex, + candidateMetrics, + sourceRun: selection.sourceRun, + sourceJob: selection.sourceJob, + sourceArtifact: selection.sourceArtifact, + sourceDigest: selection.sourceDigest, + }, + }; + }), +}; + +mkdirSync(root, { recursive: true }); +writeFileSync(resolvedSelectionsPath, `${JSON.stringify(resolved, null, 2)}\n`); + +const result = spawnSync('node', ['scripts/contextbench-score-five-lane-selections.mjs'], { + cwd: process.cwd(), + env: { ...process.env, SELECTIONS_PATH: resolvedSelectionsPath }, + stdio: 'inherit', + timeout: 60 * 60 * 1000, +}); +if (result.error) throw result.error; +process.exitCode = typeof result.status === 'number' ? result.status : 1; diff --git a/scripts/contextbench-score-five-lane-selections.mjs b/scripts/contextbench-score-five-lane-selections.mjs new file mode 100644 index 0000000..5c36bfc --- /dev/null +++ b/scripts/contextbench-score-five-lane-selections.mjs @@ -0,0 +1,325 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-five-lane-score'; +const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; +const selectionsPath = process.env.SELECTIONS_PATH || 'scripts/contextbench-five-lane-selections.json'; +const requiredLanes = (process.env.REQUIRED_LANES || 'raw-native,codebase-context,codebase-memory-mcp,grepai,codegraphcontext') + .split(',') + .map((lane) => lane.trim()) + .filter(Boolean); +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +if (!officialContextBench) throw new Error('OFFICIAL_CONTEXTBENCH is required'); +if (!existsSync(selectionsPath)) throw new Error(`selection file missing: ${selectionsPath}`); +const selections = JSON.parse(readFileSync(selectionsPath, 'utf8')); +const laneSelections = selections.laneSelections || []; +if (laneSelections.length === 0) throw new Error('selection file has no laneSelections'); + +const laneCounts = new Map(); +for (const selection of laneSelections) { + const lane = selection.lane_id || selection.lane; + laneCounts.set(lane, (laneCounts.get(lane) || 0) + 1); +} +const missingLanes = requiredLanes.filter((lane) => !laneCounts.has(lane)); +const duplicateLanes = [...laneCounts.entries()].filter(([, count]) => count > 1).map(([lane]) => lane); +const extraLanes = [...laneCounts.keys()].filter((lane) => !requiredLanes.includes(lane)); +if (missingLanes.length > 0 || duplicateLanes.length > 0 || extraLanes.length > 0) { + throw new Error( + `lane selection set invalid: missing=${missingLanes.join(',') || 'none'} duplicate=${duplicateLanes.join(',') || 'none'} extra=${extraLanes.join(',') || 'none'}`, + ); +} + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function cleanPath(path) { + return String(path || '').replaceAll('\\', '/').replace(/^\.\//, ''); +} + +function addSpan(map, file, start, end) { + const clean = cleanPath(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = map.get(clean) || []; + list.push({ start: s, end: e }); + map.set(clean, list); +} + +function finiteNumber(value) { + if (value === null || value === undefined || value === '') return null; + const numeric = Number(value); + return Number.isFinite(numeric) ? numeric : null; +} + +function estimateTokensFromBytes(bytes) { + if (!Number.isFinite(bytes)) return null; + return Math.ceil(bytes / 4); +} + +function measuredNumber(value, unit, source, unavailableReason = 'not captured in source artifact') { + const numeric = finiteNumber(value); + if (numeric !== null) return { value: numeric, unit, source }; + return { value: null, unit, source, unavailableReason }; +} + +function byteCount(text) { + return Buffer.byteLength(String(text || ''), 'utf8'); +} + +function buildTimeMetrics(readiness, evaluator, rowWallDurationMs, evaluatorSkippedReason = null) { + const setupIndex = readiness.setupIndex || {}; + return { + setupDurationMs: measuredNumber(setupIndex.setupDurationMs, 'ms', 'lane readiness setupIndex', 'readiness artifact did not report setupDurationMs'), + indexDurationMs: measuredNumber(setupIndex.indexDurationMs, 'ms', 'lane readiness setupIndex', 'readiness artifact did not report indexDurationMs'), + queryDurationMs: measuredNumber(setupIndex.queryDurationMs, 'ms', 'lane readiness setupIndex', 'readiness artifact did not report queryDurationMs'), + selectorDurationMs: measuredNumber(null, 'ms', 'selector stage', 'selector ran before scoring and did not emit wall-clock telemetry'), + evaluatorDurationMs: evaluator + ? measuredNumber(evaluator.durationMs, 'ms', 'official ContextBench evaluator command') + : measuredNumber(null, 'ms', 'official ContextBench evaluator command', evaluatorSkippedReason || 'evaluator did not run'), + rowWallDurationMs: measuredNumber(rowWallDurationMs, 'ms', 'scorer per-lane wall clock'), + }; +} + +function buildTokenMetrics(selection, prediction) { + const candidateMetrics = selection.candidateMetrics || selection.readiness?.candidateMetrics || {}; + const candidateBytes = finiteNumber(candidateMetrics.bytes); + const candidateEstimatedTokens = finiteNumber(candidateMetrics.estimatedTokens); + const predictionBytes = byteCount(JSON.stringify(prediction || {})); + const selectorUsage = selection.selectorUsage || {}; + return { + estimator: 'ceil(utf8_bytes/4); cost estimate only, not provider billing telemetry', + candidatePack: { + candidateCount: finiteNumber(selection.readiness?.candidateCount ?? selection.candidateCount ?? candidateMetrics.candidateCount) ?? 0, + fileCount: finiteNumber(candidateMetrics.fileCount), + spanCount: finiteNumber(candidateMetrics.spanCount), + bytes: candidateBytes !== null + ? measuredNumber(candidateBytes, 'bytes', candidateMetrics.source || 'candidate pack artifact') + : measuredNumber(null, 'bytes', candidateMetrics.source || 'candidate pack artifact', candidateMetrics.unavailableReason || 'candidate pack bytes were not emitted for this lane'), + estimatedTokens: candidateEstimatedTokens !== null + ? measuredNumber(candidateEstimatedTokens, 'tokens', candidateMetrics.source || 'candidate pack artifact') + : measuredNumber(null, 'tokens', candidateMetrics.source || 'candidate pack artifact', candidateMetrics.unavailableReason || 'candidate pack token estimate was not emitted for this lane'), + }, + prediction: { + bytes: measuredNumber(predictionBytes, 'bytes', 'official evaluator prediction JSON'), + estimatedTokens: measuredNumber(estimateTokensFromBytes(predictionBytes), 'tokens', 'official evaluator prediction JSON'), + }, + selectorUsage: { + model: selection.selectorModel || selections.model || 'gpt-5.4-mini-high', + inputTokens: measuredNumber(selectorUsage.inputTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + outputTokens: measuredNumber(selectorUsage.outputTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + cachedInputTokens: measuredNumber(selectorUsage.cachedInputTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + reasoningTokens: measuredNumber(selectorUsage.reasoningTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + totalTokens: measuredNumber(selectorUsage.totalTokens, 'tokens', 'selector provider usage', 'selector usage telemetry was not captured for this proof artifact'), + }, + }; +} + +function reliabilityFor(selection, rowBase, status, scoreable) { + return { + status, + officialEvaluatorScoreable: scoreable, + setupStatus: rowBase.setupStatus, + indexStatus: rowBase.indexStatus, + toolCallable: rowBase.toolCallable, + nonEmptyPrediction: rowBase.nonEmptyPrediction, + candidateCount: rowBase.candidateCount, + sourceRun: selection.readiness?.sourceRun || selection.sourceRun || null, + sourceJob: selection.readiness?.sourceJob || selection.sourceJob || null, + sourceArtifact: selection.readiness?.sourceArtifact || selection.sourceArtifact || null, + sourceDigest: selection.readiness?.sourceDigest || selection.sourceDigest || null, + }; +} + +function resultTableRow(row) { + const final = row.score?.final || {}; + return { + lane: row.lane_id, + task: row.task_id, + fileCoverage: final.file?.coverage ?? null, + filePrecision: final.file?.precision ?? null, + symbolCoverage: final.symbol?.coverage ?? null, + symbolPrecision: final.symbol?.precision ?? null, + spanCoverage: final.span?.coverage ?? null, + spanPrecision: final.span?.precision ?? null, + lineCoverage: final.line?.coverage ?? null, + linePrecision: final.line?.precision ?? null, + editlocRecall: row.score?.editloc?.recall ?? null, + editlocPrecision: row.score?.editloc?.precision ?? null, + setupDurationMs: row.timeMetrics?.setupDurationMs?.value ?? null, + indexDurationMs: row.timeMetrics?.indexDurationMs?.value ?? null, + queryDurationMs: row.timeMetrics?.queryDurationMs?.value ?? null, + evaluatorDurationMs: row.timeMetrics?.evaluatorDurationMs?.value ?? null, + rowWallDurationMs: row.timeMetrics?.rowWallDurationMs?.value ?? null, + candidateEstimatedTokens: row.tokenMetrics?.candidatePack?.estimatedTokens?.value ?? null, + predictionEstimatedTokens: row.tokenMetrics?.prediction?.estimatedTokens?.value ?? null, + }; +} + +const runStarted = Date.now(); +const runDir = join(root, 'lane-score'); +mkdirSync(runDir, { recursive: true }); +writeFileSync(join(runDir, 'selections.json'), JSON.stringify(selections, null, 2)); + +const goldPath = join(runDir, 'gold.json'); +const gold = run( + 'node', + [ + 'scripts/contextbench-select-slice.mjs', + '--write-gold', + '--task-id', + task.instance_id, + '--out', + goldPath, + '--payloads', + process.env.TASK_PAYLOADS, + ], + { timeoutMs: 600000 }, +); +writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); +if (gold.status !== 0) throw new Error(`gold materialization failed: ${gold.stderr || gold.stdout}`); + +const rows = []; +for (const selection of laneSelections) { + const rowStarted = Date.now(); + const lane = selection.lane_id || selection.lane; + const laneDir = join(runDir, lane); + mkdirSync(laneDir, { recursive: true }); + const spans = Array.isArray(selection.spans) ? selection.spans : []; + const files = Array.isArray(selection.files) ? selection.files : []; + const spanMap = new Map(); + for (const span of spans) addSpan(spanMap, span.file, span.start, span.end); + const predFiles = [...new Set([...files, ...spans.map((span) => cleanPath(span.file))])].filter(Boolean); + const predSpans = Object.fromEntries(spanMap.entries()); + const nonEmptyPrediction = predFiles.length > 0 || spans.length > 0; + const readiness = selection.readiness || {}; + const rowBase = { + lane_id: lane, + task_id: task.instance_id, + model: selections.model || 'gpt-5.4-mini-high', + predictionSource: selection.predictionSource || selections.predictionSource || 'gpt-5.4-mini-high subagent over real lane candidate pack', + setupStatus: readiness.setupStatus || selection.setupStatus || 'unknown', + indexStatus: readiness.indexStatus || selection.indexStatus || 'unknown', + toolCallable: Boolean(readiness.toolCallable ?? selection.toolCallable), + candidateCount: finiteNumber(readiness.candidateCount ?? selection.candidateCount) ?? 0, + setupIndex: readiness.setupIndex || selection.setupIndex || null, + nonEmptyPrediction, + predFiles: predFiles.length, + predSpans: spans.length, + rationale: selection.rationale || null, + }; + + writeFileSync(join(laneDir, 'selection.json'), JSON.stringify(selection, null, 2)); + if (!nonEmptyPrediction) { + const timeMetrics = buildTimeMetrics(readiness, null, Date.now() - rowStarted, 'prediction was empty'); + const row = { + ...rowBase, + status: 'empty_prediction', + officialEvaluatorScoreable: false, + score: null, + timeMetrics, + tokenMetrics: buildTokenMetrics(selection, null), + }; + row.reliability = reliabilityFor(selection, rowBase, row.status, row.officialEvaluatorScoreable); + rows.push(row); + continue; + } + + const prediction = { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { + pred_steps: [{ files: predFiles, spans: predSpans }], + pred_files: predFiles, + pred_spans: predSpans, + }, + model_patch: '', + }; + const predictionPath = join(laneDir, 'prediction.json'); + writeFileSync(predictionPath, JSON.stringify(prediction, null, 2)); + + const scorePath = join(laneDir, 'official-score.jsonl'); + const evaluator = run( + 'python', + [ + '-m', + 'contextbench.evaluate', + '--gold', + goldPath, + '--pred', + predictionPath, + '--cache', + join(laneDir, 'repo-cache'), + '--out', + scorePath, + ], + { cwd: officialContextBench, timeoutMs: 1200000 }, + ); + writeFileSync(join(laneDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + let score = null; + if (existsSync(scorePath)) { + const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); + if (lines.length > 0) score = JSON.parse(lines.at(-1)); + } + const scoreable = evaluator.status === 0 && Boolean(score); + const row = { + ...rowBase, + status: scoreable ? 'completed' : 'judge_failed', + officialEvaluatorScoreable: scoreable, + score, + timeMetrics: buildTimeMetrics(readiness, evaluator, Date.now() - rowStarted), + tokenMetrics: buildTokenMetrics(selection, prediction), + }; + row.reliability = reliabilityFor(selection, rowBase, row.status, row.officialEvaluatorScoreable); + rows.push(row); +} + +const scoreableRows = rows.filter((row) => row.officialEvaluatorScoreable); +const summary = { + createdAt: new Date().toISOString(), + attemptedRows: rows.length, + scoreableRows: scoreableRows.length, + requiredCompetitors: requiredLanes.length, + requiredLanes, + setupIndexCostReportedSeparately: true, + officialEvaluatorQualityRowsOnly: true, + model: selections.model || 'gpt-5.4-mini-high', + predictionSource: selections.predictionSource || 'gpt-5.4-mini-high subagent selections over real lane candidate packs', + caveats: selections.caveats || [], + runMetrics: { + goldMaterializationDurationMs: gold.durationMs, + totalWallDurationMs: Date.now() - runStarted, + }, + resultsTable: scoreableRows.map(resultTableRow), + rows, +}; + +writeFileSync(join(runDir, 'summary.json'), JSON.stringify(summary, null, 2)); +writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_LANE_SCORE_JSON_START'); +console.log(JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_LANE_SCORE_JSON_END'); +if (scoreableRows.length !== rows.length || scoreableRows.length < requiredLanes.length) process.exitCode = 1; diff --git a/scripts/contextbench-score-selected-prediction.mjs b/scripts/contextbench-score-selected-prediction.mjs new file mode 100644 index 0000000..71d4da0 --- /dev/null +++ b/scripts/contextbench-score-selected-prediction.mjs @@ -0,0 +1,178 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const targetTaskId = 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT; +const officialContextBench = process.env.OFFICIAL_CONTEXTBENCH; +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const runDir = join(root, 'selected-codebase-memory-mcp'); +mkdirSync(runDir, { recursive: true }); + +const selection = { + files: ['core/metrics/prometheus.go', 'cmd/root.go', 'server/subsonic/helpers.go'], + spans: [ + { file: 'core/metrics/prometheus.go', start: 15, end: 17 }, + { file: 'core/metrics/prometheus.go', start: 19, end: 25 }, + { file: 'core/metrics/prometheus.go', start: 33, end: 38 }, + { file: 'core/metrics/prometheus.go', start: 40, end: 49 }, + { file: 'core/metrics/prometheus.go', start: 51, end: 100 }, + { file: 'core/metrics/prometheus.go', start: 102, end: 123 }, + { file: 'cmd/root.go', start: 194, end: 201 }, + { file: 'cmd/root.go', start: 204, end: 220 }, + { file: 'server/subsonic/helpers.go', start: 129, end: 187 }, + { file: 'server/subsonic/helpers.go', start: 189, end: 201 }, + { file: 'server/subsonic/helpers.go', start: 256, end: 268 }, + ], + rationale: + 'These spans cover the startup metrics writer path and the Subsonic auth-header/Bearer-token parsing path where the two reported behaviors most likely live.', +}; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 600000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: r.status, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +function addSpan(map, file, start, end) { + const s = Math.max(1, Number(start) || 1); + const e = Math.max(s, Number(end) || s); + const list = map.get(file) || []; + list.push({ start: s, end: e }); + map.set(file, list); +} + +const spanMap = new Map(); +for (const span of selection.spans) addSpan(spanMap, span.file, span.start, span.end); +const predFiles = [...new Set([...selection.files, ...selection.spans.map((span) => span.file)])]; +const predSpans = Object.fromEntries(spanMap.entries()); +const prediction = { + instance_id: task.instance_id, + repo_url: task.repo_checkout_path, + commit: task.base_commit, + traj_data: { + pred_steps: [{ files: predFiles, spans: predSpans }], + pred_files: predFiles, + pred_spans: predSpans, + }, + model_patch: '', +}; + +writeFileSync(join(runDir, 'selection.json'), JSON.stringify(selection, null, 2)); +writeFileSync(join(runDir, 'prediction.json'), JSON.stringify(prediction, null, 2)); + +const goldPath = join(runDir, 'gold.json'); +const gold = run( + 'node', + [ + 'scripts/contextbench-select-slice.mjs', + '--write-gold', + '--task-id', + task.instance_id, + '--out', + goldPath, + '--payloads', + process.env.TASK_PAYLOADS, + ], + { timeoutMs: 600000 }, +); +writeFileSync(join(runDir, 'gold-command.json'), JSON.stringify(gold, null, 2)); +if (gold.status !== 0) { + throw new Error(`gold materialization failed: ${gold.stderr || gold.stdout}`); +} + +const scorePath = join(runDir, 'official-score.jsonl'); +const evaluator = run( + 'python', + [ + '-m', + 'contextbench.evaluate', + '--gold', + goldPath, + '--pred', + join(runDir, 'prediction.json'), + '--cache', + join(runDir, 'repo-cache'), + '--out', + scorePath, + ], + { cwd: officialContextBench, timeoutMs: 1200000 }, +); +writeFileSync(join(runDir, 'evaluator-command.json'), JSON.stringify(evaluator, null, 2)); + +let score = null; +if (existsSync(scorePath)) { + const lines = readFileSync(scorePath, 'utf8').trim().split(/\n+/).filter(Boolean); + if (lines.length > 0) score = JSON.parse(lines.at(-1)); +} + +const scoreable = evaluator.status === 0 && Boolean(score); +const final = score?.final || {}; +const row = { + lane_id: 'codebase-memory-mcp', + task_id: task.instance_id, + model: 'gpt-5.4-mini-high', + predictionSource: 'gpt-5.4-mini-high subagent over real codebase-memory-mcp candidate pack', + status: scoreable ? 'completed' : 'judge_failed', + setupStatus: 'completed', + indexStatus: 'completed', + toolCallable: true, + candidateCount: 100, + nonEmptyPrediction: predFiles.length > 0 && selection.spans.length > 0, + predFiles: predFiles.length, + predSpans: selection.spans.length, + officialEvaluatorScoreable: scoreable, + setupIndex: { setupDurationMs: 4, indexDurationMs: 2038, queryDurationMs: 365 }, + score, +}; +const summary = { + createdAt: new Date().toISOString(), + attemptedRows: 1, + scoreableRows: scoreable ? 1 : 0, + setupIndexCostReportedSeparately: true, + model: row.model, + caveat: 'Prediction was selected by a Codex gpt-5.4-mini-high subagent because GitHub Actions has no OpenAI API key for direct CI inference.', + resultsTable: scoreable + ? [ + { + lane: row.lane_id, + task: row.task_id, + fileCoverage: final.file?.coverage ?? null, + filePrecision: final.file?.precision ?? null, + symbolCoverage: final.symbol?.coverage ?? null, + symbolPrecision: final.symbol?.precision ?? null, + spanCoverage: final.span?.coverage ?? null, + spanPrecision: final.span?.precision ?? null, + lineCoverage: final.line?.coverage ?? null, + linePrecision: final.line?.precision ?? null, + editlocRecall: score.editloc?.recall ?? null, + editlocPrecision: score.editloc?.precision ?? null, + }, + ] + : [], + rows: [row], +}; + +writeFileSync(join(runDir, 'row.json'), JSON.stringify(row, null, 2)); +writeFileSync(join(root, 'summary.json'), JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_SELECTED_SCORE_JSON_START'); +console.log(JSON.stringify(summary, null, 2)); +console.log('CONTEXTBENCH_SELECTED_SCORE_JSON_END'); +if (!scoreable) process.exitCode = 1; diff --git a/scripts/contextbench-single-lane-candidate-pack.mjs b/scripts/contextbench-single-lane-candidate-pack.mjs new file mode 100644 index 0000000..874b621 --- /dev/null +++ b/scripts/contextbench-single-lane-candidate-pack.mjs @@ -0,0 +1,340 @@ +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { basename, join, relative } from 'node:path'; + +const laneId = process.env.LANE_ID; +const targetTaskId = process.env.TARGET_TASK_ID || 'SWE-Bench-Pro__go__maintenance__bugfix__4df06349'; +const root = process.env.ROOT || '/tmp/contextbench-single-lane'; +const outDir = join(root, 'pack'); +const requiredLanes = ['raw-native', 'codebase-context', 'codebase-memory-mcp', 'grepai', 'codegraphcontext']; +if (!requiredLanes.includes(laneId)) throw new Error(`unsupported LANE_ID: ${laneId}`); +mkdirSync(outDir, { recursive: true }); + +const payloads = JSON.parse(readFileSync(process.env.TASK_PAYLOADS, 'utf8')); +const task = payloads.tasks.find((candidate) => candidate.instance_id === targetTaskId); +if (!task) throw new Error(`target task ${targetTaskId} missing from payloads`); +const repo = task.repo_checkout_path; + +function run(cmd, args, opts = {}) { + const started = Date.now(); + const r = spawnSync(cmd, args, { + cwd: opts.cwd || process.cwd(), + env: opts.env || process.env, + encoding: 'utf8', + timeout: opts.timeoutMs || 300000, + maxBuffer: 128 * 1024 * 1024, + }); + return { + command: [cmd, ...args].join(' '), + cwd: opts.cwd || process.cwd(), + status: typeof r.status === 'number' ? r.status : null, + signal: r.signal, + error: r.error?.message || null, + durationMs: Date.now() - started, + stdout: r.stdout || '', + stderr: r.stderr || '', + }; +} + +const durationOf = (commands) => commands.reduce((sum, command) => sum + command.durationMs, 0); + +function queryOf(text) { + const stop = new Set(['that', 'this', 'with', 'from', 'when', 'then', 'into', 'should', 'would', 'could', 'there', 'where', 'which', 'about', 'after', 'before', 'have', 'will', 'been', 'than', 'also', 'only', 'some', 'using', 'must']); + return String(text || '') + .replace(/[`*_#>\[\](){},.;:!?/\\]/g, ' ') + .split(/\s+/) + .filter((w) => w.length >= 4 && !stop.has(w.toLowerCase())) + .slice(0, 24) + .join(' '); +} + +function collectRepoFiles(dir, prefix = '', files = []) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === '.git' || entry.name === 'vendor' || entry.name === 'node_modules') continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + const abs = join(dir, entry.name); + if (entry.isDirectory()) collectRepoFiles(abs, rel, files); + else files.push(rel.replaceAll('\\', '/')); + } + return files; +} + +const repoFiles = collectRepoFiles(repo); +const repoFileSet = new Set(repoFiles); +const basenameMap = new Map(); +for (const file of repoFiles) { + const name = basename(file); + const list = basenameMap.get(name) || []; + list.push(file); + basenameMap.set(name, list); +} + +function stripAnsi(text) { + return String(text || '').replace(/\u001b\[[0-9;]*m/g, ''); +} + +function jsonish(s) { + const t = stripAnsi(s).trim(); + if (!t) return null; + try { + return JSON.parse(t); + } catch {} + for (const [a, b] of [['{', '}'], ['[', ']']]) { + const i = t.indexOf(a); + const j = t.lastIndexOf(b); + if (i >= 0 && j > i) { + try { + return JSON.parse(t.slice(i, j + 1)); + } catch {} + } + } + return null; +} + +function norm(file) { + let f = stripAnsi(file).replace(/^file:\/\//, '').replaceAll('\\', '/').trim(); + if (!f) return ''; + const repoNorm = repo.replaceAll('\\', '/'); + if (f.startsWith(repoNorm)) f = relative(repo, f).replaceAll('\\', '/'); + const tmpIdx = f.indexOf('/tmp/contextbench-checkouts/'); + if (tmpIdx >= 0) { + const parts = f.slice(tmpIdx + '/tmp/contextbench-checkouts/'.length).split('/'); + f = parts.slice(1).join('/'); + } + f = f.replace(/^\/+/, '').replace(/^\.\//, ''); + if (!f || f.includes('://') || f.includes('..') || f.startsWith('tmp/')) return ''; + if (repoFileSet.has(f)) return f; + if (existsSync(join(repo, f))) return f; + const byName = basenameMap.get(basename(f)); + if (byName?.length === 1) return byName[0]; + return ''; +} + +function add(locs, file, start = 1, end = start, source = 'tool') { + const clean = norm(file); + if (!clean) return; + const s = Math.max(1, Number(start) || 1); + locs.push({ file: clean, start: s, end: Math.max(s, Number(end) || s), source }); +} + +function walk(value, locs, source) { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + for (const item of value) walk(item, locs, source); + return; + } + add( + locs, + value.file || value.path || value.file_path || value.relative_path || value.filename || value.source_path || value.uri, + value.start_line || value.startLine || value.line || value.line_number || value.start || 1, + value.end_line || value.endLine || value.end || value.line || 1, + source, + ); + for (const item of Object.values(value)) walk(item, locs, source); +} + +function collect(text, locs, source) { + const cleaned = stripAnsi(text); + const parsed = jsonish(cleaned); + if (parsed) walk(parsed, locs, source); + const rgLine = /^(.+?\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte)):(\d+):/gm; + let m; + while ((m = rgLine.exec(cleaned)) !== null) add(locs, m[1], m[2], m[2], source); + const fileLine = /([A-Za-z0-9_.\/-]+\.(?:go|mod|sum|json|yml|yaml|md|ts|tsx|js|jsx|py|rs|java|c|cc|cpp|h|hpp|rb|php|cs|kt|swift|vue|svelte))(?::|#L|\s+line\s+)?(\d+)?/g; + while ((m = fileLine.exec(cleaned)) !== null) add(locs, m[1], m[2] || 1, m[2] || 1, source); +} + +function uniq(locs, max = 120) { + const seen = new Set(); + const out = []; + for (const loc of locs) { + const key = `${loc.file}:${loc.start}:${loc.end}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(loc); + if (out.length >= max) break; + } + return out; +} + +function writeCommands(lane, commands) { + for (const [i, command] of commands.entries()) { + writeFileSync(join(outDir, `${lane}-command-${i + 1}.json`), JSON.stringify({ ...command, stdout: command.stdout.slice(0, 200000), stderr: command.stderr.slice(0, 200000) }, null, 2)); + } +} + +function laneResult(lane, commands, locs, setupStatus, indexStatus, setupIndex) { + writeCommands(lane, commands); + const candidates = uniq(locs); + return { + lane, + ready: (setupStatus === 'completed' || setupStatus === 'not_required') && (indexStatus === 'completed' || indexStatus === 'not_required') && commands.some((command) => command.status === 0) && candidates.length > 0, + setupStatus, + indexStatus, + toolCallable: commands.some((command) => command.status === 0), + candidateCount: candidates.length, + setupIndex, + commands: commands.map((command) => ({ command: command.command, status: command.status, error: command.error, durationMs: command.durationMs })), + candidates, + }; +} + +const baseQuery = queryOf(task.problem_statement); +const queryVariants = [ + baseQuery, + 'system metrics written on start prometheus writer insights', + 'Bearer token custom authorization header authentication', + 'Subsonic authorization header token parsing', + 'startup metrics insights prometheus initialization', +]; + +function runRawNative() { + const commands = []; + const locs = []; + for (const term of ['metrics', 'prometheus', 'insights', 'startup', 'start', 'written', 'authorization', 'Bearer', 'token', 'header', 'authentication', 'subsonic']) { + const r = run('rg', ['-n', '-i', '--glob', '!.git', '--glob', '!vendor/**', '--glob', '!node_modules/**', term, '.'], { cwd: repo, timeoutMs: 60000 }); + commands.push(r); + collect(r.stdout, locs, 'raw-native'); + collect(r.stderr, locs, 'raw-native'); + } + return laneResult('raw-native', commands, locs, 'not_required', 'not_required', { setupDurationMs: 0, indexDurationMs: 0, queryDurationMs: durationOf(commands) }); +} + +function runCodebaseContext() { + const commands = []; + const locs = []; + const env = { ...process.env, CODEBASE_ROOT: repo, CODEBASE_CONTEXT_ASCII: '1' }; + const setup = run('node', ['dist/index.js', '--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run('node', ['dist/index.js', 'reindex'], { env, timeoutMs: 1200000 }); + commands.push(index); + const searches = []; + for (const q of queryVariants) { + const search = run('node', ['dist/index.js', 'search', '--query', q, '--intent', 'edit', '--limit', '40', '--json'], { env, timeoutMs: 300000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'codebase-context'); + collect(search.stderr, locs, 'codebase-context'); + } + return laneResult('codebase-context', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +function runCodebaseMemoryMcp() { + const commands = []; + const locs = []; + const env = { ...process.env, CBM_CACHE_DIR: join(outDir, 'cbm-cache'), CBM_DIAGNOSTICS: '1' }; + const setup = run(process.env.CBM_BIN, ['--version'], { env, timeoutMs: 60000 }); + commands.push(setup); + const index = run(process.env.CBM_BIN, ['cli', 'index_repository', JSON.stringify({ repo_path: repo })], { cwd: repo, env, timeoutMs: 2700000 }); + commands.push(index); + const project = (jsonish(index.stdout) || jsonish(index.stderr) || {}).project || basename(repo); + const searches = []; + for (const q of queryVariants) { + const graph = run(process.env.CBM_BIN, ['cli', 'search_graph', JSON.stringify({ project, query: q, limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(graph); + searches.push(graph); + collect(graph.stdout, locs, 'codebase-memory-mcp'); + collect(graph.stderr, locs, 'codebase-memory-mcp'); + } + for (const term of ['metrics', 'prometheus', 'authorization', 'Bearer', 'token']) { + const code = run(process.env.CBM_BIN, ['cli', 'search_code', JSON.stringify({ project, pattern: term, mode: 'compact', limit: 50 })], { cwd: repo, env, timeoutMs: 120000 }); + commands.push(code); + searches.push(code); + collect(code.stdout, locs, 'codebase-memory-mcp'); + collect(code.stderr, locs, 'codebase-memory-mcp'); + } + return laneResult('codebase-memory-mcp', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(searches) }); +} + +function runGrepai() { + const commands = []; + const locs = []; + const setup = run('grepai', ['version'], { timeoutMs: 60000 }); + commands.push(setup); + const init = run('grepai', ['init', '--yes', '--provider', 'synthetic', '--backend', 'gob'], { cwd: repo, timeoutMs: 120000 }); + commands.push(init); + const watch = run('grepai', ['watch', '--background'], { cwd: repo, timeoutMs: 120000 }); + commands.push(watch); + const statusChecks = []; + for (let i = 0; i < 12; i += 1) { + const status = run('grepai', ['status', '--no-ui'], { cwd: repo, timeoutMs: 60000 }); + commands.push(status); + statusChecks.push(status); + if (/chunks?\D+[1-9]|indexed files?\D+[1-9]/i.test(`${status.stdout}\n${status.stderr}`)) break; + run('sleep', ['5'], { timeoutMs: 10000 }); + } + const searches = []; + for (const q of queryVariants) { + const search = run('grepai', ['search', q, '--json', '--compact', '--limit', '40'], { cwd: repo, timeoutMs: 180000 }); + commands.push(search); + searches.push(search); + collect(search.stdout, locs, 'grepai'); + collect(search.stderr, locs, 'grepai'); + } + const stop = run('grepai', ['watch', '--stop'], { cwd: repo, timeoutMs: 60000 }); + commands.push(stop); + return laneResult('grepai', commands, locs, setup.status === 0 && init.status === 0 ? 'completed' : 'setup_failed', watch.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs + init.durationMs, indexDurationMs: watch.durationMs + durationOf(statusChecks), queryDurationMs: durationOf(searches), teardownDurationMs: stop.durationMs }); +} + +function runCodeGraphContext() { + const commands = []; + const locs = []; + const probe = run('codegraphcontext', ['--help'], { timeoutMs: 60000 }); + const cgcCommand = probe.status === 0 ? 'codegraphcontext' : 'cgc'; + const setup = probe.status === 0 ? probe : run(cgcCommand, ['--help'], { timeoutMs: 60000 }); + commands.push(setup); + const index = run(cgcCommand, ['index', '.'], { cwd: repo, timeoutMs: 1200000 }); + commands.push(index); + const queries = []; + for (const pattern of ['Metrics', 'Prometheus', 'Authorization', 'Bearer', 'Token', 'Subsonic', 'Header']) { + const found = run(cgcCommand, ['find', 'pattern', pattern], { cwd: repo, timeoutMs: 180000 }); + commands.push(found); + queries.push(found); + collect(found.stdout, locs, 'codegraphcontext'); + collect(found.stderr, locs, 'codegraphcontext'); + } + for (const symbol of ['main', 'init', 'WriteInitialMetrics', 'ServeHTTP', 'GetUser']) { + const callers = run(cgcCommand, ['analyze', 'callers', symbol], { cwd: repo, timeoutMs: 180000 }); + commands.push(callers); + queries.push(callers); + collect(callers.stdout, locs, 'codegraphcontext'); + collect(callers.stderr, locs, 'codegraphcontext'); + } + const complexity = run(cgcCommand, ['analyze', 'complexity', '--limit', '80'], { cwd: repo, timeoutMs: 180000 }); + commands.push(complexity); + queries.push(complexity); + collect(complexity.stdout, locs, 'codegraphcontext'); + collect(complexity.stderr, locs, 'codegraphcontext'); + return laneResult('codegraphcontext', commands, locs, setup.status === 0 ? 'completed' : 'setup_failed', index.status === 0 ? 'completed' : 'index_failed', { setupDurationMs: setup.durationMs, indexDurationMs: index.durationMs, queryDurationMs: durationOf(queries) }); +} + +const result = + laneId === 'raw-native' + ? runRawNative() + : laneId === 'codebase-context' + ? runCodebaseContext() + : laneId === 'codebase-memory-mcp' + ? runCodebaseMemoryMcp() + : laneId === 'grepai' + ? runGrepai() + : runCodeGraphContext(); + +const pack = { + createdAt: new Date().toISOString(), + targetTaskId, + task: { + instance_id: task.instance_id, + repo: task.repo, + base_commit: task.base_commit, + problem_statement: task.problem_statement, + }, + queryVariants, + readiness: result, +}; + +writeFileSync(join(outDir, `${laneId}-candidate-pack.json`), JSON.stringify(pack, null, 2)); +writeFileSync(join(outDir, `${laneId}-readiness.json`), JSON.stringify(result, null, 2)); +console.log('CONTEXTBENCH_SINGLE_LANE_READINESS_JSON_START'); +console.log(JSON.stringify(pack, null, 2)); +console.log('CONTEXTBENCH_SINGLE_LANE_READINESS_JSON_END'); +if (!result.ready) process.exitCode = 1; diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index c529a5c..6f7d357 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -11,9 +11,14 @@ const vitestEntrypoint = fileURLToPath( new URL('../node_modules/vitest/vitest.mjs', import.meta.url) ); +const sensitiveEnvName = /TOKEN|KEY|SECRET|PASSWORD|AUTH|OPENAI|ANTHROPIC|CLAUDE/i; +const testEnv = Object.fromEntries( + Object.entries(process.env).filter(([name]) => !sensitiveEnvName.test(name)) +); + const child = spawn(process.execPath, [vitestEntrypoint, 'run', ...vitestArgs], { stdio: 'inherit', - env: process.env + env: testEnv }); child.on('exit', (code, signal) => { diff --git a/src/analyzers/nestjs/index.ts b/src/analyzers/nestjs/index.ts new file mode 100644 index 0000000..844194d --- /dev/null +++ b/src/analyzers/nestjs/index.ts @@ -0,0 +1,736 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { parse, type TSESTree } from '@typescript-eslint/typescript-estree'; +import type { + AnalysisResult, + ArchitecturalLayer, + CodeChunk, + CodeComponent, + CodebaseMetadata, + ExportStatement, + FrameworkAnalyzer, + ImportStatement +} from '../../types/index.js'; +import { createChunksFromCode } from '../../utils/chunking.js'; +import { categorizeDependency } from '../../utils/dependency-detection.js'; +import { + createEmptyStatistics, + isFileNotFoundError, + loadAnalyzerIndexStatistics, + normalizeAnalyzerVersion, + readAnalyzerPackageInfo +} from '../shared/metadata.js'; + +type DetectedPattern = { category: string; name: string }; +type NestComponentType = + | 'module' + | 'controller' + | 'service' + | 'repository' + | 'provider' + | 'guard' + | 'interceptor' + | 'pipe' + | 'filter' + | 'gateway' + | 'resolver' + | 'processor' + | 'schema'; + +interface NestRouteInfo { + method: string; + path: string; + handler: string; + line?: number; +} + +interface NestModuleMetadata { + imports?: string[]; + providers?: string[]; + controllers?: string[]; + exports?: string[]; +} + +interface NestClassSummary { + component: CodeComponent; + routes: NestRouteInfo[]; +} + +const ROUTE_DECORATORS: Readonly> = { + Get: 'GET', + Post: 'POST', + Put: 'PUT', + Patch: 'PATCH', + Delete: 'DELETE', + Options: 'OPTIONS', + Head: 'HEAD', + All: 'ALL' +}; + +const NEST_LIBRARY_SIGNALS: ReadonlyArray<{ + source: string; + category: string; + name: string; +}> = [ + { source: '@nestjs/swagger', category: 'documentation', name: 'Swagger' }, + { source: '@nestjs/graphql', category: 'data', name: 'GraphQL' }, + { source: '@nestjs/mongoose', category: 'data', name: 'Mongoose' }, + { source: '@nestjs/bull', category: 'queues', name: 'Bull' }, + { source: '@nestjs/bullmq', category: 'queues', name: 'BullMQ' }, + { source: '@nestjs/schedule', category: 'scheduling', name: 'Schedule' }, + { source: '@nestjs/websockets', category: 'realtime', name: 'WebSockets' }, + { source: '@nestjs/passport', category: 'security', name: 'Passport' }, + { source: '@nestjs/terminus', category: 'health', name: 'Terminus' } +]; + +const TESTING_DEPENDENCIES: ReadonlyArray = [ + ['@nestjs/testing', 'Nest Testing'], + ['vitest', 'Vitest'], + ['jest', 'Jest'] +]; + +export class NestJsAnalyzer implements FrameworkAnalyzer { + readonly name = 'nestjs'; + readonly version = '1.0.0'; + readonly supportedExtensions = ['.ts', '.js', '.mjs', '.cjs', '.mts', '.cts']; + readonly priority = 105; + + canAnalyze(filePath: string, content?: string): boolean { + const extension = path.extname(filePath).toLowerCase(); + if (!this.supportedExtensions.includes(extension) || !content) { + return false; + } + + return /\bfrom\s+['"]@nestjs\//.test(content) || /\bNestFactory\b/.test(content); + } + + async analyze(filePath: string, content: string): Promise { + const extension = path.extname(filePath).toLowerCase(); + const language = + extension === '.ts' || extension === '.mts' || extension === '.cts' + ? 'typescript' + : 'javascript'; + const relativePath = path.relative(process.cwd(), filePath); + const imports: ImportStatement[] = []; + const exports: ExportStatement[] = []; + const dependencyNames = new Set(); + const importSources = new Set(); + const detectedPatterns: DetectedPattern[] = []; + let components: CodeComponent[] = []; + + try { + const program = parse(content, { + loc: true, + range: true, + comment: true, + sourceType: 'module', + experimentalDecorators: true + }); + + for (const statement of program.body) { + if (statement.type === 'ImportDeclaration' && typeof statement.source.value === 'string') { + const source = statement.source.value; + const packageName = getPackageName(source); + importSources.add(packageName); + imports.push({ + source, + imports: statement.specifiers.map(getImportSpecifierName), + isDefault: statement.specifiers.some( + (specifier) => specifier.type === 'ImportDefaultSpecifier' + ), + isDynamic: false, + line: statement.loc?.start.line + }); + + if (!source.startsWith('.') && !source.startsWith('/')) { + dependencyNames.add(packageName); + } + } + + appendExports(exports, statement); + } + + const summaries = summarizeNestProgram(program); + components = summaries.map((summary) => summary.component); + addProgramPatterns(detectedPatterns, summaries, importSources); + } catch (error) { + console.warn(`Failed to parse NestJS file ${filePath}:`, error); + } + + const chunks = await createChunksFromCode( + content, + filePath, + relativePath, + language, + components, + { + framework: 'nestjs', + detectedPatterns + } + ); + + return { + filePath, + language, + framework: 'nestjs', + components, + imports, + exports, + dependencies: Array.from(dependencyNames) + .sort() + .map((name) => ({ + name, + category: categorizeDependency(name) + })), + metadata: { + analyzer: this.name, + detectedPatterns + }, + chunks + }; + } + + async detectCodebaseMetadata(rootPath: string): Promise { + const metadata: CodebaseMetadata = { + name: path.basename(rootPath), + rootPath, + languages: [], + dependencies: [], + architecture: { + type: 'feature-based', + layers: createEmptyStatistics().componentsByLayer, + patterns: [] + }, + styleGuides: [], + documentation: [], + projectStructure: { + type: 'single-app' + }, + statistics: createEmptyStatistics(), + customMetadata: {} + }; + + try { + const packageInfo = await readAnalyzerPackageInfo(rootPath); + metadata.name = packageInfo.projectName; + metadata.dependencies = Object.entries(packageInfo.allDependencies).map( + ([name, version]) => ({ + name, + version, + category: categorizeDependency(name) + }) + ); + + const indicators = await detectNestIndicators(rootPath, packageInfo.allDependencies); + if (indicators.includes('dep:@nestjs/core') || indicators.includes('dep:@nestjs/common')) { + metadata.framework = { + name: 'NestJS', + version: normalizeAnalyzerVersion( + packageInfo.allDependencies['@nestjs/core'] || + packageInfo.allDependencies['@nestjs/common'] + ), + type: 'nestjs', + variant: getFrameworkVariant(packageInfo.allDependencies), + testingFrameworks: detectDependencyList( + packageInfo.allDependencies, + TESTING_DEPENDENCIES + ), + indicators + }; + } + + metadata.customMetadata = { + nestjs: { + packages: Object.keys(packageInfo.allDependencies) + .filter((name) => name.startsWith('@nestjs/')) + .sort() + } + }; + } catch (error) { + if (!isFileNotFoundError(error)) { + console.warn('Failed to read NestJS project metadata:', error); + } + } + + metadata.statistics = await loadAnalyzerIndexStatistics(rootPath); + return metadata; + } + + summarize(chunk: CodeChunk): string { + const nestMetadata = + typeof chunk.metadata.nestjs === 'object' && chunk.metadata.nestjs + ? (chunk.metadata.nestjs as { type?: unknown; routes?: unknown }) + : undefined; + + if (nestMetadata?.type === 'controller' && Array.isArray(nestMetadata.routes)) { + return `NestJS controller in ${path.basename(chunk.filePath)} with ${nestMetadata.routes.length} route handlers.`; + } + if (typeof nestMetadata?.type === 'string') { + return `NestJS ${nestMetadata.type} in ${path.basename(chunk.filePath)}: lines ${chunk.startLine}-${chunk.endLine}.`; + } + + return `NestJS code in ${path.basename(chunk.filePath)}: lines ${chunk.startLine}-${chunk.endLine}.`; + } +} + +function summarizeNestProgram(program: TSESTree.Program): NestClassSummary[] { + const summaries: NestClassSummary[] = []; + + walkAst(program, (node) => { + if (node.type !== 'ClassDeclaration' || !node.id?.name) { + return; + } + + const decorators = getDecoratorNames(node); + const type = getNestComponentType(node.id.name, decorators); + if (!type) { + return; + } + + const dependencies = extractConstructorDependencies(node); + const routes = type === 'controller' ? extractRoutes(node) : []; + const moduleMetadata = type === 'module' ? extractModuleMetadata(node) : undefined; + const component: CodeComponent = { + name: node.id.name, + type: 'class', + componentType: type, + startLine: node.loc?.start.line || 1, + endLine: node.loc?.end.line || node.loc?.start.line || 1, + layer: getComponentLayer(type), + decorators: decorators.map((name) => ({ name })), + dependencies, + metadata: { + nestjs: { + type, + decorators, + routes, + dependencies, + ...(moduleMetadata ? { moduleMetadata } : {}) + } + } + }; + + summaries.push({ component, routes }); + }); + + return dedupeSummaries(summaries); +} + +function addProgramPatterns( + patterns: DetectedPattern[], + summaries: NestClassSummary[], + importSources: Set +): void { + const patternKeys = new Set(); + const addPattern = (pattern: DetectedPattern): void => { + const key = `${pattern.category}:${pattern.name}`; + if (!patternKeys.has(key)) { + patterns.push(pattern); + patternKeys.add(key); + } + }; + + for (const summary of summaries) { + const type = summary.component.componentType; + if (type === 'module') addPattern({ category: 'modules', name: 'Nest modules' }); + if (type === 'controller') addPattern({ category: 'routing', name: 'Controllers' }); + if (type === 'gateway') addPattern({ category: 'realtime', name: 'WebSockets' }); + if (type === 'resolver') addPattern({ category: 'data', name: 'GraphQL' }); + if (summary.routes.length > 0) addPattern({ category: 'routing', name: 'REST routes' }); + if ((summary.component.dependencies || []).length > 0) { + addPattern({ category: 'dependencyInjection', name: 'Constructor injection' }); + } + const decorators = getNestMetadataDecorators(summary.component); + if (decorators.includes('UseGuards')) addPattern({ category: 'security', name: 'Guards' }); + if (decorators.some((decorator) => decorator.startsWith('Api'))) { + addPattern({ category: 'documentation', name: 'Swagger' }); + } + } + + for (const signal of NEST_LIBRARY_SIGNALS) { + if (importSources.has(signal.source)) { + addPattern({ category: signal.category, name: signal.name }); + } + } +} + +function getNestComponentType( + className: string, + decorators: readonly string[] +): NestComponentType | null { + if (decorators.includes('Module')) return 'module'; + if (decorators.includes('Controller')) return 'controller'; + if (decorators.includes('WebSocketGateway')) return 'gateway'; + if (decorators.includes('Resolver')) return 'resolver'; + if (decorators.includes('Processor')) return 'processor'; + if (decorators.includes('Schema')) return 'schema'; + if (decorators.includes('Catch')) return 'filter'; + if (!decorators.includes('Injectable')) return null; + + if (/Repository$/.test(className)) return 'repository'; + if (/Guard$/.test(className)) return 'guard'; + if (/Interceptor$/.test(className)) return 'interceptor'; + if (/Pipe$/.test(className)) return 'pipe'; + if (/Filter$/.test(className)) return 'filter'; + if (/Service$/.test(className)) return 'service'; + return 'provider'; +} + +function getComponentLayer(type: NestComponentType): ArchitecturalLayer { + if (type === 'controller' || type === 'resolver') return 'presentation'; + if (type === 'service') return 'business'; + if (type === 'repository' || type === 'schema') return 'data'; + if (type === 'module') return 'feature'; + if (type === 'gateway' || type === 'processor') return 'infrastructure'; + return 'core'; +} + +function extractRoutes(node: TSESTree.ClassDeclaration): NestRouteInfo[] { + const controllerPath = normalizeRouteSegment( + getDecoratorStringArgument(getDecorator(node, 'Controller')) || '' + ); + const routes: NestRouteInfo[] = []; + + for (const member of node.body.body) { + if (member.type !== 'MethodDefinition') continue; + const methodName = getPropertyName(member.key); + if (!methodName) continue; + + for (const decorator of getDecorators(member)) { + const decoratorName = getDecoratorName(decorator); + if (!decoratorName || !ROUTE_DECORATORS[decoratorName]) continue; + const routePath = normalizeRouteSegment(getDecoratorStringArgument(decorator) || ''); + routes.push({ + method: ROUTE_DECORATORS[decoratorName], + path: joinRoutePaths(controllerPath, routePath), + handler: methodName, + line: member.loc?.start.line + }); + } + } + + return routes; +} + +function extractConstructorDependencies(node: TSESTree.ClassDeclaration): string[] { + const dependencies = new Set(); + + for (const member of node.body.body) { + if (member.type !== 'MethodDefinition' || member.kind !== 'constructor') continue; + for (const parameter of member.value.params) { + const injectToken = getInjectToken(parameter); + if (injectToken) { + dependencies.add(injectToken); + continue; + } + + const typeName = getParameterTypeName(parameter); + if (typeName) dependencies.add(typeName); + } + } + + return Array.from(dependencies).sort(); +} + +function extractModuleMetadata(node: TSESTree.ClassDeclaration): NestModuleMetadata | undefined { + const moduleDecorator = getDecorator(node, 'Module'); + if (!moduleDecorator || moduleDecorator.expression.type !== 'CallExpression') { + return undefined; + } + + const firstArgument = moduleDecorator.expression.arguments[0]; + if (!firstArgument || firstArgument.type !== 'ObjectExpression') { + return undefined; + } + + const metadata: NestModuleMetadata = {}; + for (const property of firstArgument.properties) { + if (property.type !== 'Property') continue; + const key = getPropertyName(property.key); + if (!isModuleMetadataKey(key)) continue; + const values = extractExpressionNames(property.value); + if (values.length > 0) metadata[key] = values; + } + + return Object.keys(metadata).length > 0 ? metadata : undefined; +} + +function extractExpressionNames(expression: TSESTree.Node): string[] { + if (expression.type === 'ArrayExpression') { + return expression.elements.flatMap((element) => + element && element.type !== 'SpreadElement' ? extractExpressionNames(element) : [] + ); + } + if (expression.type === 'Identifier') return [expression.name]; + if (expression.type === 'Literal') return [String(expression.value)]; + if (expression.type === 'CallExpression') return [getCalleeName(expression.callee) || 'factory']; + if (expression.type === 'MemberExpression') { + const memberName = getMemberExpressionName(expression); + return memberName ? [memberName] : []; + } + return []; +} + +function getInjectToken(parameter: TSESTree.Parameter): string | null { + const decorators = getDecorators(parameter); + const injectDecorator = decorators.find((decorator) => getDecoratorName(decorator) === 'Inject'); + if (!injectDecorator || injectDecorator.expression.type !== 'CallExpression') { + return null; + } + + const token = injectDecorator.expression.arguments[0]; + if (!token || token.type === 'SpreadElement') return null; + const names = extractExpressionNames(token); + return names[0] || null; +} + +function getParameterTypeName(parameter: TSESTree.Parameter): string | null { + const target = parameter.type === 'TSParameterProperty' ? parameter.parameter : parameter; + if (target.type !== 'Identifier') { + return null; + } + + const annotation = target.typeAnnotation?.typeAnnotation; + if (!annotation) return null; + if (annotation.type === 'TSTypeReference') return getTypeName(annotation.typeName); + return null; +} + +function getTypeName(typeName: TSESTree.EntityName): string | null { + if (typeName.type === 'Identifier') return typeName.name; + if (typeName.type === 'TSQualifiedName') { + const left = getTypeName(typeName.left); + return left ? `${left}.${typeName.right.name}` : typeName.right.name; + } + return null; +} + +function getDecorator(node: TSESTree.Node, name: string): TSESTree.Decorator | undefined { + return getDecorators(node).find((decorator) => getDecoratorName(decorator) === name); +} + +function getDecoratorNames(node: TSESTree.Node): string[] { + return getDecorators(node) + .map(getDecoratorName) + .filter((name): name is string => Boolean(name)); +} + +function getDecorators(node: TSESTree.Node): TSESTree.Decorator[] { + const candidate = node as { decorators?: unknown }; + return Array.isArray(candidate.decorators) + ? candidate.decorators.filter((decorator): decorator is TSESTree.Decorator => + isDecoratorNode(decorator) + ) + : []; +} + +function isDecoratorNode(value: unknown): value is TSESTree.Decorator { + return Boolean( + value && typeof value === 'object' && (value as { type?: unknown }).type === 'Decorator' + ); +} + +function getDecoratorName(decorator: TSESTree.Decorator): string | null { + const expression = decorator.expression; + return expression.type === 'CallExpression' + ? getCalleeName(expression.callee) + : getCalleeName(expression); +} + +function getDecoratorStringArgument(decorator: TSESTree.Decorator | undefined): string | null { + if (!decorator || decorator.expression.type !== 'CallExpression') return null; + const firstArgument = decorator.expression.arguments[0]; + if (!firstArgument || firstArgument.type === 'SpreadElement') return null; + if (firstArgument.type === 'Literal' && typeof firstArgument.value === 'string') { + return firstArgument.value; + } + if (firstArgument.type === 'TemplateLiteral' && firstArgument.expressions.length === 0) { + return firstArgument.quasis[0]?.value.cooked || null; + } + return null; +} + +function getCalleeName(node: TSESTree.Node): string | null { + if (node.type === 'Identifier') return node.name; + if (node.type === 'MemberExpression') return getMemberExpressionName(node); + return null; +} + +function getMemberExpressionName(node: TSESTree.MemberExpression): string | null { + const objectName = + node.object.type === 'Identifier' + ? node.object.name + : node.object.type === 'MemberExpression' + ? getMemberExpressionName(node.object) + : null; + const propertyName = getPropertyName(node.property); + return objectName && propertyName ? `${objectName}.${propertyName}` : propertyName; +} + +function getPropertyName(node: TSESTree.Expression | TSESTree.PrivateIdentifier): string | null { + if (node.type === 'Identifier') return node.name; + if (node.type === 'Literal') return String(node.value); + return null; +} + +function normalizeRouteSegment(segment: string): string { + return segment.replace(/^\/+|\/+$/g, '').replace(/:$/, ''); +} + +function joinRoutePaths(base: string, child: string): string { + const joined = [base, child] + .filter((segment) => segment.length > 0) + .join('/') + .replace(/\/+/g, '/'); + return `/${joined}`.replace(/\/$/, '') || '/'; +} + +function isModuleMetadataKey(key: string | null): key is keyof NestModuleMetadata { + return key === 'imports' || key === 'providers' || key === 'controllers' || key === 'exports'; +} + +function getNestMetadataDecorators(component: CodeComponent): string[] { + const metadata = component.metadata.nestjs as { decorators?: unknown } | undefined; + return Array.isArray(metadata?.decorators) + ? metadata.decorators.filter((decorator): decorator is string => typeof decorator === 'string') + : []; +} + +async function detectNestIndicators( + rootPath: string, + allDependencies: Record +): Promise { + const indicators: string[] = []; + for (const name of Object.keys(allDependencies).sort()) { + if (name.startsWith('@nestjs/')) indicators.push(`dep:${name}`); + } + + try { + await fs.stat(path.join(rootPath, 'nest-cli.json')); + indicators.push('disk:nest-cli-json'); + } catch { + // absent + } + if (await anyExists([path.join(rootPath, 'src', 'main.ts'), path.join(rootPath, 'main.ts')])) { + indicators.push('disk:main-ts'); + } + + return indicators; +} + +async function anyExists(paths: string[]): Promise { + for (const candidatePath of paths) { + try { + await fs.stat(candidatePath); + return true; + } catch { + // Continue checking remaining candidates. + } + } + return false; +} + +function getFrameworkVariant(allDependencies: Record): string { + const hasPlatform = Boolean( + allDependencies['@nestjs/platform-express'] || allDependencies['@nestjs/platform-fastify'] + ); + const hasGraphql = Boolean(allDependencies['@nestjs/graphql']); + const hasMicroservices = Boolean(allDependencies['@nestjs/microservices']); + const hasRealtime = Boolean(allDependencies['@nestjs/websockets']); + + if (hasPlatform && (hasGraphql || hasMicroservices || hasRealtime)) return 'mixed'; + if (hasPlatform) return 'http-api'; + if (hasMicroservices) return 'microservice'; + return 'library'; +} + +function detectDependencyList( + allDependencies: Record, + candidates: ReadonlyArray +): string[] { + return candidates + .filter(([dependencyName]) => Boolean(allDependencies[dependencyName])) + .map(([, label]) => label); +} + +function appendExports(exports: ExportStatement[], statement: TSESTree.Statement): void { + if (statement.type === 'ExportNamedDeclaration') { + const declaration = statement.declaration; + if (declaration?.type === 'FunctionDeclaration' && declaration.id) { + exports.push({ name: declaration.id.name, isDefault: false, type: 'function' }); + return; + } + if (declaration?.type === 'ClassDeclaration' && declaration.id) { + exports.push({ name: declaration.id.name, isDefault: false, type: 'class' }); + return; + } + if (declaration?.type === 'VariableDeclaration') { + for (const declarator of declaration.declarations) { + if (declarator.id.type === 'Identifier') { + exports.push({ name: declarator.id.name, isDefault: false, type: 'variable' }); + } + } + return; + } + for (const specifier of statement.specifiers) { + if (specifier.exported.type === 'Identifier') { + exports.push({ name: specifier.exported.name, isDefault: false, type: 're-export' }); + } + } + return; + } + + if (statement.type === 'ExportDefaultDeclaration') { + exports.push({ name: 'default', isDefault: true, type: 'default' }); + } +} + +function getImportSpecifierName(specifier: TSESTree.ImportClause): string { + if (specifier.type === 'ImportDefaultSpecifier') return 'default'; + if (specifier.type === 'ImportNamespaceSpecifier') return '*'; + return 'value' in specifier.imported ? String(specifier.imported.value) : specifier.imported.name; +} + +function getPackageName(importSource: string): string { + if (importSource.startsWith('@')) { + const [scope, name] = importSource.split('/'); + return name ? `${scope}/${name}` : importSource; + } + return importSource.split('/')[0] || importSource; +} + +function dedupeSummaries(summaries: NestClassSummary[]): NestClassSummary[] { + const seen = new Set(); + return summaries.filter((summary) => { + const key = `${summary.component.name}:${summary.component.startLine}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function walkAst( + node: TSESTree.Node, + visit: (node: TSESTree.Node, parent: TSESTree.Node | null) => void, + parent: TSESTree.Node | null = null +): void { + visit(node, parent); + for (const key of Object.keys(node) as Array) { + if (key === 'parent') continue; + const value = node[key]; + if (!value) continue; + if (Array.isArray(value)) { + for (const child of value) { + if (isAstNode(child)) walkAst(child, visit, node); + } + } else if (isAstNode(value)) { + walkAst(value, visit, node); + } + } +} + +function isAstNode(value: unknown): value is TSESTree.Node { + return Boolean( + value && typeof value === 'object' && typeof (value as { type?: unknown }).type === 'string' + ); +} diff --git a/src/analyzers/nextjs/index.ts b/src/analyzers/nextjs/index.ts index 79fa6c5..ea99dab 100644 --- a/src/analyzers/nextjs/index.ts +++ b/src/analyzers/nextjs/index.ts @@ -21,8 +21,35 @@ import { } from '../shared/metadata.js'; type DetectedPattern = { category: string; name: string }; -type NextRouter = 'app' | 'pages' | 'unknown'; -type NextRouteKind = 'page' | 'layout' | 'route' | 'api' | 'unknown'; +type NextRouter = 'app' | 'pages' | 'proxy' | 'unknown'; +type NextRouteKind = + | 'page' + | 'layout' + | 'route' + | 'api' + | 'loading' + | 'error' + | 'not-found' + | 'template' + | 'default' + | 'forbidden' + | 'unauthorized' + | 'proxy' + | 'unknown'; + +const APP_ROUTE_FILE_KINDS: Readonly> = { + page: 'page', + layout: 'layout', + route: 'route', + loading: 'loading', + error: 'error', + 'global-error': 'error', + 'not-found': 'not-found', + template: 'template', + default: 'default', + forbidden: 'forbidden', + unauthorized: 'unauthorized' +}; interface NextRoutingInfo { router: NextRouter; @@ -44,7 +71,7 @@ export class NextJsAnalyzer implements FrameworkAnalyzer { return false; } - if (isInAppRouter(filePath) || isInPagesRouter(filePath)) { + if (isInAppRouter(filePath) || isInPagesRouter(filePath) || isProxyFile(filePath)) { return true; } @@ -83,6 +110,9 @@ export class NextJsAnalyzer implements FrameworkAnalyzer { if (routing.kind === 'api') { detectedPatterns.push({ category: 'routing', name: 'API Route' }); } + if (routing.kind === 'proxy') { + detectedPatterns.push({ category: 'routing', name: 'Proxy' }); + } if (routing.hasMetadata) { detectedPatterns.push({ category: 'metadata', name: 'Next.js metadata' }); } @@ -276,21 +306,22 @@ export class NextJsAnalyzer implements FrameworkAnalyzer { function analyzeRouting(filePath: string, content: string): NextRoutingInfo { const normalizedPath = filePath.replace(/\\/g, '/'); const baseName = path.basename(normalizedPath, path.extname(normalizedPath)); - const router: NextRouter = isInAppRouter(filePath) - ? 'app' - : isInPagesRouter(filePath) - ? 'pages' - : 'unknown'; + const router: NextRouter = isProxyFile(filePath) + ? 'proxy' + : isInAppRouter(filePath) + ? 'app' + : isInPagesRouter(filePath) + ? 'pages' + : 'unknown'; let kind: NextRouteKind = 'unknown'; - if (router === 'app') { + if (router === 'proxy') { + kind = 'proxy'; + } else if (router === 'app') { const fileName = path.basename(normalizedPath); - if (fileName.startsWith('page.')) { - kind = 'page'; - } else if (fileName.startsWith('layout.')) { - kind = 'layout'; - } else if (fileName.startsWith('route.')) { - kind = 'route'; + const appFileKind = getAppRouteFileKind(fileName); + if (appFileKind) { + kind = appFileKind; } } else if (router === 'pages') { if (normalizedPath.includes('/pages/api/') || normalizedPath.includes('/src/pages/api/')) { @@ -303,9 +334,11 @@ function analyzeRouting(filePath: string, content: string): NextRoutingInfo { return { router, kind, - routePath: router === 'unknown' ? null : computeRoutePath(router, normalizedPath), + routePath: + router === 'app' || router === 'pages' ? computeRoutePath(router, normalizedPath) : null, isClientComponent: hasUseClientDirective(content), - hasMetadata: /\bexport\s+(?:const|function)\s+(metadata|generateMetadata)\b/.test(content) + hasMetadata: + /\bexport\s+(?:const|async\s+function|function)\s+(metadata|generateMetadata)\b/.test(content) }; } @@ -320,7 +353,7 @@ function isInPagesRouter(filePath: string): boolean { } function computeRoutePath( - router: Exclude, + router: Extract, normalizedFilePath: string ): string | null { const pathSegments = normalizedFilePath.split('/').filter(Boolean); @@ -361,6 +394,16 @@ function isPagesSystemFile(baseName: string): boolean { return ['_app', '_document', '_error', '_middleware'].includes(baseName); } +function getAppRouteFileKind(fileName: string): NextRouteKind | null { + const baseName = path.basename(fileName, path.extname(fileName)); + return APP_ROUTE_FILE_KINDS[baseName] || null; +} + +function isProxyFile(filePath: string): boolean { + const normalizedPath = filePath.replace(/\\/g, '/'); + return /(^|\/)(src\/)?proxy\.(?:js|jsx|ts|tsx|mjs|cjs|mts|cts)$/.test(normalizedPath); +} + async function detectRouterPresence( rootPath: string ): Promise<{ hasAppRouter: boolean; hasPagesRouter: boolean }> { diff --git a/src/analyzers/react/index.ts b/src/analyzers/react/index.ts index 317d296..5838c09 100644 --- a/src/analyzers/react/index.ts +++ b/src/analyzers/react/index.ts @@ -37,7 +37,10 @@ const BUILTIN_HOOKS = new Set([ 'useTransition', 'useId', 'useSyncExternalStore', - 'useInsertionEffect' + 'useInsertionEffect', + 'useActionState', + 'useOptimistic', + 'useFormStatus' ]); const REACT_LIBRARY_SIGNALS: ReadonlyArray<{ @@ -59,6 +62,9 @@ interface ReactAstSummary { usesContext: boolean; usesMemoization: boolean; usesSuspense: boolean; + usesActionState: boolean; + usesOptimistic: boolean; + usesFormStatus: boolean; } export class ReactAnalyzer implements FrameworkAnalyzer { @@ -101,6 +107,7 @@ export class ReactAnalyzer implements FrameworkAnalyzer { const exports: ExportStatement[] = []; const dependencyNames = new Set(); const importSources = new Set(); + const reactUseBindings = new Set(); const detectedPatterns: DetectedPattern[] = []; let components: CodeComponent[] = []; @@ -117,6 +124,9 @@ export class ReactAnalyzer implements FrameworkAnalyzer { if (statement.type === 'ImportDeclaration' && typeof statement.source.value === 'string') { const source = statement.source.value; importSources.add(getPackageName(source)); + if (source === 'react') { + addReactUseBindings(reactUseBindings, statement.specifiers); + } imports.push({ source, imports: statement.specifiers.map(getImportSpecifierName), @@ -135,7 +145,7 @@ export class ReactAnalyzer implements FrameworkAnalyzer { appendExports(exports, statement); } - const summary = summarizeReactProgram(program); + const summary = summarizeReactProgram(program, reactUseBindings); components = summary.components; if (summary.usesContext) { @@ -144,6 +154,15 @@ export class ReactAnalyzer implements FrameworkAnalyzer { if (summary.usesSuspense) { detectedPatterns.push({ category: 'reactivity', name: 'Suspense' }); } + if (summary.usesActionState) { + detectedPatterns.push({ category: 'reactivity', name: 'Actions' }); + } + if (summary.usesOptimistic) { + detectedPatterns.push({ category: 'reactivity', name: 'Optimistic UI' }); + } + if (summary.usesFormStatus) { + detectedPatterns.push({ category: 'forms', name: 'Form status' }); + } if (summary.usesMemoization) { detectedPatterns.push({ category: 'reactivity', name: 'Memoization' }); } @@ -302,18 +321,25 @@ export class ReactAnalyzer implements FrameworkAnalyzer { } } -function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { +function summarizeReactProgram( + program: TSESTree.Program, + reactUseBindings: ReadonlySet +): ReactAstSummary { const components: CodeComponent[] = []; const builtinHooksUsed = new Set(); const customHooks = new Set(); let usesContext = false; let usesMemoization = false; let usesSuspense = false; + let usesActionState = false; + let usesOptimistic = false; + let usesFormStatus = false; walkAst(program, (node, parent) => { if (node.type === 'CallExpression') { const calleeName = getCalleeName(node.callee); - if (calleeName && BUILTIN_HOOKS.has(calleeName)) { + const usesReactUse = calleeName !== null && reactUseBindings.has(calleeName); + if (calleeName && (BUILTIN_HOOKS.has(calleeName) || usesReactUse)) { builtinHooksUsed.add(calleeName); } if (calleeName === 'createContext' || calleeName === 'useContext') { @@ -325,6 +351,18 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { if (calleeName === 'lazy') { usesSuspense = true; } + if (usesReactUse) { + usesSuspense = true; + } + if (calleeName === 'useActionState') { + usesActionState = true; + } + if (calleeName === 'useOptimistic') { + usesOptimistic = true; + } + if (calleeName === 'useFormStatus') { + usesFormStatus = true; + } if ( calleeName === 'createContext' && parent?.type === 'VariableDeclarator' && @@ -390,10 +428,30 @@ function summarizeReactProgram(program: TSESTree.Program): ReactAstSummary { customHooks: Array.from(customHooks).sort(), usesContext, usesMemoization, - usesSuspense + usesSuspense, + usesActionState, + usesOptimistic, + usesFormStatus }; } +function addReactUseBindings(bindings: Set, specifiers: TSESTree.ImportClause[]): void { + for (const specifier of specifiers) { + if ( + specifier.type === 'ImportSpecifier' && + getModuleExportedName(specifier.imported) === 'use' + ) { + bindings.add(specifier.local.name); + } + if ( + specifier.type === 'ImportDefaultSpecifier' || + specifier.type === 'ImportNamespaceSpecifier' + ) { + bindings.add(`${specifier.local.name}.use`); + } + } +} + function appendExports(exports: ExportStatement[], statement: TSESTree.Statement): void { if (statement.type === 'ExportNamedDeclaration') { const declaration = statement.declaration; diff --git a/src/cli-init.ts b/src/cli-init.ts index e9efc76..bf126cc 100644 --- a/src/cli-init.ts +++ b/src/cli-init.ts @@ -10,28 +10,54 @@ import path from 'path'; import * as fs from 'node:fs/promises'; -import { execFileSync } from 'child_process'; import { select, confirm } from '@inquirer/prompts'; export type Client = 'claude-code' | 'cursor' | 'codex' | 'opencode'; -export type McpConfigResult = - | { kind: 'file'; path: string; content: string } - | { kind: 'command'; args: string[] }; +export type McpConfigResult = { kind: 'file'; path: string; content: string }; + +export type McpConfigOptions = { + rootPath?: string; + nodeExecutable?: string; + serverEntrypoint?: string; +}; type JsonObject = Record; -export function generateMcpConfig(client: Client): McpConfigResult { +function quoteTomlString(value: string): string { + return JSON.stringify(value); +} + +export function generateMcpConfig(client: Client, options: McpConfigOptions = {}): McpConfigResult { + const rootPath = path.resolve(options.rootPath ?? process.cwd()); + const nodeExecutable = options.nodeExecutable ?? process.execPath; + const serverEntrypoint = path.resolve( + options.serverEntrypoint ?? process.argv[1] ?? path.join(rootPath, 'dist', 'index.js') + ); + const stdioArgs = [serverEntrypoint, rootPath]; + switch (client) { case 'claude-code': return { - kind: 'command', - args: ['mcp', 'add', '--transport', 'http', 'codebase-context', 'http://127.0.0.1:3100/mcp'] + kind: 'file', + path: path.join(rootPath, '.mcp.json'), + content: JSON.stringify( + { + mcpServers: { + 'codebase-context': { + command: nodeExecutable, + args: stdioArgs + } + } + }, + null, + 2 + ) }; case 'cursor': return { kind: 'file', - path: '.cursor/mcp.json', + path: path.join(rootPath, '.cursor', 'mcp.json'), content: JSON.stringify( { mcpServers: { @@ -47,13 +73,18 @@ export function generateMcpConfig(client: Client): McpConfigResult { }; case 'codex': return { - kind: 'command', - args: ['mcp', 'add', 'codebase-context', 'http://127.0.0.1:3100/mcp'] + kind: 'file', + path: path.join(rootPath, '.codex', 'config.toml'), + content: + `[mcp_servers.codebase-context]\n` + + `command = ${quoteTomlString(nodeExecutable)}\n` + + `args = [${stdioArgs.map(quoteTomlString).join(', ')}]\n` + + 'startup_timeout_sec = 120\n' }; case 'opencode': return { kind: 'file', - path: 'opencode.json', + path: path.join(rootPath, 'opencode.json'), content: JSON.stringify( { $schema: 'https://opencode.ai/config.json', @@ -71,6 +102,34 @@ export function generateMcpConfig(client: Client): McpConfigResult { } } +export function resolveInitRoot( + args: string[], + envRoot: string | undefined, + cwd = process.cwd() +): string { + let explicitRoot: string | undefined; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--root') { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error('Expected a directory path after --root.'); + } + explicitRoot = value; + index++; + } else if (arg?.startsWith('--root=')) { + const value = arg.slice('--root='.length); + if (!value) { + throw new Error('Expected a directory path after --root.'); + } + explicitRoot = value; + } + } + + return path.resolve(explicitRoot ?? (envRoot ? envRoot : cwd)); +} + export function generateInstructionBlock(): string { return ` ## Codebase Context (MCP) @@ -127,57 +186,99 @@ function isJsonObject(value: unknown): value is JsonObject { return typeof value === 'object' && value !== null && !Array.isArray(value); } +function normalizeJson(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(normalizeJson); + } + if (isJsonObject(value)) { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, normalizeJson(entry)]) + ); + } + return value; +} + +function jsonValuesEqual(left: unknown, right: unknown): boolean { + return JSON.stringify(normalizeJson(left)) === JSON.stringify(normalizeJson(right)); +} + +function isFileNotFound(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; +} + +async function readExistingConfig(filePath: string): Promise { + try { + return await fs.readFile(filePath, 'utf8'); + } catch (error) { + if (isFileNotFound(error)) return null; + throw error; + } +} + /** * Merge generated MCP config into an existing JSON config file when possible. - * Preserves unrelated keys and only updates the codebase-context server entry. + * Preserves unrelated keys and refuses to replace a different codebase-context entry. */ export async function _buildMergedMcpContent( filePath: string, generatedContent: string, - client: Extract + client: Extract ): Promise<{ content: string; mergedFromExisting: boolean }> { + const existingText = await readExistingConfig(filePath); + if (existingText === null) { + return { content: generatedContent, mergedFromExisting: false }; + } + let existing: unknown; try { - existing = JSON.parse(await fs.readFile(filePath, 'utf8')); - } catch { - return { content: generatedContent, mergedFromExisting: false }; + existing = JSON.parse(existingText) as unknown; + } catch (error) { + throw new Error(`Cannot merge ${client} config at ${filePath}: existing JSON is invalid.`, { + cause: error + }); } const generated = JSON.parse(generatedContent) as unknown; if (!isJsonObject(existing) || !isJsonObject(generated)) { - return { content: generatedContent, mergedFromExisting: false }; + throw new Error(`Cannot merge ${client} config at ${filePath}: expected a JSON object.`); } - if (client === 'cursor') { - const existingServers = isJsonObject(existing.mcpServers) ? existing.mcpServers : {}; - const generatedServers = isJsonObject(generated.mcpServers) ? generated.mcpServers : {}; - return { - content: JSON.stringify( - { - ...existing, - ...generated, - mcpServers: { - ...existingServers, - ...generatedServers - } - }, - null, - 2 - ), - mergedFromExisting: true - }; + const serverKey = client === 'opencode' ? 'mcp' : 'mcpServers'; + const existingServersValue = existing[serverKey]; + if (existingServersValue !== undefined && !isJsonObject(existingServersValue)) { + throw new Error( + `Cannot merge ${client} config at ${filePath}: ${serverKey} must be a JSON object.` + ); + } + const generatedServersValue = generated[serverKey]; + if (!isJsonObject(generatedServersValue)) { + throw new Error(`Invalid generated ${client} MCP config.`); + } + const existingServers = isJsonObject(existingServersValue) ? existingServersValue : {}; + const hasExistingCodebaseContext = Object.prototype.hasOwnProperty.call( + existingServers, + 'codebase-context' + ); + const generatedCodebaseContext = generatedServersValue['codebase-context']; + if ( + hasExistingCodebaseContext && + !jsonValuesEqual(existingServers['codebase-context'], generatedCodebaseContext) + ) { + throw new Error( + `Cannot merge ${client} config at ${filePath}: it already contains a different codebase-context configuration.` + ); } - const existingMcp = isJsonObject(existing.mcp) ? existing.mcp : {}; - const generatedMcp = isJsonObject(generated.mcp) ? generated.mcp : {}; return { content: JSON.stringify( { ...existing, ...generated, - mcp: { - ...existingMcp, - ...generatedMcp + [serverKey]: { + ...existingServers, + ...generatedServersValue } }, null, @@ -187,56 +288,232 @@ export async function _buildMergedMcpContent( }; } -export async function handleInitCli(_argv: string[]): Promise { - console.log('\nSet up codebase-context for your AI client\n'); +function normalizeTomlNewlines(value: string): string { + return value.replace(/\r\n/g, '\n'); +} - const client = await select({ - message: 'Which client?', - choices: [ - { name: 'Claude Code', value: 'claude-code' }, - { name: 'Cursor', value: 'cursor' }, - { name: 'Codex', value: 'codex' }, - { name: 'OpenCode', value: 'opencode' } - ] - }); +function tomlKeyPattern(key: string): string { + return `(?:${key}|"${key}"|'${key}')`; +} - const mcpResult = generateMcpConfig(client); +/** + * Appends the project MCP table to Codex TOML, preserving other settings and + * refusing to replace an existing codebase-context table. + */ +export async function _buildMergedCodexContent( + filePath: string, + generatedContent: string +): Promise<{ content: string; mergedFromExisting: boolean }> { + const existing = await readExistingConfig(filePath); + if (existing === null) { + return { content: generatedContent, mergedFromExisting: false }; + } - console.log('\n--- MCP Config Preview ---'); - if (mcpResult.kind === 'file') { - try { - await fs.access(mcpResult.path); - console.log(`Warning: ${mcpResult.path} already exists; existing entries will be preserved.`); - } catch { - // file does not exist + const normalizedExisting = normalizeTomlNewlines(existing); + const normalizedGenerated = normalizeTomlNewlines(generatedContent).trim(); + const mcpServersKey = tomlKeyPattern('mcp_servers'); + const codebaseContextKey = tomlKeyPattern('codebase-context'); + const entryHeader = new RegExp( + `^\\s*\\[{1,2}\\s*${mcpServersKey}\\s*\\.\\s*${codebaseContextKey}(?:\\s*\\.\\s*[^\\]]+)?\\s*\\]{1,2}\\s*(?:#.*)?$`, + 'gm' + ); + const existingEntry = entryHeader.exec(normalizedExisting); + + if (existingEntry) { + const nextTable = /^\s*\[{1,2}[^\r\n]+\]{1,2}\s*(?:#.*)?$/gm; + nextTable.lastIndex = entryHeader.lastIndex; + const nextTableMatch = nextTable.exec(normalizedExisting); + const sectionEnd = nextTableMatch?.index ?? normalizedExisting.length; + const section = normalizedExisting.slice(existingEntry.index, sectionEnd).trim(); + const exactEntryHeader = new RegExp( + `^\\[\\s*${mcpServersKey}\\s*\\.\\s*${codebaseContextKey}\\s*\\]\\s*(?:#.*)?$`, + 'm' + ); + const comparableSection = section.replace(exactEntryHeader, '[mcp_servers.codebase-context]'); + if (comparableSection === normalizedGenerated) { + return { content: existing, mergedFromExisting: true }; } - console.log(`File: ${mcpResult.path}\n${mcpResult.content}`); - } else { - console.log(`Command to run: ${mcpResult.args[0]} ${mcpResult.args.slice(1).join(' ')}`); + throw new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); } - const applyMcp = await confirm({ - message: 'Apply MCP config? [y/N]', - default: false - }); - - const instructionPath = resolveInstructionFilePath(client, process.cwd()); + const dottedEntry = new RegExp( + `^\\s*${mcpServersKey}\\s*\\.\\s*${codebaseContextKey}\\s*\\.`, + 'm' + ); + if (dottedEntry.test(normalizedExisting)) { + throw new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); + } - let applyInstruction = false; - if (instructionPath !== null) { - let fileExists = false; - try { - await fs.access(instructionPath); - fileExists = true; - } catch { - // does not exist + const parentTableHeader = new RegExp(`^\\s*\\[\\s*${mcpServersKey}\\s*\\]\\s*(?:#.*)?$`, 'gm'); + let parentTable = parentTableHeader.exec(normalizedExisting); + while (parentTable) { + const nextTable = /^\s*\[{1,2}[^\r\n]+\]{1,2}\s*(?:#.*)?$/gm; + nextTable.lastIndex = parentTableHeader.lastIndex; + const nextTableMatch = nextTable.exec(normalizedExisting); + const sectionEnd = nextTableMatch?.index ?? normalizedExisting.length; + const parentSettings = normalizedExisting.slice(parentTableHeader.lastIndex, sectionEnd); + const childEntry = new RegExp(`^\\s*${codebaseContextKey}\\s*(?:=|\\.)`, 'm'); + if (childEntry.test(parentSettings)) { + throw new Error( + `Cannot merge Codex config at ${filePath}: it already contains a different codebase-context configuration.` + ); } + parentTable = parentTableHeader.exec(normalizedExisting); + } + + const firstTable = /^\s*\[/m.exec(normalizedExisting); + const rootSettings = firstTable + ? normalizedExisting.slice(0, firstTable.index) + : normalizedExisting; + const inlineServers = new RegExp(`^\\s*${mcpServersKey}\\s*=`, 'm'); + if (inlineServers.test(rootSettings)) { + throw new Error( + `Cannot merge Codex config at ${filePath}: mcp_servers is already configured inline.` + ); + } + + const prefix = normalizedExisting.trimEnd(); + return { + content: `${prefix ? `${prefix}\n\n` : ''}${normalizedGenerated}\n`, + mergedFromExisting: true + }; +} + +type InitOptions = { + client?: Client; + rootPath: string; + yes: boolean; + help: boolean; +}; + +function parseClient(value: string): Client { + if (value === 'claude-code' || value === 'cursor' || value === 'codex' || value === 'opencode') { + return value; + } + throw new Error(`Unknown client "${value}". Choose claude-code, cursor, codex, or opencode.`); +} + +function parseInitOptions(args: string[]): InitOptions { + if (args.includes('--help') || args.includes('-h')) { + return { rootPath: process.cwd(), yes: false, help: true }; + } - if (fileExists) { - console.log(`\n--- Instruction Block Preview (will append to ${instructionPath}) ---`); + let client: Client | undefined; + let yes = false; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--yes' || arg === '-y') { + yes = true; + } else if (arg === '--client') { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error('Expected a client after --client.'); + } + client = parseClient(value); + index++; + } else if (arg?.startsWith('--client=')) { + client = parseClient(arg.slice('--client='.length)); + } else if (arg === '--root') { + index++; + } else if (arg?.startsWith('--root=')) { + // Root values are validated once by resolveInitRoot below. } else { - console.log(`\n--- Instruction Block Preview (will create ${instructionPath}) ---`); + throw new Error(`Unknown init option "${arg}".`); } + } + + return { + ...(client ? { client } : {}), + rootPath: resolveInitRoot(args, process.env.CODEBASE_ROOT, process.cwd()), + yes, + help: false + }; +} + +function printInitUsage(): void { + console.log('codebase-context init [options]'); + console.log(''); + console.log('Set up MCP config for an AI client.'); + console.log(''); + console.log('Options:'); + console.log(' --client claude-code, cursor, codex, or opencode'); + console.log(' --root Project root (default: CODEBASE_ROOT or cwd)'); + console.log(' --yes, -y Write the MCP config without prompts'); + console.log(' --help, -h Show this help'); +} + +function getCurrentCliEntrypoint(): string { + const entrypoint = process.argv[1]; + if (!entrypoint) { + throw new Error('Cannot locate the codebase-context CLI entrypoint.'); + } + return path.resolve(entrypoint); +} + +export async function handleInitCli(args: string[]): Promise { + let options: InitOptions; + try { + options = parseInitOptions(args); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + printInitUsage(); + process.exitCode = 1; + return; + } + + if (options.help) { + printInitUsage(); + return; + } + if (options.yes && !options.client) { + console.error('Error: --yes requires --client.'); + printInitUsage(); + process.exitCode = 1; + return; + } + + console.log('\nSet up codebase-context for your AI client\n'); + + const client = + options.client ?? + (await select({ + message: 'Which client?', + choices: [ + { name: 'Claude Code', value: 'claude-code' }, + { name: 'Cursor', value: 'cursor' }, + { name: 'Codex', value: 'codex' }, + { name: 'OpenCode', value: 'opencode' } + ] + })); + + const mcpResult = generateMcpConfig(client, { + rootPath: options.rootPath, + nodeExecutable: process.execPath, + serverEntrypoint: getCurrentCliEntrypoint() + }); + + console.log('\n--- MCP Config Preview ---'); + console.log( + 'Existing unrelated settings will be preserved; a different codebase-context entry will stop setup.' + ); + console.log(`File: ${mcpResult.path}\n${mcpResult.content}`); + + const applyMcp = options.yes + ? true + : await confirm({ + message: 'Apply MCP config? [y/N]', + default: false + }); + + const instructionPath = options.yes ? null : resolveInstructionFilePath(client, options.rootPath); + + let applyInstruction = false; + if (instructionPath !== null) { + console.log(`\n--- Instruction Block Preview (will append to ${instructionPath}) ---`); console.log(generateInstructionBlock()); applyInstruction = await confirm({ @@ -245,47 +522,45 @@ export async function handleInitCli(_argv: string[]): Promise { }); } - // Execute confirmed actions - - if (applyMcp) { - if (mcpResult.kind === 'file') { - const dir = path.dirname(mcpResult.path); - if (dir && dir !== '.') { - await fs.mkdir(dir, { recursive: true }); + try { + if (applyMcp) { + try { + await fs.access(options.rootPath); + } catch { + throw new Error(`Project root does not exist or is not accessible: ${options.rootPath}`); } + const dir = path.dirname(mcpResult.path); + await fs.mkdir(dir, { recursive: true }); const mergedConfig = - client === 'cursor' || client === 'opencode' - ? await _buildMergedMcpContent(mcpResult.path, mcpResult.content, client) - : { content: mcpResult.content, mergedFromExisting: false }; + client === 'codex' + ? await _buildMergedCodexContent(mcpResult.path, mcpResult.content) + : await _buildMergedMcpContent(mcpResult.path, mcpResult.content, client); await fs.writeFile(mcpResult.path, mergedConfig.content, 'utf8'); if (mergedConfig.mergedFromExisting) { console.log(`Merged: ${mcpResult.path} (existing entries preserved)`); } console.log(`Written: ${mcpResult.path}`); - } else { - const [cmd, ...rest] = mcpResult.args; - try { - execFileSync(cmd, rest, { stdio: 'inherit' }); - } catch { - console.log( - `Could not run '${cmd}' automatically. Run it yourself:\n ${cmd} ${rest.join(' ')}` - ); - } } - } - if (applyInstruction && instructionPath !== null) { - const status = await _appendInstructionBlock(instructionPath); - if (status === 'skipped') { - console.log(`Instruction block already present in ${instructionPath}, skipping.`); - } else { - console.log(`Written: ${instructionPath}`); + if (applyInstruction && instructionPath !== null) { + const status = await _appendInstructionBlock(instructionPath); + if (status === 'skipped') { + console.log(`Instruction block already present in ${instructionPath}, skipping.`); + } else { + console.log(`Written: ${instructionPath}`); + } } + } catch (error) { + console.error(`Init failed: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + return; } - console.log( - '\nNext steps:\n' + - ' Run `npx codebase-context map` to see your codebase conventions\n' + - ' Start the HTTP server: npx codebase-context --http\n' - ); + const nextSteps = + client === 'claude-code' || client === 'codex' + ? ' Restart or reopen the client in this project to load the MCP server.\n' + + ' Run `npx codebase-context map` to see your codebase conventions\n' + : ' Run `npx codebase-context map` to see your codebase conventions\n' + + ' Start the HTTP server: npx codebase-context --http\n'; + console.log(`\nNext steps:\n${nextSteps}`); } diff --git a/src/cli.ts b/src/cli.ts index 1bc89f4..2e5f17e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -21,6 +21,7 @@ import type { IndexState } from './tools/types.js'; import { analyzerRegistry } from './core/analyzer-registry.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; import { GenericAnalyzer } from './analyzers/generic/index.js'; import { formatJson } from './cli-formatters.js'; @@ -28,9 +29,11 @@ import { handleMemoryCli } from './cli-memory.js'; export { handleMemoryCli } from './cli-memory.js'; import { handleInitCli } from './cli-init.js'; import { handleMapCli } from './cli-map.js'; +import { loadProjectConfig } from './server/config.js'; analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); @@ -103,6 +106,7 @@ function resolveCliRootPath(): string { async function initToolContext(): Promise { const rootPath = resolveCliRootPath(); + const projectConfig = await loadProjectConfig(rootPath); const paths = { baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), @@ -134,6 +138,7 @@ async function initToolContext(): Promise { let lastLoggedProgress = { phase: '', percentage: -1 }; const indexer = new CodebaseIndexer({ rootPath, + ...(projectConfig?.parsing ? { config: { parsing: projectConfig.parsing } } : {}), incrementalOnly, onProgress: (progress) => { const shouldLog = @@ -251,7 +256,7 @@ function booleanFlag(flags: Flags, key: string, usage: string): boolean { export async function handleCliCommand(argv: string[]): Promise { const rawCommand = argv[0]; - if (!rawCommand || rawCommand === '--help') { + if (!rawCommand || rawCommand === '--help' || rawCommand === 'help') { printUsage(); return; } diff --git a/src/core/indexer.ts b/src/core/indexer.ts index 9530576..93aebbe 100644 --- a/src/core/indexer.ts +++ b/src/core/indexer.ts @@ -57,6 +57,7 @@ import { deriveCodebaseHealth } from '../health/derive.js'; const STAGING_DIRNAME = '.staging'; const PREVIOUS_DIRNAME = '.previous'; +const DEFAULT_MAX_CHUNKS = 5000; import { computeFileHashes, @@ -282,6 +283,7 @@ export class CodebaseIndexer { analyzers: { angular: { enabled: true, priority: 100 }, nextjs: { enabled: false, priority: 90 }, + nestjs: { enabled: false, priority: 105 }, react: { enabled: false, priority: 90 }, vue: { enabled: false, priority: 90 }, generic: { enabled: true, priority: 10 } @@ -301,6 +303,7 @@ export class CodebaseIndexer { respectGitignore: true, parsing: { maxFileSize: 1048576, + maxChunks: DEFAULT_MAX_CHUNKS, chunkSize: 50, chunkOverlap: 0, parseTests: true, @@ -748,13 +751,13 @@ export class CodebaseIndexer { const chunksForEmbedding = diff ? changedChunks : allChunks; // Memory safety: limit chunks to prevent embedding memory issues - const MAX_CHUNKS = 5000; + const maxChunks = this.config.parsing.maxChunks ?? DEFAULT_MAX_CHUNKS; let chunksToEmbed = chunksForEmbedding; - if (chunksForEmbedding.length > MAX_CHUNKS) { + if (chunksForEmbedding.length > maxChunks) { console.warn( - `WARNING: ${chunksForEmbedding.length} chunks exceed limit. Indexing first ${MAX_CHUNKS} chunks.` + `WARNING: ${chunksForEmbedding.length} chunks exceed limit. Indexing first ${maxChunks} chunks.` ); - chunksToEmbed = chunksForEmbedding.slice(0, MAX_CHUNKS); + chunksToEmbed = chunksForEmbedding.slice(0, maxChunks); } // Phase 3: Embedding (only changed/added chunks in incremental mode) @@ -894,7 +897,7 @@ export class CodebaseIndexer { const indexPath = path.join(activeContextDir, KEYWORD_INDEX_FILENAME); // Memory safety: cap keyword index too const keywordChunks = - allChunks.length > MAX_CHUNKS ? allChunks.slice(0, MAX_CHUNKS) : allChunks; + allChunks.length > maxChunks ? allChunks.slice(0, maxChunks) : allChunks; await fs.writeFile( indexPath, JSON.stringify({ diff --git a/src/core/reranker.ts b/src/core/reranker.ts index 47de513..9dabeac 100644 --- a/src/core/reranker.ts +++ b/src/core/reranker.ts @@ -9,6 +9,7 @@ */ import type { SearchResult } from '../types/index.js'; +import type { PreTrainedTokenizer } from '@huggingface/transformers'; import os from 'os'; const DEFAULT_RERANKER_MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2'; @@ -19,19 +20,11 @@ const RERANK_TOP_K = 10; /** Trigger reranking when the score gap between #1 and #3 is below this threshold */ const AMBIGUITY_THRESHOLD = 0.08; -interface CrossEncoderTokenizer { - ( - query: string, - passage: string, - options: { padding: boolean; truncation: boolean; max_length: number } - ): unknown; -} - interface CrossEncoderModel { (inputs: unknown): Promise<{ logits: { data: ArrayLike } }>; } -let cachedTokenizer: CrossEncoderTokenizer | null = null; +let cachedTokenizer: PreTrainedTokenizer | null = null; let cachedModel: CrossEncoderModel | null = null; let initPromise: Promise | null = null; @@ -152,7 +145,8 @@ async function scorePair(query: string, passage: string): Promise { throw new Error('[reranker] Model not loaded — call ensureModelLoaded() first'); } - const inputs = cachedTokenizer(query, passage, { + const inputs = cachedTokenizer(query, { + text_pair: passage, padding: true, truncation: true, max_length: 512 diff --git a/src/embeddings/transformers.ts b/src/embeddings/transformers.ts index 0a2c581..7fc62e8 100644 --- a/src/embeddings/transformers.ts +++ b/src/embeddings/transformers.ts @@ -1,5 +1,5 @@ import { EmbeddingProvider, DEFAULT_MODEL } from './types.js'; -import type { FeatureExtractionPipelineType } from '@huggingface/transformers'; +import type { FeatureExtractionPipeline } from '@huggingface/transformers'; import os from 'os'; /** @@ -43,7 +43,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { readonly modelName: string; readonly dimensions: number; - private pipeline: FeatureExtractionPipelineType | null = null; + private pipeline: FeatureExtractionPipeline | null = null; private ready = false; private initPromise: Promise | null = null; @@ -70,12 +70,12 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider { const { pipeline } = await import('@huggingface/transformers'); // TS2590: pipeline() resolves AllTasks[T] — a union too complex for TSC to represent. - // Cast to a simpler signature; the actual return type IS FeatureExtractionPipelineType. + // Cast to a simpler feature-extraction signature for both v3 and v4-compatible types. type PipelineFn = ( task: 'feature-extraction', model: string, opts: Record - ) => Promise; + ) => Promise; this.pipeline = await (pipeline as PipelineFn)('feature-extraction', this.modelName, { dtype: 'q8', // Limit ONNX Runtime to half cores by default — prevents system freeze during indexing. diff --git a/src/index.ts b/src/index.ts index 8d7386b..3e7a989 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,6 +17,7 @@ import { CodebaseIndexer } from './core/indexer.js'; import { analyzerRegistry } from './core/analyzer-registry.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; import { GenericAnalyzer } from './analyzers/generic/index.js'; import { IndexCorruptedError } from './errors/index.js'; @@ -62,6 +63,7 @@ import { analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); @@ -91,7 +93,8 @@ const CLI_SUBCOMMANDS = [ 'refs', 'cycles', 'init', - 'map' + 'map', + 'help' ]; // Check if this module is the entry point. @@ -106,6 +109,10 @@ const isDirectCliSubcommand = // Resolve optional bootstrap root with validation handled later in main(). function resolveRootPath(): string | undefined { + if (isDirectRun && directSubcommand === 'help') { + return undefined; + } + const envPath = process.env.CODEBASE_ROOT; // Walk argv starting at position 2, skip known flags and their values. @@ -1039,7 +1046,7 @@ export function registerHandlers(target: Server): void { type: 'text', text: JSON.stringify({ status: 'indexing', - message: 'Index build in progress - please retry shortly' + message: 'Index build in progress. Check indexing status, then retry.' }) } ] @@ -1066,7 +1073,7 @@ export function registerHandlers(target: Server): void { type: 'text', text: JSON.stringify({ status: 'indexing', - message: 'Index rebuild in progress - please retry shortly', + message: 'Index rebuild in progress. Check indexing status, then retry.', index: indexSignal }) } @@ -1249,10 +1256,18 @@ async function performIndexingOnce( ...(project.runtimeOverrides.extraExcludePatterns?.length ? { config: { - exclude: [...EXCLUDED_GLOB_PATTERNS, ...project.runtimeOverrides.extraExcludePatterns] + exclude: [ + ...EXCLUDED_GLOB_PATTERNS, + ...project.runtimeOverrides.extraExcludePatterns + ], + ...(project.runtimeOverrides.maxChunks + ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } + : {}) } } - : {}), + : project.runtimeOverrides.maxChunks + ? { config: { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } } + : {}), ...(project.runtimeOverrides.preferredAnalyzer || project.runtimeOverrides.extraSourceExtensions?.length ? { @@ -1636,6 +1651,10 @@ function buildProjectRuntimeOverrides(projectConfig: ProjectConfig): ProjectRunt runtimeOverrides.extraExcludePatterns = [...projectConfig.excludePatterns]; } + if (projectConfig.parsing?.maxChunks) { + runtimeOverrides.maxChunks = projectConfig.parsing.maxChunks; + } + if (projectConfig.analyzerHints?.analyzer) { runtimeOverrides.preferredAnalyzer = projectConfig.analyzerHints.analyzer.trim(); } diff --git a/src/lib.ts b/src/lib.ts index 28f4d90..01ed318 100644 --- a/src/lib.ts +++ b/src/lib.ts @@ -12,6 +12,7 @@ * analyzerRegistry, * AngularAnalyzer, * NextJsAnalyzer, + * NestJsAnalyzer, * ReactAnalyzer, * GenericAnalyzer * } from 'codebase-context'; @@ -19,6 +20,7 @@ * // Register analyzers * analyzerRegistry.register(new AngularAnalyzer()); * analyzerRegistry.register(new NextJsAnalyzer()); + * analyzerRegistry.register(new NestJsAnalyzer()); * analyzerRegistry.register(new ReactAnalyzer()); * analyzerRegistry.register(new GenericAnalyzer()); * @@ -67,6 +69,8 @@ export { AngularAnalyzer } from './analyzers/angular/index.js'; import { AngularAnalyzer } from './analyzers/angular/index.js'; export { NextJsAnalyzer } from './analyzers/nextjs/index.js'; import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +export { NestJsAnalyzer } from './analyzers/nestjs/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; export { ReactAnalyzer } from './analyzers/react/index.js'; import { ReactAnalyzer } from './analyzers/react/index.js'; export { GenericAnalyzer } from './analyzers/generic/index.js'; @@ -164,6 +168,9 @@ export function createIndexer( if (!analyzerRegistry.get('nextjs')) { analyzerRegistry.register(new NextJsAnalyzer()); } + if (!analyzerRegistry.get('nestjs')) { + analyzerRegistry.register(new NestJsAnalyzer()); + } if (!analyzerRegistry.get('react')) { analyzerRegistry.register(new ReactAnalyzer()); } diff --git a/src/project-state.ts b/src/project-state.ts index 4155017..deaa5a6 100644 --- a/src/project-state.ts +++ b/src/project-state.ts @@ -14,6 +14,8 @@ import type { ToolPaths, IndexState } from './tools/types.js'; export interface ProjectRuntimeOverrides { /** Extra glob exclusion patterns merged with the default index-time exclusions. */ extraExcludePatterns?: string[]; + /** Maximum chunks retained in semantic and keyword indexes for this project. */ + maxChunks?: number; /** Analyzer name to prefer for this project without mutating global registry order. */ preferredAnalyzer?: string; /** Additional source extensions treated as code for this project only. */ diff --git a/src/review-bin.ts b/src/review-bin.ts new file mode 100644 index 0000000..1591998 --- /dev/null +++ b/src/review-bin.ts @@ -0,0 +1,8 @@ +#!/usr/bin/env node + +import { runReviewCli } from './review-cli.js'; + +runReviewCli(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src/review-cli.ts b/src/review-cli.ts new file mode 100644 index 0000000..e4e08f7 --- /dev/null +++ b/src/review-cli.ts @@ -0,0 +1,401 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { AngularAnalyzer } from './analyzers/angular/index.js'; +import { GenericAnalyzer } from './analyzers/generic/index.js'; +import { NestJsAnalyzer } from './analyzers/nestjs/index.js'; +import { NextJsAnalyzer } from './analyzers/nextjs/index.js'; +import { ReactAnalyzer } from './analyzers/react/index.js'; +import { + CODEBASE_CONTEXT_DIRNAME, + HEALTH_FILENAME, + INTELLIGENCE_FILENAME, + KEYWORD_INDEX_FILENAME, + MEMORY_FILENAME, + VECTOR_DB_DIRNAME +} from './constants/codebase-context.js'; +import { analyzerRegistry } from './core/analyzer-registry.js'; +import { CodebaseIndexer } from './core/indexer.js'; +import { buildReviewContextPacket, parseNameStatus } from './review-context.js'; +import { loadProjectConfig } from './server/config.js'; +import { dispatchTool } from './tools/index.js'; +import type { PatternResponse, SearchResponse, ToolContext, ToolResponse } from './tools/types.js'; + +analyzerRegistry.register(new AngularAnalyzer()); +analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); +analyzerRegistry.register(new ReactAnalyzer()); +analyzerRegistry.register(new GenericAnalyzer()); + +interface ReviewCliOptions { + base: string; + head: string; + rootPath: string; + maxQueries: number; + maxResults: number; + maxIdentifiers: number; + json: boolean; + noIndex: boolean; +} + +const DEFAULT_MAX_QUERIES = 8; +const DEFAULT_MAX_RESULTS = 3; +const DEFAULT_MAX_IDENTIFIERS = 10; +const MAX_GIT_BUFFER = 16 * 1024 * 1024; + +function printUsage(): void { + console.log(`codebase-context-review --base [options] + +Build a bounded review-context packet from a committed git diff. +It does not call an LLM and it does not post review comments. + +Required: + --base PR/base ref. Compared with merge-base semantics. + +Options: + --head Head ref; must resolve to checked-out HEAD (default: HEAD) + --root Repository root (default: CODEBASE_ROOT or cwd) + --max-queries Maximum changed-file search queries (default: 8) + --max-results Maximum related results per query (default: 3) + --max-identifiers Identifier candidates retained per file (default: 10) + --no-index Do not create/refresh the index; fail if it is missing + --json Emit the complete JSON packet + --help Show this help + +The v1 command requires a clean working tree so repository search and the git diff +refer to the same committed source state. + +Example: + codebase-context-review --base origin/main --head HEAD --json +`); +} + +function parsePositiveInt(value: string | undefined, flag: string, fallback: number): number { + if (value === undefined) return fallback; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${flag} must be a positive integer`); + } + return parsed; +} + +function parseArgs(argv: string[]): ReviewCliOptions | undefined { + if (argv.includes('--help') || argv.includes('-h')) { + printUsage(); + return undefined; + } + + const values = new Map(); + const booleans = new Set(); + const valueFlags = new Set([ + '--base', + '--head', + '--root', + '--max-queries', + '--max-results', + '--max-identifiers' + ]); + const booleanFlags = new Set(['--json', '--no-index']); + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (booleanFlags.has(arg)) { + booleans.add(arg); + continue; + } + if (!valueFlags.has(arg)) { + throw new Error(`Unknown argument: ${arg}`); + } + const value = argv[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`${arg} requires a value`); + } + values.set(arg, value); + index += 1; + } + + const base = values.get('--base'); + if (!base) throw new Error('--base is required'); + + return { + base, + head: values.get('--head') ?? 'HEAD', + rootPath: path.resolve(values.get('--root') ?? process.env.CODEBASE_ROOT ?? process.cwd()), + maxQueries: parsePositiveInt(values.get('--max-queries'), '--max-queries', DEFAULT_MAX_QUERIES), + maxResults: parsePositiveInt(values.get('--max-results'), '--max-results', DEFAULT_MAX_RESULTS), + maxIdentifiers: parsePositiveInt( + values.get('--max-identifiers'), + '--max-identifiers', + DEFAULT_MAX_IDENTIFIERS + ), + json: booleans.has('--json'), + noIndex: booleans.has('--no-index') + }; +} + +function runGit(rootPath: string, args: string[], maxBuffer = MAX_GIT_BUFFER): string { + try { + return execFileSync('git', ['-c', 'core.quotepath=false', ...args], { + cwd: rootPath, + encoding: 'utf8', + input: '', + stdio: ['pipe', 'pipe', 'pipe'], + maxBuffer + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`git ${args.join(' ')} failed: ${message}`); + } +} + +function resolveCommit(rootPath: string, ref: string): string { + const commit = runGit(rootPath, ['rev-parse', '--verify', `${ref}^{commit}`]).trim(); + if (!/^[0-9a-f]{40}$/i.test(commit)) { + throw new Error(`Could not resolve ${ref} to a commit`); + } + return commit; +} + +function assertReviewSourceState(rootPath: string, headCommit: string): void { + const checkoutHead = resolveCommit(rootPath, 'HEAD'); + if (checkoutHead !== headCommit) { + throw new Error( + `--head must resolve to the checked-out HEAD in review-context-v1 (${checkoutHead.slice(0, 12)} != ${headCommit.slice(0, 12)})` + ); + } + + const dirty = runGit(rootPath, ['status', '--porcelain=v1', '--untracked-files=all']); + if (dirty.length > 0) { + throw new Error( + 'review-context-v1 requires a clean working tree so the repository index and diff describe the same committed source state' + ); + } +} + +async function createToolContext(rootPath: string, noIndex: boolean): Promise { + const projectConfig = await loadProjectConfig(rootPath); + const paths = { + baseDir: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME), + memory: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, MEMORY_FILENAME), + intelligence: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, INTELLIGENCE_FILENAME), + health: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, HEALTH_FILENAME), + keywordIndex: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME), + vectorDb: path.join(rootPath, CODEBASE_CONTEXT_DIRNAME, VECTOR_DB_DIRNAME) + }; + + let indexExists = false; + try { + await fs.access(paths.keywordIndex); + indexExists = true; + } catch { + indexExists = false; + } + + const indexState: ToolContext['indexState'] = { + status: indexExists ? 'ready' : 'idle' + }; + + const performIndexing = async (incrementalOnly?: boolean, reason?: string): Promise => { + indexState.status = 'indexing'; + console.error( + `Indexing review target (${incrementalOnly ? 'incremental' : 'full'})${reason ? ` — ${reason}` : ''}` + ); + + try { + const indexer = new CodebaseIndexer({ + rootPath, + ...(projectConfig?.parsing ? { config: { parsing: projectConfig.parsing } } : {}), + incrementalOnly + }); + indexState.indexer = indexer; + const stats = await indexer.index(); + indexState.status = 'ready'; + indexState.lastIndexed = new Date(); + indexState.stats = stats; + } catch (error) { + indexState.status = 'error'; + indexState.error = error instanceof Error ? error.message : String(error); + throw error; + } + }; + + const context: ToolContext = { indexState, paths, rootPath, performIndexing }; + + if (!indexExists) { + if (noIndex) { + throw new Error( + 'No codebase-context index exists. Run `codebase-context reindex` or omit --no-index.' + ); + } + await performIndexing(false, 'review-context'); + } else if (!noIndex) { + await performIndexing(true, 'review-context-refresh'); + } + + return context; +} + +function parseToolJson(response: ToolResponse, operation: string): T { + const text = response.content?.find((item) => item.type === 'text')?.text; + if (!text) throw new Error(`${operation} returned no JSON payload`); + + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + throw new Error(`${operation} returned invalid JSON`); + } + + if (response.isError) { + const message = + typeof parsed === 'object' && parsed !== null && 'error' in parsed + ? String((parsed as { error: unknown }).error) + : `${operation} failed`; + throw new Error(message); + } + return parsed as T; +} + +function printHuman(packet: Awaited>): void { + console.log(`Review context: ${packet.refs.base}...${packet.refs.head}`); + console.log( + `Commits: ${packet.refs.baseCommit.slice(0, 12)} -> ${packet.refs.headCommit.slice(0, 12)}` + ); + console.log(`Diff: sha256:${packet.diffSha256}`); + console.log( + `Changed: ${packet.summary.filesChanged} files, +${packet.summary.additions}/-${packet.summary.deletions}` + ); + console.log( + `Searches: ${packet.summary.queryCount}, related results: ${packet.summary.relatedResultCount}` + ); + console.log(''); + + for (const file of packet.changedFiles) { + const rename = file.previousPath ? ` (from ${file.previousPath})` : ''; + const identifiers = file.identifiers.slice(0, 5).join(', '); + console.log( + `${file.rawStatus.padEnd(4)} ${file.path}${rename} +${file.additions}/-${file.deletions}${file.binary ? ' [binary]' : ''}` + ); + if (identifiers) console.log(` signals: ${identifiers}`); + } + + if (packet.searches.length > 0) { + console.log('\nRelated context'); + for (const search of packet.searches) { + console.log(`- ${search.query} <- ${search.sourceFiles.join(', ')}`); + if (search.error) { + console.log(` error: ${search.error}`); + continue; + } + for (const result of search.results) { + console.log(` ${result.file} score=${result.score.toFixed(3)} ${result.summary}`); + } + } + } + + if (packet.warnings.length > 0) { + console.log('\nWarnings'); + for (const warning of packet.warnings) console.log(`- ${warning}`); + } + + console.log('\nUse --json for the machine-readable packet including conventions and snippets.'); +} + +export async function runReviewCli(argv: string[]): Promise { + const options = parseArgs(argv); + if (!options) return 0; + + const gitRoot = runGit(options.rootPath, ['rev-parse', '--show-toplevel']).trim(); + const rootPath = path.resolve(gitRoot); + const baseCommit = resolveCommit(rootPath, options.base); + const headCommit = resolveCommit(rootPath, options.head); + assertReviewSourceState(rootPath, headCommit); + + const range = `${baseCommit}...${headCommit}`; + const nameStatus = runGit(rootPath, [ + 'diff', + '--name-status', + '-z', + '--find-renames', + range, + '--' + ]); + const changedFiles = parseNameStatus(nameStatus); + const rawDiff = runGit(rootPath, [ + 'diff', + '--no-ext-diff', + '--binary', + '--no-color', + '--find-renames', + range, + '--' + ]); + + const patchesByPath = new Map(); + for (const file of changedFiles) { + const patch = runGit(rootPath, [ + 'diff', + '--no-ext-diff', + '--unified=0', + '--no-color', + '--find-renames', + range, + '--', + file.path + ]); + patchesByPath.set(file.path, patch); + } + + const toolContext = await createToolContext(rootPath, options.noIndex); + const packet = await buildReviewContextPacket({ + base: options.base, + head: options.head, + baseCommit, + headCommit, + rawDiff, + changedFiles, + patchesByPath, + maxQueries: options.maxQueries, + maxResultsPerQuery: options.maxResults, + maxIdentifiersPerFile: options.maxIdentifiers, + search: async (query, limit) => + parseToolJson( + await dispatchTool( + 'search_codebase', + { query, includeSnippets: true, intent: 'edit', limit }, + toolContext + ), + `search_codebase(${query})` + ), + loadConventions: async () => + parseToolJson( + await dispatchTool('get_team_patterns', { category: 'all' }, toolContext), + 'get_team_patterns' + ) + }); + + if (options.json) console.log(JSON.stringify(packet, null, 2)); + else printHuman(packet); + + return 0; +} + +const isDirectRun = (() => { + const entry = process.argv[1]; + if (!entry) return false; + try { + return path.resolve(entry) === path.resolve(fileURLToPath(import.meta.url)); + } catch { + return entry.replace(/\\/g, '/').endsWith('/review-cli.js'); + } +})(); + +if (isDirectRun) { + runReviewCli(process.argv.slice(2)).catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/src/review-context.ts b/src/review-context.ts new file mode 100644 index 0000000..f3b76a7 --- /dev/null +++ b/src/review-context.ts @@ -0,0 +1,452 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import type { + PatternResponse, + SearchQuality, + SearchResponse, + SearchResultItem +} from './tools/types.js'; + +const IDENTIFIER_RE = /[A-Za-z_$][A-Za-z0-9_$]*/g; + +const IDENTIFIER_STOPWORDS = new Set([ + 'abstract', + 'any', + 'array', + 'async', + 'await', + 'boolean', + 'break', + 'case', + 'catch', + 'class', + 'const', + 'continue', + 'debugger', + 'declare', + 'default', + 'delete', + 'else', + 'enum', + 'export', + 'extends', + 'false', + 'finally', + 'for', + 'from', + 'func', + 'function', + 'implements', + 'import', + 'interface', + 'let', + 'map', + 'namespace', + 'never', + 'new', + 'null', + 'number', + 'object', + 'package', + 'private', + 'protected', + 'public', + 'readonly', + 'return', + 'self', + 'static', + 'string', + 'struct', + 'super', + 'switch', + 'this', + 'throw', + 'true', + 'try', + 'type', + 'typeof', + 'undefined', + 'unknown', + 'using', + 'var', + 'void', + 'while', + 'with', + 'yield' +]); + +export interface ChangedFileDescriptor { + path: string; + previousPath?: string; + status: string; + rawStatus: string; +} + +export interface ChangedFileContext extends ChangedFileDescriptor { + additions: number; + deletions: number; + binary: boolean; + identifiers: string[]; +} + +export interface ReviewQuery { + query: string; + sourceFiles: string[]; + identifiers: string[]; +} + +export interface ReviewRelatedResult { + file: string; + summary: string; + score: number; + relevanceReason?: string; + type?: string; + trend?: 'Rising' | 'Declining'; + patternWarning?: string; + symbol?: string; + scope?: string; + signaturePreview?: string; + snippet?: string; + importedByCount?: number; + hasTests?: boolean; +} + +export interface ReviewSearchResult { + query: string; + sourceFiles: string[]; + searchQuality?: SearchQuality; + preflight?: SearchResponse['preflight']; + results: ReviewRelatedResult[]; + error?: string; +} + +export interface ReviewContextPacket { + schemaVersion: 'review-context-v1'; + refs: { + base: string; + head: string; + baseCommit: string; + headCommit: string; + range: string; + }; + diffSha256: string; + limits: { + maxQueries: number; + maxResultsPerQuery: number; + maxIdentifiersPerFile: number; + }; + summary: { + filesChanged: number; + additions: number; + deletions: number; + queryCount: number; + relatedResultCount: number; + }; + changedFiles: ChangedFileContext[]; + searches: ReviewSearchResult[]; + conventions?: PatternResponse; + warnings: string[]; +} + +export interface PatchStats { + additions: number; + deletions: number; + binary: boolean; +} + +export interface BuildReviewContextOptions { + base: string; + head: string; + baseCommit: string; + headCommit: string; + rawDiff: string; + changedFiles: ChangedFileDescriptor[]; + patchesByPath: ReadonlyMap; + maxQueries?: number; + maxResultsPerQuery?: number; + maxIdentifiersPerFile?: number; + search: (query: string, limit: number) => Promise; + loadConventions?: () => Promise; +} + +function descriptorFromStatus(rawStatus: string, filePath: string): ChangedFileDescriptor { + return { + path: filePath, + status: rawStatus.charAt(0), + rawStatus + }; +} + +export function parseNameStatus(output: string): ChangedFileDescriptor[] { + if (output.includes('\0')) { + const fields = output.split('\0'); + const files: ChangedFileDescriptor[] = []; + let index = 0; + + while (index < fields.length) { + const rawStatus = fields[index++]; + if (!rawStatus) continue; + + const status = rawStatus.charAt(0); + if (status === 'R' || status === 'C') { + const previousPath = fields[index++]; + const currentPath = fields[index++]; + if (previousPath && currentPath) { + files.push({ path: currentPath, previousPath, status, rawStatus }); + } + continue; + } + + const filePath = fields[index++]; + if (filePath) files.push(descriptorFromStatus(rawStatus, filePath)); + } + + return files; + } + + const files: ChangedFileDescriptor[] = []; + for (const rawLine of output.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) continue; + + const parts = line.split('\t'); + const rawStatus = parts[0] ?? ''; + const status = rawStatus.charAt(0); + if (!status) continue; + + if ((status === 'R' || status === 'C') && parts.length >= 3) { + const previousPath = parts[1]; + const currentPath = parts[2]; + if (previousPath && currentPath) { + files.push({ path: currentPath, previousPath, status, rawStatus }); + } + continue; + } + + const filePath = parts[1]; + if (filePath) files.push(descriptorFromStatus(rawStatus, filePath)); + } + + return files; +} + +export function inspectPatch(patch: string): PatchStats { + let additions = 0; + let deletions = 0; + let binary = false; + + for (const line of patch.split(/\r?\n/)) { + if (line.startsWith('Binary files ') || line === 'GIT binary patch') { + binary = true; + continue; + } + if (line.startsWith('+++') || line.startsWith('---')) continue; + if (line.startsWith('+')) additions += 1; + else if (line.startsWith('-')) deletions += 1; + } + + return { additions, deletions, binary }; +} + +export function extractChangedIdentifiers(patch: string, limit = 10): string[] { + const scores = new Map(); + + for (const line of patch.split(/\r?\n/)) { + const isAddition = line.startsWith('+') && !line.startsWith('+++'); + const isDeletion = line.startsWith('-') && !line.startsWith('---'); + if (!isAddition && !isDeletion) continue; + + for (const match of line.slice(1).matchAll(IDENTIFIER_RE)) { + const identifier = match[0]; + if (identifier.length < 3) continue; + if (IDENTIFIER_STOPWORDS.has(identifier.toLowerCase())) continue; + + let score = isAddition ? 3 : 2; + if (/^[A-Z][A-Za-z0-9_$]+$/.test(identifier)) score += 2; + if (/[a-z][A-Z]/.test(identifier)) score += 1; + if (identifier.length >= 8) score += 1; + + scores.set(identifier, (scores.get(identifier) ?? 0) + score); + } + } + + return Array.from(scores.entries()) + .sort(([nameA, scoreA], [nameB, scoreB]) => scoreB - scoreA || nameA.localeCompare(nameB)) + .slice(0, Math.max(0, limit)) + .map(([identifier]) => identifier); +} + +function fallbackIdentifiersForPath(filePath: string): string[] { + const withoutExtension = filePath.replace(/\.[^./]+$/, ''); + const basename = path.basename(withoutExtension); + const parentSegments = path + .dirname(withoutExtension) + .split(/[\\/._-]+/) + .filter(Boolean) + .reverse(); + const ordered = [basename, ...parentSegments]; + const seen = new Set(); + + return ordered.filter((part) => { + const normalized = part.toLowerCase(); + if (part.length < 3 || IDENTIFIER_STOPWORDS.has(normalized) || seen.has(normalized)) { + return false; + } + seen.add(normalized); + return true; + }); +} + +export function buildReviewQueries(files: ChangedFileContext[], maxQueries = 8): ReviewQuery[] { + const selected = [...files] + .sort((a, b) => { + const identifierDelta = b.identifiers.length - a.identifiers.length; + if (identifierDelta !== 0) return identifierDelta; + + const changeDelta = b.additions + b.deletions - (a.additions + a.deletions); + if (changeDelta !== 0) return changeDelta; + + return a.path.localeCompare(b.path); + }) + .slice(0, Math.max(0, maxQueries)); + + const queries: ReviewQuery[] = []; + const seen = new Set(); + + for (const file of selected) { + const identifiers = ( + file.identifiers.length > 0 ? file.identifiers : fallbackIdentifiersForPath(file.path) + ).slice(0, 3); + if (identifiers.length === 0) continue; + + const query = identifiers.join(' '); + const key = query.toLowerCase(); + const existing = queries.find((entry) => entry.query.toLowerCase() === key); + if (existing) { + if (!existing.sourceFiles.includes(file.path)) existing.sourceFiles.push(file.path); + continue; + } + if (seen.has(key)) continue; + + seen.add(key); + queries.push({ query, sourceFiles: [file.path], identifiers }); + } + + return queries; +} + +export function fingerprintDiff(rawDiff: string): string { + return createHash('sha256').update(rawDiff, 'utf8').digest('hex'); +} + +function compactSearchResult(result: SearchResultItem): ReviewRelatedResult { + const importedByCount = result.relationships?.importedByCount ?? result.importedByCount; + + return { + file: result.file, + summary: result.summary, + score: result.score, + ...(result.relevanceReason ? { relevanceReason: result.relevanceReason } : {}), + ...(result.type ? { type: result.type } : {}), + ...(result.trend ? { trend: result.trend } : {}), + ...(result.patternWarning ? { patternWarning: result.patternWarning } : {}), + ...(result.symbol ? { symbol: result.symbol } : {}), + ...(result.scope ? { scope: result.scope } : {}), + ...(result.signaturePreview ? { signaturePreview: result.signaturePreview } : {}), + ...(result.snippet ? { snippet: result.snippet.slice(0, 1200) } : {}), + ...(importedByCount != null ? { importedByCount } : {}), + ...(result.relationships?.hasTests != null ? { hasTests: result.relationships.hasTests } : {}) + }; +} + +export async function buildReviewContextPacket( + options: BuildReviewContextOptions +): Promise { + const maxQueries = Math.max(1, Math.floor(options.maxQueries ?? 8)); + const maxResultsPerQuery = Math.max(1, Math.floor(options.maxResultsPerQuery ?? 3)); + const maxIdentifiersPerFile = Math.max(1, Math.floor(options.maxIdentifiersPerFile ?? 10)); + const warnings: string[] = []; + + const changedFiles = options.changedFiles.map((file) => { + const patch = options.patchesByPath.get(file.path) ?? ''; + if (!patch) warnings.push(`No textual patch captured for ${file.path}`); + + const stats = inspectPatch(patch); + return { + ...file, + ...stats, + identifiers: stats.binary ? [] : extractChangedIdentifiers(patch, maxIdentifiersPerFile) + }; + }); + + const queries = buildReviewQueries(changedFiles, maxQueries); + if (queries.length === 0 && changedFiles.length > 0) { + warnings.push('No review search queries could be derived from the changed diff'); + } + if (changedFiles.length > maxQueries) { + warnings.push( + `Search was bounded to ${maxQueries} changed-file queries for ${changedFiles.length} changed files` + ); + } + + const searches: ReviewSearchResult[] = []; + for (const reviewQuery of queries) { + try { + const response = await options.search(reviewQuery.query, maxResultsPerQuery); + searches.push({ + query: reviewQuery.query, + sourceFiles: reviewQuery.sourceFiles, + searchQuality: response.searchQuality, + ...(response.preflight ? { preflight: response.preflight } : {}), + results: response.results.slice(0, maxResultsPerQuery).map(compactSearchResult) + }); + } catch (error) { + searches.push({ + query: reviewQuery.query, + sourceFiles: reviewQuery.sourceFiles, + results: [], + error: error instanceof Error ? error.message : String(error) + }); + warnings.push(`Search failed for query: ${reviewQuery.query}`); + } + } + + let conventions: PatternResponse | undefined; + if (options.loadConventions) { + try { + conventions = await options.loadConventions(); + } catch (error) { + warnings.push( + `Convention lookup failed: ${error instanceof Error ? error.message : String(error)}` + ); + } + } + + const additions = changedFiles.reduce((sum, file) => sum + file.additions, 0); + const deletions = changedFiles.reduce((sum, file) => sum + file.deletions, 0); + const relatedResultCount = searches.reduce((sum, search) => sum + search.results.length, 0); + + return { + schemaVersion: 'review-context-v1', + refs: { + base: options.base, + head: options.head, + baseCommit: options.baseCommit, + headCommit: options.headCommit, + range: `${options.base}...${options.head}` + }, + diffSha256: fingerprintDiff(options.rawDiff), + limits: { maxQueries, maxResultsPerQuery, maxIdentifiersPerFile }, + summary: { + filesChanged: changedFiles.length, + additions, + deletions, + queryCount: searches.length, + relatedResultCount + }, + changedFiles, + searches, + ...(conventions ? { conventions } : {}), + warnings + }; +} diff --git a/src/server/config.ts b/src/server/config.ts index b74ed6d..ff6c1d6 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -5,6 +5,9 @@ import path from 'node:path'; export interface ProjectConfig { root: string; excludePatterns?: string[]; + parsing?: { + maxChunks?: number; + }; analyzerHints?: { extensions?: string[]; analyzer?: string; @@ -87,6 +90,20 @@ export async function loadServerConfig(): Promise { parsedProject.excludePatterns = excludePatterns; } + if ( + typeof project.parsing === 'object' && + project.parsing !== null && + !Array.isArray(project.parsing) + ) { + const parsing = project.parsing as Record; + const maxChunks = parsing.maxChunks; + if (typeof maxChunks === 'number' && Number.isInteger(maxChunks) && maxChunks > 0) { + parsedProject.parsing = { maxChunks }; + } else if (maxChunks !== undefined) { + console.error(`[config] Ignoring invalid project parsing.maxChunks: ${maxChunks}`); + } + } + if ( typeof project.analyzerHints === 'object' && project.analyzerHints !== null && @@ -137,3 +154,9 @@ export async function loadServerConfig(): Promise { return result; } + +export async function loadProjectConfig(rootPath: string): Promise { + const serverConfig = await loadServerConfig(); + const resolvedRoot = path.resolve(rootPath); + return serverConfig?.projects?.find((project) => project.root === resolvedRoot); +} diff --git a/src/tools/search-codebase.ts b/src/tools/search-codebase.ts index ab2ccc4..d68f888 100644 --- a/src/tools/search-codebase.ts +++ b/src/tools/search-codebase.ts @@ -113,7 +113,7 @@ export const definition: Tool = { properties: { framework: { type: 'string', - description: 'Filter by framework (angular, react, nextjs, vue)' + description: 'Filter by framework (angular, react, nextjs, nestjs, vue)' }, language: { type: 'string', @@ -183,7 +183,7 @@ export async function handle( text: JSON.stringify( { status: 'indexing', - message: 'Index is still being built. Retry in a moment.', + message: 'Index build in progress. Check indexing status, then retry.', progress: ctx.indexState.indexer?.getProgress() }, null, @@ -233,7 +233,8 @@ export async function handle( text: JSON.stringify( { status: 'indexing', - message: 'Index was corrupt. Rebuild started — retry shortly.' + message: + 'Index is unavailable or not ready. Indexing has started. Check indexing status, then retry.' }, null, 2 diff --git a/src/types/index.ts b/src/types/index.ts index a792517..9010843 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -206,7 +206,7 @@ export interface CodebaseMetadata { export interface FrameworkInfo { name: string; version: string; - type: 'angular' | 'react' | 'nextjs' | 'vue' | 'svelte' | 'solid' | 'other'; + type: 'angular' | 'react' | 'nextjs' | 'nestjs' | 'vue' | 'svelte' | 'solid' | 'other'; variant?: string; // 'standalone', 'module-based', 'class-components', etc. stateManagement?: string[]; // 'ngrx', 'redux', 'zustand', 'pinia', etc. uiLibraries?: string[]; @@ -461,6 +461,7 @@ export interface CodebaseConfig { angular?: AnalyzerConfig; react?: AnalyzerConfig; nextjs?: AnalyzerConfig; + nestjs?: AnalyzerConfig; vue?: AnalyzerConfig; generic?: AnalyzerConfig; [key: string]: AnalyzerConfig | undefined; @@ -474,6 +475,7 @@ export interface CodebaseConfig { // Parsing options parsing: { maxFileSize?: number; // bytes + maxChunks?: number; chunkSize?: number; // lines chunkOverlap?: number; // lines parseTests?: boolean; diff --git a/tests/analyzer-registry.test.ts b/tests/analyzer-registry.test.ts index 73d0712..703ceb7 100644 --- a/tests/analyzer-registry.test.ts +++ b/tests/analyzer-registry.test.ts @@ -2,12 +2,14 @@ import { describe, it, expect, vi } from 'vitest'; import { analyzerRegistry, AnalyzerRegistry } from '../src/core/analyzer-registry'; import { AngularAnalyzer } from '../src/analyzers/angular/index'; import { NextJsAnalyzer } from '../src/analyzers/nextjs/index'; +import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; import { ReactAnalyzer } from '../src/analyzers/react/index'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; // Register default analyzers analyzerRegistry.register(new AngularAnalyzer()); analyzerRegistry.register(new NextJsAnalyzer()); +analyzerRegistry.register(new NestJsAnalyzer()); analyzerRegistry.register(new ReactAnalyzer()); analyzerRegistry.register(new GenericAnalyzer()); @@ -47,12 +49,13 @@ describe('AnalyzerRegistry', () => { } }); - it('should include default analyzers (Angular, Next.js, React, Generic)', () => { + it('should include default analyzers (Angular, Next.js, NestJS, React, Generic)', () => { const analyzers = analyzerRegistry.getAll(); const names = analyzers.map((a) => a.name); expect(names).toContain('angular'); expect(names).toContain('nextjs'); + expect(names).toContain('nestjs'); expect(names).toContain('react'); expect(names).toContain('generic'); }); @@ -65,8 +68,9 @@ describe('AnalyzerRegistry', () => { expect(angular?.name).toBe('angular'); }); - it('should return nextjs and react analyzers by name', () => { + it('should return nextjs, nestjs, and react analyzers by name', () => { expect(analyzerRegistry.get('nextjs')?.name).toBe('nextjs'); + expect(analyzerRegistry.get('nestjs')?.name).toBe('nestjs'); expect(analyzerRegistry.get('react')?.name).toBe('react'); }); @@ -80,16 +84,36 @@ describe('AnalyzerRegistry', () => { it('should have Angular higher priority than Generic', () => { const angular = analyzerRegistry.get('angular'); const nextjs = analyzerRegistry.get('nextjs'); + const nestjs = analyzerRegistry.get('nestjs'); const react = analyzerRegistry.get('react'); const generic = analyzerRegistry.get('generic'); expect(angular).toBeDefined(); expect(nextjs).toBeDefined(); + expect(nestjs).toBeDefined(); expect(react).toBeDefined(); expect(generic).toBeDefined(); expect(angular!.priority).toBeGreaterThan(generic!.priority); + expect(nestjs!.priority).toBeGreaterThan(angular!.priority); + expect(nestjs!.priority).toBeGreaterThan(react!.priority); expect(nextjs!.priority).toBeGreaterThan(react!.priority); expect(react!.priority).toBeGreaterThan(generic!.priority); }); + + it('should prefer NestJS when framework decorators overlap with Angular', () => { + const content = ` +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class PaymentsService {} +`; + + expect( + analyzerRegistry.findAllAnalyzers('/tmp/payments.service.ts', content).map((a) => a.name) + ).toEqual(['nestjs', 'angular', 'generic']); + expect(analyzerRegistry.findAnalyzer('/tmp/payments.service.ts', content)?.name).toBe( + 'nestjs' + ); + }); }); }); diff --git a/tests/cli-entrypoint-runtime.test.ts b/tests/cli-entrypoint-runtime.test.ts index 7bdb291..1bef25a 100644 --- a/tests/cli-entrypoint-runtime.test.ts +++ b/tests/cli-entrypoint-runtime.test.ts @@ -1,5 +1,7 @@ import { spawnSync } from 'node:child_process'; -import { resolve } from 'node:path'; +import { mkdir, mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; const root = resolve(import.meta.dirname, '..'); @@ -54,4 +56,47 @@ describe('CLI entrypoint runtime', () => { expect(parsed.rootPath).toBe(root); expect(parsed.hint).toContain('refresh_index'); }, 120_000); + + it('prints usage for help from a project containing a directory named help', async () => { + const projectRoot = await mkdtemp(join(tmpdir(), 'cbc-cli-help-')); + const helpDirectory = join(projectRoot, 'help'); + + const env = { ...process.env }; + delete env.CODEBASE_ROOT; + + try { + await mkdir(helpDirectory); + + const result = spawnSync( + process.execPath, + ['--import', import.meta.resolve('tsx'), entrypoint, 'help'], + { + cwd: projectRoot, + env, + encoding: 'utf8', + timeout: 10_000 + } + ); + + expect({ + errorCode: result.error?.code, + exitCode: result.status, + helpText: result.stdout.includes('codebase-context [options]'), + commandsSection: result.stdout.includes('Commands:'), + stderr: result.stderr, + projectEntries: await readdir(projectRoot), + helpEntries: await readdir(helpDirectory) + }).toEqual({ + errorCode: undefined, + exitCode: 0, + helpText: true, + commandsSection: true, + stderr: '', + projectEntries: ['help'], + helpEntries: [] + }); + } finally { + await rm(projectRoot, { recursive: true, force: true }); + } + }, 70_000); }); diff --git a/tests/cli-init-bootstrap.test.ts b/tests/cli-init-bootstrap.test.ts new file mode 100644 index 0000000..c16c3be --- /dev/null +++ b/tests/cli-init-bootstrap.test.ts @@ -0,0 +1,33 @@ +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { generateMcpConfig } from '../src/cli-init.js'; +import { resolveRootPath } from '../src/index.js'; + +describe('init bootstrap root', () => { + it('passes the configured project root through the MCP server argv path', () => { + const projectRoot = path.resolve('C:/projects/An App With Spaces'); + const previousArgv = process.argv; + const previousEnvRoot = process.env.CODEBASE_ROOT; + const generated = generateMcpConfig('claude-code', { + rootPath: projectRoot, + nodeExecutable: process.execPath, + serverEntrypoint: path.resolve('C:/codebase-context/dist/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected project MCP config'); + + const parsed = JSON.parse(generated.content) as { + mcpServers: { 'codebase-context': { command: string; args: string[] } }; + }; + const server = parsed.mcpServers['codebase-context']; + process.argv = [server.command, ...server.args]; + process.env.CODEBASE_ROOT = path.resolve('C:/projects/Other App'); + + try { + expect(resolveRootPath()).toBe(projectRoot); + } finally { + process.argv = previousArgv; + if (previousEnvRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousEnvRoot; + } + }); +}); diff --git a/tests/cli-init.test.ts b/tests/cli-init.test.ts index 1dc718e..8d139a2 100644 --- a/tests/cli-init.test.ts +++ b/tests/cli-init.test.ts @@ -13,55 +13,82 @@ vi.mock('node:fs/promises', async (importOriginal) => { }; }); +vi.mock('@inquirer/prompts', () => ({ + select: vi.fn(), + confirm: vi.fn() +})); + import * as fsMod from 'node:fs/promises'; +import { confirm, select } from '@inquirer/prompts'; import { generateMcpConfig, generateInstructionBlock, resolveInstructionFilePath, + resolveInitRoot, _appendInstructionBlock, - _buildMergedMcpContent + _buildMergedMcpContent, + _buildMergedCodexContent, + handleInitCli } from '../src/cli-init.js'; // --- generateMcpConfig --- describe('generateMcpConfig', () => { - it('claude-code returns a command with mcp add --transport http', () => { - const result = generateMcpConfig('claude-code'); - expect(result.kind).toBe('command'); - if (result.kind !== 'command') return; - expect(result.args[0]).toBe('mcp'); - expect(result.args).toContain('--transport'); - expect(result.args).toContain('http'); - expect(result.args).toContain('codebase-context'); - expect(result.args).toContain('http://127.0.0.1:3100/mcp'); + const projectRoot = path.resolve('C:/projects/Example Project'); + const nodeExecutable = path.resolve('C:/Program Files/nodejs/node.exe'); + const serverEntrypoint = path.resolve('C:/Program Files/codebase-context/dist/index.js'); + + it('Claude Code gets a project stdio config that passes the project root', () => { + const result = generateMcpConfig('claude-code', { + rootPath: projectRoot, + nodeExecutable, + serverEntrypoint + }); + + expect(result.kind).toBe('file'); + if (result.kind !== 'file') return; + expect(result.path).toBe(path.join(projectRoot, '.mcp.json')); + expect(JSON.parse(result.content)).toEqual({ + mcpServers: { + 'codebase-context': { + command: nodeExecutable, + args: [serverEntrypoint, projectRoot] + } + } + }); }); it('cursor returns a file at .cursor/mcp.json with http type', () => { const result = generateMcpConfig('cursor'); expect(result.kind).toBe('file'); if (result.kind !== 'file') return; - expect(result.path).toBe('.cursor/mcp.json'); + expect(result.path).toBe(path.resolve('.cursor/mcp.json')); const parsed = JSON.parse(result.content) as { mcpServers: { 'codebase-context': { type: string } }; }; expect(parsed.mcpServers['codebase-context'].type).toBe('http'); }); - it('codex returns a command with mcp add', () => { - const result = generateMcpConfig('codex'); - expect(result.kind).toBe('command'); - if (result.kind !== 'command') return; - expect(result.args).toContain('mcp'); - expect(result.args).toContain('add'); - expect(result.args).toContain('codebase-context'); - expect(result.args).toContain('http://127.0.0.1:3100/mcp'); + it('Codex gets a project TOML config that passes paths containing spaces safely', () => { + const result = generateMcpConfig('codex', { + rootPath: projectRoot, + nodeExecutable, + serverEntrypoint + }); + + expect(result.kind).toBe('file'); + if (result.kind !== 'file') return; + expect(result.path).toBe(path.join(projectRoot, '.codex', 'config.toml')); + expect(result.content).toBe( + `[mcp_servers.codebase-context]\ncommand = ${JSON.stringify(nodeExecutable)}\nargs = [${JSON.stringify(serverEntrypoint)}, ${JSON.stringify(projectRoot)}]\nstartup_timeout_sec = 120\n` + ); }); it('opencode returns a file at opencode.json with remote type', () => { const result = generateMcpConfig('opencode'); expect(result.kind).toBe('file'); if (result.kind !== 'file') return; - expect(result.path).toBe('opencode.json'); + expect(result.path).toBe(path.resolve('opencode.json')); const parsed = JSON.parse(result.content) as { mcp: { 'codebase-context': { type: string } }; }; @@ -69,6 +96,32 @@ describe('generateMcpConfig', () => { }); }); +describe('resolveInitRoot', () => { + it('uses explicit root before CODEBASE_ROOT and cwd', () => { + expect( + resolveInitRoot(['--root', 'C:/projects/Explicit Project'], 'C:/projects/Env', 'C:/other') + ).toBe(path.resolve('C:/projects/Explicit Project')); + }); + + it('uses CODEBASE_ROOT when no explicit root is supplied', () => { + expect(resolveInitRoot([], 'C:/projects/Env Project', 'C:/other')).toBe( + path.resolve('C:/projects/Env Project') + ); + }); + + it('uses cwd when neither root override is supplied', () => { + expect(resolveInitRoot([], undefined, 'C:/projects/Current Project')).toBe( + path.resolve('C:/projects/Current Project') + ); + }); + + it('treats an empty CODEBASE_ROOT as unset', () => { + expect(resolveInitRoot([], '', 'C:/projects/Current Project')).toBe( + path.resolve('C:/projects/Current Project') + ); + }); +}); + // --- generateInstructionBlock --- describe('generateInstructionBlock', () => { @@ -173,7 +226,11 @@ describe('_buildMergedMcpContent', () => { const generated = generateMcpConfig('cursor'); if (generated.kind !== 'file') throw new Error('expected file config'); - const merged = await _buildMergedMcpContent('/test/.cursor/mcp.json', generated.content, 'cursor'); + const merged = await _buildMergedMcpContent( + '/test/.cursor/mcp.json', + generated.content, + 'cursor' + ); const parsed = JSON.parse(merged.content) as { mcpServers: Record; someOtherKey: boolean; @@ -191,7 +248,72 @@ describe('_buildMergedMcpContent', () => { }); }); - it('preserves existing opencode mcp entries and updates codebase-context deterministically', async () => { + it('preserves existing Claude Code mcpServers entries and adds codebase-context', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'existing-server': { type: 'stdio', command: 'node', args: ['server.js'] } + }, + someOtherKey: true + }) as unknown as Buffer + ); + + const generated = generateMcpConfig('claude-code', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedMcpContent( + '/test/.mcp.json', + generated.content, + 'claude-code' + ); + const parsed = JSON.parse(merged.content) as { + mcpServers: Record; + someOtherKey: boolean; + }; + + expect(merged.mergedFromExisting).toBe(true); + expect(parsed.someOtherKey).toBe(true); + expect(parsed.mcpServers['existing-server']).toEqual({ + type: 'stdio', + command: 'node', + args: ['server.js'] + }); + expect(parsed.mcpServers['codebase-context']).toEqual({ + command: 'C:/node.exe', + args: [path.resolve('C:/cbc/index.js'), path.resolve('C:/projects/app')] + }); + }); + + it('refuses to overwrite a disabled Claude Code codebase-context entry', async () => { + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'codebase-context': { + type: 'stdio', + command: 'node', + args: ['server.js'], + disabled: true + } + } + }) as unknown as Buffer + ); + const generated = generateMcpConfig('claude-code', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect( + _buildMergedMcpContent('/test/.mcp.json', generated.content, 'claude-code') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); + }); + + it('refuses to overwrite an existing conflicting OpenCode codebase-context entry', async () => { readFileMock.mockResolvedValue( JSON.stringify({ $schema: 'https://opencode.ai/config.json', @@ -206,21 +328,276 @@ describe('_buildMergedMcpContent', () => { const generated = generateMcpConfig('opencode'); if (generated.kind !== 'file') throw new Error('expected file config'); - const merged = await _buildMergedMcpContent('/test/opencode.json', generated.content, 'opencode'); - const parsed = JSON.parse(merged.content) as { - mcp: Record; - extra: string; + await expect( + _buildMergedMcpContent('/test/opencode.json', generated.content, 'opencode') + ).rejects.toThrow(/already contains a different codebase-context configuration/i); + }); +}); + +describe('handleInitCli', () => { + const readFileMock = vi.mocked(fsMod.readFile); + const writeFileMock = vi.mocked(fsMod.writeFile); + const mkdirMock = vi.mocked(fsMod.mkdir); + const accessMock = vi.mocked(fsMod.access); + + beforeEach(() => { + vi.resetAllMocks(); + writeFileMock.mockResolvedValue(undefined); + mkdirMock.mockResolvedValue(undefined); + readFileMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + }); + + it('writes only the selected project config in non-interactive mode', async () => { + const projectRoot = path.resolve('C:/projects/Configured App'); + + await handleInitCli(['--client', 'claude-code', '--yes', '--root', projectRoot]); + + expect(select).not.toHaveBeenCalled(); + expect(confirm).not.toHaveBeenCalled(); + expect(writeFileMock).toHaveBeenCalledOnce(); + const [writtenPath, writtenContent] = writeFileMock.mock.calls[0] as [ + string, + string, + ...unknown[] + ]; + expect(writtenPath).toBe(path.join(projectRoot, '.mcp.json')); + const parsed = JSON.parse(writtenContent) as { + mcpServers: { 'codebase-context': { args: string[] } }; }; + expect(parsed.mcpServers['codebase-context'].args.at(-1)).toBe(projectRoot); + expect(mkdirMock).toHaveBeenCalledWith(projectRoot, { recursive: true }); + }); + + it('writes a project Codex TOML file when Codex is selected non-interactively', async () => { + const projectRoot = path.resolve('C:/projects/Codex App'); + + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + + expect(writeFileMock).toHaveBeenCalledOnce(); + const [writtenPath, writtenContent] = writeFileMock.mock.calls[0] as [ + string, + string, + ...unknown[] + ]; + expect(writtenPath).toBe(path.join(projectRoot, '.codex', 'config.toml')); + expect(writtenContent).toContain('[mcp_servers.codebase-context]'); + expect(writtenContent).toContain(`args = [`); + expect(writtenContent).toContain(JSON.stringify(projectRoot)); + expect(mkdirMock).toHaveBeenCalledWith(path.join(projectRoot, '.codex'), { recursive: true }); + }); + + it('uses cwd as the project when init is run without a root override', async () => { + const previousEnvRoot = process.env.CODEBASE_ROOT; + delete process.env.CODEBASE_ROOT; + + try { + await handleInitCli(['--client', 'codex', '--yes']); + } finally { + if (previousEnvRoot === undefined) delete process.env.CODEBASE_ROOT; + else process.env.CODEBASE_ROOT = previousEnvRoot; + } + + const [writtenPath, writtenContent] = writeFileMock.mock.calls[0] as [ + string, + string, + ...unknown[] + ]; + expect(writtenPath).toBe(path.join(process.cwd(), '.codex', 'config.toml')); + expect(writtenContent).toContain(JSON.stringify(path.resolve(process.cwd()))); + }); + + it('does not overwrite a conflicting project MCP entry', async () => { + const projectRoot = path.resolve('C:/projects/Configured App'); + readFileMock.mockResolvedValue( + JSON.stringify({ + mcpServers: { + 'codebase-context': { disabled: true, command: 'old-server' } + } + }) as unknown as Buffer + ); + const previousExitCode = process.exitCode; + + await handleInitCli(['--client', 'claude-code', '--yes', '--root', projectRoot]); + + expect(writeFileMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + process.exitCode = previousExitCode; + }); + + it('does not create a project config when the selected root is missing', async () => { + const projectRoot = path.resolve('C:/projects/Missing App'); + accessMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + const previousExitCode = process.exitCode; + + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + + expect(mkdirMock).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + process.exitCode = previousExitCode; + }); + + it('shows init help without prompting or writing files', async () => { + await handleInitCli(['--help']); + + expect(select).not.toHaveBeenCalled(); + expect(confirm).not.toHaveBeenCalled(); + expect(writeFileMock).not.toHaveBeenCalled(); + }); + + it('keeps the interactive preview read-only when the user declines', async () => { + vi.mocked(select).mockResolvedValue('claude-code'); + vi.mocked(confirm).mockResolvedValue(false); + + await handleInitCli([]); + + expect(writeFileMock).not.toHaveBeenCalled(); + }); +}); + +describe('_buildMergedCodexContent', () => { + const readFileMock = vi.mocked(fsMod.readFile); + const writeFileMock = vi.mocked(fsMod.writeFile); + const mkdirMock = vi.mocked(fsMod.mkdir); + + beforeEach(() => { + vi.resetAllMocks(); + writeFileMock.mockResolvedValue(undefined); + mkdirMock.mockResolvedValue(undefined); + readFileMock.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + }); + + const runCodexSetupWithExistingConfig = async (existingToml: string) => { + const projectRoot = path.resolve('C:/projects/Codex Existing'); + const initialBytes = Buffer.from(existingToml, 'utf8'); + let persistedBytes = Buffer.from(initialBytes); + readFileMock.mockResolvedValue(existingToml as unknown as Buffer); + writeFileMock.mockImplementation(async (_filePath, content) => { + persistedBytes = Buffer.from(content as string, 'utf8'); + }); + + const previousExitCode = process.exitCode; + process.exitCode = undefined; + try { + await handleInitCli(['--client', 'codex', '--yes', '--root', projectRoot]); + return { initialBytes, persistedBytes, exitCode: process.exitCode }; + } finally { + process.exitCode = previousExitCode; + } + }; + + it('preserves unrelated Codex TOML settings when adding the project server', async () => { + readFileMock.mockResolvedValue( + '[model]\nmodel = "gpt-example"\n\n[mcp_servers.other]\ncommand = "other"\n' as unknown as Buffer + ); + const generated = generateMcpConfig('codex', { + rootPath: 'C:/projects/app', + nodeExecutable: 'C:/node.exe', + serverEntrypoint: 'C:/cbc/index.js' + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); expect(merged.mergedFromExisting).toBe(true); - expect(parsed.extra).toBe('value'); - expect(parsed.mcp['existing-server']).toEqual({ - type: 'remote', - url: 'http://127.0.0.1:5000/mcp' + expect(merged.content).toContain('[model]\nmodel = "gpt-example"'); + expect(merged.content).toContain('[mcp_servers.other]\ncommand = "other"'); + expect(merged.content).toContain('[mcp_servers.codebase-context]'); + }); + + it('leaves the same Codex codebase-context entry unchanged on a repeated setup', async () => { + const generated = generateMcpConfig('codex', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') }); - expect(parsed.mcp['codebase-context']).toEqual({ - type: 'remote', - url: 'http://127.0.0.1:3100/mcp' + if (generated.kind !== 'file') throw new Error('expected file config'); + readFileMock.mockResolvedValue(generated.content as unknown as Buffer); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged).toEqual({ content: generated.content, mergedFromExisting: true }); + }); + + it('treats quoted table keys as the same idempotent Codex entry', async () => { + const generated = generateMcpConfig('codex', { + rootPath: path.resolve('C:/projects/app'), + nodeExecutable: 'C:/node.exe', + serverEntrypoint: path.resolve('C:/cbc/index.js') + }); + if (generated.kind !== 'file') throw new Error('expected file config'); + const existing = generated.content.replace( + '[mcp_servers.codebase-context]', + '["mcp_servers"."codebase-context"]' + ); + readFileMock.mockResolvedValue(existing as unknown as Buffer); + + const merged = await _buildMergedCodexContent(generated.path, generated.content); + + expect(merged).toEqual({ content: existing, mergedFromExisting: true }); + }); + + it('refuses a conflicting or disabled Codex codebase-context entry', async () => { + readFileMock.mockResolvedValue( + '[mcp_servers.codebase-context]\ndisabled = true\ncommand = "old"\n' as unknown as Buffer + ); + const generated = generateMcpConfig('codex', { + rootPath: 'C:/projects/app', + nodeExecutable: 'C:/node.exe', + serverEntrypoint: 'C:/cbc/index.js' }); + if (generated.kind !== 'file') throw new Error('expected file config'); + + await expect(_buildMergedCodexContent(generated.path, generated.content)).rejects.toThrow( + /already contains a different codebase-context configuration/i + ); }); + + it.each([ + { + name: 'double-quoted parent and child keys', + toml: '["mcp_servers"."codebase-context"]\ncommand = "old"\n' + }, + { + name: 'double-quoted parent and bare child key', + toml: '["mcp_servers".codebase-context]\ncommand = "old"\n' + } + ])( + 'refuses to replace a Codex entry under $name without changing its bytes', + async ({ toml }) => { + const result = await runCodexSetupWithExistingConfig(toml); + + expect(result.exitCode).toBe(1); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(result.persistedBytes).toEqual(result.initialBytes); + } + ); + + it.each([ + { + name: 'inline child in the bare parent table', + toml: '[mcp_servers]\ncodebase-context = { command = "old", args = [] }\n' + }, + { + name: 'dotted child in the double-quoted parent table', + toml: '["mcp_servers"]\ncodebase-context.command = "old"\n' + }, + { + name: 'dotted child in the single-quoted parent table', + toml: "['mcp_servers']\n'codebase-context'.command = \"old\"\n" + }, + { + name: 'inline root table under a double-quoted key', + toml: '"mcp_servers" = { "codebase-context" = { command = "old", args = [] } }\n' + } + ])( + 'refuses to replace a Codex entry under $name without changing its bytes', + async ({ toml }) => { + const result = await runCodexSetupWithExistingConfig(toml); + + expect(result.exitCode).toBe(1); + expect(writeFileMock).not.toHaveBeenCalled(); + expect(result.persistedBytes).toEqual(result.initialBytes); + } + ); }); diff --git a/tests/codebase-map.test.ts b/tests/codebase-map.test.ts index 662e5b2..c90de78 100644 --- a/tests/codebase-map.test.ts +++ b/tests/codebase-map.test.ts @@ -84,7 +84,7 @@ async function removeTempMapProject(rootPath: string): Promise { } function runMapCli(args: string[], rootPath: string) { - return spawnSync(process.execPath, ['--import', 'tsx', ENTRYPOINT, 'map', ...args], { + return spawnSync(process.execPath, ['--no-deprecation', '--import', 'tsx', ENTRYPOINT, 'map', ...args], { cwd: CURRENT_REPO_ROOT, env: { ...process.env, diff --git a/tests/contextbench-baseline-schema-gate.test.ts b/tests/contextbench-baseline-schema-gate.test.ts index ad6fc0a..69b7a93 100644 --- a/tests/contextbench-baseline-schema-gate.test.ts +++ b/tests/contextbench-baseline-schema-gate.test.ts @@ -1,5 +1,12 @@ import { execFileSync } from 'node:child_process'; -import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + chmodSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { describe, expect, it, vi } from 'vitest'; @@ -55,6 +62,10 @@ function cleanupSessionRoot(sessionRoot: string): void { } } +function normalizeFilesystemPath(filePath: string): string { + return realpathSync(filePath); +} + function tempSessionRoot(phase: 'phase40' | 'phase41' = 'phase40'): string { return path.join( mkdtempSync(path.join(tmpdir(), `contextbench-${phase}-schema-gate-`)), @@ -398,7 +409,9 @@ describe('ContextBench Phase 40 schema gate', () => { repoCheckoutPath: repoPath, verificationStrict: false }); - expect(readFileSync(cwdCapturePath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdCapturePath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); const stdin = readFileSync(stdinCapturePath, 'utf8'); expect(stdin).toContain('Problem statement:'); expect(stdin).toContain('Fix the failing ContextBench task'); @@ -584,7 +597,9 @@ describe('ContextBench Phase 40 schema gate', () => { ], { encoding: 'utf8', env } ); - expect(readFileSync(cwdPath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdPath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); } execFileSync( 'node', @@ -681,7 +696,9 @@ describe('ContextBench Phase 40 schema gate', () => { (candidate) => candidate.scoring && 'baselineArmId' in candidate.scoring ); expect(row).toMatchObject({ status: 'completed' }); - expect(readFileSync(cwdCapturePath, 'utf8')).toBe(repoPath); + expect(normalizeFilesystemPath(readFileSync(cwdCapturePath, 'utf8'))).toBe( + normalizeFilesystemPath(repoPath) + ); const stdin = readFileSync(stdinCapturePath, 'utf8'); expect(stdin).toContain('Problem statement:'); expect(stdin).toContain('Run the diagnostic arm with materialized task text.'); diff --git a/tests/indexer-large-file-skip.test.ts b/tests/indexer-large-file-skip.test.ts index 741fb68..e54e513 100644 --- a/tests/indexer-large-file-skip.test.ts +++ b/tests/indexer-large-file-skip.test.ts @@ -57,4 +57,29 @@ describe('Indexer large file skip regression', () => { expect(indexedFiles.has('big.ts')).toBe(false); expect(indexedFiles.has('big.generated.ts')).toBe(false); }); + + it('limits the keyword index to the configured maximum chunk count', async () => { + await Promise.all( + ['one.ts', 'two.ts', 'three.ts'].map((fileName, index) => + fs.writeFile(path.join(tempDir, fileName), `export const value${index} = ${index};\n`) + ) + ); + + const indexer = new CodebaseIndexer({ + rootPath: tempDir, + config: { + skipEmbedding: true, + parsing: { maxChunks: 2 } + } + }); + + await indexer.index(); + + const indexPath = path.join(tempDir, CODEBASE_CONTEXT_DIRNAME, KEYWORD_INDEX_FILENAME); + const indexRaw = JSON.parse(await fs.readFile(indexPath, 'utf-8')) as { + chunks?: unknown[]; + }; + + expect(indexRaw.chunks).toHaveLength(2); + }); }); diff --git a/tests/indexer-metadata.test.ts b/tests/indexer-metadata.test.ts index 8705447..b4c7ade 100644 --- a/tests/indexer-metadata.test.ts +++ b/tests/indexer-metadata.test.ts @@ -6,6 +6,7 @@ import { CodebaseIndexer } from '../src/core/indexer'; import { analyzerRegistry } from '../src/core/analyzer-registry'; import { AngularAnalyzer } from '../src/analyzers/angular/index'; import { NextJsAnalyzer } from '../src/analyzers/nextjs/index'; +import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; import { ReactAnalyzer } from '../src/analyzers/react/index'; import { GenericAnalyzer } from '../src/analyzers/generic/index'; @@ -15,6 +16,9 @@ if (!analyzerRegistry.get('angular')) { if (!analyzerRegistry.get('nextjs')) { analyzerRegistry.register(new NextJsAnalyzer()); } +if (!analyzerRegistry.get('nestjs')) { + analyzerRegistry.register(new NestJsAnalyzer()); +} if (!analyzerRegistry.get('react')) { analyzerRegistry.register(new ReactAnalyzer()); } @@ -230,5 +234,29 @@ describe('CodebaseIndexer.detectMetadata', () => { expect(metadata.framework?.indicators).toContain('dep:@angular/core'); expect(metadata.framework?.indicators).toContain('disk:ng-package-json'); }); + + it('detects NestJS project metadata from @nestjs dependencies', async () => { + await fs.writeFile( + path.join(tempDir, 'package.json'), + JSON.stringify({ + name: 'api', + dependencies: { + '@nestjs/common': '11.1.19', + '@nestjs/core': '11.1.19', + '@nestjs/platform-express': '11.1.19', + }, + devDependencies: { + '@nestjs/testing': '11.1.19', + }, + }) + ); + + const indexer = new CodebaseIndexer({ rootPath: tempDir }); + const metadata = await indexer.detectMetadata(); + + expect(metadata.framework?.type).toBe('nestjs'); + expect(metadata.framework?.name).toBe('NestJS'); + expect(metadata.framework?.indicators).toContain('dep:@nestjs/core'); + }); }); }); diff --git a/tests/mcp-client-templates.test.ts b/tests/mcp-client-templates.test.ts index 03cf820..05baaa8 100644 --- a/tests/mcp-client-templates.test.ts +++ b/tests/mcp-client-templates.test.ts @@ -73,18 +73,21 @@ describe('templates/mcp/http/.mcp.json', () => { }); // --------------------------------------------------------------------------- -// README references templates and all four target clients +// README links the shipped guide for advanced templates and covers target clients // --------------------------------------------------------------------------- describe('README.md client setup documentation', () => { const readme = readText('README.md'); + const clientSetup = readText('docs/client-setup.md'); it('references the stdio template path', () => { - expect(readme).toContain('templates/mcp/stdio/.mcp.json'); + expect(readme).toContain('./docs/client-setup.md'); + expect(clientSetup).toContain('templates/mcp/stdio/.mcp.json'); }); it('references the HTTP template path', () => { - expect(readme).toContain('templates/mcp/http/.mcp.json'); + expect(readme).toContain('./docs/client-setup.md'); + expect(clientSetup).toContain('templates/mcp/http/.mcp.json'); }); it('mentions Claude Code', () => { @@ -103,8 +106,11 @@ describe('README.md client setup documentation', () => { expect(readme).toContain('Windsurf'); }); - it('includes the HTTP endpoint URL', () => { - expect(readme).toContain('127.0.0.1:3100/mcp'); + it('links the advanced HTTP endpoint without making it the default setup', () => { + expect(readme).toContain('./docs/client-setup.md'); + expect(readme).toContain('stdio'); + expect(clientSetup).toContain('127.0.0.1:3100/mcp'); + expect(clientSetup).toContain('not part of the default setup proof'); }); }); @@ -128,15 +134,21 @@ describe('docs/capabilities.md transport documentation', () => { }); it('covers all four target clients', () => { - expect(caps).toContain('Claude Code'); - expect(caps).toContain('Cursor'); - expect(caps).toContain('Codex'); - expect(caps).toContain('Windsurf'); - }); - - it('states the roots-first routing fallback explicitly', () => { - expect(caps).toContain('roots-capable hosts'); - expect(caps).toContain('explicit fallback is still required'); + expect(caps).toContain('./client-setup.md'); + const clientSetup = readText('docs/client-setup.md'); + for (const client of ['Claude Code', 'Cursor', 'Codex', 'Windsurf']) { + expect(clientSetup).toContain(client); + } + }); + + it('states explicit selection as the portable default and scopes compatibility behavior', () => { + expect(caps).toContain('portable default is explicit selection'); + expect(caps).toContain('absolute path as `project`'); + expect(caps).toContain('server returns `selection_required` instead of guessing'); + expect(caps).toContain('Roots are not required for the documented setup'); + expect(caps).toContain( + 'does not establish native agent continuation or concurrent HTTP isolation' + ); }); }); @@ -144,16 +156,27 @@ describe('docs/client-setup.md multi-project guidance', () => { const clientSetup = readText('docs/client-setup.md'); it('documents the project routing contract', () => { + expect(clientSetup).toContain("current repository's absolute path as `project`"); expect(clientSetup).toContain( - 'Automatic multi-project routing is evidence-backed only when the MCP host announces workspace roots.' + 'Without a configured folder, known root or explicit selection, tools return `selection_required`' ); + expect(clientSetup).toContain("They do not guess or index the user's home directory"); + expect(clientSetup).toContain('One valid root can auto-select'); expect(clientSetup).toContain( - 'the server returns `selection_required` instead of guessing' + 'several roots with no active selection return `selection_required`' ); + expect(clientSetup).toContain('Explicit selectors must stay within announced roots'); }); - it('keeps the three verification flows aligned with the roots-first contract', () => { - expect(clientSetup).toContain('Multiple projects on a roots-capable host'); - expect(clientSetup).toContain('Ambiguous or no-roots selection'); + it('separates process-local routing proof from native and HTTP acceptance', () => { + expect(clientSetup).toContain('A to B to A in one no-roots stdio process'); + expect(clientSetup).toContain('CBC routes omitted-project calls using its selected project'); + expect(clientSetup).toContain( + 'Select again after a server restart or when changing repositories' + ); + expect(clientSetup).toContain('native retrieval and continuation did not pass'); + expect(clientSetup).toContain( + 'stdio process-local selection does not establish concurrent HTTP isolation' + ); }); }); diff --git a/tests/multi-project-routing.test.ts b/tests/multi-project-routing.test.ts index fa8f181..41a81f1 100644 --- a/tests/multi-project-routing.test.ts +++ b/tests/multi-project-routing.test.ts @@ -367,6 +367,30 @@ describe('multi-project routing', () => { } }); + it('reports status-check guidance when an index build is already running', async () => { + const { server } = await import('../src/index.js'); + const { getOrCreateProject } = await import('../src/project-state.js'); + const handler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); + if (!handler) throw new Error('tools/call handler not registered'); + + const project = getOrCreateProject(primaryRoot); + const originalStatus = project.indexState.status; + project.indexState.status = 'indexing'; + try { + const response = await callTool(handler, 20, 'search_codebase', { + query: 'feature', + project: primaryRoot + }); + const payload = parsePayload(response) as { status: string; message: string }; + + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe('Index build in progress. Check indexing status, then retry.'); + expect(searchMocks.search).not.toHaveBeenCalled(); + } finally { + project.indexState.status = originalStatus; + } + }); + it('triggers a background rebuild for a corrupted explicit project without falling back to cwd', async () => { delete process.env.CODEBASE_ROOT; delete process.argv[2]; @@ -398,7 +422,7 @@ describe('multi-project routing', () => { }; expect(payload.status).toBe('indexing'); - expect(payload.message).toContain('retry shortly'); + expect(payload.message).toBe('Index rebuild in progress. Check indexing status, then retry.'); expect(payload.index?.action).toBe('rebuild-started'); expect(String(payload.index?.reason || '')).toContain('Index meta'); } finally { @@ -591,7 +615,9 @@ describe('multi-project routing', () => { it('lists bounded and full context resources for active and project-scoped flows', async () => { const { server } = await import('../src/index.js'); const toolHandler = (server as unknown as TestServer)._requestHandlers.get('tools/call'); - const resourcesHandler = (server as unknown as TestServer)._requestHandlers.get('resources/list'); + const resourcesHandler = (server as unknown as TestServer)._requestHandlers.get( + 'resources/list' + ); if (!toolHandler || !resourcesHandler) { throw new Error('required handlers not registered'); diff --git a/tests/nestjs-analyzer.test.ts b/tests/nestjs-analyzer.test.ts new file mode 100644 index 0000000..bd08d8e --- /dev/null +++ b/tests/nestjs-analyzer.test.ts @@ -0,0 +1,214 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { promises as fs } from 'fs'; +import path from 'path'; +import { randomUUID } from 'crypto'; +import { NestJsAnalyzer } from '../src/analyzers/nestjs/index'; + +const analyzer = new NestJsAnalyzer(); +const tempRoots: string[] = []; + +afterEach(async () => { + await Promise.all( + tempRoots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })) + ); +}); + +describe('NestJsAnalyzer', () => { + it('detects NestJS files without claiming generic decorator files', () => { + expect( + analyzer.canAnalyze( + '/tmp/app.controller.ts', + 'import { Controller } from "@nestjs/common";\n@Controller("health") export class HealthController {}' + ) + ).toBe(true); + expect( + analyzer.canAnalyze( + '/tmp/main.ts', + 'import { NestFactory } from "@nestjs/core";\nawait NestFactory.create(AppModule);' + ) + ).toBe(true); + expect( + analyzer.canAnalyze( + '/tmp/angular.service.ts', + 'import { Injectable } from "@angular/core";\n@Injectable() export class AngularService {}' + ) + ).toBe(false); + }); + + it('extracts controller routes, DI dependencies, guards, and Swagger patterns', async () => { + const content = ` +import { Body, Controller, Delete, Get, Inject, Param, Post, UseGuards } from "@nestjs/common"; +import { AuthGuard } from "@nestjs/passport"; +import { ApiOAuth2, ApiTags } from "@nestjs/swagger"; +import { PermissionsGuard } from "./permissions.guard"; +import { PaymentsService } from "./payments.service"; + +@ApiTags("Payments") +@Controller("payments") +@UseGuards(AuthGuard("jwt"), PermissionsGuard) +export class PaymentsController { + constructor(@Inject(PaymentsService) private readonly paymentsService: PaymentsService) {} + + @Get(":id") + findOne(@Param("id") id: string) { + return this.paymentsService.findOne(id); + } + + @Post() + create(@Body() dto: CreatePaymentDto) { + return this.paymentsService.create(dto); + } + + @Delete(":id") + remove(@Param("id") id: string) { + return this.paymentsService.remove(id); + } +} +`; + + const result = await analyzer.analyze('/tmp/payments.controller.ts', content); + const controller = result.components.find( + (component) => component.name === 'PaymentsController' + ); + + expect(result.framework).toBe('nestjs'); + expect(controller).toMatchObject({ + componentType: 'controller', + layer: 'presentation', + dependencies: ['PaymentsService'] + }); + expect(controller?.metadata.nestjs).toMatchObject({ + type: 'controller', + decorators: ['ApiTags', 'Controller', 'UseGuards'], + routes: [ + { method: 'GET', path: '/payments/:id', handler: 'findOne' }, + { method: 'POST', path: '/payments', handler: 'create' }, + { method: 'DELETE', path: '/payments/:id', handler: 'remove' } + ] + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'routing', + name: 'REST routes' + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'security', + name: 'Guards' + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'documentation', + name: 'Swagger' + }); + }); + + it('extracts modules, services, gateways, and provider metadata', async () => { + const content = ` +import { Inject, Injectable, Module } from "@nestjs/common"; +import { ConfigModule } from "@nestjs/config"; +import { WebSocketGateway } from "@nestjs/websockets"; + +@Injectable() +export class WidgetBootstrapService { + constructor(@Inject("API_CONFIG") private readonly config: ApiConfigService) {} +} + +@WebSocketGateway({ cors: true }) +export class RealtimeGateway {} + +@Module({ + imports: [ConfigModule], + providers: [WidgetBootstrapService, RealtimeGateway], + exports: [WidgetBootstrapService], +}) +export class WidgetBootstrapModule {} +`; + + const result = await analyzer.analyze('/tmp/widget-bootstrap.module.ts', content); + + expect(result.components.map((component) => component.componentType)).toEqual([ + 'service', + 'gateway', + 'module' + ]); + expect( + result.components.find((component) => component.name === 'WidgetBootstrapModule')?.metadata + .nestjs + ).toMatchObject({ + type: 'module', + moduleMetadata: { + imports: ['ConfigModule'], + providers: ['WidgetBootstrapService', 'RealtimeGateway'], + exports: ['WidgetBootstrapService'] + } + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'realtime', + name: 'WebSockets' + }); + expect(result.metadata.detectedPatterns).toContainEqual({ + category: 'modules', + name: 'Nest modules' + }); + }); + + it('detects NestJS 11 metadata from package dependencies', async () => { + const root = await createTempProject({ + name: 'nestjs-api', + dependencies: { + '@nestjs/common': '11.1.19', + '@nestjs/core': '11.1.19', + '@nestjs/platform-express': '11.1.19', + '@nestjs/swagger': '11.4.2', + '@nestjs/graphql': '13.3.0', + '@nestjs/bullmq': '^11.0.4' + }, + devDependencies: { + '@nestjs/testing': '11.1.19', + vitest: '^3.0.0' + } + }); + + const metadata = await analyzer.detectCodebaseMetadata(root); + + expect(metadata.framework).toMatchObject({ + name: 'NestJS', + version: '11.1.19', + type: 'nestjs', + variant: 'mixed' + }); + expect(metadata.framework?.indicators).toEqual( + expect.arrayContaining([ + 'dep:@nestjs/common', + 'dep:@nestjs/core', + 'dep:@nestjs/platform-express' + ]) + ); + expect(metadata.framework?.testingFrameworks).toEqual(['Nest Testing', 'Vitest']); + expect(metadata.customMetadata.nestjs).toMatchObject({ + packages: expect.arrayContaining(['@nestjs/common', '@nestjs/core', '@nestjs/graphql']) + }); + }); + + it('detects older NestJS versions without version-specific assumptions', async () => { + const root = await createTempProject({ + name: 'legacy-nest-api', + dependencies: { + '@nestjs/common': '^8.4.0', + '@nestjs/core': '^8.4.0', + '@nestjs/platform-express': '^8.4.0' + } + }); + + const metadata = await analyzer.detectCodebaseMetadata(root); + + expect(metadata.framework?.type).toBe('nestjs'); + expect(metadata.framework?.version).toBe('8.4.0'); + }); +}); + +async function createTempProject(packageJson: Record): Promise { + const root = path.join(process.cwd(), 'tests', '.tmp', `nestjs-${randomUUID()}`); + tempRoots.push(root); + await fs.mkdir(root, { recursive: true }); + await fs.writeFile(path.join(root, 'package.json'), JSON.stringify(packageJson)); + return root; +} diff --git a/tests/nextjs-analyzer.test.ts b/tests/nextjs-analyzer.test.ts index 1882dbb..4f6d9e6 100644 --- a/tests/nextjs-analyzer.test.ts +++ b/tests/nextjs-analyzer.test.ts @@ -59,6 +59,40 @@ export default function Page() { return
; } }); }); + it('classifies current App Router file conventions and proxy files', async () => { + const analyzer = new NextJsAnalyzer(); + + const loading = await analyzer.analyze( + path.join(process.cwd(), 'app', 'dashboard', 'loading.tsx'), + 'export default function Loading() { return
; }' + ); + expect(loading.metadata.nextjs).toMatchObject({ + router: 'app', + kind: 'loading', + routePath: '/dashboard' + }); + + const notFound = await analyzer.analyze( + path.join(process.cwd(), 'src', 'app', 'shop', 'not-found.tsx'), + 'export default function NotFound() { return
; }' + ); + expect(notFound.metadata.nextjs).toMatchObject({ + router: 'app', + kind: 'not-found', + routePath: '/shop' + }); + + const proxy = await analyzer.analyze( + path.join(process.cwd(), 'src', 'proxy.ts'), + 'export function proxy() { return null; }' + ); + expect(proxy.metadata.nextjs).toMatchObject({ + router: 'proxy', + kind: 'proxy', + routePath: null + }); + }); + it('does not treat _app as a pages route and infers metadata variants from disk', async () => { const analyzer = new NextJsAnalyzer(); const appShell = await analyzer.analyze( diff --git a/tests/proof-truth-surfaces.test.ts b/tests/proof-truth-surfaces.test.ts index cabb7a5..778f838 100644 --- a/tests/proof-truth-surfaces.test.ts +++ b/tests/proof-truth-surfaces.test.ts @@ -49,6 +49,7 @@ const gateArtifact = readJson('results/gate-evaluation.json'); const comparatorEvidence = readJson('results/comparator-evidence.json'); const benchmarkDoc = readText('docs/benchmark.md'); +const discoveryDoc = readText('docs/benchmark-prior-public-report.md'); const comparisonDoc = readText('docs/comparison-table.md'); const registryChecklist = readText('docs/registry-sync-checklist.md'); const readme = readText('README.md'); @@ -72,17 +73,19 @@ describe('proof truth surfaces', () => { expect(comparatorEvidence['codebase-memory-mcp']).toBeDefined(); }); - it('keeps the proof docs aligned to the current gate artifact', () => { - expectContains(benchmarkDoc, [ + it('keeps archived discovery proof aligned to its retained gate artifact', () => { + expectContains(discoveryDoc, [ 'discovery benchmark', `\`${gateArtifact.gate.status}\``, '`claimAllowed`' ]); - expectContains(comparisonDoc, [ - 'Comparator Summary', - `\`${gateArtifact.gate.status}\``, - `claimAllowed\` stays \`${String(gateArtifact.gate.claimAllowed)}\`` - ]); + expect(discoveryDoc).toContain( + `\`claimAllowed\` remains \`${String(gateArtifact.gate.claimAllowed)}\`` + ); + for (const currentDoc of [benchmarkDoc, comparisonDoc]) { + expect(currentDoc).toContain('./benchmark-prior-public-report.md'); + expect(currentDoc).toContain('separate 24-task discovery report'); + } expectContains(registryChecklist, [ `claimAllowed: ${String(gateArtifact.gate.claimAllowed)}`, gateArtifact.gate.status @@ -96,8 +99,8 @@ describe('proof truth surfaces', () => { if (rawClaude.averageFirstRelevantHit === null) { expect(rawClaudeGate.status).toBe('pending_evidence'); expect(rawClaudeGate.missingMetrics ?? []).toContain('averageFirstRelevantHit'); - expect(benchmarkDoc).toMatch(/raw Claude Code[\s\S]*averageFirstRelevantHit[\s\S]*null/i); - expect(comparisonDoc).toMatch(/raw Claude Code[\s\S]*pending_evidence/i); + expect(discoveryDoc).toMatch(/raw Claude Code[\s\S]*averageFirstRelevantHit[\s\S]*null/i); + expect(discoveryDoc).toMatch(/raw Claude Code[\s\S]*pending_evidence/i); expect(registryChecklist).toContain('averageFirstRelevantHit: null'); } }); @@ -108,8 +111,7 @@ describe('proof truth surfaces', () => { ); if (codebaseMemoryGate?.status === 'failed') { - expect(benchmarkDoc).toMatch(/codebase-memory-mcp[\s\S]*gate: `failed`/i); - expect(comparisonDoc).toMatch(/codebase-memory-mcp[\s\S]*gate: `failed`/i); + expect(discoveryDoc).toMatch(/codebase-memory-mcp[\s\S]*gate: `failed`/i); expect(registryChecklist).toContain('comparator artifact `ok` but gate `failed`'); } @@ -118,19 +120,35 @@ describe('proof truth surfaces', () => { .map(([name]) => name); for (const comparatorName of setupFailedComparators) { - expect(benchmarkDoc).toContain(`\`${comparatorName}\``); - expect(comparisonDoc).toContain(`\`${comparatorName}\``); + expect(discoveryDoc).toContain(`\`${comparatorName}\``); } }); - it('keeps package-facing proof mentions secondary and discovery-only', () => { - expectContains(readme, ['discovery-only proof', gateArtifact.gate.status, 'claimAllowed']); + it('keeps package-facing evidence scoped to retrieval and historical CLI observations', () => { + expectContains(readme, [ + '/docs/benchmark.md', + '/docs/benchmark-prior-public-report.md', + 'retrieval observations, not a winner or a coding-quality claim', + 'does not establish patch correctness or end-to-end coding quality' + ]); expectContains(capabilities, [ - 'discovery-only', - gateArtifact.gate.status, - `claimAllowed: ${String(gateArtifact.gate.claimAllowed)}` + 'docs/benchmark.md', + 'does not measure patch correctness or end-to-end task completion', + 'missing or failed setup and comparator runs remain visible' + ]); + expectContains(demo, [ + 'does not verify the current package or an MCP connection', + 'machine-specific repository root is omitted', + 'not a universal performance or coding result' + ]); + expectContains(benchmarkDoc, [ + 'does not measure patch correctness', + 'does not combine the four evidence families into a pooled score' + ]); + expectContains(comparisonDoc, [ + 'do not establish patch correctness', + 'failed attempt is counted as zero' ]); - expectContains(demo, [gateArtifact.gate.status, 'claimAllowed']); }); it('keeps shared planning summaries aligned to the same proof posture', () => { diff --git a/tests/react-analyzer.test.ts b/tests/react-analyzer.test.ts index b3b9ef4..2cb7939 100644 --- a/tests/react-analyzer.test.ts +++ b/tests/react-analyzer.test.ts @@ -80,6 +80,90 @@ export class LegacyWidget extends Component { expect(patterns).toContainEqual({ category: 'styling', name: 'tailwind' }); }); + it('detects React 19 action and optimistic hooks', async () => { + const analyzer = new ReactAnalyzer(); + const filePath = path.join(process.cwd(), 'src', 'components', 'CheckoutForm.tsx'); + + const code = ` +import { use, useActionState, useOptimistic } from "react"; +import { useFormStatus } from "react-dom"; + +export function CheckoutForm({ cartPromise }: { cartPromise: Promise }) { + const cart = use(cartPromise); + const [state, submitAction] = useActionState(async () => ({ ok: true }), { ok: false }); + const [optimisticCart] = useOptimistic(cart); + const status = useFormStatus(); + + return
{state.ok && optimisticCart.length && status.pending}
; +} +`; + + const result = await analyzer.analyze(filePath, code); + const patterns = (result.metadata.detectedPatterns || []) as Array<{ category: string; name: string }>; + + expect(patterns).toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Actions' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Optimistic UI' }); + expect(patterns).toContainEqual({ category: 'forms', name: 'Form status' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Suspense' }); + }); + + it('does not treat unrelated functions named use as React hooks or Suspense', async () => { + const analyzer = new ReactAnalyzer(); + const filePath = path.join(process.cwd(), 'src', 'components', 'UtilityWidget.tsx'); + const samples = [ + ` +function use(value: string) { + return value; +} + +export function UtilityWidget() { + return
{use("local value")}
; +} +`, + ` +import { use } from "other-library"; + +export function UtilityWidget() { + const value = use("plain value"); + return
{value}
; +} +` + ]; + + for (const code of samples) { + const result = await analyzer.analyze(filePath, code); + const patterns = (result.metadata.detectedPatterns || []) as Array<{ + category: string; + name: string; + }>; + + expect(patterns).not.toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); + expect(patterns).not.toContainEqual({ category: 'reactivity', name: 'Suspense' }); + } + }); + + it('recognizes aliased use imports from React', async () => { + const analyzer = new ReactAnalyzer(); + const filePath = path.join(process.cwd(), 'src', 'components', 'AliasedUseWidget.tsx'); + const code = ` +import { use as readPromise } from "react"; + +export function AliasedUseWidget({ value }: { value: Promise }) { + return
{readPromise(value)}
; +} +`; + + const result = await analyzer.analyze(filePath, code); + const patterns = (result.metadata.detectedPatterns || []) as Array<{ + category: string; + name: string; + }>; + + expect(patterns).toContainEqual({ category: 'reactHooks', name: 'Built-in hooks' }); + expect(patterns).toContainEqual({ category: 'reactivity', name: 'Suspense' }); + }); + it('does not claim React framework when react dependency is absent', async () => { const analyzer = new ReactAnalyzer(); const tempRoot = path.join(process.cwd(), 'tests', '.tmp', `react-${randomUUID()}`); diff --git a/tests/release-truth-surfaces.test.ts b/tests/release-truth-surfaces.test.ts index c6d8e60..593c096 100644 --- a/tests/release-truth-surfaces.test.ts +++ b/tests/release-truth-surfaces.test.ts @@ -73,6 +73,7 @@ function extractMarkdownLinks(markdown: string): string[] { describe('release truth surfaces', () => { const packageJson = readJson('package.json'); const releaseManifest = readJson('.release-please-manifest.json'); + const expectedVersion = packageJson.version; const changelog = readText('CHANGELOG.md'); const readme = readText('README.md'); const workflow = readText('.github/workflows/publish-npm-on-release.yml'); @@ -80,10 +81,9 @@ describe('release truth surfaces', () => { const visualsDoc = readOptionalText('docs/visuals.md'); const packagedPaths = ['README.md', 'LICENSE', ...(packageJson.files ?? [])]; - it('keeps package metadata, release manifest, and changelog on 2.2.0', () => { - expect(packageJson.version).toBe('2.2.0'); - expect(releaseManifest['.']).toBe('2.2.0'); - expect(changelog).toContain('## [2.2.0]'); + it('keeps package metadata, release manifest, and changelog aligned', () => { + expect(releaseManifest['.']).toBe(expectedVersion); + expect(changelog).toContain(`## [${expectedVersion}]`); expect(changelog).not.toContain('## Unreleased'); }); @@ -114,8 +114,80 @@ describe('release truth surfaces', () => { expect(visualsDoc).toContain('Historical snapshot'); }); - it('keeps the manual publish fallback aligned to v2.2.0', () => { - expect(workflow).toContain("description: 'Tag to publish (e.g. v2.2.0)'"); - expect(workflow).toContain("default: 'v2.2.0'"); + it('requires an explicit release tag for the manual publish fallback', () => { + const manualTagInput = workflow.match(/^ {6}tag:\r?\n((?: {8}[^\r\n]*\r?\n)+)/m)?.[1] ?? ''; + + expect(manualTagInput).toMatch(/^\s*required:\s*true\s*$/m); + expect(manualTagInput).toMatch(/^\s*type:\s*string\s*$/m); + expect(manualTagInput).not.toMatch(/^\s*default:/m); + expect(workflow).toMatch(/^ release:\r?\n {4}types: \[published\]/m); + expect(workflow).toContain( + 'ref: ${{ github.event.inputs.tag || github.event.release.tag_name }}' + ); + }); + + it('keeps pinned install commands on the published npm version', () => { + const publishedVersion = '2.2.0'; + const userFacingDocs = [ + 'README.md', + 'docs/capabilities.md', + 'docs/cli.md', + 'docs/client-setup.md', + 'docs/demo.md', + 'docs/review-context.md', + 'docs/templates.md' + ]; + const pinnedCommands = userFacingDocs.flatMap((path) => { + const page = readOptionalText(path); + if (!page) return []; + return [...page.matchAll(/codebase-context@(\d+\.\d+\.\d+)/g)] + .filter((match) => { + const version = match[1]; + if (version === publishedVersion) { + return false; + } + if ( + version === '2.4.0' && + (path === 'docs/review-context.md' || path === 'docs/client-setup.md') + ) { + const index = match.index ?? 0; + const surroundingText = page.slice( + Math.max(0, index - 300), + Math.min(page.length, index + 300) + ); + if (/requires\s+(?:the\s+unpublished\s+)?2\.4\.0/i.test(surroundingText)) { + return false; + } + } + return true; + }) + .map((match) => `${path}: ${match[0]}`); + }); + + expect(pinnedCommands).toEqual([]); + expect(readText('README.md')).toContain('npx -y codebase-context'); + expect(readText('docs/client-setup.md')).toContain('npx -y codebase-context'); + expect(readText('docs/client-setup.md')).toContain('codebase-context@2.4.0 init'); + expect(readText('docs/review-context.md')).toContain('--package=codebase-context@2.4.0'); + }); + + it('does not describe the unpublished 2.4.0 preview as current or published', () => { + const capabilities = readText('docs/capabilities.md'); + + expect(capabilities).not.toContain('what `codebase-context` ships today'); + expect(capabilities).not.toContain('Published 2.4.0 has'); + }); + + it('keeps release wording, analyzer claims and setup checks accurate', () => { + const capabilities = readText('docs/capabilities.md'); + const setup = readText('docs/client-setup.md'); + const beforeFirstClient = setup.slice(0, setup.indexOf('## Claude Code')); + + expect(capabilities).not.toMatch(/\b2\.4\.0 (?:setup )?preview\b/i); + expect(`${readme}\n${capabilities}`).not.toMatch(/opt-in NestJS|NestJS \(opt-in\)/i); + expect(capabilities).not.toContain('analyzers: { nestjs: { enabled: true } }'); + expect(capabilities).not.toContain('generates HTTP setup'); + expect(setup).not.toContain('package still declares Node 18+'); + expect(beforeFirstClient).toContain('`2.2.0`'); }); }); diff --git a/tests/reranker-recovery.test.ts b/tests/reranker-recovery.test.ts index 7cb77ca..6bf3325 100644 --- a/tests/reranker-recovery.test.ts +++ b/tests/reranker-recovery.test.ts @@ -63,7 +63,12 @@ describe('reranker corruption recovery', () => { await fs.mkdir(modelCacheDir, { recursive: true }); await fs.writeFile(path.join(modelCacheDir, 'model.onnx'), 'corrupt'); - const tokenizer = vi.fn((query: string, passage: string) => ({ query, passage })); + const tokenizer = vi.fn( + (query: string, options?: { text_pair?: string | null }) => ({ + query, + passage: options?.text_pair ?? '' + }) + ); const model = vi.fn(async (inputs: { passage: string }) => { if (inputs.passage.includes('/a.ts')) return { logits: { data: [1] } }; if (inputs.passage.includes('/b.ts')) return { logits: { data: [3] } }; @@ -89,6 +94,33 @@ describe('reranker corruption recovery', () => { const reranked = await rerank('auth token', results); expect(transformersMocks.tokenizerFromPretrained).toHaveBeenCalledTimes(2); expect(transformersMocks.modelFromPretrained).toHaveBeenCalledTimes(2); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/a.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/b.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); + expect(tokenizer).toHaveBeenCalledWith( + 'auth token', + expect.objectContaining({ + text_pair: expect.stringContaining('/c.ts'), + padding: true, + truncation: true, + max_length: 512 + }) + ); expect(reranked.map((result) => result.filePath)).toEqual(['/b.ts', '/c.ts', '/a.ts']); expect(getRerankerStatus()).toBe('active'); }); diff --git a/tests/review-cli-entrypoint-runtime.test.ts b/tests/review-cli-entrypoint-runtime.test.ts new file mode 100644 index 0000000..1e3c131 --- /dev/null +++ b/tests/review-cli-entrypoint-runtime.test.ts @@ -0,0 +1,21 @@ +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(import.meta.dirname, '..'); +const entrypoint = resolve(root, 'src', 'review-bin.ts'); + +describe('review CLI entrypoint runtime', () => { + it('executes the package wrapper and prints help without loading a review target', () => { + const result = spawnSync(process.execPath, ['--import', 'tsx', entrypoint, '--help'], { + cwd: root, + encoding: 'utf8', + timeout: 30_000 + }); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(''); + expect(result.stdout).toContain('codebase-context-review --base '); + expect(result.stdout).toContain('It does not call an LLM'); + }, 30_000); +}); diff --git a/tests/review-context.test.ts b/tests/review-context.test.ts new file mode 100644 index 0000000..52dca41 --- /dev/null +++ b/tests/review-context.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from 'vitest'; +import { + buildReviewContextPacket, + buildReviewQueries, + extractChangedIdentifiers, + fingerprintDiff, + inspectPatch, + parseNameStatus, + type ChangedFileContext +} from '../src/review-context.js'; +import type { SearchResponse } from '../src/tools/types.js'; + +describe('review context', () => { + it('parses modified, added, deleted and renamed files', () => { + expect( + parseNameStatus(['M\tsrc/a.ts', 'A\tsrc/b.ts', 'D\tsrc/c.ts', 'R100\tsrc/old.ts\tsrc/new.ts'].join('\n')) + ).toEqual([ + { path: 'src/a.ts', status: 'M', rawStatus: 'M' }, + { path: 'src/b.ts', status: 'A', rawStatus: 'A' }, + { path: 'src/c.ts', status: 'D', rawStatus: 'D' }, + { + path: 'src/new.ts', + previousPath: 'src/old.ts', + status: 'R', + rawStatus: 'R100' + } + ]); + }); + + it('parses NUL-delimited git output without corrupting unusual paths', () => { + expect(parseNameStatus('M\0src/tab\tname.ts\0R100\0old\nname.ts\0new name.ts\0')).toEqual([ + { path: 'src/tab\tname.ts', status: 'M', rawStatus: 'M' }, + { + path: 'new name.ts', + previousPath: 'old\nname.ts', + status: 'R', + rawStatus: 'R100' + } + ]); + }); + + it('counts textual changes without treating diff headers as source', () => { + const patch = [ + 'diff --git a/src/auth.ts b/src/auth.ts', + '--- a/src/auth.ts', + '+++ b/src/auth.ts', + '@@ -1,2 +1,3 @@', + '-const oldToken = loadToken();', + '+const refreshToken = loadRefreshToken();', + '+return refreshToken;' + ].join('\n'); + + expect(inspectPatch(patch)).toEqual({ additions: 2, deletions: 1, binary: false }); + }); + + it('detects binary patches', () => { + expect(inspectPatch('diff --git a/a.png b/a.png\nBinary files a/a.png and b/a.png differ')).toEqual({ + additions: 0, + deletions: 0, + binary: true + }); + }); + + it('ranks semantic identifiers from changed lines and drops language noise', () => { + const patch = [ + '-const token = auth.load();', + '+const refreshToken = sessionService.loadRefreshToken();', + '+return sessionService.rotateRefreshToken(refreshToken);', + '+const SessionService = createSessionService();' + ].join('\n'); + + const identifiers = extractChangedIdentifiers(patch, 8); + + expect(identifiers).toContain('sessionService'); + expect(identifiers).toContain('refreshToken'); + expect(identifiers).toContain('rotateRefreshToken'); + expect(identifiers).not.toContain('const'); + expect(identifiers).not.toContain('return'); + }); + + it('bounds review queries and falls back to file path signals', () => { + const files: ChangedFileContext[] = [ + { + path: 'src/auth/session-service.ts', + status: 'M', + rawStatus: 'M', + additions: 10, + deletions: 2, + binary: false, + identifiers: ['SessionService', 'refreshToken', 'rotateToken'] + }, + { + path: 'src/http/request-guard.ts', + status: 'M', + rawStatus: 'M', + additions: 2, + deletions: 1, + binary: false, + identifiers: [] + } + ]; + + expect(buildReviewQueries(files, 1)).toEqual([ + { + query: 'SessionService refreshToken rotateToken', + sourceFiles: ['src/auth/session-service.ts'], + identifiers: ['SessionService', 'refreshToken', 'rotateToken'] + } + ]); + + expect(buildReviewQueries(files, 2)[1]).toEqual({ + query: 'request-guard http src', + sourceFiles: ['src/http/request-guard.ts'], + identifiers: ['request-guard', 'http', 'src'] + }); + }); + + it('fingerprints exact diff bytes, including the final newline', () => { + expect(fingerprintDiff('diff body\n')).not.toBe(fingerprintDiff('diff body')); + }); + + it('builds a bounded packet and keeps failed searches explicit', async () => { + const rawDiff = [ + 'diff --git a/src/auth.ts b/src/auth.ts', + '--- a/src/auth.ts', + '+++ b/src/auth.ts', + '@@ -1 +1 @@', + '-oldSession();', + '+rotateSessionToken();' + ].join('\n'); + + const searchCalls: string[] = []; + const search = async (query: string): Promise => { + searchCalls.push(query); + return { + status: 'ok', + searchQuality: { status: 'ok', confidence: 0.91 }, + results: [ + { + file: 'src/session/store.ts:10-24', + summary: 'Session token storage', + score: 0.88, + snippet: 'export function rotateSessionToken() { return true; }' + } + ] + }; + }; + + const packet = await buildReviewContextPacket({ + base: 'origin/main', + head: 'HEAD', + baseCommit: 'a'.repeat(40), + headCommit: 'b'.repeat(40), + rawDiff, + changedFiles: [{ path: 'src/auth.ts', status: 'M', rawStatus: 'M' }], + patchesByPath: new Map([['src/auth.ts', rawDiff]]), + maxQueries: 2, + maxResultsPerQuery: 1, + search + }); + + expect(packet.schemaVersion).toBe('review-context-v1'); + expect(packet.diffSha256).toBe(fingerprintDiff(rawDiff)); + expect(packet.summary).toEqual({ + filesChanged: 1, + additions: 1, + deletions: 1, + queryCount: 1, + relatedResultCount: 1 + }); + expect(searchCalls).toEqual(['rotateSessionToken oldSession']); + expect(packet.searches[0]?.results[0]?.file).toBe('src/session/store.ts:10-24'); + }); +}); diff --git a/tests/search-codebase-auto-heal.test.ts b/tests/search-codebase-auto-heal.test.ts index 13cd1a0..2be3bb2 100644 --- a/tests/search-codebase-auto-heal.test.ts +++ b/tests/search-codebase-auto-heal.test.ts @@ -143,33 +143,71 @@ describe('search_codebase auto-heal', () => { } }); - it('fires background re-index and returns retry message when IndexCorruptedError is thrown', async () => { - const { IndexCorruptedError } = await import('../src/errors/index.js'); + it.each([ + [ + 'missing initial index metadata', + 'Index meta missing or unreadable (rebuild required): ENOENT' + ], + ['genuine index corruption', 'LanceDB index corrupted: missing vector column'] + ])( + 'starts background re-index and returns retry guidance for %s', + async (_scenario, errorMessage) => { + const { IndexCorruptedError } = await import('../src/errors/index.js'); - searchMocks.search.mockRejectedValueOnce( - new IndexCorruptedError('LanceDB index corrupted: missing vector column') - ); + searchMocks.search.mockRejectedValueOnce(new IndexCorruptedError(errorMessage)); - const { server } = await import('../src/index.js'); - const handler = (server as any)._requestHandlers.get('tools/call'); + const { server } = await import('../src/index.js'); + const handler = (server as any)._requestHandlers.get('tools/call'); - const response = await handler({ - jsonrpc: '2.0', - id: 1, - method: 'tools/call', - params: { - name: 'search_codebase', - arguments: { - query: 'test' + const response = await handler({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'search_codebase', + arguments: { + query: 'test' + } } + }); + + const payload = JSON.parse(response.content[0].text); + expect(payload.status).toBe('indexing'); + expect(payload.message).toBe( + 'Index is unavailable or not ready. Indexing has started. Check indexing status, then retry.' + ); + expect(searchMocks.search).toHaveBeenCalledTimes(1); + }, + 15000 + ); + + it('returns status-check guidance without searching while an index build is active', async () => { + if (!tempRoot) throw new Error('tempRoot not initialized'); + + const { handle } = await import('../src/tools/search-codebase.js'); + const baseDir = path.join(tempRoot, CODEBASE_CONTEXT_DIRNAME); + const response = await handle( + { query: 'search something' }, + { + indexState: { status: 'indexing' }, + paths: { + baseDir, + memory: path.join(baseDir, 'memory.json'), + intelligence: path.join(baseDir, 'intelligence.json'), + health: path.join(baseDir, 'health.json'), + keywordIndex: path.join(baseDir, KEYWORD_INDEX_FILENAME), + vectorDb: path.join(baseDir, VECTOR_DB_DIRNAME) + }, + rootPath: tempRoot, + performIndexing: vi.fn() } - }); + ); - const payload = JSON.parse(response.content[0].text); + const payload = JSON.parse(response.content?.[0]?.text ?? '{}'); expect(payload.status).toBe('indexing'); - expect(payload.message).toContain('retry'); - expect(searchMocks.search).toHaveBeenCalledTimes(1); - }, 15000); + expect(payload.message).toBe('Index build in progress. Check indexing status, then retry.'); + expect(searchMocks.search).not.toHaveBeenCalled(); + }); it('returns invalid_params when search_codebase query is missing', async () => { const { server } = await import('../src/index.js'); diff --git a/tests/search-health-scope.test.ts b/tests/search-health-scope.test.ts index 2b5e932..0e175bf 100644 --- a/tests/search-health-scope.test.ts +++ b/tests/search-health-scope.test.ts @@ -28,6 +28,7 @@ describe('search_codebase health and scoped memories', () => { beforeEach(async () => { searchMocks.search.mockReset(); tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'search-health-scope-')); + const currentDate = new Date().toISOString(); const contextDir = path.join(tempRoot, '.codebase-context'); await fs.mkdir(contextDir, { recursive: true }); @@ -56,7 +57,7 @@ describe('search_codebase health and scoped memories', () => { category: 'architecture', memory: 'Avoid direct token reads', reason: 'They bypass AuthService refresh logic.', - date: '2026-04-17T00:00:00.000Z', + date: currentDate, scope: { kind: 'symbol', file: 'src/auth/auth.service.ts', @@ -69,7 +70,7 @@ describe('search_codebase health and scoped memories', () => { category: 'architecture', memory: 'Use auth interceptors', reason: 'They keep HTTP token injection consistent.', - date: '2026-04-17T00:00:00.000Z' + date: currentDate } ], null, diff --git a/tests/search-scoped-memory.test.ts b/tests/search-scoped-memory.test.ts index 37e4b62..0f6df64 100644 --- a/tests/search-scoped-memory.test.ts +++ b/tests/search-scoped-memory.test.ts @@ -28,6 +28,7 @@ describe('search_codebase scoped memories', () => { beforeEach(async () => { searchMocks.search.mockReset(); tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'search-scoped-memory-')); + const currentDate = new Date().toISOString(); const contextDir = path.join(tempRoot, '.codebase-context'); await fs.mkdir(contextDir, { recursive: true }); @@ -55,7 +56,7 @@ describe('search_codebase scoped memories', () => { category: 'architecture', memory: 'Avoid direct token reads', reason: 'They bypass AuthService refresh logic.', - date: '2026-04-17T00:00:00.000Z', + date: currentDate, scope: { kind: 'symbol', file: 'src/auth/auth.service.ts', @@ -68,7 +69,7 @@ describe('search_codebase scoped memories', () => { category: 'architecture', memory: 'Use auth interceptors', reason: 'They keep HTTP token injection consistent.', - date: '2026-04-17T00:00:00.000Z' + date: currentDate } ], null, diff --git a/tests/server-config.test.ts b/tests/server-config.test.ts index de44feb..eb7f6c6 100644 --- a/tests/server-config.test.ts +++ b/tests/server-config.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, afterEach, vi } from 'vitest'; import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { loadServerConfig } from '../src/server/config.js'; +import { loadProjectConfig, loadServerConfig } from '../src/server/config.js'; // Helper: write a temp config file and set CODEBASE_CONTEXT_CONFIG_PATH async function withTempConfig(content: string, fn: (filePath: string) => Promise) { @@ -158,6 +158,56 @@ describe('loadServerConfig', () => { }); }); + it('parses a positive per-project maxChunks value', async () => { + const config = JSON.stringify({ + projects: [{ root: '~/large-repo', parsing: { maxChunks: 25000 } }] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const result = await loadServerConfig(); + + expect(result?.projects?.[0].parsing).toEqual({ maxChunks: 25000 }); + }); + }); + + it('loads parsing configuration for the requested CLI project root', async () => { + const projectRoot = path.join(os.tmpdir(), 'ccc-large-cli-project'); + const config = JSON.stringify({ + projects: [ + { root: path.join(os.tmpdir(), 'ccc-other-project') }, + { root: projectRoot, parsing: { maxChunks: 25000 } } + ] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const projectConfig = await loadProjectConfig(projectRoot); + + expect(projectConfig).toEqual({ + root: projectRoot, + parsing: { maxChunks: 25000 } + }); + }); + }); + + it('ignores an invalid per-project maxChunks value', async () => { + const errorSpy = vi.spyOn(console, 'error'); + const config = JSON.stringify({ + projects: [{ root: '~/large-repo', parsing: { maxChunks: 0 } }] + }); + + await withTempConfig(config, async (filePath) => { + process.env.CODEBASE_CONTEXT_CONFIG_PATH = filePath; + const result = await loadServerConfig(); + + expect(result?.projects?.[0].parsing).toBeUndefined(); + expect(errorSpy).toHaveBeenCalledWith( + '[config] Ignoring invalid project parsing.maxChunks: 0' + ); + }); + }); + it('drops empty analyzerHints objects after parsing', async () => { const config = JSON.stringify({ projects: [ diff --git a/vitest.config.mjs b/vitest.config.mjs index 6b6958c..fd318e9 100644 --- a/vitest.config.mjs +++ b/vitest.config.mjs @@ -2,6 +2,8 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { + // File-heavy subprocess fixtures compete for native resources on Windows. + ...(process.platform === 'win32' ? { maxWorkers: 1 } : {}), include: ['tests/**/*.test.ts'], environment: 'node', testTimeout: 15000