From e5789e1de09be72c41fee759619458247b18e52d Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Tue, 10 Nov 2026 21:06:39 +0300 Subject: [PATCH] =?UTF-8?q?test(bench):=20the=20benchmark=20on=20an=20exte?= =?UTF-8?q?rnal=20repo=20=E2=80=94=20plain=20grep=20wins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B1 lives in our own repository, so B2 runs the same construction on psf/requests: the repo, its history, its commit messages and the gold all come from other people. Corpus pinned at e331a288 (2026-03-24), 64 files, 786 chunks. 25 commits (40 asked; no merges, <=20 files, >=1 .py outside tests/docs) arm hit@1 hit@5 95% CI grep 6 19 [0.57, 0.89] search 5 15 [0.41, 0.77] neighbours 9 13 [0.33, 0.70] tool 8 10 [0.23, 0.59] On a 64-file repository a term-count ranking over file text is simply good: messages name modules and functions, and there are few files to confuse it. The tool's demotions, gate and merge add nothing here and take something away. This corrects an earlier claim. "We beat grep 157/160 vs 147/160" was about symbol lookup on the generated panel; it says nothing about commit localization and does not generalise to a small external repo. Caveats, stated rather than buried: n=25 (one task = 4 points, intervals overlap between search and tool, but grep's lower bound is above the tool's upper); the grep baseline reads corpus files by path from the repo's ls-files, so files created after the base are empty for it - that biases grep down and it still wins; one repository and one base are not a claim about small repos in general. The harness is `experiments/bench_2026_11/external.py`; the result file records repo, base, task count and the table, with no commit messages stored. --- AGENT_DIARY.md | 3 + EXPERIMENTS_LOG.md | 23 +++ experiments/bench_2026_11/REPORT.md | 38 ++++- experiments/bench_2026_11/external.py | 144 ++++++++++++++++++ .../results_external_requests.json | 24 +++ 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 experiments/bench_2026_11/external.py create mode 100644 experiments/bench_2026_11/results_external_requests.json diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index a4b34400..f727f299 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -57,6 +57,9 @@ Пункты «RU→EN словарь» и «concept→symbol словарь» сняты: их заменяет контракт. - **Урок:** не учить сервер языкам, которые знает хост; отдавать `hints` и честно просить их, когда в вопросе нет code-имени. +- **E70 (внешний репо):** на `psf/requests` grep берёт 19/25 hit@5, search 15, + neighbours 13, наш tool 10. На маленьком корпусе оркестрация проигрывает обычному + поиску по тексту; «мы лучше grep» верно только для символьного поиска на нашей панели. - **E69 (вариант C):** отказ остаётся, но печатает `next: investigate(... include_neighbours=true)`. Замер на B1 dev: дефолт 19/71 не изменился, подсказка на 36/36 отказах, вместе 37/71 [0.41, 0.63] против 38 у neighbours-руки. diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index f2b6593a..e56295d6 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -1,4 +1,27 @@ # EXPERIMENTS_LOG.md — Audit Verification (2026-07-22) +## [2026-11-10] — E70: external repo (psf/requests) — plain grep wins + +**Why:** B1 still lives in our own repository. Here the repo, history, messages +and gold are all external; corpus pinned at `e331a288` (2026-03-24), 64 files, +786 chunks. + +``` +arm hit@1 hit@5 95% CI (25 commits) +grep 6 19 [0.57, 0.89] +search 5 15 [0.41, 0.77] +neighbours 9 13 [0.33, 0.70] +tool 8 10 [0.23, 0.59] +``` + +**Verdict:** on a small repository the orchestration is last and grep is first. +Our earlier "we beat grep" was about symbol lookup on a generated panel, not +about commit localization; it does not generalise. + +**Caveats:** n=25 (one task = 4 points); the grep baseline reads corpus files by +path from the repo's `ls-files`, so files created after the base are empty for +it — that biases grep down, and it still wins; one repo and one base are not a +claim about small repos in general. + ## [2026-11-10] — E69: an abstain that names the next call (option C) **Decision (owner):** keep `abstain` — never present an unverified row as an diff --git a/experiments/bench_2026_11/REPORT.md b/experiments/bench_2026_11/REPORT.md index b11971a5..761a621a 100644 --- a/experiments/bench_2026_11/REPORT.md +++ b/experiments/bench_2026_11/REPORT.md @@ -141,8 +141,44 @@ visible in the log before the crash; no code, arm or metric changed after that (the arms were frozen in Amendments 1-3 before the sealed run), so this is disclosed rather than "clean". -### Re-measured after E67 +## B2 — external repository (`psf/requests`) + +Nothing in this run is ours: the repository, its history, its commit messages and +the gold all come from other people. Same rule as B1; corpus at +`e331a288` (2026-03-24), 64 files / 786 chunks, one index build (16 s). + +25 commits survived the filter (40 asked; no merges, ≤20 files, ≥1 `.py` outside +`tests/` and `docs/`). + +``` +arm hit@1 hit@5 95% CI +grep 6 19 [0.57, 0.89] +search 5 15 [0.41, 0.77] +neighbours 9 13 [0.33, 0.70] +tool 8 10 [0.23, 0.59] +``` +**Our orchestration is last; plain grep is first.** On a 64-file repository a +term-count ranking over file text is simply good: commit messages name functions +and modules, and there are few enough files that counting works. The tool's +demotions, gate and merge add nothing here and take something away. + +This is the most useful result of the day for honesty: the earlier claim "we +beat grep on the generated panel (147/160 vs 157/160)" was about *symbol +lookup*, not about localization, and it does not generalise to a small external +repo. + +Caveats, stated rather than buried: + +- n = 25: one task is 4 points; the intervals overlap heavily between `search` + and `tool`, but grep's interval is above the tool's upper bound. +- The grep baseline reads corpus files by path from the *repo's* `ls-files`, so + files created after the corpus commit are empty for it — that biases grep + **down**, and it still wins. +- One repository, one pinned base. A second external repo is needed before this + is a claim about "small repos" in general. + +### Re-measured after E67 The test-bucket fix (E67, merged as `092418a`) changed a bucket rule, so the dev split was re-run on the merged build: **identical** (`tool` 19, `neighbours` 38, `search` 22, `grep` 2; paired +16). The fix does not touch this benchmark, which diff --git a/experiments/bench_2026_11/external.py b/experiments/bench_2026_11/external.py new file mode 100644 index 00000000..7bbde1c1 --- /dev/null +++ b/experiments/bench_2026_11/external.py @@ -0,0 +1,144 @@ +"""B2: the same commit-localization benchmark on an external repository. + +Nothing here is ours: the repo, its commit messages and its history were written +by other people. Same rule as B1 (PLAN.md): gold = the files the commit changed, +corpus at `--base` so the fix is not indexed. + +Run: + python experiments/bench_2026_11/external.py --repo D:\\Project\\bench-ext\\requests \ + --corpus D:\\Project\\bench-ext\\req-B --base e331a288 --limit 40 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11")) +sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11" / "evidence_gate")) +sys.stdout.reconfigure(encoding="utf-8") + +from run_gate import _files, _invalid, _norm, _tools_for # noqa: E402 +from wilson import wilson # noqa: E402 + +STOP = {"the", "and", "for", "with", "that", "this", "from", "into", "when", + "what", "where", "which", "have", "has", "not", "are", "was", "were"} + + +def _git(repo: Path, *args: str) -> str: + out = subprocess.run(["git", *args], cwd=repo, capture_output=True, + text=True, encoding="utf-8", errors="replace") + return out.stdout if out.returncode == 0 else "" + + +def _tasks(repo: Path, base: str, limit: int) -> list[dict]: + raw = _git(repo, "log", f"{base}..HEAD", "--no-merges", + "--format=%H%x00%ad%x00%B%x00", "--date=short") + parts = raw.split("\x00") + tasks = [] + for sha, date, message in zip(parts[0::3], parts[1::3], parts[2::3]): + sha, message = sha.strip(), message.strip() + files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] + if not files or len(files) > 20 or len(message) < 20: + continue + gold = sorted(f for f in files + if f.endswith(".py") and not f.startswith(("tests/", "docs/"))) + if not gold: + continue + tasks.append({"id": sha[:12], "sha": sha, "date": date, + "question": message, "gold": gold}) + tasks.sort(key=lambda t: (t["date"], t["id"])) + return tasks[:limit] if limit else tasks + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", required=True) + parser.add_argument("--corpus", required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--limit", type=int, default=40) + args = parser.parse_args() + + repo, corpus = Path(args.repo), Path(args.corpus) + tasks = _tasks(repo, args.base, args.limit) + print(f"repo={repo.name} base={args.base[:12]} tasks={len(tasks)}", flush=True) + + tool = _tools_for({corpus})[str(corpus)] + searcher = tool.resolve_searcher(explicit_project_root=str(corpus)) + texts = {} + for path in _git(repo, "ls-files", "*.py").splitlines(): + try: + texts[path] = (corpus / path).read_text(encoding="utf-8", errors="ignore").lower() + except OSError: + texts[path] = "" + + def grep_rank(message: str) -> list[str]: + terms = {w for w in re.findall(r"[a-zA-Z_]{4,}", message.lower()) if w not in STOP} + if not terms: + return [] + scored = [(-sum(1 for t in terms if t in text), path) + for path, text in texts.items()] + return [p for score, p in sorted(scored) if score] + + def hits(files, gold, k): + return any(any(g in f for f in files[:k]) for g in gold) + + rows = [] + for index, task in enumerate(tasks, 1): + text = asyncio.run(tool.execute(query=task["question"], limit=8, + project_root=str(corpus))) + files = _norm(_files(text), corpus) + nb = _norm(_files(asyncio.run(tool.execute( + query=task["question"], limit=8, include_neighbours=True, + project_root=str(corpus)))), corpus) + raw = asyncio.run(searcher.hybrid_search_async(task["question"], limit=50, rerank=False)) + search = [] + for row in raw or []: + meta = row.get("metadata") or {} + f = str(meta.get("file") or row.get("file") or "").replace("\\", "/") + if f.startswith(str(corpus).replace("\\", "/")): + f = f[len(str(corpus).replace("\\", "/")).rstrip("/"):].lstrip("/") + if f: + search.append(f) + grep = grep_rank(task["question"]) + rows.append({ + "id": task["id"], + "valid": not _invalid(text), + "tool": (hits(files, task["gold"], 1), hits(files, task["gold"], 5)), + "neighbours": (hits(nb, task["gold"], 1), hits(nb, task["gold"], 5)), + "search": (hits(search, task["gold"], 1), hits(search, task["gold"], 5)), + "grep": (hits(grep, task["gold"], 1), hits(grep, task["gold"], 5)), + }) + print(f" {index:>3}/{len(tasks)} {task['id']} " + f"tool@5={rows[-1]['tool'][1]} nb@5={rows[-1]['neighbours'][1]}", flush=True) + + n = len(rows) + print(f"\n{'arm':<12} {'hit@1':>6} {'hit@5':>6} hit@5 95% CI") + table = {} + for arm in ("tool", "neighbours", "search", "grep"): + top1 = sum(1 for r in rows if r[arm][0]) + top5 = sum(1 for r in rows if r[arm][1]) + lo, hi = wilson(top5, n) + table[arm] = {"top1": top1, "top5": top5} + print(f"{arm:<12} {top1:>6} {top5:>6} [{lo:.2f}, {hi:.2f}]") + invalid = sum(1 for r in rows if not r["valid"]) + print(f"invalid runs: {invalid}") + + out = HERE / f"results_external_{repo.name}.json" + out.write_text(json.dumps({"repo": repo.name, "base": args.base, + "tasks": n, "table": table, + "invalid": invalid}, ensure_ascii=False, indent=1), + encoding="utf-8", newline="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/bench_2026_11/results_external_requests.json b/experiments/bench_2026_11/results_external_requests.json new file mode 100644 index 00000000..9257c24a --- /dev/null +++ b/experiments/bench_2026_11/results_external_requests.json @@ -0,0 +1,24 @@ +{ + "repo": "requests", + "base": "e331a288f369973f5de0ec8901c94cae4fa87286", + "tasks": 25, + "table": { + "tool": { + "top1": 8, + "top5": 10 + }, + "neighbours": { + "top1": 9, + "top5": 13 + }, + "search": { + "top1": 5, + "top5": 15 + }, + "grep": { + "top1": 6, + "top5": 19 + } + }, + "invalid": 0 +} \ No newline at end of file