diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index a4b34400..f727f299 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -57,6 +57,9 @@ Пункты «RU→EN словарь» и «concept→symbol словарь» сняты: их заменяет контракт. - **Урок:** не учить сервер языкам, которые знает хост; отдавать `hints` и честно просить их, когда в вопросе нет code-имени. +- **E70 (внешний репо):** на `psf/requests` grep берёт 19/25 hit@5, search 15, + neighbours 13, наш tool 10. На маленьком корпусе оркестрация проигрывает обычному + поиску по тексту; «мы лучше grep» верно только для символьного поиска на нашей панели. - **E69 (вариант C):** отказ остаётся, но печатает `next: investigate(... include_neighbours=true)`. Замер на B1 dev: дефолт 19/71 не изменился, подсказка на 36/36 отказах, вместе 37/71 [0.41, 0.63] против 38 у neighbours-руки. diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index f2b6593a..e56295d6 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -1,4 +1,27 @@ # EXPERIMENTS_LOG.md — Audit Verification (2026-07-22) +## [2026-11-10] — E70: external repo (psf/requests) — plain grep wins + +**Why:** B1 still lives in our own repository. Here the repo, history, messages +and gold are all external; corpus pinned at `e331a288` (2026-03-24), 64 files, +786 chunks. + +``` +arm hit@1 hit@5 95% CI (25 commits) +grep 6 19 [0.57, 0.89] +search 5 15 [0.41, 0.77] +neighbours 9 13 [0.33, 0.70] +tool 8 10 [0.23, 0.59] +``` + +**Verdict:** on a small repository the orchestration is last and grep is first. +Our earlier "we beat grep" was about symbol lookup on a generated panel, not +about commit localization; it does not generalise. + +**Caveats:** n=25 (one task = 4 points); the grep baseline reads corpus files by +path from the repo's `ls-files`, so files created after the base are empty for +it — that biases grep down, and it still wins; one repo and one base are not a +claim about small repos in general. + ## [2026-11-10] — E69: an abstain that names the next call (option C) **Decision (owner):** keep `abstain` — never present an unverified row as an diff --git a/experiments/bench_2026_11/REPORT.md b/experiments/bench_2026_11/REPORT.md index b11971a5..761a621a 100644 --- a/experiments/bench_2026_11/REPORT.md +++ b/experiments/bench_2026_11/REPORT.md @@ -141,8 +141,44 @@ visible in the log before the crash; no code, arm or metric changed after that (the arms were frozen in Amendments 1-3 before the sealed run), so this is disclosed rather than "clean". -### Re-measured after E67 +## B2 — external repository (`psf/requests`) + +Nothing in this run is ours: the repository, its history, its commit messages and +the gold all come from other people. Same rule as B1; corpus at +`e331a288` (2026-03-24), 64 files / 786 chunks, one index build (16 s). + +25 commits survived the filter (40 asked; no merges, ≤20 files, ≥1 `.py` outside +`tests/` and `docs/`). + +``` +arm hit@1 hit@5 95% CI +grep 6 19 [0.57, 0.89] +search 5 15 [0.41, 0.77] +neighbours 9 13 [0.33, 0.70] +tool 8 10 [0.23, 0.59] +``` +**Our orchestration is last; plain grep is first.** On a 64-file repository a +term-count ranking over file text is simply good: commit messages name functions +and modules, and there are few enough files that counting works. The tool's +demotions, gate and merge add nothing here and take something away. + +This is the most useful result of the day for honesty: the earlier claim "we +beat grep on the generated panel (147/160 vs 157/160)" was about *symbol +lookup*, not about localization, and it does not generalise to a small external +repo. + +Caveats, stated rather than buried: + +- n = 25: one task is 4 points; the intervals overlap heavily between `search` + and `tool`, but grep's interval is above the tool's upper bound. +- The grep baseline reads corpus files by path from the *repo's* `ls-files`, so + files created after the corpus commit are empty for it — that biases grep + **down**, and it still wins. +- One repository, one pinned base. A second external repo is needed before this + is a claim about "small repos" in general. + +### Re-measured after E67 The test-bucket fix (E67, merged as `092418a`) changed a bucket rule, so the dev split was re-run on the merged build: **identical** (`tool` 19, `neighbours` 38, `search` 22, `grep` 2; paired +16). The fix does not touch this benchmark, which diff --git a/experiments/bench_2026_11/external.py b/experiments/bench_2026_11/external.py new file mode 100644 index 00000000..7bbde1c1 --- /dev/null +++ b/experiments/bench_2026_11/external.py @@ -0,0 +1,144 @@ +"""B2: the same commit-localization benchmark on an external repository. + +Nothing here is ours: the repo, its commit messages and its history were written +by other people. Same rule as B1 (PLAN.md): gold = the files the commit changed, +corpus at `--base` so the fix is not indexed. + +Run: + python experiments/bench_2026_11/external.py --repo D:\\Project\\bench-ext\\requests \ + --corpus D:\\Project\\bench-ext\\req-B --base e331a288 --limit 40 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11")) +sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11" / "evidence_gate")) +sys.stdout.reconfigure(encoding="utf-8") + +from run_gate import _files, _invalid, _norm, _tools_for # noqa: E402 +from wilson import wilson # noqa: E402 + +STOP = {"the", "and", "for", "with", "that", "this", "from", "into", "when", + "what", "where", "which", "have", "has", "not", "are", "was", "were"} + + +def _git(repo: Path, *args: str) -> str: + out = subprocess.run(["git", *args], cwd=repo, capture_output=True, + text=True, encoding="utf-8", errors="replace") + return out.stdout if out.returncode == 0 else "" + + +def _tasks(repo: Path, base: str, limit: int) -> list[dict]: + raw = _git(repo, "log", f"{base}..HEAD", "--no-merges", + "--format=%H%x00%ad%x00%B%x00", "--date=short") + parts = raw.split("\x00") + tasks = [] + for sha, date, message in zip(parts[0::3], parts[1::3], parts[2::3]): + sha, message = sha.strip(), message.strip() + files = [f for f in _git(repo, "show", "--name-only", "--format=", sha).splitlines() if f.strip()] + if not files or len(files) > 20 or len(message) < 20: + continue + gold = sorted(f for f in files + if f.endswith(".py") and not f.startswith(("tests/", "docs/"))) + if not gold: + continue + tasks.append({"id": sha[:12], "sha": sha, "date": date, + "question": message, "gold": gold}) + tasks.sort(key=lambda t: (t["date"], t["id"])) + return tasks[:limit] if limit else tasks + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo", required=True) + parser.add_argument("--corpus", required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--limit", type=int, default=40) + args = parser.parse_args() + + repo, corpus = Path(args.repo), Path(args.corpus) + tasks = _tasks(repo, args.base, args.limit) + print(f"repo={repo.name} base={args.base[:12]} tasks={len(tasks)}", flush=True) + + tool = _tools_for({corpus})[str(corpus)] + searcher = tool.resolve_searcher(explicit_project_root=str(corpus)) + texts = {} + for path in _git(repo, "ls-files", "*.py").splitlines(): + try: + texts[path] = (corpus / path).read_text(encoding="utf-8", errors="ignore").lower() + except OSError: + texts[path] = "" + + def grep_rank(message: str) -> list[str]: + terms = {w for w in re.findall(r"[a-zA-Z_]{4,}", message.lower()) if w not in STOP} + if not terms: + return [] + scored = [(-sum(1 for t in terms if t in text), path) + for path, text in texts.items()] + return [p for score, p in sorted(scored) if score] + + def hits(files, gold, k): + return any(any(g in f for f in files[:k]) for g in gold) + + rows = [] + for index, task in enumerate(tasks, 1): + text = asyncio.run(tool.execute(query=task["question"], limit=8, + project_root=str(corpus))) + files = _norm(_files(text), corpus) + nb = _norm(_files(asyncio.run(tool.execute( + query=task["question"], limit=8, include_neighbours=True, + project_root=str(corpus)))), corpus) + raw = asyncio.run(searcher.hybrid_search_async(task["question"], limit=50, rerank=False)) + search = [] + for row in raw or []: + meta = row.get("metadata") or {} + f = str(meta.get("file") or row.get("file") or "").replace("\\", "/") + if f.startswith(str(corpus).replace("\\", "/")): + f = f[len(str(corpus).replace("\\", "/")).rstrip("/"):].lstrip("/") + if f: + search.append(f) + grep = grep_rank(task["question"]) + rows.append({ + "id": task["id"], + "valid": not _invalid(text), + "tool": (hits(files, task["gold"], 1), hits(files, task["gold"], 5)), + "neighbours": (hits(nb, task["gold"], 1), hits(nb, task["gold"], 5)), + "search": (hits(search, task["gold"], 1), hits(search, task["gold"], 5)), + "grep": (hits(grep, task["gold"], 1), hits(grep, task["gold"], 5)), + }) + print(f" {index:>3}/{len(tasks)} {task['id']} " + f"tool@5={rows[-1]['tool'][1]} nb@5={rows[-1]['neighbours'][1]}", flush=True) + + n = len(rows) + print(f"\n{'arm':<12} {'hit@1':>6} {'hit@5':>6} hit@5 95% CI") + table = {} + for arm in ("tool", "neighbours", "search", "grep"): + top1 = sum(1 for r in rows if r[arm][0]) + top5 = sum(1 for r in rows if r[arm][1]) + lo, hi = wilson(top5, n) + table[arm] = {"top1": top1, "top5": top5} + print(f"{arm:<12} {top1:>6} {top5:>6} [{lo:.2f}, {hi:.2f}]") + invalid = sum(1 for r in rows if not r["valid"]) + print(f"invalid runs: {invalid}") + + out = HERE / f"results_external_{repo.name}.json" + out.write_text(json.dumps({"repo": repo.name, "base": args.base, + "tasks": n, "table": table, + "invalid": invalid}, ensure_ascii=False, indent=1), + encoding="utf-8", newline="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/bench_2026_11/results_external_requests.json b/experiments/bench_2026_11/results_external_requests.json new file mode 100644 index 00000000..9257c24a --- /dev/null +++ b/experiments/bench_2026_11/results_external_requests.json @@ -0,0 +1,24 @@ +{ + "repo": "requests", + "base": "e331a288f369973f5de0ec8901c94cae4fa87286", + "tasks": 25, + "table": { + "tool": { + "top1": 8, + "top5": 10 + }, + "neighbours": { + "top1": 9, + "top5": 13 + }, + "search": { + "top1": 5, + "top5": 15 + }, + "grep": { + "top1": 6, + "top5": 19 + } + }, + "invalid": 0 +} \ No newline at end of file