From 70156a4db81734bacb0013b8d025eaf88c7f1784 Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Tue, 10 Nov 2026 16:03:51 +0300 Subject: [PATCH 1/2] test(experiment): generate a 281-task panel from the index and calibrate it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 41-task panel makes one task worth 6 points, and every rate we publish is a point estimate with a wide interval. This adds a panel built by a generator instead of by hand: symbols are taken from the index, the sample is seeded by `sha256(commit)`, and a symbol is used only if its gold file literally defines it. 160 `name` + 40 `qualified` + 40 `negative` + the 41 existing anchors. Frozen predictions all pass: bucket n top1 any chars/top1 name 160 154 157 2163 qualified 40 38 40 2669 negative 40 40 40 332 anchor:xerj/positive 16 6 10 5894 anchor:mcp_audit/pos 17 11 14 3442 S3 is the point: the anchors reproduce E65's `weights` arm exactly (6/16, 11/17), so the instrument is stable enough to compare experiments against. The generated rate is not a quality headline - the questions contain the identifier, so they measure the symbol channel; prose is `hints` (E64). Building the panel surfaced two index defects, both recorded rather than silently absorbed into the rate: the graph returns nodes whose file is not in this project (`src/sources/manifest/extract.py`), and it labels a call expression as a `Function` (`callable` in `di_container.py:160`). The evidence gate survives the second - the row gets no `✓def` and the verdict is `caveat (0.40)` - but the symbol channel ranks a non-definition. `phantom_gold = 0` on the shipped panel: every gold was checked against the source at build time, so a miss is a retrieval miss, not a broken task. --- AGENT_DIARY.md | 7 + EXPERIMENTS_LOG.md | 33 +++ experiments/scale_2026_11/REPORT.md | 80 +++++++ experiments/scale_2026_11/frozen/PLAN.md | 56 +++++ experiments/scale_2026_11/generate.py | 173 ++++++++++++++ experiments/scale_2026_11/results.json | 55 +++++ experiments/scale_2026_11/run_scale.py | 152 ++++++++++++ experiments/scale_2026_11/tasks.jsonl | 281 +++++++++++++++++++++++ experiments/xerj_2026_11/wilson.py | 4 + 9 files changed, 841 insertions(+) create mode 100644 experiments/scale_2026_11/REPORT.md create mode 100644 experiments/scale_2026_11/frozen/PLAN.md create mode 100644 experiments/scale_2026_11/generate.py create mode 100644 experiments/scale_2026_11/results.json create mode 100644 experiments/scale_2026_11/run_scale.py create mode 100644 experiments/scale_2026_11/tasks.jsonl diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index 4a04311d..caf59cb2 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -57,6 +57,13 @@ Пункты «RU→EN словарь» и «concept→symbol словарь» сняты: их заменяет контракт. - **Урок:** не учить сервер языкам, которые знает хост; отдавать `hints` и честно просить их, когда в вопросе нет code-имени. +- **E66 (панель из индекса):** 281 задача, сид = sha256(commit). name 154/160, + qualified 38/40, negative 40/40; якоря воспроизвели E65 **точно** (6/16 и 11/17) — + инструмент откалиброван. Это детектор регрессий, а не «поиск на 96%»: вопросы + содержат идентификатор, проза закрыта hints (E64). +- **Найдено генератором:** в графе есть узлы из чужого проекта и фантомные + `Function` для вызовов (`callable` в `di_container.py:160`). Гейт не обманут + (нет `✓def`, вердикт `caveat 0.40`), но символьный канал такой узел поднимает. - **E65 (fusion A/B):** три руки в одном бинарнике (`MSCODEBASE_FUSION`). top1 weights 21 / rrf 15 / norm 20 из 53; F1/F2/F3 провалены, шипим weights, ранжирование закрыто. Три попытки замера убиты до публикации: (1) кап демоушена + тай-брейк по diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index b76ee52d..ed3e4ba9 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -1,4 +1,37 @@ # EXPERIMENTS_LOG.md — Audit Verification (2026-07-22) +## [2026-11-10] — E66: generated 281-task panel — the instrument is calibrated + +**Design** (frozen before the run): tasks generated from the index, seed = +`sha256(commit)`, so the panel is reproducible and changes only with the code. +160 `name` + 40 `qualified` + 40 `negative` + the 41 existing anchors = 281. +Symbols must have exactly one definition and a gold file that literally defines +them (two index defects were found this way, see below). + +``` +bucket n top1 any chars/top1 +name 160 154 157 2163 +qualified 40 38 40 2669 +negative 40 40 40 332 +anchor:xerj/positive 16 6 10 5894 +anchor:mcp_audit/pos 17 11 14 3442 +``` + +**Verdict:** S1-S4 all PASS. S3 is the claim that matters: the anchors +reproduced E65's `weights` arm **exactly** (6/16 and 11/17), so the instrument +is stable enough to compare experiments against. + +**What it does not say:** generated questions contain the identifier, so they +measure the symbol channel, not prose. 0.963 is a regression detector ("exact +names resolve"), not a quality headline; prose is covered by `hints` (E64). + +**Index defects found by the generator's own gold check:** the graph returns +nodes whose file is not in the project (`src/sources/manifest/extract.py`), and +it labels a call expression as a `Function` (`callable` in +`di_container.py:160`). The evidence gate survives the second one — the row gets +no `✓def` and the verdict is `caveat (0.40)` — but the symbol channel ranks a +non-definition. `KNOWN_ISSUES.md` is at its 300-line limit, so both are recorded +in the task state; the rotation is a separate task. + ## [2026-11-10] — E65: fusion A/B — hand weights beat RRF by six tasks **Design** (frozen before the run, plus Amendment 1: latency is not a diff --git a/experiments/scale_2026_11/REPORT.md b/experiments/scale_2026_11/REPORT.md new file mode 100644 index 00000000..82ca830c --- /dev/null +++ b/experiments/scale_2026_11/REPORT.md @@ -0,0 +1,80 @@ +# Scale panel — result (E66) + +Frozen before the run: `frozen/PLAN.md`. Panel generated from commit +`e81e0b0` (`sha256(commit)` seed), 281 tasks, one arm (shipped defaults). + +## Result + +``` +bucket n top1 any chars/top1 95% CI top1 +name 160 154 157 2163 [0.92, 0.98] +qualified 40 38 40 2669 [0.83, 0.99] +negative 40 40 40 332 [0.91, 1.00] +anchor:xerj/positive 16 6 10 5894 [0.18, 0.61] +anchor:xerj/ambiguity 4 0 2 n/a [0.00, 0.49] +anchor:xerj/negative 4 4 4 359 [0.51, 1.00] +anchor:mcp_audit/pos 17 11 14 3442 [0.41, 0.83] + invalid runs: 0 +``` + +`chars/top1` is answer characters divided by the number of top-1 hits — an +estimate of answer cost per correct answer, **not** a tokenizer measurement. + +## Frozen predictions + +| # | Prediction | Result | +|---|---|---| +| S1 | generated `name` top-1 >= 0.60 | **PASS** — 154/160 = 0.963 | +| S2 | negatives yield no rows on >= 0.95 | **PASS** — 40/40 | +| S3 | anchors reproduce E65 `weights` within +/- 2 | **PASS, exactly** — xerj 6/16 and mcp_audit 11/17, the same two numbers | +| S4 | `qualified` top-1 <= `name` top-1 | **PASS** — 0.950 <= 0.963 | + +## What this does and does not mean + +**It means the instrument is calibrated.** The anchors did not merely stay within +two tasks — they reproduced E65 exactly (6/16, 11/17). Two independent +experiments, two different runners, same panel: the measurement is stable enough +to compare against. + +**It does not mean "search is 96% accurate".** The generated questions contain +the identifier (`where is \`resolve_llama_role_bin\` defined`), so they measure +the symbol channel and the ranking around it. A tool that greps the name passes +them. The number is a **regression detector**, not a quality claim: it says +"exact names resolve, 154 of 160 times", and nothing about prose questions +(`hints`, E64), ambiguous golds (XERJ A01-A04) or docs routing. + +The cost column shows the same thing from another side: a generated task costs +2163 chars per correct answer, an XERJ prose task 5894. The difference is task +difficulty, not an improvement in the tool. + +## The 6 misses, honestly + +Six `name` tasks and two `qualified` tasks did not put the gold file first +(`any@8`: 3 and 0 misses). They were not inspected individually in this run — +that is the next step if the number has to move: the generator records the id, +so a miss list is reproducible from the same commit (`tasks.jsonl` + seed). + +## Instrument findings (found while building the panel, not fixed here) + +1. **Foreign graph nodes.** `graph.find_nodes()` returned a node whose file + (`src/sources/manifest/extract.py`) does not exist in this project. The + generator now requires the file to exist; the tool still enumerates such + nodes. +2. **Phantom `Function` nodes for calls.** `callable` in + `src/core/di_container.py:160` is `if callable(close_method):`, but the graph + labels it `Function`. The evidence gate is **not** fooled — `investigate` + returns that row without `✓def` and the verdict is `caveat (0.40) — no + verified declaration` — but the symbol channel still ranks a non-definition. + Follow-up: parser filter plus a regression test. + +Both would have silently depressed the generated rate (a task whose gold does +not define the symbol can never be answered), which is why the generator now +verifies gold at build time: `phantom_gold = 0` on the shipped panel. + +## Reproduce + +``` +python experiments/scale_2026_11/generate.py # same commit -> same 281 tasks +python experiments/scale_2026_11/run_scale.py # ~15 min, 3 s per task +python experiments/xerj_2026_11/wilson.py # intervals +``` diff --git a/experiments/scale_2026_11/frozen/PLAN.md b/experiments/scale_2026_11/frozen/PLAN.md new file mode 100644 index 00000000..01f42e83 --- /dev/null +++ b/experiments/scale_2026_11/frozen/PLAN.md @@ -0,0 +1,56 @@ +# Scale panel — frozen plan (before the first run) + +Date: 2026-11-10. Question: **does the 41-task panel generalize, or is it a +hand-picked sample that our numbers overfit?** + +A panel of 16 tasks makes one task worth 6 points. Every rate we publish is a +point estimate with a huge interval (`REPORT.md`, Wilson). The way out is size, +but not size bought by writing more questions by hand: those would come from the +same head that wrote the hypotheses. + +## Design + +- **Generator, not author.** Tasks are derived from the index itself, seeded by + the commit (`sha256(commit) -> random.Random`), so the same commit yields the + same panel and a different commit yields a different one. +- **Buckets:** + - `name` (160): "where is `` defined" for a symbol with **exactly one** + definition in the repo. + - `qualified` (40): same, for `Class.method` names with a unique definition. + - `negative` (40): the same question for a symbol that does not exist + (`zz_<8 hex>`); the correct answer is "no rows". +- **Anchors (41):** the existing XERJ 24 + mcp_audit 17 panel, unchanged. They + are the calibration: if the instrument drifts, the anchors move first. + +Total: 281 tasks. + +## Metrics + +`top1`, `any@8`, `tokens-per-correct` (answer characters / 4 / number of top-1 +hits), per bucket, with Wilson 95% intervals. Tokens are an estimate, not a +tokenizer measurement, and are labelled as such. + +## Predictions (frozen) + +| # | Prediction | Falsifier | +|---|---|---| +| S1 | generated `name` top-1 >= 0.60 | below 0.60 | +| S2 | `negative` yields no rows on >= 0.95 | any 3+ negatives return rows | +| S3 | anchors reproduce E65 `weights` (xerj 6/16, mcp_audit 11/17) within +/- 2 tasks | drift > 2 tasks | +| S4 | `qualified` top-1 <= `name` top-1 | qualified beats name | + +S3 is the kill switch: if the anchors drift, the instrument is broken or the +code moved under it, and **no number from this run is published**. + +## What this measures, and what it does not + +Generated questions **contain the identifier**, so they measure the symbol +channel and the ranking around it. They say nothing about prose questions — that +is the `hints` contract (E64) and the metamorphic panel. A high generated rate +is not "search is good"; it is "exact names resolve", which is a necessary but +not sufficient property. Stating this here, before the run, so the result cannot +be oversold later. + +The generator also cannot detect a symbol channel that is *too* eager: a +generated `name` task is passed by any tool that finds the name. The anchors and +the negative bucket are what keep it honest. diff --git a/experiments/scale_2026_11/generate.py b/experiments/scale_2026_11/generate.py new file mode 100644 index 00000000..ee6824c8 --- /dev/null +++ b/experiments/scale_2026_11/generate.py @@ -0,0 +1,173 @@ +"""Generate the scale panel from the index, deterministically. + +Frozen spec: `frozen/PLAN.md`. Same commit -> same panel; a different commit +gets a different sample. Gold is a file that must define the symbol exactly once: +a symbol with two definitions is an ambiguity task, and this generator does not +create those (XERJ A01-A04 already cover that class). + +Run: + python experiments/scale_2026_11/generate.py +""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.stdout.reconfigure(encoding="utf-8") + +OUT = HERE / "tasks.jsonl" +COUNTS = {"name": 160, "qualified": 40, "negative": 40} +SKIP_FILES = ("tests/", "docs/", "experiments/", "scripts/", ".venv/") +NAME_RE = re.compile(r"^[a-z_][a-z0-9_]{4,40}$") +QUALIFIED_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*\.[a-z_][a-z0-9_]{3,40}$") + + +def _commit() -> str: + out = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, + capture_output=True, text=True, encoding="utf-8") + if out.returncode != 0: + raise SystemExit("not a git repo: pin the panel to a commit") + return out.stdout.strip() + + +def _definitions() -> dict[str, list[tuple[str, int]]]: + """symbol -> [(relative file, line)] from the property graph. + + The symbol index is a view over the graph (`SymbolIndexAdapter`), so its + in-memory dicts are empty until a query; the graph is the source of truth. + + Two filters keep the panel honest, both measured on 2026-11-10: + - the graph can hold nodes whose file is not in this project + (`src/sources/manifest/extract.py`), so the file must exist under ROOT; + - the graph can hold phantom "functions" that are really calls + (`callable` in `di_container.py`), so the source line must literally + define the name. + """ + from src.core.di_container import ( + IndexerFactoryKey, + ProjectIndexerRegistry, + create_service_collection, + ) + + services = create_service_collection(ROOT) + indexer = services.resolve(ProjectIndexerRegistry).get_indexer( + ROOT, factory=services.resolve(IndexerFactoryKey) + ) + graph = indexer._symbol_index.graph + base = str(ROOT).replace("\\", "/").rstrip("/") + + found: dict[str, list[tuple[str, int]]] = {} + for node in graph.find_nodes(limit=50000): + if node.label not in ("Function", "Class"): + continue + path = str(node.file_path or "").replace("\\", "/") + if not path.startswith(base): + continue + path = path[len(base):].lstrip("/") + source = ROOT / path + if not source.is_file(): + continue + if not _defines(source, node.name): + continue + line = int((node.properties or {}).get("line") or 0) + found.setdefault(node.name, []).append((path, line)) + return found + + +def _defines(source: Path, name: str) -> bool: + short = name.split(".")[-1] + try: + text = source.read_text(encoding="utf-8", errors="ignore") + except OSError: + return False + return bool(re.search(rf"^\s*(async\s+def|def|class)\s+{re.escape(short)}\b", + text, re.M)) + + +def main() -> int: + commit = _commit() + rng = random.Random(int(hashlib.sha256(commit.encode()).hexdigest(), 16)) + + defs = _definitions() + usable = { + symbol: refs + for symbol, refs in defs.items() + if len(refs) == 1 + and not any(part in refs[0][0] for part in SKIP_FILES) + } + names = sorted(s for s in usable if NAME_RE.match(s)) + qualified = sorted(s for s in usable if QUALIFIED_RE.match(s)) + rng.shuffle(names) + rng.shuffle(qualified) + + tasks: list[dict] = [] + for symbol in names[:COUNTS["name"]]: + file, _line = usable[symbol][0] + tasks.append({ + "id": f"G-N-{symbol}", + "bucket": "name", + "kind": "positive", + "question": f"where is `{symbol}` defined", + "gold": [file], + "anchor": False, + }) + for symbol in qualified[:COUNTS["qualified"]]: + file, _line = usable[symbol][0] + tasks.append({ + "id": f"G-Q-{symbol}", + "bucket": "qualified", + "kind": "positive", + "question": f"where is `{symbol}` defined", + "gold": [file], + "anchor": False, + }) + for index in range(COUNTS["negative"]): + token = f"zz_{rng.getrandbits(32):08x}" + tasks.append({ + "id": f"G-X-{index:03d}", + "bucket": "negative", + "kind": "negative", + "question": f"where is `{token}` defined", + "gold": [], + "anchor": False, + }) + + # Anchors: the existing 41-task panel, unchanged, for calibration (S3). + sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11" / "evidence_gate")) + from run_gate import _panel_tasks # noqa: E402 + + for task in _panel_tasks(): + tasks.append({ + "id": task["id"], + "bucket": f"anchor:{task['panel']}/{task['kind']}", + "kind": task["kind"], + "question": task["question"], + "gold": task["gold"], + "anchor": True, + "root": str(task["root"]), + }) + + if not (COUNTS["name"] == sum(t["bucket"] == "name" for t in tasks)): + raise SystemExit("generator produced fewer name tasks than requested") + + OUT.write_text( + "\n".join(json.dumps(t, ensure_ascii=False) for t in tasks) + "\n", + encoding="utf-8", newline="", + ) + print(f"commit={commit[:12]} tasks={len(tasks)} -> {OUT.name}") + for bucket in sorted({t["bucket"] for t in tasks}): + print(f" {bucket:<22} {sum(t['bucket'] == bucket for t in tasks)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/scale_2026_11/results.json b/experiments/scale_2026_11/results.json new file mode 100644 index 00000000..c281739a --- /dev/null +++ b/experiments/scale_2026_11/results.json @@ -0,0 +1,55 @@ +{ + "panel": "tasks.jsonl", + "tasks": 281, + "invalid": 0, + "summary": { + "anchor:mcp_audit/positive": { + "n": 17, + "top1": 11, + "any": 14, + "chars_per_top1": 3441.7272727272725 + }, + "anchor:xerj/ambiguity": { + "n": 4, + "top1": 0, + "any": 2, + "chars_per_top1": NaN + }, + "anchor:xerj/negative": { + "n": 4, + "top1": 4, + "any": 4, + "chars_per_top1": 358.75 + }, + "anchor:xerj/positive": { + "n": 16, + "top1": 6, + "any": 10, + "chars_per_top1": 5894.0 + }, + "name": { + "n": 160, + "top1": 154, + "any": 157, + "chars_per_top1": 2162.935064935065 + }, + "negative": { + "n": 40, + "top1": 40, + "any": 40, + "chars_per_top1": 332.0 + }, + "qualified": { + "n": 40, + "top1": 38, + "any": 40, + "chars_per_top1": 2668.9473684210525 + } + }, + "predictions": { + "S1 name top1 >= 0.60": true, + "S2 negatives empty >= 0.95": true, + "S3 anchors within +/-2": true, + "S4 qualified <= name": true + } +} \ No newline at end of file diff --git a/experiments/scale_2026_11/run_scale.py b/experiments/scale_2026_11/run_scale.py new file mode 100644 index 00000000..c367400b --- /dev/null +++ b/experiments/scale_2026_11/run_scale.py @@ -0,0 +1,152 @@ +"""Run the scale panel (frozen: `frozen/PLAN.md`). + +One arm (shipped defaults), N tasks, metrics per bucket plus the calibration +check S3: if the anchors drift from E65, the run is not publishable. + +Run: + python experiments/scale_2026_11/run_scale.py # full panel + python experiments/scale_2026_11/run_scale.py --limit 8 # smoke +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from collections import defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +EXP = HERE.parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(EXP / "xerj_2026_11")) +sys.path.insert(0, str(EXP / "xerj_2026_11" / "evidence_gate")) +sys.stdout.reconfigure(encoding="utf-8") + +from run_baseline import score as canon_score # noqa: E402 +from run_gate import VERDICT_RE, _files, _invalid, _norm, _tools_for # noqa: E402 +from wilson import wilson # noqa: E402 + +TASKS = HERE / "tasks.jsonl" +# Frozen calibration targets from E65 (`fusion/REPORT.md`), weights arm. +ANCHOR_EXPECTED = {"anchor:xerj/positive": 6, "anchor:mcp_audit/positive": 11} + + +def _load() -> list[dict]: + tasks = [json.loads(line) for line in TASKS.read_text(encoding="utf-8").splitlines() if line.strip()] + if not tasks: + raise SystemExit("empty panel: run generate.py first") + return tasks + + +def _run(tool, task: dict) -> dict: + started = time.perf_counter() + text = asyncio.run( + tool.execute(query=task["question"], limit=8, + project_root=task.get("root") or str(ROOT)) + ) + elapsed = time.perf_counter() - started + files = _norm(_files(text), Path(task.get("root") or ROOT)) + valid = not _invalid(text) + verdict = VERDICT_RE.search(text) + return { + "id": task["id"], + "bucket": task["bucket"], + "kind": task["kind"], + "top1": None if not valid else canon_score(task, files[:1]), + "any": None if not valid else canon_score(task, files[:8]), + "valid": valid, + "chars": len(text), + "verdict": verdict.group(1) if verdict else "", + "elapsed_s": round(elapsed, 3), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--limit", type=int, default=0) + args = parser.parse_args() + + tasks = _load() + if args.limit: + tasks = tasks[:args.limit] + print(f"tasks={len(tasks)}", flush=True) + + tools = _tools_for({Path(t.get("root") or ROOT) for t in tasks}) + rows = [] + for index, task in enumerate(tasks, 1): + root = Path(task.get("root") or ROOT) + row = _run(tools[str(root)], task) + rows.append(row) + print(f" {index:>3}/{len(tasks)} {row['bucket']:<22} " + f"top1={row['top1']} any={row['any']} " + f"{row['verdict'] or '-':<9} {row['chars']:>5}c " + f"{row['elapsed_s']:5.2f}s", flush=True) + + invalid = sum(1 for r in rows if not r["valid"]) + print(f"\ninvalid runs: {invalid}") + + by_bucket: dict[str, list[dict]] = defaultdict(list) + for row in rows: + if row["valid"]: + by_bucket[row["bucket"]].append(row) + + print(f"\n{'bucket':<24} {'n':>3} {'top1':>4} {'any':>4} " + f"{'chars/top1':>10} 95% CI top1") + summary = {} + for bucket in sorted(by_bucket): + subset = by_bucket[bucket] + top1 = sum(bool(r["top1"]) for r in subset) + any8 = sum(bool(r["any"]) for r in subset) + chars = sum(r["chars"] for r in subset) + per_correct = chars / top1 if top1 else float("nan") + lo, hi = wilson(top1, len(subset)) + summary[bucket] = {"n": len(subset), "top1": top1, "any": any8, + "chars_per_top1": per_correct} + print(f"{bucket:<24} {len(subset):>3} {top1:>4} {any8:>4} " + f"{per_correct:>10.0f} [{lo:.2f}, {hi:.2f}]") + + preds = {} + name = summary.get("name") + neg = summary.get("negative") + qual = summary.get("qualified") + if args.limit: + print("\nSMOKE: no predictions, no results.json") + return 0 + if name: + preds["S1 name top1 >= 0.60"] = (name["top1"] / name["n"]) >= 0.60 + if neg: + preds["S2 negatives empty >= 0.95"] = (neg["top1"] / neg["n"]) >= 0.95 + drift = {b: summary.get(b, {}).get("top1") for b in ANCHOR_EXPECTED} + preds["S3 anchors within +/-2"] = all( + drift[b] is not None and abs(drift[b] - ANCHOR_EXPECTED[b]) <= 2 + for b in ANCHOR_EXPECTED + ) + if name and qual: + preds["S4 qualified <= name"] = (qual["top1"] / qual["n"]) <= (name["top1"] / name["n"]) + + print("\nfrozen predictions") + for label, passed in preds.items(): + print(f" {label:<32} {'PASS' if passed else 'FAIL'}") + + out = HERE / "results.json" + if len(tasks) == len(_load()): + out.write_text(json.dumps({ + "panel": TASKS.name, + "tasks": len(tasks), + "invalid": invalid, + "summary": summary, + "predictions": preds, + }, ensure_ascii=False, indent=1), encoding="utf-8", newline="") + + if invalid or not preds.get("S3 anchors within +/-2", False): + print("ABORT: instrument is not calibrated (S3) or a task did not run.") + return 3 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/scale_2026_11/tasks.jsonl b/experiments/scale_2026_11/tasks.jsonl new file mode 100644 index 00000000..4e56b9a9 --- /dev/null +++ b/experiments/scale_2026_11/tasks.jsonl @@ -0,0 +1,281 @@ +{"id": "G-N-risk_level", "bucket": "name", "kind": "positive", "question": "where is `risk_level` defined", "gold": ["src/core/test_impact.py"], "anchor": false} +{"id": "G-N-apply", "bucket": "name", "kind": "positive", "question": "where is `apply` defined", "gold": ["src/core/silent_subprocess.py"], "anchor": false} +{"id": "G-N-_url_hash", "bucket": "name", "kind": "positive", "question": "where is `_url_hash` defined", "gold": ["src/sources/git_url/__init__.py"], "anchor": false} +{"id": "G-N-run_quiet_break_gate", "bucket": "name", "kind": "positive", "question": "where is `run_quiet_break_gate` defined", "gold": ["src/core/quiet_break_gate.py"], "anchor": false} +{"id": "G-N-_collect_tools", "bucket": "name", "kind": "positive", "question": "where is `_collect_tools` defined", "gold": ["src/plugins/loader.py"], "anchor": false} +{"id": "G-N-_get_models_dir", "bucket": "name", "kind": "positive", "question": "where is `_get_models_dir` defined", "gold": ["src/providers/reranker/llama_install.py"], "anchor": false} +{"id": "G-N-on_step", "bucket": "name", "kind": "positive", "question": "where is `on_step` defined", "gold": ["tools/factory/cli.py"], "anchor": false} +{"id": "G-N-_enable_ansi", "bucket": "name", "kind": "positive", "question": "where is `_enable_ansi` defined", "gold": ["install.py"], "anchor": false} +{"id": "G-N-find_duplicates", "bucket": "name", "kind": "positive", "question": "where is `find_duplicates` defined", "gold": ["src/core/duplication.py"], "anchor": false} +{"id": "G-N-_make_ack_token", "bucket": "name", "kind": "positive", "question": "where is `_make_ack_token` defined", "gold": ["src/core/modification_guard.py"], "anchor": false} +{"id": "G-N-install_all", "bucket": "name", "kind": "positive", "question": "where is `install_all` defined", "gold": ["src/providers/reranker/llama_install.py"], "anchor": false} +{"id": "G-N-_resolve_trust", "bucket": "name", "kind": "positive", "question": "where is `_resolve_trust` defined", "gold": ["src/plugins/loader.py"], "anchor": false} +{"id": "G-N-references", "bucket": "name", "kind": "positive", "question": "where is `references` defined", "gold": ["tools/factory/plan.py"], "anchor": false} +{"id": "G-N-format_incident", "bucket": "name", "kind": "positive", "question": "where is `format_incident` defined", "gold": ["src/utils/ui_formatter.py"], "anchor": false} +{"id": "G-N-_prune_project_dirs", "bucket": "name", "kind": "positive", "question": "where is `_prune_project_dirs` defined", "gold": ["src/core/artifact_gc.py"], "anchor": false} +{"id": "G-N-parse_jsonc", "bucket": "name", "kind": "positive", "question": "where is `parse_jsonc` defined", "gold": ["adapters/zed/zed_config.py"], "anchor": false} +{"id": "G-N-run_bootstrap_pipeline", "bucket": "name", "kind": "positive", "question": "where is `run_bootstrap_pipeline` defined", "gold": ["src/core/bootstrap_pipeline.py"], "anchor": false} +{"id": "G-N-load_plugin", "bucket": "name", "kind": "positive", "question": "where is `load_plugin` defined", "gold": ["src/plugins/loader.py"], "anchor": false} +{"id": "G-N-get_onnx_models_base", "bucket": "name", "kind": "positive", "question": "where is `get_onnx_models_base` defined", "gold": ["src/core/artifact_paths.py"], "anchor": false} +{"id": "G-N-_extract_zip", "bucket": "name", "kind": "positive", "question": "where is `_extract_zip` defined", "gold": ["src/sources/upload/__init__.py"], "anchor": false} +{"id": "G-N-_normalize", "bucket": "name", "kind": "positive", "question": "where is `_normalize` defined", "gold": ["src/core/bootstrap_tests.py"], "anchor": false} +{"id": "G-N-_file_fingerprint", "bucket": "name", "kind": "positive", "question": "where is `_file_fingerprint` defined", "gold": ["src/core/modification_guard.py"], "anchor": false} +{"id": "G-N-_get_symbol_range", "bucket": "name", "kind": "positive", "question": "where is `_get_symbol_range` defined", "gold": ["src/mcp/tools/lsp_tools.py"], "anchor": false} +{"id": "G-N-_extract_uv_lock", "bucket": "name", "kind": "positive", "question": "where is `_extract_uv_lock` defined", "gold": ["src/core/deps/manifest/extract.py"], "anchor": false} +{"id": "G-N-_lsh_bands", "bucket": "name", "kind": "positive", "question": "where is `_lsh_bands` defined", "gold": ["src/core/duplication.py"], "anchor": false} +{"id": "G-N-get_counters", "bucket": "name", "kind": "positive", "question": "where is `get_counters` defined", "gold": ["src/core/runtime_coordinator.py"], "anchor": false} +{"id": "G-N-keep_in_range", "bucket": "name", "kind": "positive", "question": "where is `keep_in_range` defined", "gold": ["tools/knowledge/repoint_stale_refs.py"], "anchor": false} +{"id": "G-N-has_instruction_risk", "bucket": "name", "kind": "positive", "question": "where is `has_instruction_risk` defined", "gold": ["src/core/instruction_scan.py"], "anchor": false} +{"id": "G-N-get_verifier", "bucket": "name", "kind": "positive", "question": "where is `get_verifier` defined", "gold": ["src/core/intelligence/verify_on_read.py"], "anchor": false} +{"id": "G-N-_parse_extensions", "bucket": "name", "kind": "positive", "question": "where is `_parse_extensions` defined", "gold": ["src/config/settings.py"], "anchor": false} +{"id": "G-N-install_git_hooks", "bucket": "name", "kind": "positive", "question": "where is `install_git_hooks` defined", "gold": ["src/mcp/tools/dev_tools.py"], "anchor": false} +{"id": "G-N-_norm_tokens", "bucket": "name", "kind": "positive", "question": "where is `_norm_tokens` defined", "gold": ["src/core/duplication.py"], "anchor": false} +{"id": "G-N-_iter_nodes", "bucket": "name", "kind": "positive", "question": "where is `_iter_nodes` defined", "gold": ["src/core/language_imports.py"], "anchor": false} +{"id": "G-N-_get_max_chunk_preview_len", "bucket": "name", "kind": "positive", "question": "where is `_get_max_chunk_preview_len` defined", "gold": ["src/providers/reranker/multi_provider.py"], "anchor": false} +{"id": "G-N-_extract_composer_lock", "bucket": "name", "kind": "positive", "question": "where is `_extract_composer_lock` defined", "gold": ["src/core/deps/manifest/extract.py"], "anchor": false} +{"id": "G-N-_digest", "bucket": "name", "kind": "positive", "question": "where is `_digest` defined", "gold": ["tools/factory/runtime.py"], "anchor": false} +{"id": "G-N-set_idle_vor_callback", "bucket": "name", "kind": "positive", "question": "where is `set_idle_vor_callback` defined", "gold": ["src/core/task_queue.py"], "anchor": false} +{"id": "G-N-_is_symbol_definition", "bucket": "name", "kind": "positive", "question": "where is `_is_symbol_definition` defined", "gold": ["src/core/search/engine.py"], "anchor": false} +{"id": "G-N-_dedup_sections", "bucket": "name", "kind": "positive", "question": "where is `_dedup_sections` defined", "gold": ["src/mcp/tools/context_tool.py"], "anchor": false} +{"id": "G-N-_incoming_callers", "bucket": "name", "kind": "positive", "question": "where is `_incoming_callers` defined", "gold": ["src/core/quiet_break_gate.py"], "anchor": false} +{"id": "G-N-reset_counters", "bucket": "name", "kind": "positive", "question": "where is `reset_counters` defined", "gold": ["src/core/runtime_coordinator.py"], "anchor": false} +{"id": "G-N-_module_names", "bucket": "name", "kind": "positive", "question": "where is `_module_names` defined", "gold": ["src/core/language_imports.py"], "anchor": false} +{"id": "G-N-module_of", "bucket": "name", "kind": "positive", "question": "where is `module_of` defined", "gold": ["src/core/test_impact.py"], "anchor": false} +{"id": "G-N-load_model", "bucket": "name", "kind": "positive", "question": "where is `load_model` defined", "gold": ["src/core/embedder/onnx_server.py"], "anchor": false} +{"id": "G-N-get_commit_memory_dir", "bucket": "name", "kind": "positive", "question": "where is `get_commit_memory_dir` defined", "gold": ["src/core/artifact_paths.py"], "anchor": false} +{"id": "G-N-_count_lines", "bucket": "name", "kind": "positive", "question": "where is `_count_lines` defined", "gold": ["src/core/code_health.py"], "anchor": false} +{"id": "G-N-classify", "bucket": "name", "kind": "positive", "question": "where is `classify` defined", "gold": ["src/core/trust_boundary.py"], "anchor": false} +{"id": "G-N-_default_client_caps", "bucket": "name", "kind": "positive", "question": "where is `_default_client_caps` defined", "gold": ["src/core/lsp_client.py"], "anchor": false} +{"id": "G-N-_patch_dll_imports", "bucket": "name", "kind": "positive", "question": "where is `_patch_dll_imports` defined", "gold": ["src/providers/reranker/llama_install.py"], "anchor": false} +{"id": "G-N-expand_ru_variants", "bucket": "name", "kind": "positive", "question": "where is `expand_ru_variants` defined", "gold": ["src/core/search/synonyms_ru.py"], "anchor": false} +{"id": "G-N-_pkg_in_pool", "bucket": "name", "kind": "positive", "question": "where is `_pkg_in_pool` defined", "gold": ["src/core/intelligence/verify_on_read.py"], "anchor": false} +{"id": "G-N-grep_fallback", "bucket": "name", "kind": "positive", "question": "where is `grep_fallback` defined", "gold": ["src/core/utils/grep_fallback.py"], "anchor": false} +{"id": "G-N-_on_heartbeat", "bucket": "name", "kind": "positive", "question": "where is `_on_heartbeat` defined", "gold": ["src/mcp/server_factory.py"], "anchor": false} +{"id": "G-N-validate", "bucket": "name", "kind": "positive", "question": "where is `validate` defined", "gold": ["tools/factory/plan.py"], "anchor": false} +{"id": "G-N-exception_fields", "bucket": "name", "kind": "positive", "question": "where is `exception_fields` defined", "gold": ["src/core/reindex_ledger.py"], "anchor": false} +{"id": "G-N-_ensure_data_root", "bucket": "name", "kind": "positive", "question": "where is `_ensure_data_root` defined", "gold": ["src/core/artifact_paths.py"], "anchor": false} +{"id": "G-N-is_error_envelope", "bucket": "name", "kind": "positive", "question": "where is `is_error_envelope` defined", "gold": ["src/mcp/error_results.py"], "anchor": false} +{"id": "G-N-get_config", "bucket": "name", "kind": "positive", "question": "where is `get_config` defined", "gold": ["src/config/settings.py"], "anchor": false} +{"id": "G-N-set_locale", "bucket": "name", "kind": "positive", "question": "where is `set_locale` defined", "gold": ["src/utils/i18n.py"], "anchor": false} +{"id": "G-N-register_intelligence_tools", "bucket": "name", "kind": "positive", "question": "where is `register_intelligence_tools` defined", "gold": ["src/core/intelligence/tools_reg.py"], "anchor": false} +{"id": "G-N-_log_project_resolution_failure", "bucket": "name", "kind": "positive", "question": "where is `_log_project_resolution_failure` defined", "gold": ["src/core/project_resolution.py"], "anchor": false} +{"id": "G-N-get_alert_store", "bucket": "name", "kind": "positive", "question": "where is `get_alert_store` defined", "gold": ["src/core/intelligence/alert_store.py"], "anchor": false} +{"id": "G-N-is_whitelisted", "bucket": "name", "kind": "positive", "question": "where is `is_whitelisted` defined", "gold": ["src/core/quiet_break_gate.py"], "anchor": false} +{"id": "G-N-is_external_or_stdlib", "bucket": "name", "kind": "positive", "question": "where is `is_external_or_stdlib` defined", "gold": ["src/core/search/graph_resolver.py"], "anchor": false} +{"id": "G-N-flatten", "bucket": "name", "kind": "positive", "question": "where is `flatten` defined", "gold": ["tools/factory/plan.py"], "anchor": false} +{"id": "G-N-_line_starts", "bucket": "name", "kind": "positive", "question": "where is `_line_starts` defined", "gold": ["src/mcp/tools/grep_tool.py"], "anchor": false} +{"id": "G-N-iter_import_candidate_nodes", "bucket": "name", "kind": "positive", "question": "where is `iter_import_candidate_nodes` defined", "gold": ["src/core/language_imports.py"], "anchor": false} +{"id": "G-N-_extract_key_terms", "bucket": "name", "kind": "positive", "question": "where is `_extract_key_terms` defined", "gold": ["src/core/search/utils.py"], "anchor": false} +{"id": "G-N-is_active", "bucket": "name", "kind": "positive", "question": "where is `is_active` defined", "gold": ["src/core/embedder_lease.py"], "anchor": false} +{"id": "G-N-_is_dataclass_decorator", "bucket": "name", "kind": "positive", "question": "where is `_is_dataclass_decorator` defined", "gold": ["src/core/bootstrap_entities.py"], "anchor": false} +{"id": "G-N-register_all_tools", "bucket": "name", "kind": "positive", "question": "where is `register_all_tools` defined", "gold": ["src/mcp/server_tools.py"], "anchor": false} +{"id": "G-N-_server_bin", "bucket": "name", "kind": "positive", "question": "where is `_server_bin` defined", "gold": ["src/core/lsp_client.py"], "anchor": false} +{"id": "G-N-_safe_join", "bucket": "name", "kind": "positive", "question": "where is `_safe_join` defined", "gold": ["src/sources/upload/__init__.py"], "anchor": false} +{"id": "G-N-_ledger_autosync_enabled", "bucket": "name", "kind": "positive", "question": "where is `_ledger_autosync_enabled` defined", "gold": ["src/core/auto_doc_updater.py"], "anchor": false} +{"id": "G-N-expect", "bucket": "name", "kind": "positive", "question": "where is `expect` defined", "gold": ["tools/verification/g5_denominator.py"], "anchor": false} +{"id": "G-N-get_bridge_dir", "bucket": "name", "kind": "positive", "question": "where is `get_bridge_dir` defined", "gold": ["src/core/lsp_project_bridge.py"], "anchor": false} +{"id": "G-N-notify_change", "bucket": "name", "kind": "positive", "question": "where is `notify_change` defined", "gold": ["src/mcp/server_tools.py"], "anchor": false} +{"id": "G-N-log_incident", "bucket": "name", "kind": "positive", "question": "where is `log_incident` defined", "gold": ["src/core/intelligence/tools_reg.py"], "anchor": false} +{"id": "G-N-op_git", "bucket": "name", "kind": "positive", "question": "where is `op_git` defined", "gold": ["tools/factory/ops.py"], "anchor": false} +{"id": "G-N-_project_name", "bucket": "name", "kind": "positive", "question": "where is `_project_name` defined", "gold": ["src/core/bootstrap_tests.py"], "anchor": false} +{"id": "G-N-_llama_bin", "bucket": "name", "kind": "positive", "question": "where is `_llama_bin` defined", "gold": ["src/providers/reranker/llama_install.py"], "anchor": false} +{"id": "G-N-_safe_shutdown", "bucket": "name", "kind": "positive", "question": "where is `_safe_shutdown` defined", "gold": ["src/cli.py"], "anchor": false} +{"id": "G-N-_post", "bucket": "name", "kind": "positive", "question": "where is `_post` defined", "gold": ["tools/factory/brain.py"], "anchor": false} +{"id": "G-N-format_eta", "bucket": "name", "kind": "positive", "question": "where is `format_eta` defined", "gold": ["src/utils/ui_formatter.py"], "anchor": false} +{"id": "G-N-release", "bucket": "name", "kind": "positive", "question": "where is `release` defined", "gold": ["src/core/embedder_lease.py"], "anchor": false} +{"id": "G-N-_extract_diff", "bucket": "name", "kind": "positive", "question": "where is `_extract_diff` defined", "gold": ["tools/microtask_pipeline/engine.py"], "anchor": false} +{"id": "G-N-_with_flags", "bucket": "name", "kind": "positive", "question": "where is `_with_flags` defined", "gold": ["src/core/silent_subprocess.py"], "anchor": false} +{"id": "G-N-format_hotspots", "bucket": "name", "kind": "positive", "question": "where is `format_hotspots` defined", "gold": ["src/utils/ui_formatter.py"], "anchor": false} +{"id": "G-N-run_bounded", "bucket": "name", "kind": "positive", "question": "where is `run_bounded` defined", "gold": ["src/core/run_bounded.py"], "anchor": false} +{"id": "G-N-is_supported", "bucket": "name", "kind": "positive", "question": "where is `is_supported` defined", "gold": ["src/core/extensions.py"], "anchor": false} +{"id": "G-N-step_zed", "bucket": "name", "kind": "positive", "question": "where is `step_zed` defined", "gold": ["install.py"], "anchor": false} +{"id": "G-N-_reject_self_index_target", "bucket": "name", "kind": "positive", "question": "where is `_reject_self_index_target` defined", "gold": ["src/core/project_resolution.py"], "anchor": false} +{"id": "G-N-_ast_blocks", "bucket": "name", "kind": "positive", "question": "where is `_ast_blocks` defined", "gold": ["tools/microtask_pipeline/engine.py"], "anchor": false} +{"id": "G-N-user_brief", "bucket": "name", "kind": "positive", "question": "where is `user_brief` defined", "gold": ["tools/factory/brain.py"], "anchor": false} +{"id": "G-N-verdict_from_results", "bucket": "name", "kind": "positive", "question": "where is `verdict_from_results` defined", "gold": ["src/core/action_receipt.py"], "anchor": false} +{"id": "G-N-_delayed_auto_index", "bucket": "name", "kind": "positive", "question": "where is `_delayed_auto_index` defined", "gold": ["src/mcp/server_factory.py"], "anchor": false} +{"id": "G-N-format_receipt", "bucket": "name", "kind": "positive", "question": "where is `format_receipt` defined", "gold": ["src/core/action_receipt.py"], "anchor": false} +{"id": "G-N-apply_bucket_weights", "bucket": "name", "kind": "positive", "question": "where is `apply_bucket_weights` defined", "gold": ["src/core/search/scoring.py"], "anchor": false} +{"id": "G-N-_parse_iso_datetime", "bucket": "name", "kind": "positive", "question": "where is `_parse_iso_datetime` defined", "gold": ["src/core/search/utils.py"], "anchor": false} +{"id": "G-N-_sanitize_error_message", "bucket": "name", "kind": "positive", "question": "where is `_sanitize_error_message` defined", "gold": ["src/core/error_handler.py"], "anchor": false} +{"id": "G-N-_job_key", "bucket": "name", "kind": "positive", "question": "where is `_job_key` defined", "gold": ["src/mcp/tools/indexing_tools.py"], "anchor": false} +{"id": "G-N-should_skip_file", "bucket": "name", "kind": "positive", "question": "where is `should_skip_file` defined", "gold": ["tools/stale_detector/stale_check.py"], "anchor": false} +{"id": "G-N-_ips_are_global", "bucket": "name", "kind": "positive", "question": "where is `_ips_are_global` defined", "gold": ["src/sources/git_url/__init__.py"], "anchor": false} +{"id": "G-N-apply_unified_diff", "bucket": "name", "kind": "positive", "question": "where is `apply_unified_diff` defined", "gold": ["tools/microtask_pipeline/engine.py"], "anchor": false} +{"id": "G-N-_git_diff", "bucket": "name", "kind": "positive", "question": "where is `_git_diff` defined", "gold": ["src/core/quiet_break_gate.py"], "anchor": false} +{"id": "G-N-normalize_tool_name", "bucket": "name", "kind": "positive", "question": "where is `normalize_tool_name` defined", "gold": ["src/plugins/registry.py"], "anchor": false} +{"id": "G-N-g3_generalization", "bucket": "name", "kind": "positive", "question": "where is `g3_generalization` defined", "gold": ["tools/verification/gates.py"], "anchor": false} +{"id": "G-N-_import_entrypoint", "bucket": "name", "kind": "positive", "question": "where is `_import_entrypoint` defined", "gold": ["src/plugins/loader.py"], "anchor": false} +{"id": "G-N-_ext_dir_from_install_path", "bucket": "name", "kind": "positive", "question": "where is `_ext_dir_from_install_path` defined", "gold": ["adapters/zed/zed_config.py"], "anchor": false} +{"id": "G-N-_verify_ack_token", "bucket": "name", "kind": "positive", "question": "where is `_verify_ack_token` defined", "gold": ["src/core/modification_guard.py"], "anchor": false} +{"id": "G-N-code_first_plan", "bucket": "name", "kind": "positive", "question": "where is `code_first_plan` defined", "gold": ["tools/microtask_pipeline/engine.py"], "anchor": false} +{"id": "G-N-get_system_summary", "bucket": "name", "kind": "positive", "question": "where is `get_system_summary` defined", "gold": ["src/providers/reranker/llama_install.py"], "anchor": false} +{"id": "G-N-_prepend_code_name_matches", "bucket": "name", "kind": "positive", "question": "where is `_prepend_code_name_matches` defined", "gold": ["src/core/search/engine.py"], "anchor": false} +{"id": "G-N-check_engine_compat", "bucket": "name", "kind": "positive", "question": "where is `check_engine_compat` defined", "gold": ["src/plugins/manifest.py"], "anchor": false} +{"id": "G-N-check_disk_space", "bucket": "name", "kind": "positive", "question": "where is `check_disk_space` defined", "gold": ["src/core/artifact_paths.py"], "anchor": false} +{"id": "G-N-_proxy", "bucket": "name", "kind": "positive", "question": "where is `_proxy` defined", "gold": ["src/plugins/registry.py"], "anchor": false} +{"id": "G-N-get_shared_models_dir", "bucket": "name", "kind": "positive", "question": "where is `get_shared_models_dir` defined", "gold": ["src/core/artifact_paths.py"], "anchor": false} +{"id": "G-N-format_report", "bucket": "name", "kind": "positive", "question": "where is `format_report` defined", "gold": ["src/core/doc_reference_l1.py"], "anchor": false} +{"id": "G-N-resolve_mcp_exe", "bucket": "name", "kind": "positive", "question": "where is `resolve_mcp_exe` defined", "gold": ["src/core/process_titles.py"], "anchor": false} +{"id": "G-N-get_actual_version", "bucket": "name", "kind": "positive", "question": "where is `get_actual_version` defined", "gold": ["tools/stale_detector/stale_check.py"], "anchor": false} +{"id": "G-N-_start_zed_parent_watchdog", "bucket": "name", "kind": "positive", "question": "where is `_start_zed_parent_watchdog` defined", "gold": ["src/mcp/server_factory.py"], "anchor": false} +{"id": "G-N-current_platform", "bucket": "name", "kind": "positive", "question": "where is `current_platform` defined", "gold": ["src/plugins/manifest.py"], "anchor": false} +{"id": "G-N-_extract_identifier_tokens", "bucket": "name", "kind": "positive", "question": "where is `_extract_identifier_tokens` defined", "gold": ["src/core/search/engine.py"], "anchor": false} +{"id": "G-N-_extract_package_json", "bucket": "name", "kind": "positive", "question": "where is `_extract_package_json` defined", "gold": ["src/core/deps/manifest/extract.py"], "anchor": false} +{"id": "G-N-failing_test_id", "bucket": "name", "kind": "positive", "question": "where is `failing_test_id` defined", "gold": ["tools/microtask_pipeline/engine.py"], "anchor": false} +{"id": "G-N-run_task", "bucket": "name", "kind": "positive", "question": "where is `run_task` defined", "gold": ["tools/microtask_pipeline/engine.py"], "anchor": false} +{"id": "G-N-referenced_modules", "bucket": "name", "kind": "positive", "question": "where is `referenced_modules` defined", "gold": ["src/core/test_impact.py"], "anchor": false} +{"id": "G-N-_clear_memory", "bucket": "name", "kind": "positive", "question": "where is `_clear_memory` defined", "gold": ["src/mcp/server_factory.py"], "anchor": false} +{"id": "G-N-_cache_key", "bucket": "name", "kind": "positive", "question": "where is `_cache_key` defined", "gold": ["src/core/search/engine.py"], "anchor": false} +{"id": "G-N-get_runtime_status", "bucket": "name", "kind": "positive", "question": "where is `get_runtime_status` defined", "gold": ["src/core/intelligence/tools_reg.py"], "anchor": false} +{"id": "G-N-_extract_bun_lock", "bucket": "name", "kind": "positive", "question": "where is `_extract_bun_lock` defined", "gold": ["src/core/deps/manifest/extract.py"], "anchor": false} +{"id": "G-N-_sigmoid", "bucket": "name", "kind": "positive", "question": "where is `_sigmoid` defined", "gold": ["src/providers/reranker/multi_provider.py"], "anchor": false} +{"id": "G-N-_build_minimal_env", "bucket": "name", "kind": "positive", "question": "where is `_build_minimal_env` defined", "gold": ["src/core/sandbox/executor.py"], "anchor": false} +{"id": "G-N-section", "bucket": "name", "kind": "positive", "question": "where is `section` defined", "gold": ["src/utils/ui_formatter.py"], "anchor": false} +{"id": "G-N-_parse_log_line", "bucket": "name", "kind": "positive", "question": "where is `_parse_log_line` defined", "gold": ["src/core/log_manager.py"], "anchor": false} +{"id": "G-N-_extract_cargo_lock", "bucket": "name", "kind": "positive", "question": "where is `_extract_cargo_lock` defined", "gold": ["src/core/deps/manifest/extract.py"], "anchor": false} +{"id": "G-N-_ensure_lsp", "bucket": "name", "kind": "positive", "question": "where is `_ensure_lsp` defined", "gold": ["src/mcp/tools/lsp_tools.py"], "anchor": false} +{"id": "G-N-_build_vocabulary", "bucket": "name", "kind": "positive", "question": "where is `_build_vocabulary` defined", "gold": ["src/core/doc_reference_l1.py"], "anchor": false} +{"id": "G-N-record_tool_result", "bucket": "name", "kind": "positive", "question": "where is `record_tool_result` defined", "gold": ["src/core/error_handler.py"], "anchor": false} +{"id": "G-N-predict_root_cause", "bucket": "name", "kind": "positive", "question": "where is `predict_root_cause` defined", "gold": ["src/core/intelligence/tools_reg.py"], "anchor": false} +{"id": "G-N-_open_table_calls_in", "bucket": "name", "kind": "positive", "question": "where is `_open_table_calls_in` defined", "gold": ["src/core/bootstrap_entities.py"], "anchor": false} +{"id": "G-N-_print_tree", "bucket": "name", "kind": "positive", "question": "where is `_print_tree` defined", "gold": ["tools/factory/cli.py"], "anchor": false} +{"id": "G-N-_trunc", "bucket": "name", "kind": "positive", "question": "where is `_trunc` defined", "gold": ["install.py"], "anchor": false} +{"id": "G-N-platform_label", "bucket": "name", "kind": "positive", "question": "where is `platform_label` defined", "gold": ["src/core/platform_utils.py"], "anchor": false} +{"id": "G-N-step_models", "bucket": "name", "kind": "positive", "question": "where is `step_models` defined", "gold": ["install.py"], "anchor": false} +{"id": "G-N-_extract_go_mod", "bucket": "name", "kind": "positive", "question": "where is `_extract_go_mod` defined", "gold": ["src/core/deps/manifest/extract.py"], "anchor": false} +{"id": "G-N-_prog", "bucket": "name", "kind": "positive", "question": "where is `_prog` defined", "gold": ["install.py"], "anchor": false} +{"id": "G-N-_parse_entry", "bucket": "name", "kind": "positive", "question": "where is `_parse_entry` defined", "gold": ["src/core/bootstrap_tests.py"], "anchor": false} +{"id": "G-N-_filter_by_time", "bucket": "name", "kind": "positive", "question": "where is `_filter_by_time` defined", "gold": ["src/core/search/utils.py"], "anchor": false} +{"id": "G-N-schema_check", "bucket": "name", "kind": "positive", "question": "where is `schema_check` defined", "gold": ["src/core/search/cypher_schema.py"], "anchor": false} +{"id": "G-N-apply_mcp_process_title", "bucket": "name", "kind": "positive", "question": "where is `apply_mcp_process_title` defined", "gold": ["src/core/process_titles.py"], "anchor": false} +{"id": "G-N-_sha256_text", "bucket": "name", "kind": "positive", "question": "where is `_sha256_text` defined", "gold": ["src/mcp/tools/write_tools.py"], "anchor": false} +{"id": "G-N-ensure_mcp_hardlink", "bucket": "name", "kind": "positive", "question": "where is `ensure_mcp_hardlink` defined", "gold": ["src/core/process_titles.py"], "anchor": false} +{"id": "G-N-_run_trace", "bucket": "name", "kind": "positive", "question": "where is `_run_trace` defined", "gold": ["src/core/bootstrap_pipeline.py"], "anchor": false} +{"id": "G-N-run_plan", "bucket": "name", "kind": "positive", "question": "where is `run_plan` defined", "gold": ["tools/factory/runtime.py"], "anchor": false} +{"id": "G-N-_is_up_to_date", "bucket": "name", "kind": "positive", "question": "where is `_is_up_to_date` defined", "gold": ["install.py"], "anchor": false} +{"id": "G-N-_is_complex_query", "bucket": "name", "kind": "positive", "question": "where is `_is_complex_query` defined", "gold": ["src/mcp/tools/search_tools.py"], "anchor": false} +{"id": "G-N-get_hotspots", "bucket": "name", "kind": "positive", "question": "where is `get_hotspots` defined", "gold": ["src/core/intelligence/tools_reg.py"], "anchor": false} +{"id": "G-N-_manifest_fallback", "bucket": "name", "kind": "positive", "question": "where is `_manifest_fallback` defined", "gold": ["src/sources/git_url/__init__.py"], "anchor": false} +{"id": "G-N-_is_broken", "bucket": "name", "kind": "positive", "question": "where is `_is_broken` defined", "gold": ["src/core/doc_reference_l1.py"], "anchor": false} +{"id": "G-Q-OnnxEmbedderClient.__init__", "bucket": "qualified", "kind": "positive", "question": "where is `OnnxEmbedderClient.__init__` defined", "gold": ["src/core/embedder/onnx_client.py"], "anchor": false} +{"id": "G-Q-PluginProcess.__exit__", "bucket": "qualified", "kind": "positive", "question": "where is `PluginProcess.__exit__` defined", "gold": ["src/plugins/proxy.py"], "anchor": false} +{"id": "G-Q-ProjectContext._capture_memory", "bucket": "qualified", "kind": "positive", "question": "where is `ProjectContext._capture_memory` defined", "gold": ["src/core/intelligence/project_context.py"], "anchor": false} +{"id": "G-Q-FTS5IndexManager.search_names", "bucket": "qualified", "kind": "positive", "question": "where is `FTS5IndexManager.search_names` defined", "gold": ["src/core/search/fts5_index.py"], "anchor": false} +{"id": "G-Q-GraphQueryTool._execute_query", "bucket": "qualified", "kind": "positive", "question": "where is `GraphQueryTool._execute_query` defined", "gold": ["src/mcp/tools/graph_tools.py"], "anchor": false} +{"id": "G-Q-RemoteEmbedder.set_circuit_breaker", "bucket": "qualified", "kind": "positive", "question": "where is `RemoteEmbedder.set_circuit_breaker` defined", "gold": ["src/providers/embedder/remote_embedder.py"], "anchor": false} +{"id": "G-Q-PropertyGraph.import_compressed", "bucket": "qualified", "kind": "positive", "question": "where is `PropertyGraph.import_compressed` defined", "gold": ["src/core/graph.py"], "anchor": false} +{"id": "G-Q-VerifyOnRead.__init__", "bucket": "qualified", "kind": "positive", "question": "where is `VerifyOnRead.__init__` defined", "gold": ["src/core/intelligence/verify_on_read.py"], "anchor": false} +{"id": "G-Q-ResourceMonitor._start_auto_logger", "bucket": "qualified", "kind": "positive", "question": "where is `ResourceMonitor._start_auto_logger` defined", "gold": ["src/core/indexing/resource_monitor.py"], "anchor": false} +{"id": "G-Q-CodeParser._init_tree_sitter", "bucket": "qualified", "kind": "positive", "question": "where is `CodeParser._init_tree_sitter` defined", "gold": ["src/core/indexing/parser.py"], "anchor": false} +{"id": "G-Q-ChangePreview._make_worktree", "bucket": "qualified", "kind": "positive", "question": "where is `ChangePreview._make_worktree` defined", "gold": ["src/core/change_preview.py"], "anchor": false} +{"id": "G-Q-ProjectIntelligenceLayer._collect_signals", "bucket": "qualified", "kind": "positive", "question": "where is `ProjectIntelligenceLayer._collect_signals` defined", "gold": ["src/core/intelligence/layer.py"], "anchor": false} +{"id": "G-Q-PluginProcess.call", "bucket": "qualified", "kind": "positive", "question": "where is `PluginProcess.call` defined", "gold": ["src/plugins/proxy.py"], "anchor": false} +{"id": "G-Q-CypherParser.parse", "bucket": "qualified", "kind": "positive", "question": "where is `CypherParser.parse` defined", "gold": ["src/core/search/cypher_parser.py"], "anchor": false} +{"id": "G-Q-CommitMemory.get_hotspots", "bucket": "qualified", "kind": "positive", "question": "where is `CommitMemory.get_hotspots` defined", "gold": ["src/core/commit_memory.py"], "anchor": false} +{"id": "G-Q-PropertyGraph.get_meta", "bucket": "qualified", "kind": "positive", "question": "where is `PropertyGraph.get_meta` defined", "gold": ["src/core/graph.py"], "anchor": false} +{"id": "G-Q-CypherParser._parse_return_items", "bucket": "qualified", "kind": "positive", "question": "where is `CypherParser._parse_return_items` defined", "gold": ["src/core/search/cypher_parser.py"], "anchor": false} +{"id": "G-Q-_InterProcessLock.__exit__", "bucket": "qualified", "kind": "positive", "question": "where is `_InterProcessLock.__exit__` defined", "gold": ["src/providers/reranker/llama_runner.py"], "anchor": false} +{"id": "G-Q-FTS5IndexManager.__init__", "bucket": "qualified", "kind": "positive", "question": "where is `FTS5IndexManager.__init__` defined", "gold": ["src/core/search/fts5_index.py"], "anchor": false} +{"id": "G-Q-SymbolIndexAdapter._hybrid_add_references", "bucket": "qualified", "kind": "positive", "question": "where is `SymbolIndexAdapter._hybrid_add_references` defined", "gold": ["src/core/search/graph_adapter.py"], "anchor": false} +{"id": "G-Q-CommitMemory.get_significant_commits", "bucket": "qualified", "kind": "positive", "question": "where is `CommitMemory.get_significant_commits` defined", "gold": ["src/core/commit_memory.py"], "anchor": false} +{"id": "G-Q-GetActionReceiptTool.__init__", "bucket": "qualified", "kind": "positive", "question": "where is `GetActionReceiptTool.__init__` defined", "gold": ["src/mcp/tools/lifecycle_tools.py"], "anchor": false} +{"id": "G-Q-SymbolIndexAdapter.get_repo_map", "bucket": "qualified", "kind": "positive", "question": "where is `SymbolIndexAdapter.get_repo_map` defined", "gold": ["src/core/search/graph_adapter.py"], "anchor": false} +{"id": "G-Q-WorkspaceSource.watch", "bucket": "qualified", "kind": "positive", "question": "where is `WorkspaceSource.watch` defined", "gold": ["src/core/interfaces/workspace_source.py"], "anchor": false} +{"id": "G-Q-ProjectIndexerRegistry.find_first_non_self_indexing", "bucket": "qualified", "kind": "positive", "question": "where is `ProjectIndexerRegistry.find_first_non_self_indexing` defined", "gold": ["src/core/indexing/project_indexer_registry.py"], "anchor": false} +{"id": "G-Q-LspClient.document_symbols", "bucket": "qualified", "kind": "positive", "question": "where is `LspClient.document_symbols` defined", "gold": ["src/core/lsp_client.py"], "anchor": false} +{"id": "G-Q-GenerateChunkSummariesTool.__init__", "bucket": "qualified", "kind": "positive", "question": "where is `GenerateChunkSummariesTool.__init__` defined", "gold": ["src/mcp/tools/analysis_tools.py"], "anchor": false} +{"id": "G-Q-GrepCodeTool._format", "bucket": "qualified", "kind": "positive", "question": "where is `GrepCodeTool._format` defined", "gold": ["src/mcp/tools/grep_tool.py"], "anchor": false} +{"id": "G-Q-MCPTool.execute", "bucket": "qualified", "kind": "positive", "question": "where is `MCPTool.execute` defined", "gold": ["src/mcp/tools/base.py"], "anchor": false} +{"id": "G-Q-LanceDBManager.to_pandas_async", "bucket": "qualified", "kind": "positive", "question": "where is `LanceDBManager.to_pandas_async` defined", "gold": ["src/core/indexing/db_manager.py"], "anchor": false} +{"id": "G-Q-LlamaRunner.start_reranker", "bucket": "qualified", "kind": "positive", "question": "where is `LlamaRunner.start_reranker` defined", "gold": ["src/providers/reranker/llama_runner.py"], "anchor": false} +{"id": "G-Q-ConsistencyTracker.mark_stale", "bucket": "qualified", "kind": "positive", "question": "where is `ConsistencyTracker.mark_stale` defined", "gold": ["src/core/consistency.py"], "anchor": false} +{"id": "G-Q-GenerateChunkSummariesTool.execute", "bucket": "qualified", "kind": "positive", "question": "where is `GenerateChunkSummariesTool.execute` defined", "gold": ["src/mcp/tools/analysis_tools.py"], "anchor": false} +{"id": "G-Q-SystemArtifacts.is_artifact", "bucket": "qualified", "kind": "positive", "question": "where is `SystemArtifacts.is_artifact` defined", "gold": ["src/core/system_artifacts.py"], "anchor": false} +{"id": "G-Q-PureGraphMixin._graph_call_chain", "bucket": "qualified", "kind": "positive", "question": "where is `PureGraphMixin._graph_call_chain` defined", "gold": ["src/core/search/graph_adapter_pure.py"], "anchor": false} +{"id": "G-Q-RuntimeCoordinator.__init__", "bucket": "qualified", "kind": "positive", "question": "where is `RuntimeCoordinator.__init__` defined", "gold": ["src/core/runtime_coordinator.py"], "anchor": false} +{"id": "G-Q-MultiProjectSearcher.cross_repo_search", "bucket": "qualified", "kind": "positive", "question": "where is `MultiProjectSearcher.cross_repo_search` defined", "gold": ["src/core/multi_project_searcher.py"], "anchor": false} +{"id": "G-Q-Indexer.to_pandas_async", "bucket": "qualified", "kind": "positive", "question": "where is `Indexer.to_pandas_async` defined", "gold": ["src/core/indexing/indexer.py"], "anchor": false} +{"id": "G-Q-JobHistoryStore.load_history", "bucket": "qualified", "kind": "positive", "question": "where is `JobHistoryStore.load_history` defined", "gold": ["src/core/intelligence/store.py"], "anchor": false} +{"id": "G-Q-LiveBuffer.keys", "bucket": "qualified", "kind": "positive", "question": "where is `LiveBuffer.keys` defined", "gold": ["src/sync/live_buffer.py"], "anchor": false} +{"id": "G-X-000", "bucket": "negative", "kind": "negative", "question": "where is `zz_a81a24a0` defined", "gold": [], "anchor": false} +{"id": "G-X-001", "bucket": "negative", "kind": "negative", "question": "where is `zz_22291929` defined", "gold": [], "anchor": false} +{"id": "G-X-002", "bucket": "negative", "kind": "negative", "question": "where is `zz_65292eea` defined", "gold": [], "anchor": false} +{"id": "G-X-003", "bucket": "negative", "kind": "negative", "question": "where is `zz_18b93f5d` defined", "gold": [], "anchor": false} +{"id": "G-X-004", "bucket": "negative", "kind": "negative", "question": "where is `zz_6bdedec1` defined", "gold": [], "anchor": false} +{"id": "G-X-005", "bucket": "negative", "kind": "negative", "question": "where is `zz_13d1081b` defined", "gold": [], "anchor": false} +{"id": "G-X-006", "bucket": "negative", "kind": "negative", "question": "where is `zz_9f02e2ea` defined", "gold": [], "anchor": false} +{"id": "G-X-007", "bucket": "negative", "kind": "negative", "question": "where is `zz_6b887ea5` defined", "gold": [], "anchor": false} +{"id": "G-X-008", "bucket": "negative", "kind": "negative", "question": "where is `zz_1bac076b` defined", "gold": [], "anchor": false} +{"id": "G-X-009", "bucket": "negative", "kind": "negative", "question": "where is `zz_08cbed4a` defined", "gold": [], "anchor": false} +{"id": "G-X-010", "bucket": "negative", "kind": "negative", "question": "where is `zz_75dcb0a0` defined", "gold": [], "anchor": false} +{"id": "G-X-011", "bucket": "negative", "kind": "negative", "question": "where is `zz_5d6e4f66` defined", "gold": [], "anchor": false} +{"id": "G-X-012", "bucket": "negative", "kind": "negative", "question": "where is `zz_157e8328` defined", "gold": [], "anchor": false} +{"id": "G-X-013", "bucket": "negative", "kind": "negative", "question": "where is `zz_8acc73df` defined", "gold": [], "anchor": false} +{"id": "G-X-014", "bucket": "negative", "kind": "negative", "question": "where is `zz_6bb3df64` defined", "gold": [], "anchor": false} +{"id": "G-X-015", "bucket": "negative", "kind": "negative", "question": "where is `zz_17a64366` defined", "gold": [], "anchor": false} +{"id": "G-X-016", "bucket": "negative", "kind": "negative", "question": "where is `zz_a1a79c56` defined", "gold": [], "anchor": false} +{"id": "G-X-017", "bucket": "negative", "kind": "negative", "question": "where is `zz_003eee6c` defined", "gold": [], "anchor": false} +{"id": "G-X-018", "bucket": "negative", "kind": "negative", "question": "where is `zz_6212927d` defined", "gold": [], "anchor": false} +{"id": "G-X-019", "bucket": "negative", "kind": "negative", "question": "where is `zz_4adc6132` defined", "gold": [], "anchor": false} +{"id": "G-X-020", "bucket": "negative", "kind": "negative", "question": "where is `zz_0d6ce32c` defined", "gold": [], "anchor": false} +{"id": "G-X-021", "bucket": "negative", "kind": "negative", "question": "where is `zz_25164ba3` defined", "gold": [], "anchor": false} +{"id": "G-X-022", "bucket": "negative", "kind": "negative", "question": "where is `zz_28b1baa3` defined", "gold": [], "anchor": false} +{"id": "G-X-023", "bucket": "negative", "kind": "negative", "question": "where is `zz_4c813e44` defined", "gold": [], "anchor": false} +{"id": "G-X-024", "bucket": "negative", "kind": "negative", "question": "where is `zz_87920bdd` defined", "gold": [], "anchor": false} +{"id": "G-X-025", "bucket": "negative", "kind": "negative", "question": "where is `zz_1614b192` defined", "gold": [], "anchor": false} +{"id": "G-X-026", "bucket": "negative", "kind": "negative", "question": "where is `zz_f428c8c3` defined", "gold": [], "anchor": false} +{"id": "G-X-027", "bucket": "negative", "kind": "negative", "question": "where is `zz_7d71074c` defined", "gold": [], "anchor": false} +{"id": "G-X-028", "bucket": "negative", "kind": "negative", "question": "where is `zz_4732fe71` defined", "gold": [], "anchor": false} +{"id": "G-X-029", "bucket": "negative", "kind": "negative", "question": "where is `zz_780f87c5` defined", "gold": [], "anchor": false} +{"id": "G-X-030", "bucket": "negative", "kind": "negative", "question": "where is `zz_dcecd4d0` defined", "gold": [], "anchor": false} +{"id": "G-X-031", "bucket": "negative", "kind": "negative", "question": "where is `zz_03b28aee` defined", "gold": [], "anchor": false} +{"id": "G-X-032", "bucket": "negative", "kind": "negative", "question": "where is `zz_f59feb3e` defined", "gold": [], "anchor": false} +{"id": "G-X-033", "bucket": "negative", "kind": "negative", "question": "where is `zz_2891965f` defined", "gold": [], "anchor": false} +{"id": "G-X-034", "bucket": "negative", "kind": "negative", "question": "where is `zz_1f418482` defined", "gold": [], "anchor": false} +{"id": "G-X-035", "bucket": "negative", "kind": "negative", "question": "where is `zz_b12ab15f` defined", "gold": [], "anchor": false} +{"id": "G-X-036", "bucket": "negative", "kind": "negative", "question": "where is `zz_e11c4137` defined", "gold": [], "anchor": false} +{"id": "G-X-037", "bucket": "negative", "kind": "negative", "question": "where is `zz_7638e5cc` defined", "gold": [], "anchor": false} +{"id": "G-X-038", "bucket": "negative", "kind": "negative", "question": "where is `zz_b86f12b7` defined", "gold": [], "anchor": false} +{"id": "G-X-039", "bucket": "negative", "kind": "negative", "question": "where is `zz_ef763cc1` defined", "gold": [], "anchor": false} +{"id": "P01", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где определяется `build_call_graph` и что оно возвращает?", "gold": ["src/core/indexing/symbol_index.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P02", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Кто вызывает `get_impact_analysis`?", "gold": ["src/core/graph_rag.py", "src/core/modification_guard.py", "src/mcp/tools/investigate_tool.py", "src/mcp/tools/search_tools.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P03", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Что сломается, если изменить сигнатуру `ModificationGuard`?", "gold": ["src/mcp/server_tools.py", "src/mcp/tools/write_tools.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P04", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где решается, нужно ли переиндексировать файл?", "gold": ["src/core/indexing/freshness.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P05", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где вычисляется путь `progress.json` вне проекта?", "gold": ["src/core/artifact_paths.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P06", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Какая функция возвращает путь прогресса?", "gold": ["src/core/artifact_paths.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P07", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где выдаётся и проверяется impact-токен?", "gold": ["src/core/modification_guard.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P08", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где в `investigate` появляется строка `degraded`?", "gold": ["src/mcp/tools/investigate_tool.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P09", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где определяется allowlist/профиль инструментов?", "gold": ["src/mcp/server_tools.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P10", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где задан профиль ETA для `impact_analysis`?", "gold": ["src/core/eta_predictor.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P11", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где объявлен `SymbolIndexAdapter`?", "gold": ["src/core/search/graph_adapter.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P12", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где считается call chain?", "gold": ["src/core/indexing/symbol_index.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P13", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где хранится схема `file_hash` / `file_mtime_ns`?", "gold": ["src/core/indexing/db_manager.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P14", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Какой документ является каноном публичных чисел?", "gold": ["docs/FACTS.md"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P15", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где LSP-инструмент отправляет за ссылками?", "gold": ["src/mcp/tools/lsp_tools.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "P16", "bucket": "anchor:xerj/positive", "kind": "positive", "question": "Где описан контракт «артефакты вне проекта»?", "gold": ["src/core/artifact_paths.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "A01", "bucket": "anchor:xerj/ambiguity", "kind": "ambiguity", "question": "Где реализован `build_call_graph`?", "gold": ["src/core/indexing/symbol_index.py", "src/core/search/graph_adapter.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "A02", "bucket": "anchor:xerj/ambiguity", "kind": "ambiguity", "question": "Где реализован `get_call_chain`?", "gold": ["src/core/indexing/symbol_index.py", "src/core/search/graph_adapter_pure.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "A03", "bucket": "anchor:xerj/ambiguity", "kind": "ambiguity", "question": "Где считается «устаревание» (staleness)?", "gold": ["src/core/auto_doc_updater.py", "src/core/indexing/freshness.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "A04", "bucket": "anchor:xerj/ambiguity", "kind": "ambiguity", "question": "Где определён `Indexer`?", "gold": ["src/core/indexing/indexer.py", "src/core/indexing/indexer_table.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "N01", "bucket": "anchor:xerj/negative", "kind": "negative", "question": "Где реализовано «согласие партнёрской модели» (`partner_agreement`)?", "gold": [], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "N02", "bucket": "anchor:xerj/negative", "kind": "negative", "question": "Где читается `XERJ_FACTS_MODEL_DIR`?", "gold": [], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "N03", "bucket": "anchor:xerj/negative", "kind": "negative", "question": "Где ключ подписи Corpus Hub?", "gold": [], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "N04", "bucket": "anchor:xerj/negative", "kind": "negative", "question": "Где `consensus_partner_agreement`?", "gold": [], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC01", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is the on-demand rerank resolved", "gold": ["src/core/search/engine.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC02", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where does search_code dispatch by mode", "gold": ["src/mcp/tools/search_tools.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC03", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is the project state machine", "gold": ["src/core/indexing/project_indexer_registry.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC04", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "how is the knowledge gate K3 checked", "gold": ["tools/knowledge/check_knowledge.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC05", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where are public claims verified", "gold": ["tools/verification/verify_public_claims.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC06", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is install sync implemented", "gold": ["install.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC07", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "где решается включать ли реранк", "gold": ["src/core/search/engine.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC08", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "где проверяются публичные числа", "gold": ["tools/verification/verify_public_claims.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "MSC09", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "где переключается проект", "gold": ["src/mcp/tools/indexing_tools.py"], "anchor": true, "root": "D:\\Project\\MSCodeBase"} +{"id": "BLA01", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is the default line length defined", "gold": ["src/black/const.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\black"} +{"id": "BLA02", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "how is source parsed into an ast", "gold": ["src/black/parsing.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\black"} +{"id": "BLA03", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is the cache directory chosen", "gold": ["src/black/cache.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\black"} +{"id": "BLA04", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where does line wrapping decide parentheses", "gold": ["src/black/linegen.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\black"} +{"id": "HTT01", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is the /headers response built", "gold": ["httpbin/core.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\httpbin"} +{"id": "HTT02", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "how does digest auth response work", "gold": ["httpbin/helpers.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\httpbin"} +{"id": "HTT03", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where is the user agent endpoint defined", "gold": ["httpbin/core.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\httpbin"} +{"id": "HTT04", "bucket": "anchor:mcp_audit/positive", "kind": "positive", "question": "where are cookies parsed", "gold": ["httpbin/core.py"], "anchor": true, "root": "D:\\Project\\bench_projects\\httpbin"} diff --git a/experiments/xerj_2026_11/wilson.py b/experiments/xerj_2026_11/wilson.py index 90a639b9..1e300ff7 100644 --- a/experiments/xerj_2026_11/wilson.py +++ b/experiments/xerj_2026_11/wilson.py @@ -47,6 +47,10 @@ def wilson(k: int, n: int, z: float = Z) -> tuple[float, float]: ("E65 metamorphic top1, rrf", 4, 12), ("E65 metamorphic top1, norm", 5, 12), ("E65 negatives empty, weights", 4, 4), + ("E66 generated name top1", 154, 160), + ("E66 generated name any@8", 157, 160), + ("E66 generated qualified top1", 38, 40), + ("E66 generated negative empty", 40, 40), ) From 6f26315bed035d2f175fca17e0137cb41d581518 Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Tue, 10 Nov 2026 16:16:27 +0300 Subject: [PATCH 2/2] test(experiment): red-team the scale panel before trusting its number Three controls on the E66 panel, and one of them changes what the number may be used for. Grep baseline (a file that literally defines the symbol, first in path order): bucket grep top1 tool top1 grep any tool any name 147/160 154/160 160/160 157/160 qualified 23/40 38/40 33/40 40/40 The `name` bucket is close to a grep test (91.9% vs 96.3%); `qualified` is where the tool actually separates (57.5% vs 95%). So 0.963 is a regression detector, not a quality claim, and the report now says that next to the number. The three `any` misses are one bug, not three: `risk_level`, `module_of` and `referenced_modules` live in `src/core/test_impact.py`, and the bucket classifier penalises it by -2.0 because the file name starts with `test_`. Blast radius: 926 tracked .py files, 271 match the test regex, 260 of them are real tests, 1 production module is misclassified. `src/core/test_impact.py` is exactly the file that computes test impact, so the penalty is backwards. Fixing it is E67 with its own frozen bar, not a silent edit inside this PR. Determinism: 24 tasks x 3 repeats are stable, but one task (`MultiProjectSearcher.cross_repo_search`) scored False in the original run and True in all three repeats afterwards. Single-run rates carry session noise, so published numbers from this panel need 3 repeats, as in E65. Also recorded: selection bias (791 eligible name symbols, 160 sampled, short and dunder names dropped) and the leakage the panel cannot see (gold comes from the index, so indexing gaps are invisible). --- experiments/scale_2026_11/REDTEAM.md | 96 ++++++++++++++++++++ experiments/scale_2026_11/REPORT.md | 48 ++++++++-- experiments/scale_2026_11/controls.py | 121 ++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 9 deletions(-) create mode 100644 experiments/scale_2026_11/REDTEAM.md create mode 100644 experiments/scale_2026_11/controls.py diff --git a/experiments/scale_2026_11/REDTEAM.md b/experiments/scale_2026_11/REDTEAM.md new file mode 100644 index 00000000..b03a0ec0 --- /dev/null +++ b/experiments/scale_2026_11/REDTEAM.md @@ -0,0 +1,96 @@ +# Scale panel — red team (after the first run) + +Attacks on the E66 result. Two of them are uncomfortable, and both change what +the number may be used for. + +## A1. Triviality: does plain grep solve this panel? + +Control: for each positive task, find every `.py` file whose source literally +defines the symbol (`^\s*(async def|def|class) \b`) and take the first hit +in path order. No index, no ranking. + +``` +bucket grep top1 tool top1 grep any tool any +name 147/160 154/160 160/160 157/160 +qualified 23/40 38/40 33/40 40/40 +``` + +**Verdict: the `name` bucket is a grep test.** A one-line `grep` with no +retrieval at all reaches 91.9% top-1 and 100% recall, against the tool's 96.3% +and 98.1%. The difference is real but small, and the tool *loses recall*: three +tasks that grep finds are missed entirely. + +`qualified` is the opposite: grep 57.5% vs tool 95%, so that bucket does +discriminate. Twelve points of the headline rate come from 40 tasks, not 160. + +**Consequence:** the 0.963 `name` rate must not be published as search quality. +It is a regression detector, as the frozen plan said — but now with a number +behind the caveat. + +## A2. A real bug the panel found: `test_impact.py` is treated as a test file + +The three `any` misses (tasks 1, 43, 127) are all definitions in +`src/core/test_impact.py` — `risk_level:178`, `module_of:55`, +`referenced_modules:63`. The tool returns `abstain` or never lists the file, +while grep finds it at line level. + +Cause: `_bucket_penalty` classifies by path, and `_TEST_PATH_RE` matches any +`test_*.py`, including a production module named for *test impact prediction*: + +``` +_bucket_penalty('src/core/test_impact.py', 'where is risk_level defined') -> -2.0 +_bucket_penalty('tests/test_x.py', 'where is X defined') -> -2.0 +_bucket_penalty('src/core/artifact_paths.py', ...) -> 0.0 +``` + +Blast radius, measured: 926 tracked `.py` files; 271 match the regex; 260 of +them also contain `def test_` or import a test framework (real tests); **1 +production module is misclassified** (`src/core/test_impact.py`). The other +matches are under `experiments/` and `scripts/`, which the panel does not sample. + +This is the most valuable output of the run: a defect the 41-task panel never +touched, because none of its questions pointed at that file. + +## A3. Selection bias: who is in the panel + +``` +graph definitions 5958 symbols +with exactly one definition 5809 +after code-only filter 2584 +matching the `name` regex 791 -> 160 sampled (20%) +matching `qualified` regex 1483 -> 40 sampled (3%) +dropped by regex (short/dunder) 310 +``` + +The panel samples distinctive names only: 5-40 characters, no dunders, no +one-letter helpers. That is a defensible choice (a question about `x` is not a +question), but it makes the panel easier than the corpus, and the rate inherits +that. + +## A4. Determinism: one run is not enough + +24 positive tasks x 3 repeats: `unstable_metric=0`, `unstable_top3_rows=0`. +But task 197 (`MultiProjectSearcher.cross_repo_search`) scored `top1=False` in +the original run and `top1=True` in all three repeats afterwards, with the same +code and the same panel. So the per-task result can move *between sessions* +(index freshness), even though it is stable within one. + +**Consequence:** a single-run rate is a point with unmeasured session noise. +Every published number from this panel must be a 3-repeat majority, as in E65. + +## A5. Leakage the panel cannot see + +Gold comes from the index, so by construction the gold file is indexed. The panel +therefore cannot detect indexing gaps (a file the indexer skipped is never a +gold). It measures ranking and name resolution, not coverage. + +## What changes because of this + +1. E66 stays as **instrument calibration** (the anchors reproduced E65 exactly), + not as a quality claim, and `REPORT.md` now says so with the grep control in + the same table. +2. The `test_impact.py` misclassification becomes its own frozen experiment + (E67), not a silent edit: bar is `any@8` +3 on this panel with no loss on the + anchors, 3 repeats. +3. Published rates from this panel require 3 repeats and the grep control next + to them. diff --git a/experiments/scale_2026_11/REPORT.md b/experiments/scale_2026_11/REPORT.md index 82ca830c..9ddcca9b 100644 --- a/experiments/scale_2026_11/REPORT.md +++ b/experiments/scale_2026_11/REPORT.md @@ -37,22 +37,52 @@ experiments, two different runners, same panel: the measurement is stable enough to compare against. **It does not mean "search is 96% accurate".** The generated questions contain -the identifier (`where is \`resolve_llama_role_bin\` defined`), so they measure -the symbol channel and the ranking around it. A tool that greps the name passes -them. The number is a **regression detector**, not a quality claim: it says -"exact names resolve, 154 of 160 times", and nothing about prose questions -(`hints`, E64), ambiguous golds (XERJ A01-A04) or docs routing. +the identifier, and the grep control below shows it: a plain `grep` reaches +91.9% on the same `name` tasks. The number is a **regression detector**, not a +quality claim — it says "exact names resolve, 154 of 160 times", and nothing +about prose questions (`hints`, E64), ambiguous golds (XERJ A01-A04) or docs +routing. The cost column shows the same thing from another side: a generated task costs 2163 chars per correct answer, an XERJ prose task 5894. The difference is task difficulty, not an improvement in the tool. +## Controls (red team, `REDTEAM.md`) + +**grep baseline** — every `.py` file that literally defines the symbol, first +hit in path order: + +``` +bucket grep top1 tool top1 grep any tool any +name 147/160 154/160 160/160 157/160 +qualified 23/40 38/40 33/40 40/40 +``` + +So the `name` bucket is close to a grep test (91.9% vs 96.3%), while `qualified` +discriminates strongly (57.5% vs 95%). Read the headline rate with that in mind. + +**Selection bias** — 5958 graph symbols -> 5809 unique -> 2584 code-only -> +791 eligible `name` names -> 160 sampled (20%). The panel uses distinctive names +(5-40 chars, no dunders), which makes it easier than the corpus. + +**Determinism** — 24 tasks x 3 repeats: metric stable. But one task +(`MultiProjectSearcher.cross_repo_search`) differed between the original run and +later repeats, so a single-run rate carries session noise: published numbers +from this panel need 3 repeats, as in E65. + ## The 6 misses, honestly -Six `name` tasks and two `qualified` tasks did not put the gold file first -(`any@8`: 3 and 0 misses). They were not inspected individually in this run — -that is the next step if the number has to move: the generator records the id, -so a miss list is reproducible from the same commit (`tasks.jsonl` + seed). +Three of the six are one bug, not three retrieval misses: `risk_level`, +`module_of` and `referenced_modules` all live in `src/core/test_impact.py`, and +the bucket classifier penalises that production module by `-2.0` because its +name starts with `test_`. Measured blast radius: 926 `.py` files, 271 match the +test regex, 1 production module is misclassified. That is E67, with its own +frozen bar. + +The remaining three are ranking (`expect`, `CommitMemory.get_hotspots` found at +row 2) and name resolution. The generator records ids, so the list is +reproducible from the same commit. + ## Instrument findings (found while building the panel, not fixed here) diff --git a/experiments/scale_2026_11/controls.py b/experiments/scale_2026_11/controls.py new file mode 100644 index 00000000..f7f1d7ba --- /dev/null +++ b/experiments/scale_2026_11/controls.py @@ -0,0 +1,121 @@ +"""Red-team controls for the E66 panel. + +Three questions, all adversarial: + +1. **Triviality** — would `grep "def "` score the same? If yes, the panel + measures grep, not search, and the number must not be sold as quality. +2. **Correctness of the misses** — for every top-1 miss, is the gold really a + definition, and what did the tool return instead? +3. **Selection bias** — how many symbols did the generator drop, and why? A + panel built only from distinctive, unique names is easier than the corpus. + +Run: python experiments/scale_2026_11/controls.py +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.stdout.reconfigure(encoding="utf-8") + +TASKS = HERE / "tasks.jsonl" +RESULTS = HERE / "results.json" + + +def _tasks() -> list[dict]: + return [json.loads(x) for x in TASKS.read_text(encoding="utf-8").splitlines() if x.strip()] + + +def _py_files() -> list[str]: + out = subprocess.run(["git", "ls-files", "*.py"], cwd=ROOT, + capture_output=True, text=True, encoding="utf-8") + return sorted(x for x in out.stdout.splitlines() if x.strip()) + + +def _symbol(question: str) -> str: + match = re.search(r"`([^`]+)`", question) + return match.group(1).split(".")[-1] if match else "" + + +def main() -> int: + tasks = _tasks() + files = _py_files() + cache: dict[str, str] = {} + + def text_of(path: str) -> str: + if path not in cache: + try: + cache[path] = (ROOT / path).read_text(encoding="utf-8", errors="ignore") + except OSError: + cache[path] = "" + return cache[path] + + print("== 1. grep control (definition lines, path order) ==") + rows = [] + for task in tasks: + if task["kind"] != "positive": + continue + name = _symbol(task["question"]) + pattern = re.compile( + rf"^\s*(async\s+def|def|class)\s+{re.escape(name)}\b", re.M + ) + hits = [path for path in files if pattern.search(text_of(path))] + gold = task["gold"][0] + top1 = bool(hits) and hits[0].endswith(gold) + any8 = any(h.endswith(gold) for h in hits[:8]) + rows.append({"bucket": task["bucket"], "top1": top1, "any": any8}) + + for bucket in ("name", "qualified"): + subset = [r for r in rows if r["bucket"] == bucket] + if subset: + print(f" grep {bucket:<10} top1={sum(r['top1'] for r in subset)}/{len(subset)} " + f"any={sum(r['any'] for r in subset)}/{len(subset)}") + + result_data = json.loads(RESULTS.read_text(encoding="utf-8")) + summary = result_data["summary"] + for bucket in ("name", "qualified"): + cell = summary.get(bucket) + if cell: + print(f" tool {bucket:<10} top1={cell['top1']}/{cell['n']} " + f"any={cell['any']}/{cell['n']}") + + print("\n== 2. the misses, one by one ==") + # Per-task rows are not stored; re-derive the list from the run log if present. + log = Path(r"C:\Users\misha\AppData\Local\Temp\opencode\scale_run2.log") + misses = [] + if log.is_file(): + for line in log.read_text(encoding="utf-8", errors="ignore").splitlines(): + match = re.search(r"^\s+\d+/\d+ (name|qualified).*top1=False", line) + if match: + misses.append(line.strip()) + print(f" misses found in the run log: {len(misses)}") + for line in misses[:10]: + print(" ", line[:120]) + + print("\n== 3. what the generator dropped ==") + from generate import _definitions, SKIP_FILES, NAME_RE, QUALIFIED_RE # noqa: E402 + + defs = _definitions() + unique = {s: r for s, r in defs.items() if len(r) == 1} + code = {s: r for s, r in unique.items() + if not any(part in r[0][0] for part in SKIP_FILES)} + names = [s for s in code if NAME_RE.match(s)] + qualified = [s for s in code if QUALIFIED_RE.match(s)] + print(f" graph definitions: {len(defs)} symbols, {len(unique)} with one definition") + print(f" after code-only filter: {len(code)}") + print(f" matching `name` regex: {len(names)} (panel uses 160)") + print(f" matching `qualified` regex: {len(qualified)} (panel uses 40)") + print(f" dropped by regex (short/dunder/generic names): " + f"{len(code) - len(set(names) | set(qualified))}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())