From 2f8da195d141f2ecbf70f4879f6d8ebf0c1466a5 Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Wed, 11 Nov 2026 00:05:14 +0300 Subject: [PATCH 1/3] =?UTF-8?q?test(search):=20measured=20file-dedup=20?= =?UTF-8?q?=E2=80=94=20removes=20repeats,=20moves=20no=20score?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The external benchmark said the tool answers with chunks while grep ranks whole files, and 22-29% of the answer rows repeated a file already shown. This tests whether that is the gap: one row per file first, leftover rows fill the tail, behind MSCODEBASE_FILE_DEDUP=file, off by default. Frozen spec, written before the code: experiments/file_dedup_2026_11/frozen/PLAN.md sqlalchemy, 40 commits hit@5 flag off 11 neighbours 19 · search 18 · grep 20 flag on 11 (bar >=14: FAILED) B1 dev, 71 commits 19/71 both ways (bar 19: PASS) usage probes (P3) 5/20 kept (bar 3 of 4 groups = 15/20: FAILED) flag off, 10 answers sha256-identical to HEAD (P4: PASS) The flag did work: on the same 20 commits duplicate rows went 25 -> 0, and 10 of those 25 sat at ranks 1-5. The swaps were real and neutral. What limits the tool is which distinct files rank where, not that one file filled several slots. The cost is real for "where is X used": once 8 distinct files are found the second pass never runs, so same-file call sites collapse to one row. Side finding, recorded rather than fixed: 6 of 20 sqlalchemy commits get an empty answer (0 rows) with and without the flag. Verdict: negative result. The switch stays experimental and off and is documented as such in all three CONFIGURATION.md files. "Dedup by file" is not the explanation for the grep gap and must not be shipped as one. P4 needed a second run: the first pair was measured while the sqlalchemy benchmark held the same SQLite index, one baseline answer carried "degraded: impact:InterfaceError", and the two files differed - the harness, not the code. --- AGENT_DIARY.md | 5 + EXPERIMENTS_LOG.md | 33 +++++ docs/en/CONFIGURATION.md | 1 + docs/ru/CONFIGURATION.md | 1 + docs/zh/CONFIGURATION.md | 1 + experiments/file_dedup_2026_11/REPORT.md | 120 ++++++++++++++++++ .../file_dedup_2026_11/duplicate_positions.py | 87 +++++++++++++ experiments/file_dedup_2026_11/frozen/PLAN.md | 76 +++++++++++ experiments/file_dedup_2026_11/p4_identity.py | 82 ++++++++++++ experiments/file_dedup_2026_11/usage_probe.py | 110 ++++++++++++++++ src/mcp/tools/investigate_tool.py | 39 ++++++ tests/test_file_dedup.py | 65 ++++++++++ 12 files changed, 620 insertions(+) create mode 100644 experiments/file_dedup_2026_11/REPORT.md create mode 100644 experiments/file_dedup_2026_11/duplicate_positions.py create mode 100644 experiments/file_dedup_2026_11/frozen/PLAN.md create mode 100644 experiments/file_dedup_2026_11/p4_identity.py create mode 100644 experiments/file_dedup_2026_11/usage_probe.py create mode 100644 tests/test_file_dedup.py diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index 2bcddfc8..f5111c36 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -57,6 +57,11 @@ Пункты «RU→EN словарь» и «concept→symbol словарь» сняты: их заменяет контракт. - **Урок:** не учить сервер языкам, которые знает хост; отдавать `hints` и честно просить их, когда в вопросе нет code-имени. +- **E72 (file dedup):** `MSCODEBASE_FILE_DEDUP=file` убрал все повторы файлов + (25→0 на 20 коммитах), но hit@5 остался 11/40 на sqlalchemy и 19/71 на B1 dev. + 10 из 25 повторов стояли в рангах 1–5, то есть замена была реальной и всё равно + нейтральной. Цена: «где используется X» теряет строки одного файла (5/20 проб). + Флаг оставлен экспериментальным и выключенным; дедуп не объясняет разрыв с grep. - **E71 (Change-Id):** хеш из Gerrit занимал первый слот идентификаторов, гейт считал `0` вхождений и отказывал на каждом коммите sqlalchemy (0/40). Фильтр хешей и трейлеров поднял tool до 11/40, B1 dev не изменился (19/38/22/2). diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index dbd33cc8..b3553ecd 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -1,4 +1,37 @@ # EXPERIMENTS_LOG.md — Audit Verification (2026-07-22) +## [2026-11-11] — E72: file-level dedup changes the shape, not the score + +**Why:** on external repos the tool answers with chunks while grep ranks whole +files, and 22-29% of answer rows repeated a file already shown. Hypothesis: that +is the gap. Frozen spec written before the code: +`experiments/file_dedup_2026_11/frozen/PLAN.md`. + +**Change under test:** `MSCODEBASE_FILE_DEDUP=file` — one row per file first, +leftover rows fill the tail; off by default. + +``` +sqlalchemy, 40 commits hit@5 + flag off 11 neighbours 19 · search 18 · grep 20 + flag on 11 (>=14 was the bar: FAILED) +B1 dev, 71 commits 19/71 both ways (bar 19: PASS) +usage probes (P3) 5/20 kept (bar 3 of 4 groups = 15/20: FAILED) +flag off, 10 answers sha256 identical to HEAD (P4: PASS) +``` + +The flag did work: on the same 20 commits duplicates went 25 -> 0 +(`experiments/file_dedup_2026_11/duplicate_positions.py`). 10 of those 25 repeats +sat at ranks 1-5, so the swaps were real and still neutral: 11/40 either way. +The limiting factor is which distinct files rank where, not that one file filled +several slots. + +**Side finding:** 6 of 20 sqlalchemy commits get an empty answer (0 rows), with +and without the flag. On that repo the abstain is the bigger gap, and it is not +addressed by this experiment. + +**Verdict:** negative result. The switch stays experimental and off; it is +documented as such in all three `CONFIGURATION.md` files. "Dedup by file" must +not be shipped, or claimed, as the explanation for the grep gap. + ## [2026-11-10] — E71: a Gerrit `Change-Id` made the gate abstain on every commit **Found by** the sqlalchemy run of B2: tool 0/40 at hit@5 while neighbours got diff --git a/docs/en/CONFIGURATION.md b/docs/en/CONFIGURATION.md index 5ad33d66..e2783941 100644 --- a/docs/en/CONFIGURATION.md +++ b/docs/en/CONFIGURATION.md @@ -58,6 +58,7 @@ are gitignored. | `MSCODEBASE_EVIDENCE_GATE` | `true` | Verify each `investigate` row (definition/ast/fresh/graph) and print a deterministic verdict; `0` restores the pre-gate answer | | `MSCODEBASE_ASK_REWRITE` | `smart` | `on` — ask for an identifier whenever a Cyrillic question has no code token; `smart` — only when it also names a proper-noun-like Latin word (XERJ N03); `off` — return neighbours with a verdict. Measured: `experiments/xerj_2026_11/evidence_gate/REPORT_REWRITE.md` | | `MSCODEBASE_FUSION` | `weights` | merge formula for `investigate`: `weights` — channel score + hand weight (shipped); `rrf` — reciprocal-rank sum over the channels that found a row; `norm` — both terms on a [0, 1] scale. Unknown or empty falls back to `weights`. This is an experiment switch, not a tuning knob: `experiments/xerj_2026_11/fusion/` | +| `MSCODEBASE_FILE_DEDUP` | *(off)* | answer ordering for `investigate`: `file` — one row per file first, the leftover rows fill the tail; off — the shipped order. Unknown or empty is off. Measured in E72: no hit@5 change on `sqlalchemy` (11/40) or B1 dev (19/71), and "where is X used" lost its same-file rows (5 of 20 probes kept) — an experiment switch, not a default: `experiments/file_dedup_2026_11/` | | `MSCODEBASE_SCRIPTS_DEMOTE` | `true` | Penalise `scripts/`, `examples/`, `benches/`, `fuzz/` rows for code questions, like tests and docs. Measured: XERJ P05 top-1 recovered, no `any` lost (`experiments/xerj_2026_11/probe_scripts_demote.py`) | `EXPERIMENTS/**` is excluded from indexing by `FileGuard` regardless of these flags: diff --git a/docs/ru/CONFIGURATION.md b/docs/ru/CONFIGURATION.md index 495ab146..bfab46e8 100644 --- a/docs/ru/CONFIGURATION.md +++ b/docs/ru/CONFIGURATION.md @@ -61,6 +61,7 @@ | `MSCODEBASE_EVIDENCE_GATE` | `true` | Проверять каждую строку `investigate` (definition/ast/fresh/graph) и печатать детерминированный вердикт; `0` возвращает ответ до гейта | | `MSCODEBASE_ASK_REWRITE` | `smart` | `on` — просить идентификатор для любого кириллического вопроса без code-токена; `smart` — только если в нём есть proper-noun-подобное латинское слово (XERJ N03); `off` — возвращать соседей с вердиктом. Замер: `experiments/xerj_2026_11/evidence_gate/REPORT_REWRITE.md` | | `MSCODEBASE_FUSION` | `weights` | формула слияния для `investigate`: `weights` — score канала + ручной вес (шипится); `rrf` — сумма обратных рангов по каналам, нашедшим строку; `norm` — оба слагаемых в [0, 1]. Неизвестное или пустое значение откатывается к `weights`. Это переключатель эксперимента, а не ручка настройки: `experiments/xerj_2026_11/fusion/` | +| `MSCODEBASE_FILE_DEDUP` | *(выкл.)* | порядок строк ответа `investigate`: `file` — сначала по одной строке на файл, остаток добивает хвост; выкл. — шипящийся порядок. Неизвестное или пустое значение = выкл. Замер E72: hit@5 не изменился ни на `sqlalchemy` (11/40), ни на B1 dev (19/71), а вопрос «где используется X» терял строки одного файла (сохранились 5 из 20 проб) — переключатель эксперимента, не дефолт: `experiments/file_dedup_2026_11/` | | `MSCODEBASE_SCRIPTS_DEMOTE` | `true` | Понижать строки из `scripts/`, `examples/`, `benches/`, `fuzz/` на code-вопросах, как tests и docs. Замер: XERJ P05 вернулся в top-1, `any` не потерян (`experiments/xerj_2026_11/probe_scripts_demote.py`) | `EXPERIMENTS/**` исключён из индексации через `FileGuard` независимо от этих флагов: diff --git a/docs/zh/CONFIGURATION.md b/docs/zh/CONFIGURATION.md index 21c82f5c..3b2a4f7c 100644 --- a/docs/zh/CONFIGURATION.md +++ b/docs/zh/CONFIGURATION.md @@ -57,6 +57,7 @@ | `MSCODEBASE_EVIDENCE_GATE` | `true` | 校验 `investigate` 的每一行（definition/ast/fresh/graph）并打印确定性结论；`0` 恢复门控前行为 | | `MSCODEBASE_ASK_REWRITE` | `smart` | `on` —— 任何没有代码形态标识符的西里尔文问题都要求给出标识符；`smart` —— 仅当问题还包含类专有名词的拉丁词（XERJ N03）；`off` —— 返回邻居并带结论。实测：`experiments/xerj_2026_11/evidence_gate/REPORT_REWRITE.md` | | `MSCODEBASE_FUSION` | `weights` | `investigate` 的合并公式：`weights` —— 通道得分 + 手工权重（默认发布）；`rrf` —— 对命中该行的通道求倒数秩之和；`norm` —— 两项都归一化到 [0, 1]。未知或空值回退到 `weights`。这是实验开关，不是调参旋钮：`experiments/xerj_2026_11/fusion/` | +| `MSCODEBASE_FILE_DEDUP` | *（关）* | `investigate` 的作答顺序：`file` —— 每个文件先出一行，剩余行补到末尾；关 —— 发布顺序。未知或空值即为关。E72 实测：`sqlalchemy`（11/40）与 B1 dev（19/71）的 hit@5 均未变化，而“X 在哪里被使用”丢失了同一文件的行（20 个探针中保留 5 个）—— 这是实验开关，不是默认行为：`experiments/file_dedup_2026_11/` | | `MSCODEBASE_SCRIPTS_DEMOTE` | `true` | 对代码问题降低 `scripts/`、`examples/`、`benches/`、`fuzz/` 行的排名，与 tests、docs 相同。实测：XERJ P05 回到 top-1，`any` 无损失（`experiments/xerj_2026_11/probe_scripts_demote.py`） | 无论这些开关如何设置，`EXPERIMENTS/**` 都会被 `FileGuard` 排除在索引之外： diff --git a/experiments/file_dedup_2026_11/REPORT.md b/experiments/file_dedup_2026_11/REPORT.md new file mode 100644 index 00000000..044e6911 --- /dev/null +++ b/experiments/file_dedup_2026_11/REPORT.md @@ -0,0 +1,120 @@ +# E72 — file-level dedup: negative result + +Date: 2026-11-10/11. Frozen spec: `frozen/PLAN.md` (written before any code +change). Switch: `MSCODEBASE_FILE_DEDUP=file`, **off by default**. + +## What was asked + +The tool answers with chunks; a "which file" question wants files. On 10 +sqlalchemy tasks 72 rows covered 51 files (0.71 files per row, worst case 8 rows += 1 file). Hypothesis: 29% of the answer slots repeat a file, so grep — which +ranks whole files — wins on external repos. + +The repeat share on a larger sample (20 commits, below) is 22% of rows, not 29%; +the direction is the same, the number is not. Both samples are reported rather +than the rounder one. + +## What was measured + +Corpora: `sqlalchemy` at `8b62343` (668 files / 34 984 chunks, 40 commits) and +B1 dev (71 commits, calibrated in E65/E67/E71). + +``` +sqlalchemy, 40 tasks hit@1 hit@5 + tool, flag off 8 11 + tool, flag on 8 11 <- P1 (>=14) FAILED + neighbours 11 19 + search 12 18 + grep 7 20 +``` + +``` +B1 dev, 71 tasks tool neighbours search grep + flag off 19/71 38 22 2 + flag on 19/71 38 22 2 +``` + +**The flag changed no hit@5 number on either corpus**, and it did work: with the +flag on, the same answers carry **0 duplicate rows instead of 25**. So the +repeats were real and were removed — the score simply did not move. + +`duplicate_positions.py`, 20 sqlalchemy commits, flag off: + +``` +rows=112 duplicate_rows=25 (0.22 of rows) +duplicate row positions (1-based in the answer): + rank 2: 1 rank 3: 2 rank 4: 4 rank 5: 3 + rank 6: 5 rank 7: 3 rank 8: 7 +duplicates at ranks 1-5: 10 of 25 +``` + +So 10 of the 25 repeats sat inside the top-5 that hit@5 counts, and replacing +them with distinct files still produced 11/40 — the swaps were neutral: the +duplicate was not what kept the gold file out. What limits the tool is **which +distinct files rank where**, not that one file took several slots. + +One more fact from the same run, unrelated to the flag but larger than it: +**6 of 20** sqlalchemy commits get an empty answer (0 rows) both with and +without the flag. On that repo the abstain is a bigger gap than dedup, and it is +not addressed here. + +## The harm the flag does (P3) + +20 probes built from the symbol index: functions with >= 2 distinct call lines +inside one file, asked as `where is used`. + +``` +probes kept (file still contributes >= 2 rows, before and after): 5 / 20 +``` + +Examples: + +``` +select_from lib/sqlalchemy/dialects/mssql/base.py before=3 after=1 +_join_fixture_compound_expression_1 test/orm/test_rel_fn.py before=8 after=1 +_get_custom_operator_dispatch lib/sqlalchemy/sql/compiler.py before=7 after=1 +extend_string (kept) lib/sqlalchemy/dialects/mysql/base.py before=8 after=3 +``` + +When the answer has >= 8 distinct files, the first pass fills all 8 slots and +the same-file rows are pushed out entirely — which is exactly the wrong answer +for a "where is X used" question. Two probes (`le_`, `to_instance`) never +contained the file before either; they are not losses, they are probes whose +answer was already wrong. + +## Gates + +| # | Gate | Result | +|---|---|---| +| P1 | sqlalchemy tool hit@5 >= 14/40 | **FAIL** (11, identical to flag off) | +| P2 | B1 dev >= 19/71 | PASS (19, identical) | +| P3 | >= 3 of 4 probe groups keep same-file rows | **FAIL** (5/20 = 25%; the frozen "3 of 4 groups" is 75%, i.e. 15 of 20 probes) | +| P4 | flag off byte-identical on 10 tasks | PASS (`sha256` equal, sequential rerun) | + +P4 needed a second run: the first pair was measured while the sqlalchemy +benchmark held the same SQLite index, one baseline answer carried +`degraded: impact:InterfaceError`, and the two files differed. The difference +was the harness, not the code — with both runs sequential the two files are +`sha256`-identical. + +## Verdict + +The gap to grep is **not** a chunk-vs-file artifact. Dedup removes every +duplicate row (25 -> 0 on the same 20 commits) and hit@5 stays at 11/40, so the +repeated files were not what pushed the gold out of the top-5. The cost is real +though: "where is X used" loses its same-file rows (5 of 20 probes kept), because +once 8 distinct files are found the second pass never runs. + +Consequence for the product: keep the switch experimental and off. Do not claim, +and do not ship, "dedup by file" as the explanation for the grep gap. The next +honest question is the one this run points at: why 6 of 20 external commits get +an empty answer at all, and why the distinct files rank below grep's. + +## Reproduce + +``` +python experiments/file_dedup_2026_11/p4_identity.py --src ... +$env:MSCODEBASE_FILE_DEDUP="file"; python experiments/bench_2026_11/external.py ... +python experiments/file_dedup_2026_11/usage_probe.py --corpus D:\Project\bench-ext\sa-B +python experiments/file_dedup_2026_11/duplicate_positions.py --corpus ... --repo ... +``` diff --git a/experiments/file_dedup_2026_11/duplicate_positions.py b/experiments/file_dedup_2026_11/duplicate_positions.py new file mode 100644 index 00000000..18561940 --- /dev/null +++ b/experiments/file_dedup_2026_11/duplicate_positions.py @@ -0,0 +1,87 @@ +"""Where do the repeated files sit in the answer? + +E72 asked whether rank-5 localization loses to grep because a file can fill +several of the 8 slots. If the repeats sit at ranks 6-8, reordering cannot move +hit@5 - which is what the sqlalchemy run showed (11/40 either way). +""" + +from __future__ import annotations + +import argparse +import asyncio +import re +import subprocess +import sys +from collections import Counter +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.stdout.reconfigure(encoding="utf-8") + +from src.core.di_container import ( # noqa: E402 + IndexerFactoryKey, + ProjectIndexerRegistry, + create_service_collection, +) +from src.mcp.tools.investigate_tool import InvestigateTool # noqa: E402 + +ROW_RE = re.compile(r"^\d+\. (.+?):(\d+) ", re.M) + + +def _git(repo: Path, *args: str) -> str: + out = subprocess.run(["git", *args], cwd=repo, capture_output=True, + text=True, encoding="utf-8", errors="replace") + return out.stdout if out.returncode == 0 else "" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", required=True) + parser.add_argument("--repo", required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--tasks", type=int, default=20) + args = parser.parse_args() + + corpus = Path(args.corpus) + raw = _git(Path(args.repo), "log", f"{args.base}..HEAD", "--no-merges", + "--format=%H%x00%B%x00").split("\x00") + pairs = [(raw[i].strip(), raw[i + 1].strip()) for i in range(0, len(raw) - 1, 2)] + tasks = [{"id": s[:12], "question": m} for s, m in pairs if s][: args.tasks] + + services = create_service_collection(corpus) + services.resolve(ProjectIndexerRegistry).get_indexer( + corpus, factory=services.resolve(IndexerFactoryKey) + ) + tool = InvestigateTool(services) + + positions: Counter = Counter() + duplicates = 0 + rows_total = 0 + for task in tasks: + text = asyncio.run(tool.execute(query=task["question"], limit=8, + project_root=str(corpus))) + files = [f for f, _ in ROW_RE.findall(text)] + seen: set = set() + for index, file in enumerate(files, 1): + rows_total += 1 + if file in seen: + duplicates += 1 + positions[index] += 1 + else: + seen.add(file) + print(f" {task['id']} rows={len(files)} distinct={len(set(files))}", flush=True) + + print(f"\nrows={rows_total} duplicate_rows={duplicates} " + f"({duplicates / max(rows_total, 1):.2f} of rows)") + print("duplicate row positions (1-based) in the answer:") + for pos in sorted(positions): + print(f" rank {pos}: {positions[pos]}") + ranked = sum(positions[p] for p in positions if p <= 5) + print(f"duplicates at ranks 1-5: {ranked} of {duplicates}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/file_dedup_2026_11/frozen/PLAN.md b/experiments/file_dedup_2026_11/frozen/PLAN.md new file mode 100644 index 00000000..e62b84b7 --- /dev/null +++ b/experiments/file_dedup_2026_11/frozen/PLAN.md @@ -0,0 +1,76 @@ +# File-level dedup — frozen plan (written before the first run) + +Date: 2026-11-10. Question this answers: *the tool answers with chunks; a +file-level question wants files. Is that why grep wins on external repos?* + +The hypothesis was formed **after** the numbers (`psf/requests` 10 vs 19, +`sqlalchemy` 11 vs 20) and before any code change. The bar below is written +now and is not edited after a result is seen. + +## What was measured to motivate it + +10 sqlalchemy tasks, up to 8 rows each: + +``` +rows=72 distinct_files=51 -> 0.71 files per row +worst case: 8 rows = 1 file +``` + +29% of the answer slots repeat a file that is already shown. + +## Change under test + +One env switch, `MSCODEBASE_FILE_DEDUP`: + +- empty / unknown -> shipped behaviour, byte-identical answer (P4); +- `file` -> after ranking, a **two-pass fill**: first one row per file (best + rank wins), then the remaining rows fill the tail up to `limit`. + +The "usage" question ("where is X used") is the reason for the second pass: on +such a question several rows of one file are the answer, so they must survive +when there are fewer distinct files than `limit`. Nothing is dropped that a +shorter answer could not already drop. + +## Corpus (neither questions nor gold are ours) + +| corpus | base | tasks | note | +|---|---|---|---| +| `sqlalchemy` | `8b62343` | 40 | 668 files / 34 984 chunks | +| B1 dev (this repo) | `7c2c7a3` | 71 | calibrated corpus, anchors from E65 | + +Same rule as B1/B2: query = commit message, gold = files the commit changed, +corpus pinned at the base so the fix is not indexed. + +## Bar (all four, frozen) + +| # | Gate | Pass | +|---|---|---| +| P1 | sqlalchemy `tool` hit@5 | **>= 14 / 40** (from 11) | +| P2 | B1 dev `tool` hit@5 | **>= 19 / 71** (no loss against E71) | +| P3 | usage probes: answers that kept their same-file rows | **>= 3 of 4** probe groups | +| P4 | default off | answer text byte-identical on 10 tasks | + +## P3 protocol (usage questions, because the flag can hurt them) + +Probe built from the graph, not written by hand: take functions that the index +records with >= 2 distinct call lines **inside one file**, ask +`where is used`, and count whether that file still contributes >= 2 rows. + +Frozen: 20 probes drawn by hash order from the graph, first 20 with a +same-file multi-call site; a probe counts as kept if the file appears twice in +the answer both **before** and **after** the flag. + +## Decision rule + +- P1..P4 pass -> ship `file` as the default in a follow-up commit (its own PR, + with the numbers above). +- P1 or P2 fails -> keep the flag experimental, write the negative result in + `EXPERIMENTS_LOG.md`, do not touch the default. +- P3 fails -> dedup only when the question is a file-level intent; that is a + new design, so stop and ask the owner. + +## What would make this invalid + +- Any tuning of `limit`, the bar, or the probe list after the first result. +- Running P3 probes on tasks that were used to form the hypothesis. +- Reporting the flag-on number without the flag-off number from the same code. diff --git a/experiments/file_dedup_2026_11/p4_identity.py b/experiments/file_dedup_2026_11/p4_identity.py new file mode 100644 index 00000000..6c94e305 --- /dev/null +++ b/experiments/file_dedup_2026_11/p4_identity.py @@ -0,0 +1,82 @@ +"""P4 of E72: with the flag off, the answer text must not change at all. + +Two source trees are compared on the same corpus, the same 10 commits and the +same flag state (off): `--src` pins the tree, `--out` receives the answers, and +`diff` decides. This compares text, not metrics. + + python experiments/file_dedup_2026_11/p4_identity.py --src D:\\Project\\bench-p4 \ + --corpus D:\\Project\\bench-ext\\sa-B --repo D:\\Project\\bench-ext\\sqlalchemy \ + --base 8b62343 --out C:\\...\\p4_baseline.json +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import re +import subprocess +import sys +from pathlib import Path + +ROW_RE = re.compile(r"^\d+\. (.+?):(\d+) ", re.M) + + +def _git(repo: Path, *args: str) -> str: + out = subprocess.run(["git", *args], cwd=repo, capture_output=True, + text=True, encoding="utf-8", errors="replace") + return out.stdout if out.returncode == 0 else "" + + +def _tasks(repo: Path, base: str, limit: int) -> list[dict]: + raw = _git(repo, "log", f"{base}..HEAD", "--no-merges", + "--format=%H%x00%B%x00").split("\x00") + pairs = [(raw[i].strip(), raw[i + 1].strip()) for i in range(0, len(raw) - 1, 2)] + return [{"id": sha[:12], "sha": sha, "question": message} + for sha, message in pairs if sha][:limit] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--src", required=True) + parser.add_argument("--corpus", required=True) + parser.add_argument("--repo", required=True) + parser.add_argument("--base", required=True) + parser.add_argument("--out", required=True) + parser.add_argument("--tasks", type=int, default=10) + args = parser.parse_args() + + sys.path.insert(0, args.src) + sys.stdout.reconfigure(encoding="utf-8") + from src.core.di_container import ( # noqa: E402 + IndexerFactoryKey, + ProjectIndexerRegistry, + create_service_collection, + ) + from src.mcp.tools.investigate_tool import InvestigateTool # noqa: E402 + + assert str(Path(args.src).resolve()) in str( + Path(sys.modules["src"].__file__).resolve()), "src did not come from --src" + + corpus = Path(args.corpus) + services = create_service_collection(corpus) + services.resolve(ProjectIndexerRegistry).get_indexer( + corpus, factory=services.resolve(IndexerFactoryKey) + ) + tool = InvestigateTool(services) + + answers = {} + for task in _tasks(Path(args.repo), args.base, args.tasks): + answers[task["id"]] = asyncio.run(tool.execute( + query=task["question"], limit=8, project_root=str(corpus))) + print(f" {task['id']} rows={len(ROW_RE.findall(answers[task['id']]))}", + flush=True) + + Path(args.out).write_text(json.dumps(answers, ensure_ascii=False, indent=1), + encoding="utf-8", newline="") + print(f"wrote {len(answers)} answers to {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/file_dedup_2026_11/usage_probe.py b/experiments/file_dedup_2026_11/usage_probe.py new file mode 100644 index 00000000..b8626336 --- /dev/null +++ b/experiments/file_dedup_2026_11/usage_probe.py @@ -0,0 +1,110 @@ +"""P3 of E72: file dedup must not break "where is X used". + +Probes are built from the symbol index, not written by hand: a function with at +least two distinct call lines inside one file is a question where two rows of +one file are the correct answer. Frozen in `frozen/PLAN.md`. + +Run: + python experiments/file_dedup_2026_11/usage_probe.py \ + --corpus D:\\Project\\bench-ext\\sa-B --need 20 +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import os +import re +import subprocess +import sys +from collections import Counter, defaultdict +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11")) +sys.path.insert(0, str(ROOT / "experiments" / "xerj_2026_11" / "evidence_gate")) +sys.stdout.reconfigure(encoding="utf-8") + +from run_gate import _files, _norm, _tools_for # noqa: E402 + +DEF_RE = re.compile(r"^\s*(?:async\s+)?def\s+([A-Za-z_]\w*)", re.M) + + +def _def_names(corpus: Path) -> list[str]: + listing = subprocess.run(["git", "ls-files", "*.py"], cwd=corpus, + capture_output=True, text=True, + encoding="utf-8").stdout.splitlines() + names = set() + for rel in listing: + try: + text = (corpus / rel).read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + names.update(DEF_RE.findall(text)) + return sorted(names) + + +def _probes(tool, corpus: Path, need: int) -> list[dict]: + index = tool.resolve_symbol_index(explicit_project_root=str(corpus)) + out = [] + for name in sorted(_def_names(corpus), key=lambda n: hashlib.sha1(n.encode()).hexdigest()): + try: + refs = index.get_callers(name) or [] + except Exception: # noqa: BLE001 - a missing call graph is a skip, not a crash + continue + per_file: dict[str, set] = defaultdict(set) + for ref in refs: + if getattr(ref, "is_definition", False): + continue + file = str(getattr(ref, "file_path", "") or "").replace("\\", "/") + if file: + per_file[file].add(int(getattr(ref, "line", 0) or 0)) + file = next((f for f, lines in sorted(per_file.items()) if len(lines) >= 2), "") + if not file: + continue + out.append({"name": name, "file": _norm([file], corpus)[0]}) + if len(out) >= need: + break + return out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--corpus", required=True) + parser.add_argument("--need", type=int, default=20) + parser.add_argument("--limit", type=int, default=8) + args = parser.parse_args() + + corpus = Path(args.corpus) + tool = _tools_for({corpus})[str(corpus)] + probes = _probes(tool, corpus, args.need) + print(f"corpus={corpus.name} probes={len(probes)}", flush=True) + if not probes: + print("no probes: the call graph returned nothing, so P3 is not measured", flush=True) + return 2 + + kept = 0 + for probe in probes: + query = f"where is {probe['name']} used" + os.environ.pop("MSCODEBASE_FILE_DEDUP", None) + before = Counter(_norm(_files(asyncio.run(tool.execute( + query=query, limit=args.limit, project_root=str(corpus)))), corpus)) + os.environ["MSCODEBASE_FILE_DEDUP"] = "file" + after = Counter(_norm(_files(asyncio.run(tool.execute( + query=query, limit=args.limit, project_root=str(corpus)))), corpus)) + os.environ.pop("MSCODEBASE_FILE_DEDUP", None) + ok = before[probe["file"]] >= 2 and after[probe["file"]] >= 2 + kept += ok + print(f" {probe['name']:<34} {probe['file']:<44} " + f"before={before[probe['file']]} after={after[probe['file']]} " + f"{'kept' if ok else 'LOST'}", flush=True) + + print(f"\nP3 kept {kept}/{len(probes)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/mcp/tools/investigate_tool.py b/src/mcp/tools/investigate_tool.py index fa0ab79b..2a529dc5 100644 --- a/src/mcp/tools/investigate_tool.py +++ b/src/mcp/tools/investigate_tool.py @@ -512,6 +512,43 @@ def _fusion_rank(item: Dict[str, Any], idx: int, mode: str) -> float: return score + weight +_FILE_DEDUP_MODES = ("file",) + + +def _file_dedup_mode() -> str: + """Answer with files before repeating a file, switchable for E72. + + The answer ranks chunks, but a "which file" question wants files: on + sqlalchemy 72 rows covered 51 files (0.71 files per row), so 29% of the + slots repeated a file already shown. Empty or unknown -> shipped order. + """ + mode = (os.environ.get("MSCODEBASE_FILE_DEDUP") or "").strip().lower() + return mode if mode in _FILE_DEDUP_MODES else "" + + +def _dedup_by_file( + ranked: List[Dict[str, Any]], limit: int +) -> List[Dict[str, Any]]: + """One row per file first, then the leftover rows fill the tail. + + The second pass is what keeps "where is X used" intact: when a question has + fewer distinct files than `limit`, every row survives as before. + """ + head: List[Dict[str, Any]] = [] + tail: List[Dict[str, Any]] = [] + seen: set = set() + for row in ranked: + file = row.get("file") or "" + if file in seen: + tail.append(row) + continue + seen.add(file) + head.append(row) + if len(head) >= limit: + return head[:limit] + return head + tail[: limit - len(head)] + + def _merge( items: Iterable[Dict[str, Any]], limit: int, @@ -617,6 +654,8 @@ def _merge( # in every arm (a stable sort keeps it inside each rank group). ranked = sorted(merged.values(), key=lambda r: (r["file"], r["line"])) ranked.sort(key=lambda r: (r["rank"], len(r["channels"])), reverse=True) + if _file_dedup_mode(): + return _dedup_by_file(ranked, limit) return ranked[:limit] diff --git a/tests/test_file_dedup.py b/tests/test_file_dedup.py new file mode 100644 index 00000000..aeb5aca1 --- /dev/null +++ b/tests/test_file_dedup.py @@ -0,0 +1,65 @@ +"""E72: file-level dedup keeps a "which file" answer from repeating one file. + +The flag is experimental and off by default; these tests pin both sides (the +shipped order must not move) and the reason the second pass exists (a "where is +X used" question with few files keeps all its rows). +""" + +from __future__ import annotations + +import pytest + +from src.mcp.tools.investigate_tool import _dedup_by_file, _merge + +ITEMS = [ + {"file": "a.py", "line": 1, "score": 3.0, "channels": ["search"]}, + {"file": "a.py", "line": 9, "score": 2.5, "channels": ["search"]}, + {"file": "a.py", "line": 20, "score": 2.4, "channels": ["grep"]}, + {"file": "b.py", "line": 5, "score": 2.0, "channels": ["search"]}, +] + + +def _where(rows): + return [(r["file"], r["line"]) for r in rows] + + +@pytest.fixture(autouse=True) +def _no_flag(monkeypatch): + monkeypatch.delenv("MSCODEBASE_FILE_DEDUP", raising=False) + + +def test_one_row_per_file_comes_first(): + head = [{"file": "a.py", "line": 1}, {"file": "b.py", "line": 5}] + tail = [{"file": "a.py", "line": 9}, {"file": "a.py", "line": 20}] + + assert _where(_dedup_by_file(head + tail, 3)) == [ + ("a.py", 1), + ("b.py", 5), + ("a.py", 9), + ] + + +def test_a_short_list_keeps_every_row(): + # "where is X used": three rows, two files - the second pass must not drop + # the same-file call sites. + rows = [{"file": "a.py", "line": 1}, {"file": "a.py", "line": 9}, + {"file": "b.py", "line": 5}] + + assert len(_dedup_by_file(rows, 8)) == 3 + + +def test_flag_off_keeps_the_shipped_order(): + assert _where(_merge(ITEMS, 3)) == [("a.py", 1), ("a.py", 9), ("a.py", 20)] + + +def test_flag_on_leads_with_files(monkeypatch): + monkeypatch.setenv("MSCODEBASE_FILE_DEDUP", "file") + + assert _where(_merge(ITEMS, 3)) == [("a.py", 1), ("b.py", 5), ("a.py", 9)] + + +def test_unknown_value_is_off(monkeypatch): + # A typo in an env var must never silently change the shipped answer. + monkeypatch.setenv("MSCODEBASE_FILE_DEDUP", "files") + + assert _where(_merge(ITEMS, 3)) == [("a.py", 1), ("a.py", 9), ("a.py", 20)] From a6d8b02a6b1b6b460b7de5fe1024ac5789b1bddf Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Wed, 11 Nov 2026 00:43:54 +0300 Subject: [PATCH 2/3] test(bench): correct the population behind the empty-answer and repeat-rate numbers The report's "6 of 20 empty answers" and "22-29% repeated rows" came from duplicate_positions.py, which at the time sampled raw commits from git log. That sample includes docs-only commits with no .py gold - a different population than the benchmark's 40-task panel, where the bar is measured. Re-measured on the benchmark's own task list, flag off: rows=216 duplicate_rows=70 (0.32) empty_answers=13 duplicates at ranks 1-5: 34 of 70 The E72 verdict does not change (the flag removes every duplicate and hit@5 stays at 11/40), but the motivation is now stated on the right denominator, and the 13 empty answers point at E73 rather than at "a small-repo effect". duplicate_positions.py now applies the same task filter as the benchmark and exits 2 when the population size does not match, so the share cannot be printed against a different denominator again. --- AGENT_DIARY.md | 11 ++++++++--- EXPERIMENTS_LOG.md | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index f5111c36..d5c78ec1 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -59,9 +59,14 @@ просить их, когда в вопросе нет code-имени. - **E72 (file dedup):** `MSCODEBASE_FILE_DEDUP=file` убрал все повторы файлов (25→0 на 20 коммитах), но hit@5 остался 11/40 на sqlalchemy и 19/71 на B1 dev. - 10 из 25 повторов стояли в рангах 1–5, то есть замена была реальной и всё равно - нейтральной. Цена: «где используется X» теряет строки одного файла (5/20 проб). - Флаг оставлен экспериментальным и выключенным; дедуп не объясняет разрыв с grep. + На решаемых 40: 34 из 70 повторов стояли в рангах 1–5, то есть замена была + реальной и всё равно нейтральной. Цена: «где используется X» теряет строки одного + файла (5/20 проб). Флаг оставлен экспериментальным; дедуп не объясняет разрыв. +- **Урок знаменателя (E72):** долю нельзя брать с одного сэмплера и сравнивать с + планом на другом. «6 из 20 пустых» пришло из скрипта без фильтра задач (там + docs-only коммиты, им отвечать нечего); на решаемых 40 пустых 13, и это уже другой + вывод. Guard: `duplicate_positions.py` теперь применяет фильтр бенчмарка и падает + с кодом 2 при несовпадении популяции. - **E71 (Change-Id):** хеш из Gerrit занимал первый слот идентификаторов, гейт считал `0` вхождений и отказывал на каждом коммите sqlalchemy (0/40). Фильтр хешей и трейлеров поднял tool до 11/40, B1 dev не изменился (19/38/22/2). diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index b3553ecd..1dce520e 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -24,9 +24,14 @@ sat at ranks 1-5, so the swaps were real and still neutral: 11/40 either way. The limiting factor is which distinct files rank where, not that one file filled several slots. -**Side finding:** 6 of 20 sqlalchemy commits get an empty answer (0 rows), with -and without the flag. On that repo the abstain is the bigger gap, and it is not -addressed by this experiment. +**Side finding, corrected:** 13 of the 40 solvable sqlalchemy commits get an +empty answer (0 rows), with and without the flag. E73 traced that to a false +refusal, not to an absent subject. + +The first draft said "6 of 20". That count came from the unfiltered raw-commit +sample in `duplicate_positions.py`, which includes docs-only commits whose empty +answer is correct; the script now applies the benchmark task filter and refuses +to print a share on a different denominator. **Verdict:** negative result. The switch stays experimental and off; it is documented as such in all three `CONFIGURATION.md` files. "Dedup by file" must From c2d6fce8cf39c8d42147d216ee4320fc5efff11d Mon Sep 17 00:00:00 2001 From: MSCodeBase Agent Date: Wed, 11 Nov 2026 00:45:19 +0300 Subject: [PATCH 3/3] test(bench): finish the population correction - report text and script filter --- experiments/file_dedup_2026_11/REPORT.md | 44 +++++++++-------- .../file_dedup_2026_11/duplicate_positions.py | 47 ++++++++++--------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/experiments/file_dedup_2026_11/REPORT.md b/experiments/file_dedup_2026_11/REPORT.md index 044e6911..4f9f5e5c 100644 --- a/experiments/file_dedup_2026_11/REPORT.md +++ b/experiments/file_dedup_2026_11/REPORT.md @@ -5,14 +5,14 @@ change). Switch: `MSCODEBASE_FILE_DEDUP=file`, **off by default**. ## What was asked -The tool answers with chunks; a "which file" question wants files. On 10 -sqlalchemy tasks 72 rows covered 51 files (0.71 files per row, worst case 8 rows -= 1 file). Hypothesis: 29% of the answer slots repeat a file, so grep — which -ranks whole files — wins on external repos. +The tool answers with chunks; a "which file" question wants files. On the +benchmark's own 40-commit task list, **70 of 216 rows (32%) repeat a file**, +worst case 8 rows from 1 file. Hypothesis: repeated slots are why grep, which +ranks whole files, wins on external repos. -The repeat share on a larger sample (20 commits, below) is 22% of rows, not 29%; -the direction is the same, the number is not. Both samples are reported rather -than the rounder one. +An earlier draft used 22-29% from a raw `git log` sample; that sample is not the +task population (it includes docs-only commits), so the number here is the one +measured with the benchmark's task filter applied. ## What was measured @@ -34,29 +34,35 @@ B1 dev, 71 tasks tool neighbours search grep flag on 19/71 38 22 2 ``` -**The flag changed no hit@5 number on either corpus**, and it did work: with the -flag on, the same answers carry **0 duplicate rows instead of 25**. So the -repeats were real and were removed — the score simply did not move. +**The flag changed no hit@5 number on either corpus**, and it did work: on the +20-commit sample used to check its effect, the same answers carry **0 duplicate +rows instead of 25**. So the repeats were real and were removed — the score +simply did not move. -`duplicate_positions.py`, 20 sqlalchemy commits, flag off: +`duplicate_positions.py`, the benchmark's 40 sqlalchemy commits, flag off: ``` -rows=112 duplicate_rows=25 (0.22 of rows) +rows=216 duplicate_rows=70 (0.32 of rows) empty_answers=13 duplicate row positions (1-based in the answer): - rank 2: 1 rank 3: 2 rank 4: 4 rank 5: 3 - rank 6: 5 rank 7: 3 rank 8: 7 -duplicates at ranks 1-5: 10 of 25 + rank 2: 9 rank 3: 9 rank 4: 4 rank 5: 12 + rank 6: 12 rank 7: 14 rank 8: 10 +duplicates at ranks 1-5: 34 of 70 ``` -So 10 of the 25 repeats sat inside the top-5 that hit@5 counts, and replacing +So 34 of the 70 repeats sat inside the top-5 that hit@5 counts, and replacing them with distinct files still produced 11/40 — the swaps were neutral: the duplicate was not what kept the gold file out. What limits the tool is **which distinct files rank where**, not that one file took several slots. One more fact from the same run, unrelated to the flag but larger than it: -**6 of 20** sqlalchemy commits get an empty answer (0 rows) both with and -without the flag. On that repo the abstain is a bigger gap than dedup, and it is -not addressed here. +**13 of the 40** solvable commits get an empty answer (0 rows) with and without +the flag. E73 traced that to a false refusal, not to an absent subject. + +An earlier draft of this report said "6 of 20". That count came from +`duplicate_positions.py`, which then sampled raw commits without the benchmark's +task filter, so it included docs-only commits that have no `.py` gold and whose +empty answer is correct. The script now applies the same filter and exits with +code 2 instead of printing a share on a different denominator. ## The harm the flag does (P3) diff --git a/experiments/file_dedup_2026_11/duplicate_positions.py b/experiments/file_dedup_2026_11/duplicate_positions.py index 18561940..d2022903 100644 --- a/experiments/file_dedup_2026_11/duplicate_positions.py +++ b/experiments/file_dedup_2026_11/duplicate_positions.py @@ -1,16 +1,15 @@ -"""Where do the repeated files sit in the answer? +"""Where do the repeated files sit in the answer, ON THE TASK POPULATION? -E72 asked whether rank-5 localization loses to grep because a file can fill -several of the 8 slots. If the repeats sit at ranks 6-8, reordering cannot move -hit@5 - which is what the sqlalchemy run showed (11/40 either way). +E72's motivation used a raw `git log` sample, which includes docs-only commits +with no `.py` gold - a different population than the 40-task panel. This script +applies the same task filter as the benchmark and asserts that the population +matches, so the number cannot be quoted against the wrong denominator again. """ from __future__ import annotations import argparse import asyncio -import re -import subprocess import sys from collections import Counter from pathlib import Path @@ -18,8 +17,12 @@ HERE = Path(__file__).resolve().parent ROOT = HERE.parents[1] sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "experiments" / "bench_2026_11")) sys.stdout.reconfigure(encoding="utf-8") +import re # noqa: E402 + +from external import _tasks # noqa: E402 from src.core.di_container import ( # noqa: E402 IndexerFactoryKey, ProjectIndexerRegistry, @@ -30,39 +33,37 @@ ROW_RE = re.compile(r"^\d+\. (.+?):(\d+) ", re.M) -def _git(repo: Path, *args: str) -> str: - out = subprocess.run(["git", *args], cwd=repo, capture_output=True, - text=True, encoding="utf-8", errors="replace") - return out.stdout if out.returncode == 0 else "" - - def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--corpus", required=True) parser.add_argument("--repo", required=True) parser.add_argument("--base", required=True) - parser.add_argument("--tasks", type=int, default=20) + parser.add_argument("--tasks", type=int, default=40) args = parser.parse_args() corpus = Path(args.corpus) - raw = _git(Path(args.repo), "log", f"{args.base}..HEAD", "--no-merges", - "--format=%H%x00%B%x00").split("\x00") - pairs = [(raw[i].strip(), raw[i + 1].strip()) for i in range(0, len(raw) - 1, 2)] - tasks = [{"id": s[:12], "question": m} for s, m in pairs if s][: args.tasks] + tasks = _tasks(Path(args.repo), args.base, args.tasks) + if len(tasks) != args.tasks: + print(f"population mismatch: asked {args.tasks}, benchmark rule kept " + f"{len(tasks)} - refusing to report a share on a different denominator") + return 2 + print(f"corpus={corpus.name} tasks={len(tasks)} (benchmark filter applied)") services = create_service_collection(corpus) services.resolve(ProjectIndexerRegistry).get_indexer( - corpus, factory=services.resolve(IndexerFactoryKey) - ) + corpus, factory=services.resolve(IndexerFactoryKey)) tool = InvestigateTool(services) positions: Counter = Counter() duplicates = 0 rows_total = 0 + empty = 0 for task in tasks: text = asyncio.run(tool.execute(query=task["question"], limit=8, project_root=str(corpus))) files = [f for f, _ in ROW_RE.findall(text)] + if not files: + empty += 1 seen: set = set() for index, file in enumerate(files, 1): rows_total += 1 @@ -74,12 +75,12 @@ def main() -> int: print(f" {task['id']} rows={len(files)} distinct={len(set(files))}", flush=True) print(f"\nrows={rows_total} duplicate_rows={duplicates} " - f"({duplicates / max(rows_total, 1):.2f} of rows)") - print("duplicate row positions (1-based) in the answer:") + f"({duplicates / max(rows_total, 1):.2f} of rows) empty_answers={empty}") + print("duplicate row positions (1-based):") for pos in sorted(positions): print(f" rank {pos}: {positions[pos]}") - ranked = sum(positions[p] for p in positions if p <= 5) - print(f"duplicates at ranks 1-5: {ranked} of {duplicates}") + print(f"duplicates at ranks 1-5: " + f"{sum(positions[p] for p in positions if p <= 5)} of {duplicates}") return 0