diff --git a/AGENT_DIARY.md b/AGENT_DIARY.md index 36ebb6cf..a4b34400 100644 --- a/AGENT_DIARY.md +++ b/AGENT_DIARY.md @@ -57,6 +57,11 @@ Пункты «RU→EN словарь» и «concept→symbol словарь» сняты: их заменяет контракт. - **Урок:** не учить сервер языкам, которые знает хост; отдавать `hints` и честно просить их, когда в вопросе нет code-имени. +- **E69 (вариант C):** отказ остаётся, но печатает `next: investigate(... + include_neighbours=true)`. Замер на B1 dev: дефолт 19/71 не изменился, подсказка + на 36/36 отказах, вместе 37/71 [0.41, 0.63] против 38 у neighbours-руки. +- **Урок:** порог и метрика — разные вещи; «18» было «среди 36 с подсказкой», а не + «из 71». Порог не двигал, чинил счёт — и добавил assert против двойного счёта. - **E67 (бакет тестов):** `src/core/test_impact.py` штрафовался на −2.0 как тестовый файл, потому что начинается с `test_`. Исправлено проверкой содержимого (тесты или импорт фреймворка), дефолт переключён; откат `MSCODEBASE_TEST_BUCKET=path`. diff --git a/EXPERIMENTS_LOG.md b/EXPERIMENTS_LOG.md index 027180de..f2b6593a 100644 --- a/EXPERIMENTS_LOG.md +++ b/EXPERIMENTS_LOG.md @@ -1,4 +1,23 @@ # EXPERIMENTS_LOG.md — Audit Verification (2026-07-22) +## [2026-11-10] — E69: an abstain that names the next call (option C) + +**Decision (owner):** keep `abstain` — never present an unverified row as an +answer — but make the continuation machine-readable instead of a sentence. + +**Change:** on a withheld answer the tool prints +`next: investigate(query="...", include_neighbours=true, limit=8)`; the rewrite +gate prints the same shape with `hints=[""]`. No row is added, the +verdict is unchanged. + +**Measurement (B1 dev, 71):** hint emitted 36/36 abstains; default `hit@5` stays +19/71 (no leak); following the hint recovers 18; panel total 37/71 `[0.41, 0.63]` +against the neighbours arm's 38. + +**Caught before publishing:** the first run compared "18" against a panel-level +threshold — a metric bug (18 is "among the 36 hinted", not "of 71"). The +threshold was not moved; the metric was fixed and an assertion now prevents +double counting. + ## [2026-11-10] — E67: a production module was penalised as a test file **Bug (found by the E66 generator):** `_bucket_penalty` classifies by path, and diff --git a/experiments/retry_hint_2026_11/REPORT.md b/experiments/retry_hint_2026_11/REPORT.md new file mode 100644 index 00000000..e913321a --- /dev/null +++ b/experiments/retry_hint_2026_11/REPORT.md @@ -0,0 +1,53 @@ +# E69 — retry hint (option C): result + +Frozen: `frozen/PLAN.md`. Owner decision: keep `abstain`, make the continuation +explicit instead of prose. + +## Change + +When the gate withholds rows, the answer ends with: + +``` +Neighbours withheld: 8 (unverified). +next: investigate(query="...", include_neighbours=true, limit=8) +``` + +`ask_rewrite` carries the same shape with `hints=[""]`. No row is +added to the default answer; the verdict stays `abstain`. + +## Measurement (B1 dev, 71 solvable tasks) + +``` +hint emitted 36/71 (every abstain) +default hit@5 19/71 (E68 baseline: 19) +recovered by hint 18/36 +default + hint total 37/71 [0.41, 0.63] (E68 neighbours arm: 38) +``` + +Frozen predictions: + +| # | Prediction | Result | +|---|---|---| +| R1 | default stays 19/71 | **PASS** — no rows leaked | +| R2 | every withheld answer carries the hint | **PASS** — 36/36 | +| R3 | default + hint >= 36/71 | **PASS** — 37 | + +The one-task gap to the neighbours arm (37 vs 38) is the `abstain` task that +`include_neighbours` would answer and the default path scores differently; it is +inside the interval. + +## A metric bug caught before publishing + +The first run printed "after hint = 18/71" and failed R3. That was the metric, +not the product: 18 is the count *among the 36 hinted tasks*, and the other 35 +were counted as misses even though the default had already answered them. The +correct panel-level number is `default + recovered = 37`. The threshold was not +changed; the metric was fixed, and an assertion now enforces that every hinted +task has `default hit@5 = False` (otherwise the sum would double-count). + +## What C still does not do + +The default answer remains empty on those 36 tasks. Recovering them requires the +caller to act on the `next:` line — that is the contract of option C, and it is +why the hint is machine-readable rather than a sentence. A host that ignores it +sees the same 19/71 as before, which is exactly what R1 measures. diff --git a/experiments/retry_hint_2026_11/agent_loop.py b/experiments/retry_hint_2026_11/agent_loop.py new file mode 100644 index 00000000..bfdf8771 --- /dev/null +++ b/experiments/retry_hint_2026_11/agent_loop.py @@ -0,0 +1,107 @@ +"""E69: does following the retry hint recover the withheld neighbours? + +Frozen: `frozen/PLAN.md`. For every solvable dev task of the B1 benchmark: + +1. call `investigate` with defaults, score it, and look for the `next:` line; +2. if the hint names `include_neighbours=true`, replay with it and score again. + +Run: python experiments/retry_hint_2026_11/agent_loop.py +""" + +from __future__ import annotations + +import asyncio +import json +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +EXP = HERE.parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(EXP / "bench_2026_11")) +sys.path.insert(0, str(EXP / "xerj_2026_11")) +sys.path.insert(0, str(EXP / "xerj_2026_11" / "evidence_gate")) +sys.stdout.reconfigure(encoding="utf-8") + +from run import CORPUS, _hits, _kept_gold, _tasks # noqa: E402 +from run_gate import _files, _norm, _tools_for # noqa: E402 +from wilson import wilson # noqa: E402 + +HINT_RE = re.compile(r"^next: investigate\((.*)\)$", re.M) + + +def _has_next(text: str) -> bool: + return bool(HINT_RE.search(text)) and "include_neighbours=true" in text + + +def main() -> int: + tasks = [] + for task in _tasks("dev"): + gold = _kept_gold(task) + if gold: + tasks.append({**task, "gold": gold}) + print(f"dev solvable tasks={len(tasks)}", flush=True) + + tool = _tools_for({CORPUS})[str(CORPUS)] + rows = [] + for index, task in enumerate(tasks, 1): + text = asyncio.run(tool.execute(query=task["question"], limit=8, + project_root=str(CORPUS))) + files = _norm(_files(text), CORPUS) + hinted = _has_next(text) + second = [] + if hinted: + more = asyncio.run(tool.execute(query=task["question"], limit=8, + include_neighbours=True, + project_root=str(CORPUS))) + second = _norm(_files(more), CORPUS) + rows.append({ + "id": task["id"], + "default_top1": _hits(files, task["gold"], 1), + "default_top5": _hits(files, task["gold"], 5), + "hinted": hinted, + "hinted_top1": _hits(second, task["gold"], 1) if hinted else None, + "hinted_top5": _hits(second, task["gold"], 5) if hinted else None, + }) + print(f" {index:>3}/{len(tasks)} {task['id']} " + f"hint={hinted} default@5={rows[-1]['default_top5']} " + f"hinted@5={rows[-1]['hinted_top5']}", flush=True) + + def rate(key: str) -> int: + return sum(1 for r in rows if r[key]) + + emitted = sum(1 for r in rows if r["hinted"]) + default5 = rate("default_top5") + hinted5 = rate("hinted_top5") + # A hinted task abstained by definition, so its default hit@5 is False; the + # panel-level result is "default answer + recovered", not "recovered alone". + assert all(not r["default_top5"] for r in rows if r["hinted"]) + after_total = default5 + hinted5 + lo, hi = wilson(after_total, len(rows)) + print(f"\nhint emitted on {emitted}/{len(rows)} tasks") + print(f"default hit@5 = {default5}/{len(rows)} (E68 baseline: 19)") + print(f"recovered by hint = {hinted5}/{emitted}") + print(f"default + hint total = {after_total}/{len(rows)} " + f"[{lo:.2f}, {hi:.2f}] (E68 neighbours arm: 38)") + + preds = { + "R1 default unchanged (19)": default5 == 19, + "R2 hint coverage 100% of abstains": emitted >= 36, + "R3 default+hint >= 36": after_total >= 36, + } + print("\nfrozen predictions") + for label, passed in preds.items(): + print(f" {label:<34} {'PASS' if passed else 'FAIL'}") + + (HERE / "results.json").write_text(json.dumps( + {"tasks": len(rows), "emitted": emitted, "default_hit5": default5, + "recovered_hit5": hinted5, "after_total_hit5": after_total, + "predictions": preds}, + ensure_ascii=False, indent=1), encoding="utf-8", newline="") + return 0 if all(preds.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/retry_hint_2026_11/frozen/PLAN.md b/experiments/retry_hint_2026_11/frozen/PLAN.md new file mode 100644 index 00000000..88cf19c9 --- /dev/null +++ b/experiments/retry_hint_2026_11/frozen/PLAN.md @@ -0,0 +1,44 @@ +# Retry hint — frozen plan (option C, before the code change) + +Date: 2026-11-10. Owner decision: **C** — keep `abstain` (never present an +unverified row as an answer), but make the continuation explicit instead of +leaving it in prose that a single-shot caller does not act on. + +Measured before this change (E68): 36 of 52 default misses on the commit +benchmark are `abstain` with zero rows, and the neighbours that were withheld +carry 19 of 71 hit@5 (dev) and 14 of 37 (sealed). + +## Change + +When the gate withholds rows, the answer ends with a machine-readable block: + +``` +Neighbours withheld: 8 (unverified). +next: investigate(query="...", include_neighbours=true, limit=8) +``` + +`ask_rewrite` gets the same shape (`next: investigate(query="...", hints=[...])`) +so a caller never has to parse prose. No row is added to the default answer; the +verdict stays `abstain`. + +## Predictions (frozen) + +| # | Prediction | Falsifier | +|---|---|---| +| R1 | B1 dev `hit@5` of the **default** arm stays 19/71 | any change (rows leaked) | +| R2 | every answer with withheld rows carries a `next:` line that names `include_neighbours=true` | any withheld answer without it | +| R3 | following the hint reaches `hit@5` >= 36/71 (the neighbours arm measured 38) | below 36 | +| R4 | `ask_rewrite` answers carry a `next:` line with a `hints=[...]` placeholder | any without it | + +R1 is the honesty check: C is supposed to change the text, not the evidence. + +## Measurement + +`experiments/retry_hint_2026_11/agent_loop.py` on the B1 dev split: + +1. call `investigate` with defaults, record `top1`/`hit@5` and whether a `next:` + line with `include_neighbours=true` is present; +2. if present, replay with `include_neighbours=true` and score again; +3. report default, after-hint, and the count of emitted/absent hints. + +No new panel: the tasks, corpus (`B = 7c2c7a3`) and gold are E68's, unchanged. diff --git a/experiments/retry_hint_2026_11/results.json b/experiments/retry_hint_2026_11/results.json new file mode 100644 index 00000000..76442a51 --- /dev/null +++ b/experiments/retry_hint_2026_11/results.json @@ -0,0 +1,12 @@ +{ + "tasks": 71, + "emitted": 36, + "default_hit5": 19, + "recovered_hit5": 18, + "after_total_hit5": 37, + "predictions": { + "R1 default unchanged (19)": true, + "R2 hint coverage 100% of abstains": true, + "R3 default+hint >= 36": true + } +} \ No newline at end of file diff --git a/src/mcp/tools/investigate_tool.py b/src/mcp/tools/investigate_tool.py index 4a506c02..5a508fdd 100644 --- a/src/mcp/tools/investigate_tool.py +++ b/src/mcp/tools/investigate_tool.py @@ -16,6 +16,7 @@ from __future__ import annotations import asyncio +import json import os import re import threading @@ -215,6 +216,29 @@ def _needs_identifier_rewrite(query: str, identifiers: List[str]) -> str | None: " Соседи не выдаются, чтобы не принять их за ответ. " "Переформулируйте: `where is defined` " 'или передайте hints=[""] — хост знает язык, сервер нет.' + + _rewrite_hint(query) + ) + + +def _retry_hint(query: str, withheld: int, limit: int) -> str: + """Machine-readable continuation for a withheld answer (option C, E69). + + The verdict stays `abstain`: unverified rows are never presented as an + answer. But the caller gets the exact call to make instead of a sentence it + has to parse — measured in E68, a single-shot caller does not act on prose. + """ + return ( + f"\nNeighbours withheld: {withheld} (unverified).\n" + f"next: investigate(query={json.dumps(query, ensure_ascii=False)}, " + f"include_neighbours=true, limit={limit})" + ) + + +def _rewrite_hint(query: str) -> str: + """Same shape for the rewrite gate: name the identifiers and the call.""" + return ( + f"\nnext: investigate(query={json.dumps(query, ensure_ascii=False)}, " + 'hints=[""])' ) @@ -1029,10 +1053,7 @@ def _memory() -> None: query, identifiers, merged, symbol_lines, degraded, exact, report ) if withheld: - text += ( - f"\nNeighbours withheld: {withheld}. " - "Pass include_neighbours=true to see them anyway." - ) + text += _retry_hint(query, withheld, limit) return text def _format( diff --git a/tests/test_retry_hint.py b/tests/test_retry_hint.py new file mode 100644 index 00000000..228fdd07 --- /dev/null +++ b/tests/test_retry_hint.py @@ -0,0 +1,55 @@ +"""Retry hint (option C, E69): an abstain that names the next call. + +Frozen: `experiments/retry_hint_2026_11/frozen/PLAN.md`. +""" + +from __future__ import annotations + +import asyncio +import json + +from src.mcp.tools.investigate_tool import _retry_hint, _rewrite_hint + + +def test_retry_hint_names_the_parameter_and_the_limit(): + hint = _retry_hint("where is x", 8, 8) + + assert "Neighbours withheld: 8 (unverified)." in hint + assert "include_neighbours=true" in hint + assert "limit=8" in hint + assert hint.startswith("\nnext: investigate(") or "\nnext: investigate(" in hint + + +def test_retry_hint_round_trips_the_query_verbatim(): + query = 'где "прогресс" и \\ бэкслеш' + hint = _retry_hint(query, 3, 5) + + line = next(x for x in hint.splitlines() if x.startswith("next:")) + call = line[len("next: investigate("):-1] + args = dict( + part.split("=", 1) for part in call.split(", ") + ) + assert json.loads(args["query"]) == query + assert args["include_neighbours"] == "true" + + +def test_rewrite_hint_has_the_hints_placeholder(): + hint = _rewrite_hint("где вычисляется путь") + + assert 'hints=[""]' in hint + assert "next: investigate(" in hint + + +def test_rewrite_gate_prints_the_machine_readable_next_line(tmp_path, monkeypatch): + from tests.test_investigate import _fake_tool + + monkeypatch.setenv("MSCODEBASE_ASK_REWRITE", "1") + tool = _fake_tool(tmp_path) + + out = asyncio.run( + tool.execute("где вычисляется путь прогресса", project_root=str(tmp_path)) + ) + + assert "verdict: ask_rewrite" in out + assert "next: investigate(" in out + assert 'hints=[""]' in out