From 1c55aecaffb73ab79ba4d00691be36892025a1b0 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 11:33:52 -0400 Subject: [PATCH] ci: helper to move a branch onto the split claim, command and inventory files After #1342, #1344 and #1345 land, every open branch that touched claims.yaml, the CAPABILITIES.md command list or public-artifacts.json conflicts once with main. scripts/migrate_branch_to_split_files.py runs during that merge. It compares the branch with the merge base, writes the branch's claim and command changes to claims/.yaml and docs/commands//.md, takes main's side only for conflict hunks inside the command list, and regenerates the report, the command list and the path-only inventory. It never overwrites a claim or command main also changed, and it does not commit. Co-Authored-By: Claude Opus 5.5 --- scripts/migrate_branch_to_split_files.py | 406 +++++++++++++++++++++++ tests/test_migrate_split_files.py | 92 +++++ 2 files changed, 498 insertions(+) create mode 100644 scripts/migrate_branch_to_split_files.py create mode 100644 tests/test_migrate_split_files.py diff --git a/scripts/migrate_branch_to_split_files.py b/scripts/migrate_branch_to_split_files.py new file mode 100644 index 000000000..cfe071329 --- /dev/null +++ b/scripts/migrate_branch_to_split_files.py @@ -0,0 +1,406 @@ +"""Finish merging main into a branch that predates the split files. + +Three shared files used to make parallel pull requests conflict: + +* packages/flow/claims.yaml, now one file per claim in packages/flow/claims/ + (#1344), with docs/VERIFICATION.md and docs/verification.json regenerated; +* the "Every command, by capability" list in CAPABILITIES.md, now one file per + command in packages/flow/docs/commands// (#1345), rendered by + scripts/build_command_map.py; +* packages/flow/public-artifacts.json, now a path-only list (#1342). + +Run it on your branch while the merge of main is still in progress: + + git fetch origin + git merge origin/main # stops with conflicts in those files + python scripts/migrate_branch_to_split_files.py + git status # review, resolve anything it lists + git commit + +It compares your branch with the merge base, so it moves only what your branch +changed: claims it added, changed or removed go to claims/.yaml, commands +it added, moved or removed go to docs/commands/, and the generated files are +regenerated from the result. It never overwrites a claim or command that main +also changed; it lists those for you to merge by hand. It stages what it +resolved and does not commit. + +Each step runs only when the merged main already has that format, so the +script works whichever of the three changes have landed. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +FLOW = "packages/flow" +CLAIMS_FILE = f"{FLOW}/claims.yaml" +CLAIMS_DIR = f"{FLOW}/claims" +COMMANDS_DIR = f"{FLOW}/docs/commands" +INVENTORY = f"{FLOW}/public-artifacts.json" +CAPABILITIES = "CAPABILITIES.md" +COMMANDS_HEADING = "## Every command, by capability" +GENERATED_BEGIN = "" +GENERATED_END = "" +_CLAIM_START = re.compile(r"^ - id:\s*(\S+)\s*$") +_GROUP_LINE = re.compile(r"^- \*\*(?P.+?):\*\*\s+(?P<body>.*)$") +_VERB = re.compile(r"`([a-z][a-z0-9-]*)`(?:\s*\(([^)]*)\))?") + + +def git(root: Path, *args: str, check: bool = True) -> str: + result = subprocess.run( + ["git", "-C", str(root), *args], capture_output=True, text=True, check=False + ) + if check and result.returncode != 0: + raise SystemExit(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout + + +def show(root: Path, rev: str, path: str) -> str | None: + result = subprocess.run( + ["git", "-C", str(root), "show", f"{rev}:{path}"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout if result.returncode == 0 else None + + +def has_tree(root: Path, rev: str, path: str) -> bool: + return bool(git(root, "ls-tree", "--name-only", rev, f"{path}/", check=False)) + + +# --------------------------------------------------------------------------- # +# claims.yaml -> claims/<id>.yaml +# --------------------------------------------------------------------------- # +def split_claims(text: str) -> dict[str, str]: + """Split a claims.yaml into {id: one-claim YAML text}, keeping its wording. + + Each claim block starts at `` - id: <id>``; its lines lose the four-space + list indent. Section separator comments between claims are dropped. + """ + import yaml + + lines = text.splitlines(keepends=True) + try: + start = next(i for i, line in enumerate(lines) if line.rstrip() == "claims:") + except StopIteration: + raise SystemExit("claims.yaml has no top-level 'claims:' list") from None + blocks: dict[str, list[str]] = {} + current: list[str] | None = None + for line in lines[start + 1 :]: + match = _CLAIM_START.match(line) + if match: + current = [match.group(0)[4:].rstrip() + "\n"] + blocks[match.group(1)] = current + elif line.startswith(" #") or current is None: + continue + elif not line.strip(): + current.append("\n") + elif line.startswith(" "): + current.append(line[4:]) + else: + raise SystemExit(f"unexpected line in claims.yaml: {line!r}") + parsed = {c["id"]: c for c in (yaml.safe_load(text) or {}).get("claims", [])} + result: dict[str, str] = {} + for claim_id, block in blocks.items(): + body = "".join(block).rstrip() + "\n" + if yaml.safe_load(body) != parsed.get(claim_id): + # The text split did not round-trip; fall back to a plain dump. + body = yaml.safe_dump(parsed[claim_id], sort_keys=False, width=79) + result[claim_id] = body + return result + + +def _same_claim(left: str | None, right: str | None) -> bool: + import yaml + + if left is None or right is None: + return left is right + return yaml.safe_load(left) == yaml.safe_load(right) + + +def migrate_claims(root: Path, base: str, report: list[str], manual: list[str]) -> bool: + ours_text = show(root, "HEAD", CLAIMS_FILE) + if ours_text is None: + return False + base_text = show(root, base, CLAIMS_FILE) + ours = split_claims(ours_text) + before = split_claims(base_text) if base_text else {} + for claim_id in sorted(set(ours) | set(before)): + mine, original = ours.get(claim_id), before.get(claim_id) + if _same_claim(mine, original): + continue + target = root / CLAIMS_DIR / f"{claim_id}.yaml" + current = target.read_text("utf-8") if target.exists() else None + if _same_claim(current, mine): + continue + if not _same_claim(current, original): + manual.append( + f"{CLAIMS_DIR}/{claim_id}.yaml: main also changed this claim; " + "merge your version by hand" + ) + continue + if mine is None: + git(root, "rm", "-q", "--", f"{CLAIMS_DIR}/{claim_id}.yaml") + report.append(f"removed claim {claim_id}") + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(mine, "utf-8") + git(root, "add", "--", f"{CLAIMS_DIR}/{claim_id}.yaml") + report.append( + f"{'added' if original is None else 'updated'} " + f"{CLAIMS_DIR}/{claim_id}.yaml" + ) + git(root, "rm", "-q", "--cached", "--ignore-unmatch", "--", CLAIMS_FILE) + (root / CLAIMS_FILE).unlink(missing_ok=True) + return True + + +def regenerate_claims_report(root: Path, report: list[str]) -> None: + run(root / FLOW, [sys.executable, "scripts/validate_claims.py", "--report"]) + git( + root, + "add", + "--", + f"{FLOW}/docs/VERIFICATION.md", + f"{FLOW}/docs/verification.json", + ) + report.append("regenerated docs/VERIFICATION.md and docs/verification.json") + + +# --------------------------------------------------------------------------- # +# CAPABILITIES.md command list -> docs/commands/<group>/<verb>.md +# --------------------------------------------------------------------------- # +def parse_command_list(text: str | None) -> dict[str, tuple[str, str]]: + """Read the one-line-per-capability list: {verb: (capability, note)}.""" + if not text or COMMANDS_HEADING not in text: + return {} + section = text[text.index(COMMANDS_HEADING) + len(COMMANDS_HEADING) :] + section = section.split("\n## ", 1)[0] + commands: dict[str, tuple[str, str]] = {} + for line in section.splitlines(): + match = _GROUP_LINE.match(line) + if not match: + continue + for verb, note in _VERB.findall(match.group("body")): + commands[verb] = (match.group("title"), note.strip()) + return commands + + +def _group_dirs(root: Path) -> dict[str, Path]: + groups: dict[str, Path] = {} + for directory in sorted(p for p in (root / COMMANDS_DIR).iterdir() if p.is_dir()): + title = (directory / "README.md").read_text("utf-8").strip() + groups[title.removeprefix("# ")] = directory + return groups + + +def _group_dir(root: Path, title: str, report: list[str]) -> Path: + groups = _group_dirs(root) + if title in groups: + return groups[title] + numbers = [int(p.name.split("-", 1)[0]) for p in groups.values()] + slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") + directory = root / COMMANDS_DIR / f"{max(numbers, default=0) + 1:02d}-{slug}" + directory.mkdir(parents=True) + (directory / "README.md").write_text(f"# {title}\n", "utf-8") + report.append(f"new capability group {directory.relative_to(root)}") + return directory + + +def _note_text(note: str) -> str: + if not note: + return "" + return note[0].upper() + note[1:].rstrip(".") + ".\n" + + +def migrate_commands( + root: Path, base: str, report: list[str], manual: list[str] +) -> bool: + ours = parse_command_list(show(root, "HEAD", CAPABILITIES)) + before = parse_command_list(show(root, base, CAPABILITIES)) + if not ours: + return False + for verb in sorted(set(ours) | set(before)): + mine, original = ours.get(verb), before.get(verb) + if mine == original: + continue + found = sorted((root / COMMANDS_DIR).glob(f"*/{verb}.md")) + existing = found[0] if found else None + groups = _group_dirs(root) + main_unchanged = ( + existing is not None + and original is not None + and existing.parent == groups.get(original[0]) + and existing.read_text("utf-8") == _note_text(original[1]) + ) + if mine is None: + if existing is None: + continue + if main_unchanged: + git(root, "rm", "-q", "--", str(existing.relative_to(root))) + report.append(f"removed command {verb}") + else: + manual.append(f"{verb}: your branch removed it but main changed it") + continue + target = _group_dir(root, mine[0], report) / f"{verb}.md" + if existing is not None and not main_unchanged: + if existing != target or existing.read_text("utf-8") != _note_text(mine[1]): + manual.append( + f"{verb}: main also changed {existing.relative_to(root)}; " + f"your branch has it under '{mine[0]}'" + + (f" with the note '{mine[1]}'" if mine[1] else "") + ) + continue + if existing is not None and existing != target: + git( + root, + "mv", + str(existing.relative_to(root)), + str(target.relative_to(root)), + ) + target.write_text(_note_text(mine[1]), "utf-8") + git(root, "add", "--", str(target.relative_to(root))) + report.append( + f"{'added' if existing is None else 'updated'} {target.relative_to(root)}" + ) + return True + + +def resolve_command_hunks(text: str) -> str: + """Take main's side of conflict hunks inside the command-list section. + + Hunks elsewhere in the file are left in place for a person to resolve. + """ + out: list[str] = [] + lines = text.splitlines(keepends=True) + section = "" + i = 0 + while i < len(lines): + line = lines[i] + if not line.startswith("<<<<<<< "): + if line.startswith("## "): + section = line.strip() + out.append(line) + i += 1 + continue + ours: list[str] = [] + theirs: list[str] = [] + side = ours + end = i + 1 + while end < len(lines) and not lines[end].startswith(">>>>>>> "): + current = lines[end] + if current.startswith("||||||| "): + side = [] # diff3 base: discard + elif current.rstrip("\n") == "=======": + side = theirs + else: + side.append(current) + end += 1 + in_section = section == COMMANDS_HEADING or any( + COMMANDS_HEADING in part or GENERATED_BEGIN in part or GENERATED_END in part + for part in ours + theirs + ) + if in_section and not any( + part.startswith("## ") and part.strip() != COMMANDS_HEADING + for part in ours + theirs + ): + out.extend(theirs) + for part in theirs: + if part.startswith("## "): + section = part.strip() + else: + out.extend(lines[i : end + 1]) + i = end + 1 + return "".join(out) + + +def regenerate_command_list(root: Path, report: list[str], manual: list[str]) -> None: + path = root / CAPABILITIES + text = resolve_command_hunks(path.read_text("utf-8")) + path.write_text(text, "utf-8") + if GENERATED_BEGIN not in text or GENERATED_END not in text: + manual.append(f"{CAPABILITIES}: the generated command block is missing") + return + run(root, [sys.executable, "scripts/build_command_map.py"]) + if re.search(r"^(<<<<<<<|>>>>>>>) ", path.read_text("utf-8"), re.MULTILINE): + manual.append( + f"{CAPABILITIES}: conflicts outside the command list remain; resolve " + "them, then run python scripts/build_command_map.py" + ) + return + git(root, "add", "--", CAPABILITIES) + report.append("regenerated the command list in CAPABILITIES.md") + + +# --------------------------------------------------------------------------- # +# public-artifacts.json -> path-only list +# --------------------------------------------------------------------------- # +def regenerate_inventory(root: Path, report: list[str]) -> None: + run( + root / FLOW, + [ + sys.executable, + "scripts/check_release_consistency.py", + "--write-public-artifact-inventory", + ], + ) + git(root, "add", "--", INVENTORY) + report.append( + "rewrote public-artifacts.json as the path-only list (review its diff: " + "it lists every artifact file in your working tree)" + ) + + +def run(cwd: Path, command: list[str]) -> None: + result = subprocess.run(command, cwd=cwd, capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit( + f"{' '.join(command)} failed in {cwd}:\n{result.stdout}{result.stderr}" + ) + + +def _inventory_schema(text: str | None) -> int | None: + match = re.search(r'"schema_version":\s*(\d+)', text or "") + return int(match.group(1)) if match else None + + +def main(argv: list[str]) -> int: + root = Path(git(Path.cwd(), "rev-parse", "--show-toplevel").strip()) + if git(root, "rev-parse", "-q", "--verify", "MERGE_HEAD", check=False) == "": + print( + "No merge in progress. Run `git fetch origin && git merge origin/main` " + "first, then run this script before committing." + ) + return 2 + base = git(root, "merge-base", "HEAD", "MERGE_HEAD").strip() + report: list[str] = [] + manual: list[str] = [] + + if has_tree(root, "MERGE_HEAD", CLAIMS_DIR): + if migrate_claims(root, base, report, manual): + regenerate_claims_report(root, report) + if has_tree(root, "MERGE_HEAD", COMMANDS_DIR): + if migrate_commands(root, base, report, manual): + regenerate_command_list(root, report, manual) + if _inventory_schema(show(root, "MERGE_HEAD", INVENTORY)) == 2: + regenerate_inventory(root, report) + + for line in report: + print(f" done: {line}") + for line in manual: + print(f" BY HAND: {line}") + unmerged = git(root, "diff", "--name-only", "--diff-filter=U").split() + if unmerged: + print("Still conflicted (resolve, then `git add`):") + for path in unmerged: + print(f" {path}") + print("Nothing was committed. Review with `git diff --cached`, then `git commit`.") + return 1 if manual or unmerged else 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/test_migrate_split_files.py b/tests/test_migrate_split_files.py new file mode 100644 index 000000000..11ea526d0 --- /dev/null +++ b/tests/test_migrate_split_files.py @@ -0,0 +1,92 @@ +"""scripts/migrate_branch_to_split_files.py moves a branch onto the split files.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +SCRIPT = ( + Path(__file__).resolve().parents[1] / "scripts/migrate_branch_to_split_files.py" +) +_spec = importlib.util.spec_from_file_location("migrate_split_files", SCRIPT) +assert _spec and _spec.loader +migrate = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(migrate) + +CLAIMS = """\ +# header comment +version: 1 + +claims: + # ---------------------------------------------- web + - id: web-supported + claim: >- + Records and replays. + tier: supported + evidence: + # why this test + - path: tests/test_web.py + proves: it works + + # ---------------------------------------------- new + - id: new-claim + claim: A new one. + tier: roadmap +""" + +OLD_LIST = """\ +## Every command, by capability + +Intro. + +- **Record and build a program:** `record`, `compile`. +- **Judge any executor's write:** `verify-run`, `serve-reward` (scores agent episodes). + +## Measured results +""" + + +def test_split_claims_keeps_each_claims_wording() -> None: + pytest.importorskip("yaml") + split = migrate.split_claims(CLAIMS) + assert list(split) == ["web-supported", "new-claim"] + assert split["web-supported"].startswith("id: web-supported\nclaim: >-\n") + assert "# why this test" in split["web-supported"] + assert "# ----" not in split["web-supported"] + assert split["new-claim"] == "id: new-claim\nclaim: A new one.\ntier: roadmap\n" + + +def test_parse_command_list_reads_groups_and_notes() -> None: + commands = migrate.parse_command_list(OLD_LIST) + assert commands == { + "record": ("Record and build a program", ""), + "compile": ("Record and build a program", ""), + "verify-run": ("Judge any executor's write", ""), + "serve-reward": ("Judge any executor's write", "scores agent episodes"), + } + assert migrate._note_text("scores agent episodes") == "Scores agent episodes.\n" + assert migrate.parse_command_list("# no list here\n") == {} + + +def test_resolve_takes_main_only_inside_the_command_list() -> None: + text = ( + "## Map\n" + "<<<<<<< HEAD\n| branch row |\n=======\n| main row |\n>>>>>>> origin/main\n" + "## Every command, by capability\n" + "<<<<<<< HEAD\n" + "- **Run:** `run`, `new-verb`.\n" + "||||||| base\n" + "- **Run:** `run`.\n" + "=======\n" + f"{migrate.GENERATED_BEGIN}\n- `run`\n{migrate.GENERATED_END}\n" + ">>>>>>> origin/main\n" + "## Measured results\n" + ) + resolved = migrate.resolve_command_hunks(text) + assert "new-verb" not in resolved + assert migrate.GENERATED_BEGIN in resolved + # The map-row conflict is not ours to decide. + assert "<<<<<<< HEAD\n| branch row |" in resolved + assert resolved.count("<<<<<<<") == 1