diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index cd8dd3de94..1e18ac96ed 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -397,6 +397,7 @@ jobs: tests/test_effect_e2e_harness.py \ tests/test_admitted_composition_runtime.py \ tests/e2e/test_structural_action.py \ + tests/e2e/test_compile_agent_e2e.py \ --basetemp=runs/ci-critical --junitxml=runs/critical-junit.xml - name: Fail if any critical test was skipped or none ran run: | diff --git a/CAPABILITIES.md b/CAPABILITIES.md index 8259b43fe3..fdbf570a0d 100644 --- a/CAPABILITIES.md +++ b/CAPABILITIES.md @@ -220,7 +220,7 @@ improvising or retrying when something is off. Each `openadapt-flow` command, grouped by the capability it serves. A test fails when a command is added without being listed here. -- **Record and build a program:** `record`, `demo-record`, `compile`, `induce`, `disambiguate`, `for-each`, `compose`, `process`, `intake`, `visualize`, `pull`, `variants`, `route`. +- **Record and build a program:** `record`, `demo-record`, `compile`, `induce`, `disambiguate`, `for-each`, `compose`, `process`, `intake`, `visualize`, `pull`, `variants`, `route`, `compile-agent`. - **Check and admit:** `lint`, `scaffold-verifier`, `check`, `api`, `bind-record-checks`, `certify`, `qualify`, `propose-qualification`, `known-ids`, `jit`, `admit`, `approver-grant`. - **Run and answer:** `run`, `replay`, `resume`, `approve`, `item-approval`, `triage`, `business-decisions`, `explain`, `console`. - **Judge any executor's write:** `verify-run`, `serve-reward` (scores agent training episodes by reading the system of record). diff --git a/packages/flow/benchmark/compile_agent/README.md b/packages/flow/benchmark/compile_agent/README.md new file mode 100644 index 0000000000..b706c7f054 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/README.md @@ -0,0 +1,101 @@ +# compile-agent on MockMed: structure first against a screenshot every turn + +Measured 2026-10-18 with `measure.py` in this folder, engine commit +`2d5e4c15` (when the command was still named `crystallize`; the code paths +it measures are unchanged), model `claude-opus-5-5` at effort `medium`, on the operator's own +key. MockMed with its transactional system of record (`?fault=ok`), 1280x800, +synthetic data only. + +The task: "Open the referral for Jane Sample, start a Triage encounter, and +save this note: {note}". Each trial runs both arms on a freshly reset MockMed, +in alternating order, with a distinct note per trial: + +- **selective**: the harness sends the page's structural listing; the model + can ask for a screenshot, and gets one after the save (the gate). +- **every_turn**: Claude's computer toolset (`computer_toolset_20260801`), + with a screenshot after every action. + +Both arms run through the same harness, capture and compiler, with prompt +caching on (`cache_control` at the top level) in both. + +## Results (10 trials per arm, `results/`) + +| | selective | every turn | every turn / selective | +| --- | ---: | ---: | ---: | +| Run 1 passed its record check | 10 / 10 | 10 / 10 | | +| Model calls (median) | 5 | 5 | 1.0 | +| Actions performed | 6 | 14 | | +| Screenshots sent to the model | 1 | 15 | 15 | +| Prompt tokens per run (median) | 10,332 | 76,053 | **7.36** | +| Output tokens per run (median) | 418 | 576 | 1.38 | +| Cost per run (median) | $0.028 | $0.148 | **5.29** | +| Model time per run (median) | 8.9 s | 11.6 s | 1.29 | +| Run 1 wall time (median, all trials) | 41.6 s | 52.6 s | 1.26 | +| Run 1 wall time, trials 1 to 3 (host load under 200) | 14.6 to 17.6 s | 36.4 to 58.8 s | | + +"Prompt tokens" counts every input token the model read: uncached input, +cache writes and cache reads. Cost uses $4 per million input tokens, $20 +output, $5 cache write and $0.20 cache read. Token counts were nearly identical +across trials (selective 10,332 every time; every turn 76,053 to 76,079), so +the medians are representative. + +Run 1 wall time includes the recording (a before and after frame and a +system-of-record read for every action) and the compile. The host was shared +with other jobs, and its one-minute load average rose from 60 to 480 during +the run (`host_load_1m` in each row), so wall time is noisy. Model time, the +time spent waiting on the API, is less affected. + +### Later runs + +One admitted program from each arm was replayed five times with new notes: +**10 of 10 VERIFIED, 0 model calls, 0 tokens**, and every verify-run receipt +VERIFIED. Replay wall time was 15.9 to 37.3 s (median 36.0 s) at a host load +of 390 to 480. In the pilot, at lower load, the replays of the program +compiled from the selective run took 5.6 and 6.3 s. + +### Spend + +$1.77 for the 20 runs above. With the pilot ($0.17), the identical-inputs +runs ($0.12) and one aborted run (under $0.18), the total was under $2.30. + +## Other files + +- `results/pilot/`: one trial per arm and two replays per arm, run first to + check the harness and the cost. After the pilot, the fields `images_sent` + and `refused_actions` were renamed or removed in the code; the pilot rows were + edited to match (`images_in_requests`), with no value changed. +- `results/identical-inputs/runs.jsonl`: six runs that used the same note in + every trial. Later runs read earlier runs' screenshots from the prompt cache, + so the every-turn arm cost $0.026 instead of $0.148 at the same token count. + Real tasks have different inputs, so the main run uses a distinct note per + trial. Kept to show the effect. + +## Rows + +`results/runs.jsonl` has one row per run 1: the arm, the trial, the order, +the note, the host load before and after, and `run1` with the per-call token +usage and time, the harness's screenshot counts, the gate decision, and the +record check. `results/replays.jsonl` has one row per later run. +`results/summary.json` is computed from them by `measure.py`. + +## Reproduce + +```bash +PYTHONPATH=. python benchmark/compile_agent/measure.py \ + --env-file /path/to/.env --trials 10 --replays 5 --max-total-usd 15 +``` + +`--env-file` holds `ANTHROPIC_API_KEY=...`; the key is read into the process +and never printed. The script stops before a run that could pass +`--max-total-usd`. + +## What this does and does not show + +- One small web app and one task. The structural listing is short on MockMed; + on a dense page the selective arm's listing grows, and a page whose controls + have no accessible names gives the agent less to work with. +- Both arms took the same path in every trial, so this compares observation + cost, not task success. Both passed 10 of 10. +- The model asked for no screenshot in any selective run. On a canvas or a + remote desktop there is no structure, and the selective arm becomes the + every-turn arm. diff --git a/packages/flow/benchmark/compile_agent/measure.py b/packages/flow/benchmark/compile_agent/measure.py new file mode 100644 index 0000000000..1e2035cc22 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/measure.py @@ -0,0 +1,364 @@ +"""Measure compile-agent on MockMed: selective vision against screenshot-every-turn. + +Run 1 is a live Claude computer-use run (``claude-opus-5-5``) on the operator's +own key. Each trial runs both arms, in alternating order, on the same task +against a freshly reset MockMed with its transactional system of record: + +* ``selective``: the harness sends the page's structural listing; the model + asks for a screenshot when it wants one, and gets one after a save. +* ``every_turn``: Claude's computer toolset, a screenshot after every action. + +For each run it records the model's token usage per call (uncached input, +cache writes, cache reads, output), the estimated cost, wall time, the +screenshots sent, and whether run 1 passed the record check. Then one +passing program is admitted (a person's name confirms the checks; a +throwaway admitter key admits) and replayed ``--replays`` times, to measure +later runs: model calls, tokens and time. + + PYTHONPATH=. python benchmark/compile_agent/measure.py --env-file /path/to/.env --trials 3 + +The key is read from ``--env-file`` (``ANTHROPIC_API_KEY=...``) into this +process only, and never printed. ``--max-total-usd`` stops before a run that +could pass it. All data is synthetic (MockMed). +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +RESULTS = HERE / "results" +PROMPT = ( + "Open the referral for Jane Sample, start a Triage encounter, and save " + "this note: {note}" +) +NOTE = "Synthetic follow-up in two weeks" +REPLAY_NOTES = [ + "Recheck knee in ten days", + "Synthetic note for replay two", + "Ice and rest, review in a week", + "Synthetic note for replay four", + "Physio referral, synthetic", +] + + +def _load_key(env_file: Path) -> None: + for line in env_file.read_text().splitlines(): + name, sep, value = line.strip().partition("=") + if sep and name.strip() == "ANTHROPIC_API_KEY": + os.environ["ANTHROPIC_API_KEY"] = value.strip().strip('"').strip("'") + return + raise SystemExit(f"no ANTHROPIC_API_KEY in {env_file}") + + +def _verify_yaml(dirpath: Path, base: str) -> Path: + path = dirpath / "verify.yaml" + path.write_text( + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/db\n" + " records_key: records\n" + " timeout_s: 3.0\n" + " poll_interval_s: 0.1\n" + ) + return path + + +def _compiler(work: Path, base: str, task_id: str) -> Any: + from openadapt_flow.compile_agent import FirstRunCompiler + from openadapt_flow.jit.keys import load_customer_model_key + from openadapt_flow.tutorial import _http_json + + return FirstRunCompiler( + PROMPT, + params=["note"], + url=f"{base}/?fault=ok&idempotency=demo#tasks", + verify=_verify_yaml(work, base), + expect={"patient_id": "p1", "type": "Triage"}, + on_screen=["Jane Sample", "MRN P1"], + task_id=task_id, + cache_dir=work / "cache", + key_dir=work / "keys", + reset=lambda: _http_json(f"{base}/api/reset", method="POST", body={}), + on_warning=lambda text: None, + model_key=load_customer_model_key("anthropic"), + ) + + +def _clean_run1(run1: dict[str, Any]) -> dict[str, Any]: + """Run 1's record without local paths or gate screenshots.""" + + gates = [ + {k: v for k, v in g.items() if k not in ("screenshot", "url", "at")} + for g in run1.get("gates") or [] + ] + return { + "jit_status": run1.get("jit_status"), + "seconds": run1.get("seconds"), + "agent": run1.get("agent"), + "harness": run1.get("harness"), + "gates": gates, + "record_check": run1.get("record_check"), + "agent_error": run1.get("agent_error"), + } + + +def _admitter(work: Path) -> Any: + from openadapt_flow.jit import authority + + os.environ.setdefault(authority.KEY_PASSPHRASE_ENV, "measurement-only") + pem = work / "keyfiles" / "admitter.pem" + authority.register_principal( + work / "keys", + role="admitter_key", + principal_id="ci:measurement", + public_key_hex=authority.generate_encrypted_key(pem), + ) + return authority.principal_from_key( + authority.load_encrypted_key(pem), work / "keys" + ) + + +def _median(rows: list[dict[str, Any]], key: str) -> float | None: + values = [float(r[key]) for r in rows if r.get(key) is not None] + return round(statistics.median(values), 3) if values else None + + +def _summary( + runs: list[dict[str, Any]], replays: list[dict[str, Any]] +) -> dict[str, Any]: + arms: dict[str, Any] = {} + for arm in ("selective", "every_turn"): + rows = [] + for run in runs: + if run["arm"] != arm: + continue + agent = run["run1"].get("agent") or {} + harness = run["run1"].get("harness") or {} + rows.append( + { + "passed": run["status"] == "draft", + "prompt_tokens": agent.get("prompt_tokens"), + "input_tokens": agent.get("input_tokens"), + "cache_read_input_tokens": agent.get("cache_read_input_tokens"), + "cache_creation_input_tokens": agent.get( + "cache_creation_input_tokens" + ), + "output_tokens": agent.get("output_tokens"), + "cost_usd": agent.get("cost_usd"), + "model_calls": agent.get("model_calls"), + "run_seconds": run["run1"].get("seconds"), + "model_seconds": agent.get("model_seconds"), + "screenshots_sent": ( + harness.get("screenshots_requested", 0) + + harness.get("screenshots_at_gate", 0) + + harness.get("screenshots_every_turn", 0) + ), + } + ) + arms[arm] = { + "runs": len(rows), + "passed": sum(1 for r in rows if r["passed"]), + **{ + f"median_{key}": _median(rows, key) + for key in ( + "prompt_tokens", + "input_tokens", + "cache_read_input_tokens", + "cache_creation_input_tokens", + "output_tokens", + "cost_usd", + "model_calls", + "run_seconds", + "model_seconds", + "screenshots_sent", + ) + }, + "total_cost_usd": round(sum(r["cost_usd"] or 0 for r in rows), 4), + } + sel, full = arms["selective"], arms["every_turn"] + + def ratio(key: str) -> float | None: + a, b = sel.get(key), full.get(key) + return round(b / a, 2) if a and b else None + + return { + "arms": arms, + "every_turn_over_selective": { + key: ratio(key) + for key in ( + "median_prompt_tokens", + "median_output_tokens", + "median_cost_usd", + "median_run_seconds", + "median_model_seconds", + ) + }, + "replays": { + "runs": len(replays), + "verified": sum(1 for r in replays if r["verified"]), + "model_calls_total": sum(r["model_calls"] or 0 for r in replays), + "median_seconds": _median(replays, "seconds"), + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--env-file", type=Path, required=True) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--replays", type=int, default=5) + parser.add_argument("--max-run-usd", type=float, default=3.0) + parser.add_argument("--max-total-usd", type=float, default=15.0) + parser.add_argument("--effort", default="medium") + parser.add_argument("--out", type=Path, default=RESULTS) + args = parser.parse_args() + _load_key(args.env_file) + + from openadapt_flow.compile_agent import ClaudeAgent + from openadapt_flow.mockmed.fault_server import serve + + args.out.mkdir(parents=True, exist_ok=True) + runs_path = args.out / "runs.jsonl" + replays_path = args.out / "replays.jsonl" + runs: list[dict[str, Any]] = [] + replays: list[dict[str, Any]] = [] + spent = 0.0 + work_root = Path(tempfile.mkdtemp(prefix="compile-agent-measure-")) + base, _db, stop = serve() + base = base.rstrip("/") + passing: list[tuple[str, Any]] = [] + try: + for trial in range(1, args.trials + 1): + order = ["selective", "every_turn"] + if trial % 2 == 0: + order.reverse() + for arm in order: + if spent + args.max_run_usd > args.max_total_usd: + print( + f"stopping: ${spent:.2f} spent, cap ${args.max_total_usd:.2f}" + ) + raise StopIteration + work = work_root / f"t{trial}-{arm}" + work.mkdir(parents=True) + compiler = _compiler(work, base, f"mockmed.measure-{arm}-t{trial}") + agent = ClaudeAgent( + vision=arm, effort=args.effort, max_cost_usd=args.max_run_usd + ) + started = time.time() + load_before = os.getloadavg()[0] + # A distinct note per trial, as real tasks have distinct inputs: + # with one note for every trial, later runs read earlier runs' + # screenshots from the prompt cache and cost less than a real run. + note = f"{NOTE}, trial {trial}" + result = compiler.record(agent, {"note": note}) + run1 = _clean_run1(result.run1) + cost = float(((run1.get("agent") or {}).get("cost_usd")) or 0.0) + spent += cost + row = { + "trial": trial, + "arm": arm, + "order": order.index(arm), + "started_at": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(started) + ), + "status": result.status, + "note": note, + "detail": result.detail if result.status != "draft" else None, + # The host is shared; wall time moves with its load. + "host_load_1m": [ + round(load_before, 1), + round(os.getloadavg()[0], 1), + ], + "run1": run1, + } + runs.append(row) + with runs_path.open("a") as fh: + fh.write(json.dumps(row, sort_keys=True) + "\n") + agent_row = run1.get("agent") or {} + print( + f"trial {trial} {arm}: {result.status}; " + f"{agent_row.get('model_calls')} calls, " + f"{agent_row.get('prompt_tokens')} prompt tokens, " + f"${cost:.3f}, {run1.get('seconds')} s (total ${spent:.2f})", + flush=True, + ) + if result.status == "draft": + passing.append((arm, compiler)) + except StopIteration: + pass + try: + # Admit one passing program from each arm and replay it. + for arm in ("selective", "every_turn"): + found = [c for a, c in passing if a == arm] + if not found: + continue + compiler = found[0] + compiler.confirm_checks( + principal_id="measurement.operator", drop_fields=["key", "source"] + ) + admitted = compiler.admit( + admitter=_admitter(compiler.dir.parent.parent), + canary_params=[{"note": "Canary test note 1"}], + ) + print(f"admit {arm}: {admitted.status} {admitted.detail or ''}", flush=True) + if not admitted.admitted: + continue + compiler.emit() + for i, note in enumerate(REPLAY_NOTES[: args.replays], start=1): + replay = compiler.run(note=note) + row = { + "program_from": arm, + "replay": i, + "status": replay.status, + "verified": replay.verified, + "model_calls": replay.model_calls, + "model_tokens": 0 if replay.model_calls == 0 else None, + "seconds": replay.seconds, + "receipt_outcome": replay.receipt_outcome, + "host_load_1m": round(os.getloadavg()[0], 1), + } + replays.append(row) + with replays_path.open("a") as fh: + fh.write(json.dumps(row, sort_keys=True) + "\n") + print(f"replay {arm} {i}: {row}", flush=True) + finally: + stop() + summary = _summary(runs, replays) + summary["environment"] = { + "model": "claude-opus-5-5", + "effort": args.effort, + "computer_tool": "computer_toolset_20260801 (every_turn arm)", + "app": "MockMed fault server, ?fault=ok&idempotency=demo, 1280x800", + "engine_commit": subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + cwd=HERE, + ).stdout.strip(), + "python": sys.version.split()[0], + "platform": platform.platform(), + "measured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "total_cost_usd": round(spent, 4), + "prices_per_mtok": "input 4.00, output 20.00, cache write 5.00, cache read 0.20", + } + (args.out / "summary.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n" + ) + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/compile_agent/results/identical-inputs/runs.jsonl b/packages/flow/benchmark/compile_agent/results/identical-inputs/runs.jsonl new file mode 100644 index 0000000000..c2e37a422e --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/identical-inputs/runs.jsonl @@ -0,0 +1,6 @@ +{"arm": "selective", "detail": null, "host_load_1m": [116.9, 96.1], "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3716, "cache_read_input_tokens": 6561, "calls": [{"cache_creation_input_tokens": 1288, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.916, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1288, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.323, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1453, "images_in_request": 0, "input_tokens": 2, "output_tokens": 127, "seconds": 1.941, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 510, "cache_read_input_tokens": 1655, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.415, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1551, "cache_read_input_tokens": 2165, "images_in_request": 1, "input_tokens": 2, "output_tokens": 102, "seconds": 1.894, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02734, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.489, "output_tokens": 370, "prompt_tokens": 10289, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with note \"Synthetic follow-up in two weeks\"; confirmation shown and encounter listed.", "tool_errors": 0, "vision": "selective", "wall_seconds": 10.604}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 17.532}, "started_at": "2026-10-18T17:30:07Z", "status": "draft", "trial": 1} +{"arm": "every_turn", "detail": null, "host_load_1m": [96.1, 72.3], "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 0, "cache_read_input_tokens": 76006, "calls": [{"cache_creation_input_tokens": 0, "cache_read_input_tokens": 5712, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.784, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 9904, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 7.062, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 14094, "images_in_request": 7, "input_tokens": 2, "output_tokens": 149, "seconds": 3.159, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 21053, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.961, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 25243, "images_in_request": 15, "input_tokens": 2, "output_tokens": 100, "seconds": 2.915, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.025989, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 18.881, "output_tokens": 537, "prompt_tokens": 76018, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with note \"Synthetic follow-up in two weeks\"; it now appears in her encounter list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 25.154}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 37.582}, "started_at": "2026-10-18T17:30:25Z", "status": "draft", "trial": 1} +{"arm": "every_turn", "detail": null, "host_load_1m": [72.3, 121.6], "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 0, "cache_read_input_tokens": 76006, "calls": [{"cache_creation_input_tokens": 0, "cache_read_input_tokens": 5712, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.161, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 9904, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.698, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 14094, "images_in_request": 7, "input_tokens": 2, "output_tokens": 149, "seconds": 3.056, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 21053, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.587, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 25243, "images_in_request": 15, "input_tokens": 2, "output_tokens": 99, "seconds": 2.528, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.025969, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 13.029, "output_tokens": 536, "prompt_tokens": 76018, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks\"; the app confirmed the save.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 20.233}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 38.253}, "started_at": "2026-10-18T17:31:02Z", "status": "draft", "trial": 2} +{"arm": "selective", "detail": null, "host_load_1m": [121.6, 100.4], "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 0, "cache_read_input_tokens": 10277, "calls": [{"cache_creation_input_tokens": 0, "cache_read_input_tokens": 1288, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.793, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 1453, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.367, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 1655, "images_in_request": 0, "input_tokens": 2, "output_tokens": 127, "seconds": 1.807, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 2165, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.294, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 3716, "images_in_request": 1, "input_tokens": 2, "output_tokens": 106, "seconds": 1.905, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.009583, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.166, "output_tokens": 374, "prompt_tokens": 10289, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with note \"Synthetic follow-up in two weeks\"; confirmation shown and it appears in her encounter list.", "tool_errors": 0, "vision": "selective", "wall_seconds": 10.586}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 14.778}, "started_at": "2026-10-18T17:31:41Z", "status": "draft", "trial": 2} +{"arm": "selective", "detail": null, "host_load_1m": [100.4, 111.0], "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 0, "cache_read_input_tokens": 10277, "calls": [{"cache_creation_input_tokens": 0, "cache_read_input_tokens": 1288, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.792, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 1453, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.328, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 1655, "images_in_request": 0, "input_tokens": 2, "output_tokens": 127, "seconds": 1.824, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 2165, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.311, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 3716, "images_in_request": 1, "input_tokens": 2, "output_tokens": 101, "seconds": 1.759, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.009483, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.015, "output_tokens": 369, "prompt_tokens": 10289, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks\"; it now appears in her encounter list.", "tool_errors": 0, "vision": "selective", "wall_seconds": 10.677}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 13.313}, "started_at": "2026-10-18T17:31:55Z", "status": "draft", "trial": 3} +{"arm": "every_turn", "detail": null, "host_load_1m": [111.0, 72.4], "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 0, "cache_read_input_tokens": 76006, "calls": [{"cache_creation_input_tokens": 0, "cache_read_input_tokens": 5712, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.254, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 9904, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 3.065, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 14094, "images_in_request": 7, "input_tokens": 2, "output_tokens": 149, "seconds": 3.085, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 21053, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.555, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 0, "cache_read_input_tokens": 25243, "images_in_request": 15, "input_tokens": 2, "output_tokens": 80, "seconds": 2.764, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.025589, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 13.723, "output_tokens": 517, "prompt_tokens": 76018, "summary": "Saved Triage encounter for Jane Sample with note \"Synthetic follow-up in two weeks\".", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 20.678}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 37.697}, "started_at": "2026-10-18T17:32:09Z", "status": "draft", "trial": 3} diff --git a/packages/flow/benchmark/compile_agent/results/pilot/replays.jsonl b/packages/flow/benchmark/compile_agent/results/pilot/replays.jsonl new file mode 100644 index 0000000000..1c65b0dad6 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/pilot/replays.jsonl @@ -0,0 +1,4 @@ +{"model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 1, "seconds": 6.296, "status": "VERIFIED", "verified": true} +{"model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 2, "seconds": 5.634, "status": "VERIFIED", "verified": true} +{"model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 1, "seconds": 10.048, "status": "VERIFIED", "verified": true} +{"model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 2, "seconds": 35.765, "status": "VERIFIED", "verified": true} diff --git a/packages/flow/benchmark/compile_agent/results/pilot/runs.jsonl b/packages/flow/benchmark/compile_agent/results/pilot/runs.jsonl new file mode 100644 index 0000000000..612fede426 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/pilot/runs.jsonl @@ -0,0 +1,2 @@ +{"arm": "selective", "detail": null, "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3716, "cache_read_input_tokens": 6561, "calls": [{"cache_creation_input_tokens": 1288, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 2.146, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1288, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.323, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1453, "images_in_request": 0, "input_tokens": 2, "output_tokens": 127, "seconds": 1.901, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 510, "cache_read_input_tokens": 1655, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.29, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1551, "cache_read_input_tokens": 2165, "images_in_request": 1, "input_tokens": 2, "output_tokens": 101, "seconds": 1.895, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02732, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.554, "output_tokens": 369, "prompt_tokens": 10289, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks\"; it now shows in her encounter list.", "tool_errors": 0, "vision": "selective", "wall_seconds": 12.199}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 24.119}, "started_at": "2026-10-18T17:25:46Z", "status": "draft", "trial": 1} +{"arm": "every_turn", "detail": null, "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25243, "cache_read_input_tokens": 50763, "calls": [{"cache_creation_input_tokens": 5712, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.812, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5712, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.705, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9904, "images_in_request": 7, "input_tokens": 2, "output_tokens": 149, "seconds": 3.038, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6959, "cache_read_input_tokens": 14094, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 3.192, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21053, "images_in_request": 15, "input_tokens": 2, "output_tokens": 101, "seconds": 2.992, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.147176, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 14.739, "output_tokens": 538, "prompt_tokens": 76018, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks\"; it now appears in her encounters list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 21.485}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 37.653}, "started_at": "2026-10-18T17:26:10Z", "status": "draft", "trial": 1} diff --git a/packages/flow/benchmark/compile_agent/results/pilot/summary.json b/packages/flow/benchmark/compile_agent/results/pilot/summary.json new file mode 100644 index 0000000000..df2d8e4a95 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/pilot/summary.json @@ -0,0 +1,59 @@ +{ + "arms": { + "every_turn": { + "median_cache_creation_input_tokens": 25243.0, + "median_cache_read_input_tokens": 50763.0, + "median_cost_usd": 0.147, + "median_input_tokens": 12.0, + "median_model_calls": 5.0, + "median_model_seconds": 14.739, + "median_output_tokens": 538.0, + "median_prompt_tokens": 76018.0, + "median_run_seconds": 37.653, + "median_screenshots_sent": 15.0, + "passed": 1, + "runs": 1, + "total_cost_usd": 0.1472 + }, + "selective": { + "median_cache_creation_input_tokens": 3716.0, + "median_cache_read_input_tokens": 6561.0, + "median_cost_usd": 0.027, + "median_input_tokens": 12.0, + "median_model_calls": 5.0, + "median_model_seconds": 8.554, + "median_output_tokens": 369.0, + "median_prompt_tokens": 10289.0, + "median_run_seconds": 24.119, + "median_screenshots_sent": 1.0, + "passed": 1, + "runs": 1, + "total_cost_usd": 0.0273 + } + }, + "environment": { + "app": "MockMed fault server, ?fault=ok&idempotency=demo, 1280x800", + "computer_tool": "computer_toolset_20260801 (every_turn arm)", + "effort": "medium", + "engine_commit": "3140e831", + "measured_at": "2026-10-18T17:28:09Z", + "model": "claude-opus-5-5", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "prices_per_mtok": "input 4.00, output 20.00, cache write 5.00, cache read 0.20", + "python": "3.12.7", + "total_cost_usd": 0.1745 + }, + "every_turn_over_selective": { + "median_cost_usd": 5.44, + "median_model_seconds": 1.72, + "median_output_tokens": 1.46, + "median_prompt_tokens": 7.39, + "median_run_seconds": 1.56 + }, + "replays": { + "median_seconds": 8.172, + "model_calls_total": 0, + "runs": 4, + "verified": 4 + } +} diff --git a/packages/flow/benchmark/compile_agent/results/replays.jsonl b/packages/flow/benchmark/compile_agent/results/replays.jsonl new file mode 100644 index 0000000000..747ada0cc8 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/replays.jsonl @@ -0,0 +1,10 @@ +{"host_load_1m": 481.9, "model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 1, "seconds": 36.265, "status": "VERIFIED", "verified": true} +{"host_load_1m": 481.4, "model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 2, "seconds": 37.299, "status": "VERIFIED", "verified": true} +{"host_load_1m": 461.1, "model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 3, "seconds": 36.045, "status": "VERIFIED", "verified": true} +{"host_load_1m": 443.6, "model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 4, "seconds": 36.021, "status": "VERIFIED", "verified": true} +{"host_load_1m": 413.9, "model_calls": 0, "model_tokens": 0, "program_from": "selective", "receipt_outcome": "VERIFIED", "replay": 5, "seconds": 36.821, "status": "VERIFIED", "verified": true} +{"host_load_1m": 441.3, "model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 1, "seconds": 36.6, "status": "VERIFIED", "verified": true} +{"host_load_1m": 435.1, "model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 2, "seconds": 26.609, "status": "VERIFIED", "verified": true} +{"host_load_1m": 409.0, "model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 3, "seconds": 15.867, "status": "VERIFIED", "verified": true} +{"host_load_1m": 391.0, "model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 4, "seconds": 19.247, "status": "VERIFIED", "verified": true} +{"host_load_1m": 392.9, "model_calls": 0, "model_tokens": 0, "program_from": "every_turn", "receipt_outcome": "VERIFIED", "replay": 5, "seconds": 30.113, "status": "VERIFIED", "verified": true} diff --git a/packages/flow/benchmark/compile_agent/results/runs.jsonl b/packages/flow/benchmark/compile_agent/results/runs.jsonl new file mode 100644 index 0000000000..d1a951ff30 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/runs.jsonl @@ -0,0 +1,20 @@ +{"arm": "selective", "detail": null, "host_load_1m": [59.9, 64.9], "note": "Synthetic follow-up in two weeks, trial 1", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.842, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.311, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.812, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.33, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 139, "seconds": 2.208, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02829, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.503, "output_tokens": 412, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 1\"; her encounter list shows the full note, though the save banner cut off the final \"1\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 11.306}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 17.597}, "started_at": "2026-10-18T17:33:12Z", "status": "draft", "trial": 1} +{"arm": "every_turn", "detail": null, "host_load_1m": [64.9, 47.4], "note": "Synthetic follow-up in two weeks, trial 1", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.792, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.588, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.838, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.952, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 136, "seconds": 3.164, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.148031, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 14.334, "output_tokens": 578, "prompt_tokens": 76053, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 1\"; it now shows in her encounter list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 20.97}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 36.351}, "started_at": "2026-10-18T17:33:29Z", "status": "draft", "trial": 1} +{"arm": "every_turn", "detail": null, "host_load_1m": [47.4, 85.0], "note": "Synthetic follow-up in two weeks, trial 2", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.909, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.003, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.367, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.186, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 134, "seconds": 2.331, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.147991, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 10.797, "output_tokens": 576, "prompt_tokens": 76053, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 2\"; the encounter list shows the full note.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 17.104}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 38.832}, "started_at": "2026-10-18T17:34:06Z", "status": "draft", "trial": 2} +{"arm": "selective", "detail": null, "host_load_1m": [85.0, 111.7], "note": "Synthetic follow-up in two weeks, trial 2", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.956, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.459, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.883, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.264, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 160, "seconds": 2.429, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02871, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.991, "output_tokens": 433, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with note \"Synthetic follow-up in two weeks, trial 2\"; the encounter list shows the full note, but the confirmation banner cuts off the final \"2\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 11.236}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 14.558}, "started_at": "2026-10-18T17:34:44Z", "status": "draft", "trial": 2} +{"arm": "selective", "detail": null, "host_load_1m": [111.7, 95.6], "note": "Synthetic follow-up in two weeks, trial 3", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.786, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.253, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.811, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.288, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 148, "seconds": 2.423, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02847, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.56, "output_tokens": 421, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 3\"; the encounter list shows the full note, though the save banner cuts off the final \"3\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 10.537}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 15.34}, "started_at": "2026-10-18T17:34:59Z", "status": "draft", "trial": 3} +{"arm": "every_turn", "detail": null, "host_load_1m": [95.6, 199.4], "note": "Synthetic follow-up in two weeks, trial 3", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.589, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 1.834, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.052, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 1.861, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 128, "seconds": 2.331, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.147871, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 9.667, "output_tokens": 570, "prompt_tokens": 76053, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with note \"Synthetic follow-up in two weeks, trial 3\"; it appears in her encounter list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 17.378}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 58.772}, "started_at": "2026-10-18T17:35:14Z", "status": "draft", "trial": 3} +{"arm": "every_turn", "detail": null, "host_load_1m": [199.4, 313.5], "note": "Synthetic follow-up in two weeks, trial 4", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.103, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.298, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.706, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.593, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 145, "seconds": 2.983, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.148211, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 12.684, "output_tokens": 587, "prompt_tokens": 76053, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 4\"; it is listed under her Encounters, though the green save banner cuts off the final \"4\".", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 19.388}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 42.293}, "started_at": "2026-10-18T17:36:13Z", "status": "draft", "trial": 4} +{"arm": "selective", "detail": null, "host_load_1m": [313.5, 348.1], "note": "Synthetic follow-up in two weeks, trial 4", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.769, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.531, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 2.754, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.587, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 138, "seconds": 2.692, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02827, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 10.333, "output_tokens": 411, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 4\", shown in full in her encounter list (the save banner cuts off the final \"4\").", "tool_errors": 0, "vision": "selective", "wall_seconds": 13.172}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 36.589}, "started_at": "2026-10-18T17:36:55Z", "status": "draft", "trial": 4} +{"arm": "selective", "detail": null, "host_load_1m": [348.1, 311.7], "note": "Synthetic follow-up in two weeks, trial 5", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.888, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.321, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.959, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.287, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 146, "seconds": 2.425, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02843, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.879, "output_tokens": 419, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 5\"; the encounters list shows it, though the save banner cuts off the final \"5\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 12.19}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 27.829}, "started_at": "2026-10-18T17:37:32Z", "status": "draft", "trial": 5} +{"arm": "every_turn", "detail": null, "host_load_1m": [311.7, 327.8], "note": "Synthetic follow-up in two weeks, trial 5", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 3.544, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.342, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.727, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.596, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 133, "seconds": 2.804, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.147971, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 14.012, "output_tokens": 575, "prompt_tokens": 76053, "summary": "Saved a Triage encounter on Jane Sample's referral (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 5\"; it appears in her encounter list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 21.225}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 52.825}, "started_at": "2026-10-18T17:38:00Z", "status": "draft", "trial": 5} +{"arm": "every_turn", "detail": null, "host_load_1m": [327.8, 381.1], "note": "Synthetic follow-up in two weeks, trial 6", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25278, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.192, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.276, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.345, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 123, "seconds": 2.442, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4215, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 136, "seconds": 2.692, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.148656, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 11.947, "output_tokens": 603, "prompt_tokens": 76078, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 6\"; the encounter list shows the full note.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 19.131}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 55.774}, "started_at": "2026-10-18T17:38:53Z", "status": "draft", "trial": 6} +{"arm": "selective", "detail": null, "host_load_1m": [381.1, 397.4], "note": "Synthetic follow-up in two weeks, trial 6", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.814, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.342, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.819, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.322, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 150, "seconds": 2.502, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02851, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.799, "output_tokens": 423, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 6\"; her encounter list shows it in full, though the save banner cuts off the final \"6\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 11.112}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 49.431}, "started_at": "2026-10-18T17:39:48Z", "status": "draft", "trial": 6} +{"arm": "selective", "detail": null, "host_load_1m": [397.4, 412.3], "note": "Synthetic follow-up in two weeks, trial 7", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.862, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.267, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.863, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.336, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 140, "seconds": 2.347, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02831, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 8.675, "output_tokens": 413, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 7\"; it shows in full in her encounter list (the save banner cuts off the final \"7\").", "tool_errors": 0, "vision": "selective", "wall_seconds": 11.059}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 48.553}, "started_at": "2026-10-18T17:40:38Z", "status": "draft", "trial": 7} +{"arm": "every_turn", "detail": null, "host_load_1m": [412.3, 450.0], "note": "Synthetic follow-up in two weeks, trial 7", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25279, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.029, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.125, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.109, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 124, "seconds": 2.114, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4216, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 106, "seconds": 1.836, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.148081, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 10.213, "output_tokens": 574, "prompt_tokens": 76079, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 7\"; the encounter list shows the full note.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 16.994}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 52.307}, "started_at": "2026-10-18T17:41:26Z", "status": "draft", "trial": 7} +{"arm": "every_turn", "detail": null, "host_load_1m": [450.0, 465.7], "note": "Synthetic follow-up in two weeks, trial 8", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 2.279, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.573, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.978, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 3.608, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 132, "seconds": 4.444, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.147951, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 15.882, "output_tokens": 574, "prompt_tokens": 76053, "summary": "Saved a Triage encounter on Jane Sample's referral (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 8\"; the full note shows in her encounter list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 23.122}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 58.368}, "started_at": "2026-10-18T17:42:19Z", "status": "draft", "trial": 8} +{"arm": "selective", "detail": null, "host_load_1m": [465.7, 450.1], "note": "Synthetic follow-up in two weeks, trial 8", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.476, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 2.017, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 2.027, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.38, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 139, "seconds": 2.779, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02829, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 9.679, "output_tokens": 412, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 8\". The encounter list shows the full note, though the save banner cuts off the final \"8\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 12.226}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 47.129}, "started_at": "2026-10-18T17:43:17Z", "status": "draft", "trial": 8} +{"arm": "selective", "detail": null, "host_load_1m": [450.1, 450.9], "note": "Synthetic follow-up in two weeks, trial 9", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.661, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.412, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 2.062, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.478, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 143, "seconds": 2.557, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02837, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 9.17, "output_tokens": 416, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample (MRN P1); the encounter list shows the full note \"Synthetic follow-up in two weeks, trial 9\" (the save banner cuts off the final \"9\").", "tool_errors": 0, "vision": "selective", "wall_seconds": 11.753}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 46.606}, "started_at": "2026-10-18T17:44:04Z", "status": "draft", "trial": 9} +{"arm": "every_turn", "detail": null, "host_load_1m": [450.9, 442.1], "note": "Synthetic follow-up in two weeks, trial 9", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25278, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.942, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 2.06, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.381, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 123, "seconds": 2.371, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4215, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 117, "seconds": 2.408, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.148276, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 11.163, "output_tokens": 584, "prompt_tokens": 76078, "summary": "Saved a Triage encounter for Jane Sample (MRN P1); the encounter list shows the full note \"Synthetic follow-up in two weeks, trial 9\" (the save banner cuts off the trailing \"9\").", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 18.57}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 50.81}, "started_at": "2026-10-18T17:44:51Z", "status": "draft", "trial": 9} +{"arm": "every_turn", "detail": null, "host_load_1m": [442.1, 452.8], "note": "Synthetic follow-up in two weeks, trial 10", "order": 0, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 25253, "cache_read_input_tokens": 50788, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.914, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 1.826, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 154, "seconds": 2.266, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6964, "cache_read_input_tokens": 14099, "images_in_request": 12, "input_tokens": 2, "output_tokens": 98, "seconds": 2.191, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 21063, "images_in_request": 15, "input_tokens": 2, "output_tokens": 130, "seconds": 2.276, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.147911, "finished": true, "images_in_requests": 39, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 10.473, "output_tokens": 572, "prompt_tokens": 76053, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 10\"; it appears in her encounter list.", "tool_errors": 0, "vision": "every_turn", "wall_seconds": 17.079}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 14, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 0, "screenshots_at_gate": 0, "screenshots_every_turn": 15, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 57.257}, "started_at": "2026-10-18T17:45:42Z", "status": "draft", "trial": 10} +{"arm": "selective", "detail": null, "host_load_1m": [452.8, 480.5], "note": "Synthetic follow-up in two weeks, trial 10", "order": 1, "run1": {"agent": {"agent": "claude", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.74, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.577, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 2.116, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.431, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 150, "seconds": 2.793, "stop_reason": "tool_use", "turn": 5}], "cost_usd": 0.02851, "finished": true, "images_in_requests": 1, "input_tokens": 12, "model": "claude-opus-5-5", "model_calls": 5, "model_seconds": 9.656, "output_tokens": 423, "prompt_tokens": 10332, "summary": "Saved a Triage encounter for Jane Sample (MRN P1) with the note \"Synthetic follow-up in two weeks, trial 10\"; the encounter list shows the full text, but the confirmation banner cuts off the trailing \"10\".", "tool_errors": 0, "vision": "selective", "wall_seconds": 12.138}, "agent_error": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "harness": {"actions": 6, "gate_checks": 1, "gate_evidence_screenshots": 1, "gate_refusals": 0, "observations": 6, "screenshots_at_gate": 1, "screenshots_every_turn": 0, "screenshots_requested": 0}, "jit_status": "needs_human_admit", "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "seconds": 49.309}, "started_at": "2026-10-18T17:46:39Z", "status": "draft", "trial": 10} diff --git a/packages/flow/benchmark/compile_agent/results/summary.json b/packages/flow/benchmark/compile_agent/results/summary.json new file mode 100644 index 0000000000..e784a4f482 --- /dev/null +++ b/packages/flow/benchmark/compile_agent/results/summary.json @@ -0,0 +1,59 @@ +{ + "arms": { + "every_turn": { + "median_cache_creation_input_tokens": 25253.0, + "median_cache_read_input_tokens": 50788.0, + "median_cost_usd": 0.148, + "median_input_tokens": 12.0, + "median_model_calls": 5.0, + "median_model_seconds": 11.555, + "median_output_tokens": 575.5, + "median_prompt_tokens": 76053.0, + "median_run_seconds": 52.566, + "median_screenshots_sent": 15.0, + "passed": 10, + "runs": 10, + "total_cost_usd": 1.4809 + }, + "selective": { + "median_cache_creation_input_tokens": 3737.0, + "median_cache_read_input_tokens": 6583.0, + "median_cost_usd": 0.028, + "median_input_tokens": 12.0, + "median_model_calls": 5.0, + "median_model_seconds": 8.935, + "median_output_tokens": 417.5, + "median_prompt_tokens": 10332.0, + "median_run_seconds": 41.597, + "median_screenshots_sent": 1.0, + "passed": 10, + "runs": 10, + "total_cost_usd": 0.2842 + } + }, + "environment": { + "app": "MockMed fault server, ?fault=ok&idempotency=demo, 1280x800", + "computer_tool": "computer_toolset_20260801 (every_turn arm)", + "effort": "medium", + "engine_commit": "2d5e4c15", + "measured_at": "2026-10-18T17:54:03Z", + "model": "claude-opus-5-5", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "prices_per_mtok": "input 4.00, output 20.00, cache write 5.00, cache read 0.20", + "python": "3.12.7", + "total_cost_usd": 1.7651 + }, + "every_turn_over_selective": { + "median_cost_usd": 5.29, + "median_model_seconds": 1.29, + "median_output_tokens": 1.38, + "median_prompt_tokens": 7.36, + "median_run_seconds": 1.26 + }, + "replays": { + "median_seconds": 36.033, + "model_calls_total": 0, + "runs": 10, + "verified": 10 + } +} diff --git a/packages/flow/docs/CAPABILITIES.md b/packages/flow/docs/CAPABILITIES.md index d23c65c84b..8632d7a711 100644 --- a/packages/flow/docs/CAPABILITIES.md +++ b/packages/flow/docs/CAPABILITIES.md @@ -139,6 +139,21 @@ at replaying one. These capabilities layer onto the same $0, model-free runtime. Compiled workflows can also be emitted as Agent Skills or MCP servers (`emit-skill` / `emit-mcp`), so other agents can invoke them. +## From a prompt to a program + +`openadapt-flow compile-agent ""` lets a computer-use agent (Claude, on +your own key) do the task once in a harness that gives it the page's structure +first and a screenshot only when it asks or after a save, and that checks the +right record is on screen before every save. If the system of record shows +exactly the record the task named, the run is compiled, a person confirms the +proposed record checks, and the program is admitted under the JIT lifecycle +(by a key, by a registered agent under a signed policy, or by a person). The +command also emits an Agent Skill, an MCP tool and a one-line CLI command. +Every later run makes no model calls and gets a `verify-run` receipt. An agent +that opens the wrong patient is refused at the save, and nothing is compiled. +See [`COMPILE_AGENT.md`](COMPILE_AGENT.md), which also has the measured token and +time cost of structure-first observation against a screenshot every turn. + ## Answer a halt from a phone An attended run can project one signed operational-halt task to an diff --git a/packages/flow/docs/COMPILE_AGENT.md b/packages/flow/docs/COMPILE_AGENT.md new file mode 100644 index 0000000000..829bee3255 --- /dev/null +++ b/packages/flow/docs/COMPILE_AGENT.md @@ -0,0 +1,262 @@ +# Compile an agent's first verified run + +`compile-agent` lets a computer-use agent do a task once, checks the record +it wrote, and compiles that run into a program. Every later run executes the +program: no model calls, every save read back from the system of record. + +The agent runs against the app at `--url` and makes real entries there. Point +it at a test copy of the app with test records. + +```bash +openadapt-flow compile-agent \ + "Open the referral for Jane Sample, start a Triage encounter, and save this note: {note}" \ + --url http://localhost:8080/#tasks --verify verify.yaml \ + --param note="Follow up in two weeks" \ + --expect patient_id=p1 --on-screen "Jane Sample" --on-screen "MRN P1" \ + --canary-param note="Canary test note" \ + --admit-key agent.pem --admission-policy signed-policy.json + +openadapt-flow compile-agent run .openadapt-compile-agent/ \ + --param note="Recheck knee in ten days" +``` + +`verify.yaml` needs only a `verifier:` section that reads the system of record +(`rest`, `sql`, `fhir`, `graphql` or `file`), the same section +`record --verify` and `verify-run` read. The agent is Claude +(`claude-opus-5-5`) on your own key from `ANTHROPIC_API_KEY`; OpenAdapt holds +no key. `--agent steps.json` runs a scripted agent with no model calls. + +## What happens + +1. **Run 1, in the harness.** The agent does the task while + `openadapt_flow.jit` records every action, exactly as for any other JIT + capture ([JIT.md](JIT.md)). +2. **The record check.** The system of record is read when run 1 starts and + after it ends. Run 1 passes only when exactly one record appeared, no other + record changed or disappeared, the new record has every `--expect` field + with that value, and every `--param` value landed in one of its fields. + Otherwise the draft is set aside and nothing can be admitted. +3. **Proposed checks, confirmed by a person.** The change becomes proposed + record checks (the `record --verify` proposer). A person reads them at a + terminal, can leave out fields that change on every save + (`--drop-field`), and confirms with their name. They become the program's + `verify.yaml`. The same person confirms the parameters, which recompiles the + draft with them. A field named in `--expect` can't be dropped, and every + parameter must still be read back. +4. **Admission.** Canaries, then the JIT admission + ([JIT_ADMISSION.md](JIT_ADMISSION.md)): a registered admitter key, or a + registered agent under an admission policy a person signed. The recording + is signed with the driving agent's `principal_id`. When that is the same + registered agent that admits, it is self-admission, which the policy must + allow (`allow_self_admission`); either way the agent needs + `min_clean_canaries` clean canaries and a read-back on every write step. + Without a key, `--admit-as NAME` at a terminal shows that person every step + and the parameters, asks them to confirm the review, runs one canary, shows + its result, and asks them to admit. A canary is a real save on the record + `--expect` names, so its values come from the operator + (`--canary-param NAME=VALUE`); there is no default. Each canary also gets a + `verify-run` receipt against the confirmed checks; both must end VERIFIED. +5. **Emit.** An Agent Skill (`compiled/skill//SKILL.md`), an MCP + server with one tool (`compiled/mcp/server.py`), and the one-line + `openadapt-flow compile-agent run` command. All three call + `run_compiled`, so every call goes through the admission check, the + read-back and the receipt. Neither copies the bundle out of the task + directory. + +Every later run executes the admitted program with no model calls, and gets a +`verify-run` receipt (`receipts/run-*/effect-receipt.json`). A run that does +not end VERIFIED goes to the exception inbox, and the agent is not called. + +The command is re-entrant. Run it again with the same arguments and it +continues where it stopped: the agent runs only when the task has neither a +draft nor an admitted program. Without an interactive terminal it stops after +run 1 with the proposed checks waiting for a person (exit code 3). + +## The harness: structure first, screenshots when needed + +The agent does not get a screenshot every turn. It gets the page's structure +as short lines of text: headings, text, table rows, and every visible control +with a ref: + +``` +page: MockMed (http://127.0.0.1:8080/#encounter) +Jane Sample — MRN P1 — DOB 1980-01-01 +# New Encounter +Encounter Type +[e1] button "Triage" (selected) +[e2] button "Consult" +Note +[e3] textarea "Note" value="" +[e4] button "Save Encounter" +``` + +It acts with `click(ref)`, `type_text(text, ref)` and `press_key(key)`, and +gets a new listing after each action. A ref still has to name a control with +the same label in the same table row when it is used; otherwise the action is +refused and the agent observes again. It can call `screenshot` when the +listing is not enough. After an action that committed a write, it gets a +screenshot of the result. An action on a ref becomes an ordinary click at the +control's center, so the recording and the compiled program are the same as +for any other capture. The page itself is never modified. + +The listing is scrubbed with `openadapt_flow.privacy` (Presidio, when +`openadapt-privacy` is installed and `OPENADAPT_FLOW_SCRUB` is not `off`), +except for the values the operator put in the task: the parameters, the +`--expect` values and the `--on-screen` identity. The agent needs those to +find the right row. + +`--vision every_turn` is the baseline for comparison: Claude's computer +toolset (`computer_toolset_20260801`) with a screenshot after every action. + +### The save gate + +Every action that can commit a write is gated unless it plainly does not: + +- a click on a control whose label names a write (Save, Submit, Sign, Approve, + Confirm, Send, Post, Delete, Remove, Update, Apply, Done, Finish, Finalize, + Complete), whatever its tag or role; +- a click on an element the harness can't read; +- Enter, a typed line break, Space on anything but a text field, and any + keyboard shortcut. + +A click on a control with another label (Open, New Encounter), typing without +a line break, and navigation keys pass without a check. For a gated action, +the harness: + +1. checks that every `--on-screen` string appears, as whole words ("MRN P1" + is not "MRN P12"), in the row, dialog, form or section that holds the + control (or the focused field). A name in a sidebar or a worklist does not + count. If one is missing, the action is not performed and run 1 ends + `refused_wrong_record`: nothing is saved and nothing is compiled, even if + the agent catches the refusal and carries on; +2. keeps a screenshot as evidence (`attempts//gates/`); +3. asks the hold when there is one. At a terminal the CLI asks before every + save (`--no-hold` turns it off). A hold that says no ends the run + `refused_by_hold`. + +This is the right-record check and the hold from the guarded save design +([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)). It protects run +1 only. The compiled program carries its own identity checks and reads every +save back. + +## Measured on MockMed + +Ten live trials per arm with `claude-opus-5-5`, alternating order, a distinct +note per trial ([benchmark/compile_agent](../benchmark/compile_agent/README.md), +raw rows committed): + +| Per run 1 (median) | Structure first | Screenshot every turn | +| --- | ---: | ---: | +| Passed the record check | 10 / 10 | 10 / 10 | +| Screenshots sent to the model | 1 | 15 | +| Prompt tokens | 10,332 | 76,053 (7.4x) | +| Cost | $0.028 | $0.148 (5.3x) | +| Model time | 8.9 s | 11.6 s | + +Both arms made 5 model calls. Wall time depended mostly on the load of the +shared host; in the first three trials, at low load, run 1 took 15 to 18 s +structure first and 36 to 59 s with a screenshot every turn. Ten later runs of +the admitted programs ended VERIFIED with 0 model calls and 0 tokens. + +## Python + +```python +from openadapt_flow.compile_agent import ClaudeAgent, compile_first_run, run_compiled +from openadapt_flow.jit import load_customer_model_key + +result = compile_first_run( + "Open the referral for Jane Sample, start a Triage encounter, and save this note: {note}", + params={"note": "Follow up in two weeks"}, + url="http://localhost:8080/#tasks", + verify="verify.yaml", + expect={"patient_id": "p1", "type": "Triage"}, + on_screen=["Jane Sample", "MRN P1"], + agent=ClaudeAgent(vision="selective"), + model_key=load_customer_model_key("anthropic"), + confirm_checks="dana@clinic.example", # or a callback that shows them to a person + drop_fields=["key"], + canary_params=[{"note": "Canary test note"}], # a real save; choose safe values + admitter=agent_principal, # principal_from_key(...) +) +result.status # "admitted" +result.run1["agent"] # model calls, tokens, cost, seconds +result.emitted # {"skill": ..., "mcp_server": ..., "cli": ...} + +later = run_compiled(result.dir, note="Recheck knee in ten days") +later.verified, later.model_calls, later.receipt_outcome # True, 0, "VERIFIED" +``` + +`FirstRunCompiler` exposes the same steps one at a time (`record`, +`confirm_checks`, `admit`, `emit`, `run`) for a flow where a person confirms +later. + +Any agent works if it has a `dialect` (an `openadapt_flow.jit` action format) +and `drive(request, harness)`. `ScriptedAgent` follows a list of steps by +control name and makes no model calls. `ClaudeAgent(client=...)` takes any +object with `messages.create`; the tests pass a fake. + +## Statuses + +| Status | Meaning | +| --- | --- | +| `admitted` | Later runs execute the program with no model calls. | +| `draft` | Run 1 passed its record check; the checks or the admission wait for a person. | +| `needs_person` | The draft is outside the agent's admission policy; a person decides. | +| `refused_wrong_record` | The save gate did not see the record's identity; nothing was saved. | +| `refused_by_hold` | The hold said no; nothing was saved. | +| `record_check_failed` | The system of record did not show exactly the record the task named. | +| `canary_failed` | A canary did not end VERIFIED, or its receipt did not. | +| `capture_failed`, `unsupported` | The agent failed or used an action the recording can't hold. | + +## Files + +``` +// + compile_agent.json the task, app, verifier file, expect, on_screen + program/ the JIT draft, then the admitted bundle (JIT.md) + attempts// run 1: run1.json (tokens, time, screenshots, gate log), gates/ + compiled/ + verify.proposed.yaml the proposed checks (mode 0600) + verify.yaml the checks a person confirmed + record_checks.json who confirmed them, digests, dropped fields + admission.json canaries and who admitted + skill//SKILL.md the Agent Skill + mcp/server.py the MCP server + emitted.json the CLI line and the paths above + receipts// verify-run baseline and effect receipt per canary and run + runs/ one directory per canary and run +``` + +## Limits + +- Web apps only so far. The JIT captures and runs through a Playwright + browser, and the harness reads the page's DOM. A Windows, macOS, Linux, RDP + or Citrix target (`--backend`, or a `--config` whose `backend.kind` names + one) and any URL that is not http(s) are refused before anything runs; there + is no fallback to a browser. Those surfaces need the JIT to drive the + backends `record --backend` uses and an observer over UI Automation, + Accessibility, AT-SPI or pixels. +- The save gate recognizes a click that commits by the control's label. An + icon with no label, or a label that names no write ("OK"), is not gated in + run 1; the record check after run 1 still refuses a wrong record, but only + after the write landed. +- The identity must be shown in the same row, dialog, form or section as the + control. An app that shows the patient only in a page header outside them + is refused at every save. +- The run-1 record check compares two reads of the system of record. A change + someone else makes during run 1 makes it fail closed. +- A verifier read has no run parameters while recording, so it must name its + records with literals (the `record --verify` rule). +- On a replay that writes to the wrong record, the program halts at the save + step and goes to the inbox, but the `verify-run` receipt reports + `HALTED_BEFORE_EFFECT` because the confirmed check found no matching record. + The guarded save ([#1258](https://github.com/OpenAdaptAI/OpenAdapt/pull/1258)) + reports such a write as `RECONCILIATION_REQUIRED`. +- MockMed's demo idempotency key makes a second note on the same patient a + duplicate, so the tests and the measurement reset MockMed before each run. + A real app is never reset. +- A canary that fails its verify-run receipt is still recorded as passed in + the JIT's canary log. `compile-agent` stops on it, but an admission made + directly with `openadapt-flow jit admit` does not see the receipt. +- Screenshot-every-turn mode keeps every screenshot in the conversation, so a + long run can reach the API's request size limit. diff --git a/packages/flow/docs/JIT.md b/packages/flow/docs/JIT.md index 5af26e557e..86c7bccc4d 100644 --- a/packages/flow/docs/JIT.md +++ b/packages/flow/docs/JIT.md @@ -142,9 +142,12 @@ Create `client` with `request.model_key.secret()`, the customer key you passed to `compile_agent`. An agent that calls a model is not recorded without one. An agent that calls no model sets `requires_model_key = False`. -This sketch is not shipped or tested. The tests and the example use -`openadapt_flow.jit.recorded.RecordedTrajectoryAgent`, which replays a saved -trajectory and refuses a request with other inputs. +This sketch is not shipped. The shipped, tested live loop is +`openadapt_flow.compile_agent.ClaudeAgent`, which `openadapt-flow compile-agent` +runs inside a harness that observes structure first and checks the record +before each save ([COMPILE_AGENT.md](COMPILE_AGENT.md)). The JIT tests and the +example use `openadapt_flow.jit.recorded.RecordedTrajectoryAgent`, which +replays a saved trajectory and refuses a request with other inputs. ### Your own loop diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index b82bf2476d..146d3a0811 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -10580,6 +10580,10 @@ def _repair_store_flag(rp: argparse.ArgumentParser) -> None: _add_jit_parser(sub) + from openadapt_flow.compile_agent.cli import add_parser as _add_compile_agent_parser + + _add_compile_agent_parser(sub) + from openadapt_flow.cli_fleet import register_fleet_parser register_fleet_parser(sub) diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index 67a2926797..75e2dd6d8a 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -533,6 +533,13 @@ def __init__( def backend_kind(self) -> str: return self._kind + @property + def backend(self) -> Any: + """The live backend the recorder drives. Callers read it; actions go + through this session so they are recorded.""" + + return self._backend + @property def screen_only(self) -> bool: """True for a remote session: pixels and keys, no DOM or UIA.""" diff --git a/packages/flow/openadapt_flow/compile_agent/__init__.py b/packages/flow/openadapt_flow/compile_agent/__init__.py new file mode 100644 index 0000000000..ebcf6f593d --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/__init__.py @@ -0,0 +1,49 @@ +"""Compile a computer-use agent's first verified run into a program. + +Run 1 lets a computer-use agent do the task in a harness that gives it the +page's structure first and screenshots only on request or at a gate, and that +checks the right record is on screen before every save. If the system of +record shows exactly the record the task named, the run is compiled, a person +confirms the proposed record checks, and the program is admitted under the +JIT lifecycle. Every later run makes no model calls. See docs/COMPILE_AGENT.md. +""" + +from openadapt_flow.compile_agent.agents import ( + AgentLedger, + BudgetExceeded, + ClaudeAgent, + ScriptedAgent, +) +from openadapt_flow.compile_agent.harness import ( + GateRefused, + GateRequest, + Harness, + SaveGate, +) +from openadapt_flow.compile_agent.pipeline import ( + CompileResult, + FirstRunCompiler, + RecordCheck, + ReplayResult, + check_run_record, + compile_first_run, + run_compiled, +) + +__all__ = [ + "AgentLedger", + "BudgetExceeded", + "ClaudeAgent", + "CompileResult", + "FirstRunCompiler", + "GateRefused", + "GateRequest", + "Harness", + "RecordCheck", + "ReplayResult", + "SaveGate", + "ScriptedAgent", + "check_run_record", + "compile_first_run", + "run_compiled", +] diff --git a/packages/flow/openadapt_flow/compile_agent/agents.py b/packages/flow/openadapt_flow/compile_agent/agents.py new file mode 100644 index 0000000000..f160fd1025 --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/agents.py @@ -0,0 +1,560 @@ +"""Agents that drive the compile-agent harness on run 1. + +:class:`ClaudeAgent` is a live Claude computer-use loop on the customer's own +key, in one of two observation modes: + +``selective`` (default) + The model reads the page's structure (:mod:`.observe`) and acts on refs + with ``click``, ``type_text`` and ``press_key``. A screenshot enters its + context only when it calls ``screenshot``, and after an action that + committed a write (the verification gate). +``every_turn`` + The baseline: Claude's computer toolset (``computer_toolset_20260801``), + with a screenshot returned after every action. + +Both modes perform every action through the same harness and capture computer, +so both are recorded and compiled the same way. Each model call's token usage +and time go into an :class:`AgentLedger`. + +:class:`ScriptedAgent` follows a fixed list of steps by control name and makes +no model calls. Tests, demos and CI use it. + +Other providers plug in through ``openadapt_flow.jit`` dialects: any object +with a ``dialect`` and ``drive(request, harness)`` works. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any, ClassVar, Mapping, Optional, Sequence + +from openadapt_flow.compile_agent.harness import GateRefused, Harness, StaleRef +from openadapt_flow.jit.adapters import UnsupportedAction +from openadapt_flow.jit.compile_agent import TaskRequest + +__all__ = [ + "AgentLedger", + "BudgetExceeded", + "ClaudeAgent", + "DEFAULT_MODEL", + "ModelCall", + "PRICES_PER_MTOK", + "ScriptedAgent", +] + +DEFAULT_MODEL = "claude-opus-5-5" +#: Opus 5.5 and later take computer use only as this toolset (no beta header). +COMPUTER_TOOLSET = "computer_toolset_20260801" + +#: USD per million tokens: input, output, cache write (5 minute), cache read. +PRICES_PER_MTOK: dict[str, dict[str, float]] = { + "claude-opus-5-5": { + "input": 4.0, + "output": 20.0, + "cache_write": 5.0, + "cache_read": 0.20, + }, + "claude-sonnet-5-5": { + "input": 2.0, + "output": 10.0, + "cache_write": 2.5, + "cache_read": 0.20, + }, +} + +VISION_MODES = ("selective", "every_turn") + + +class BudgetExceeded(RuntimeError): + """The run's model spend passed its cap. The run ends with no draft.""" + + +@dataclass +class ModelCall: + """One request to the model.""" + + turn: int + input_tokens: int + output_tokens: int + cache_creation_input_tokens: int + cache_read_input_tokens: int + seconds: float + stop_reason: Optional[str] + images_in_request: int + + @property + def prompt_tokens(self) -> int: + """Every input token the model read: uncached, written and read cache.""" + + return ( + self.input_tokens + + self.cache_creation_input_tokens + + self.cache_read_input_tokens + ) + + +@dataclass +class AgentLedger: + """What one agent run cost: model calls, tokens, time, and screenshots.""" + + agent: str + model: Optional[str] = None + vision: Optional[str] = None + calls: list[ModelCall] = field(default_factory=list) + started_at: float = field(default_factory=time.monotonic) + wall_seconds: float = 0.0 + model_seconds: float = 0.0 + finished: bool = False + summary: Optional[str] = None + tool_errors: int = 0 + prices: Optional[dict[str, float]] = None + + @property + def model_calls(self) -> int: + return len(self.calls) + + def total(self, name: str) -> int: + return sum(int(getattr(call, name)) for call in self.calls) + + def cost_usd(self) -> float: + prices = self.prices or PRICES_PER_MTOK.get(self.model or "") + if prices is None: + return 0.0 + return ( + self.total("input_tokens") * prices["input"] + + self.total("output_tokens") * prices["output"] + + self.total("cache_creation_input_tokens") * prices["cache_write"] + + self.total("cache_read_input_tokens") * prices["cache_read"] + ) / 1_000_000 + + def to_dict(self) -> dict[str, Any]: + return { + "agent": self.agent, + "model": self.model, + "vision": self.vision, + "model_calls": self.model_calls, + "input_tokens": self.total("input_tokens"), + "output_tokens": self.total("output_tokens"), + "cache_creation_input_tokens": self.total("cache_creation_input_tokens"), + "cache_read_input_tokens": self.total("cache_read_input_tokens"), + "prompt_tokens": sum(call.prompt_tokens for call in self.calls), + "images_in_requests": sum(call.images_in_request for call in self.calls), + "cost_usd": round(self.cost_usd(), 6), + "wall_seconds": round(self.wall_seconds, 3), + "model_seconds": round(self.model_seconds, 3), + "finished": self.finished, + "summary": self.summary, + "tool_errors": self.tool_errors, + "calls": [dict(call.__dict__) for call in self.calls], + } + + +# -- scripted ---------------------------------------------------------------------- + + +@dataclass +class ScriptedAgent: + """Follows ``steps`` by control name; no model calls. + + Each step is one of:: + + {"click": "Open", "near": "Jane Sample"} # near: text on the same line + {"type": "Follow up in two weeks", "into": "Note"} + {"type": "{note}", "into": "Note"} # a task input by name + {"press": "Enter"} + + A control that is missing or ambiguous ends the run with no draft. + """ + + steps: Sequence[Mapping[str, Any]] + principal_id: Optional[str] = "agent:scripted" + dialect: ClassVar[str] = "generic" + requires_model_key: ClassVar[bool] = False + + def drive(self, request: TaskRequest, harness: Harness) -> AgentLedger: + ledger = AgentLedger(agent="scripted") + for step in self.steps: + obs = harness.observe() + if "click" in step: + control = obs.find(str(step["click"]), near=step.get("near")) + if control is None: + raise LookupError(f"no single control named {step['click']!r}") + harness.click(control.ref) + elif "type" in step: + text = str(step["type"]).format(**request.inputs) + ref = None + if step.get("into"): + control = obs.find(str(step["into"])) + if control is None: + raise LookupError(f"no single field named {step['into']!r}") + ref = control.ref + harness.type_text(text, ref) + elif "press" in step: + harness.press(str(step["press"])) + else: + raise ValueError(f"unknown step {dict(step)!r}") + ledger.finished = True + ledger.wall_seconds = time.monotonic() - ledger.started_at + return ledger + + +# -- Claude ----------------------------------------------------------------------- + +_SELECTIVE_SYSTEM = """\ +You operate a web application for the user through a harness. You see the \ +page as a structural listing: one line per heading, text, table row or \ +control. Controls carry refs such as [e3]; act on them with click, type_text \ +and press_key. After every action you get a new listing, and refs change, so \ +use the refs in the latest listing. Call screenshot only when the listing is \ +not enough to decide what to do. Before any save, the harness checks that the \ +right record is on screen; a save on the wrong record is refused and ends the \ +run. Do exactly the task, with the values given, and nothing else. When the \ +task is done, or cannot be done, call finish with a one-line summary.""" + +_EVERY_TURN_SYSTEM = """\ +You operate a web application for the user with the computer tools. After \ +every action you get a screenshot of the screen. Before any save, the \ +harness checks that the right record is on screen; a save on the wrong \ +record is refused and ends the run. Do exactly the task, with the values \ +given, and nothing else. When the task is done, or cannot be done, call \ +finish with a one-line summary.""" + +_FINISH_TOOL = { + "name": "finish", + "description": "End the task. Call it once, when the task is done or cannot be done.", + "input_schema": { + "type": "object", + "properties": { + "summary": {"type": "string", "description": "One line: what was done."} + }, + "required": ["summary"], + "additionalProperties": False, + }, +} + +_SELECTIVE_TOOLS = [ + { + "name": "observe", + "description": "Read the current page's structural listing again.", + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + }, + { + "name": "screenshot", + "description": ( + "Get a screenshot of the page. Costs far more than a listing; use it " + "only when the listing is not enough." + ), + "input_schema": { + "type": "object", + "properties": {}, + "additionalProperties": False, + }, + }, + { + "name": "click", + "description": "Click the control with this ref from the latest listing.", + "input_schema": { + "type": "object", + "properties": { + "ref": {"type": "string", "description": "A ref such as e3."} + }, + "required": ["ref"], + "additionalProperties": False, + }, + }, + { + "name": "type_text", + "description": ( + "Type text. With a ref, click that field first; without one, type " + "into the focused field. Typing appends to what the field holds." + ), + "input_schema": { + "type": "object", + "properties": { + "text": {"type": "string"}, + "ref": {"type": "string", "description": "Optional field ref."}, + }, + "required": ["text"], + "additionalProperties": False, + }, + }, + { + "name": "press_key", + "description": "Press one key, such as Enter, Tab or Escape.", + "input_schema": { + "type": "object", + "properties": {"key": {"type": "string"}}, + "required": ["key"], + "additionalProperties": False, + }, + }, + _FINISH_TOOL, +] + + +def _block_dict(block: Any) -> dict[str, Any]: + if isinstance(block, dict): + return block + if hasattr(block, "to_dict"): + return block.to_dict() + return block.model_dump(exclude_none=True) + + +def _get(block: Any, name: str, default: Any = None) -> Any: + if isinstance(block, dict): + return block.get(name, default) + return getattr(block, name, default) + + +def _image(png: bytes) -> dict[str, Any]: + import base64 + + return { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": base64.standard_b64encode(png).decode("ascii"), + }, + } + + +def _count_images(messages: Sequence[Mapping[str, Any]]) -> int: + count = 0 + for message in messages: + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "image": + count += 1 + inner = block.get("content") + if isinstance(inner, list): + count += sum( + 1 for b in inner if isinstance(b, dict) and b.get("type") == "image" + ) + return count + + +@dataclass +class ClaudeAgent: + """A live Claude computer-use loop on the customer's own key. + + Args: + vision: ``selective`` (structure first) or ``every_turn`` (baseline). + model: The model id. Default ``claude-opus-5-5``. + effort: ``output_config.effort``. Opus 5.5 defaults to ``medium``. + max_turns: Model calls before the run is ended. + max_cost_usd: Estimated spend before the run is ended + (:class:`BudgetExceeded`), priced from :data:`PRICES_PER_MTOK`. + client: An Anthropic client. Default: built from the request's + customer key. Tests pass a fake. + principal_id: The registered agent identity this loop records as. + """ + + vision: str = "selective" + model: str = DEFAULT_MODEL + effort: Optional[str] = "medium" + max_turns: int = 40 + max_cost_usd: float = 2.0 + max_tokens: int = 8000 + client: Any = None + principal_id: Optional[str] = None + #: USD per million tokens (input, output, cache_write, cache_read) for a + #: model not in :data:`PRICES_PER_MTOK`. The cost cap needs a price. + prices: Optional[dict[str, float]] = None + requires_model_key: ClassVar[bool] = True + + def __post_init__(self) -> None: + if self.vision not in VISION_MODES: + raise ValueError(f"vision must be one of {VISION_MODES}") + if self.prices is None and self.model not in PRICES_PER_MTOK: + raise ValueError( + f"no price for {self.model!r}, so the cost cap can't hold; pass " + "prices={'input': ..., 'output': ..., 'cache_write': ..., " + "'cache_read': ...} in USD per million tokens" + ) + if self.principal_id is None: + self.principal_id = f"agent:{self.model}" + + @property + def dialect(self) -> str: + return "generic" if self.vision == "selective" else "anthropic" + + def _client(self, request: TaskRequest) -> Any: + if self.client is not None: + return self.client + if request.model_key is None: + raise RuntimeError("ClaudeAgent needs the customer's model key") + import anthropic + + return anthropic.Anthropic(api_key=request.model_key.secret(), max_retries=4) + + def drive(self, request: TaskRequest, harness: Harness) -> AgentLedger: + client = self._client(request) + ledger = AgentLedger( + agent="claude", model=self.model, vision=self.vision, prices=self.prices + ) + #: Kept so a run that raises still reports what it spent. + self.last_ledger = ledger + selective = self.vision == "selective" + if selective: + system = _SELECTIVE_SYSTEM + tools: list[dict[str, Any]] = list(_SELECTIVE_TOOLS) + first: list[dict[str, Any]] = [ + {"type": "text", "text": f"Task: {request.instruction}"}, + {"type": "text", "text": "Current page:\n" + harness.observe().text}, + ] + else: + system = _EVERY_TURN_SYSTEM + tools = [ + {"type": COMPUTER_TOOLSET, "configs": {"zoom": {"enabled": False}}}, + _FINISH_TOOL, + ] + first = [ + {"type": "text", "text": f"Task: {request.instruction}"}, + _image(harness.screenshot("every_turn")), + ] + messages: list[dict[str, Any]] = [{"role": "user", "content": first}] + extra: dict[str, Any] = {"cache_control": {"type": "ephemeral"}} + if self.effort: + extra["output_config"] = {"effort": self.effort} + try: + for turn in range(1, self.max_turns + 1): + started = time.monotonic() + response = client.messages.create( + model=self.model, + max_tokens=self.max_tokens, + system=system, + tools=tools, + messages=messages, + extra_body=extra, + ) + elapsed = time.monotonic() - started + usage = _get(response, "usage") + ledger.calls.append( + ModelCall( + turn=turn, + input_tokens=int(_get(usage, "input_tokens", 0) or 0), + output_tokens=int(_get(usage, "output_tokens", 0) or 0), + cache_creation_input_tokens=int( + _get(usage, "cache_creation_input_tokens", 0) or 0 + ), + cache_read_input_tokens=int( + _get(usage, "cache_read_input_tokens", 0) or 0 + ), + seconds=round(elapsed, 3), + stop_reason=_get(response, "stop_reason"), + images_in_request=_count_images(messages), + ) + ) + ledger.model_seconds += elapsed + if ledger.cost_usd() > self.max_cost_usd: + raise BudgetExceeded( + f"estimated spend ${ledger.cost_usd():.2f} passed the cap " + f"${self.max_cost_usd:.2f}" + ) + content = [_block_dict(b) for b in _get(response, "content", [])] + messages.append({"role": "assistant", "content": content}) + stop = _get(response, "stop_reason") + if stop == "refusal": + raise RuntimeError( + "the model declined the task (stop_reason refusal)" + ) + calls = [b for b in content if b.get("type") == "tool_use"] + if not calls: + ledger.finished = stop == "end_turn" + ledger.summary = ( + " ".join( + str(b.get("text", "")) + for b in content + if b.get("type") == "text" + ).strip() + or None + ) + return ledger + results: list[dict[str, Any]] = [] + for call in calls: + if call.get("name") == "finish": + ledger.finished = True + ledger.summary = str( + (call.get("input") or {}).get("summary", "") + ) + return ledger + results.append( + self._run_tool(call, harness, ledger, selective=selective) + ) + messages.append({"role": "user", "content": results}) + raise RuntimeError(f"the agent did not finish in {self.max_turns} turns") + finally: + ledger.wall_seconds = time.monotonic() - ledger.started_at + + def _run_tool( + self, + call: Mapping[str, Any], + harness: Harness, + ledger: AgentLedger, + *, + selective: bool, + ) -> dict[str, Any]: + name = str(call.get("name")) + args = dict(call.get("input") or {}) + result: dict[str, Any] = {"type": "tool_result", "tool_use_id": call.get("id")} + toolset = call.get("toolset_name") + if toolset: + result["toolset_name"] = toolset + try: + if selective: + result["content"] = self._selective(name, args, harness) + else: + result["content"] = self._computer(name, args, harness) + except GateRefused: + raise + except (StaleRef, UnsupportedAction, LookupError, ValueError) as exc: + # Nothing in the action was performed; the model can try another. + ledger.tool_errors += 1 + result["content"] = [{"type": "text", "text": f"Not done: {exc}"}] + result["is_error"] = True + return result + + def _selective( + self, name: str, args: Mapping[str, Any], harness: Harness + ) -> list[dict[str, Any]]: + gates_before = harness.stats.gate_checks + if name == "observe": + return [{"type": "text", "text": harness.observe().text}] + if name == "screenshot": + return [_image(harness.screenshot("requested"))] + if name == "click": + harness.click(str(args["ref"])) + elif name == "type_text": + ref = args.get("ref") + harness.type_text(str(args["text"]), str(ref) if ref else None) + elif name == "press_key": + harness.press(str(args["key"])) + else: + raise LookupError(f"unknown tool {name!r}") + out: list[dict[str, Any]] = [{"type": "text", "text": harness.observe().text}] + if harness.stats.gate_checks > gates_before: + # A write was committed: show the screen it left (verification gate). + out.append(_image(harness.screenshot("gate"))) + return out + + def _computer( + self, name: str, args: Mapping[str, Any], harness: Harness + ) -> list[dict[str, Any]]: + obs = harness.act({"action": name, **args}) + harness.stats.screenshots_every_turn += 1 + from openadapt_flow.privacy import scrub_image_bytes + + out: list[dict[str, Any]] = [_image(scrub_image_bytes(obs.png))] + if obs.notes: + out.insert(0, {"type": "text", "text": "; ".join(obs.notes)}) + return out diff --git a/packages/flow/openadapt_flow/compile_agent/cli.py b/packages/flow/openadapt_flow/compile_agent/cli.py new file mode 100644 index 0000000000..279a4a1fcf --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/cli.py @@ -0,0 +1,337 @@ +"""``openadapt-flow compile-agent``: compile an agent's first verified run. + +A computer-use agent does the task once in the app at ``--url``. That run is +real: the agent makes the entry in that app. Point ``--url`` at a test copy +with test records. If the system of record shows exactly the record the task +named, the run is compiled into a program that later runs with no model calls. + +:: + + openadapt-flow compile-agent "Open the referral for Jane Sample and save this note: {note}" \\ + --url http://localhost:8080/#tasks --verify verify.yaml \\ + --param note="Follow up in two weeks" \\ + --expect patient_id=p1 --on-screen "Jane Sample" --on-screen "MRN P1" \\ + --admit-key agent.pem --admission-policy signed-policy.json + + openadapt-flow compile-agent run --param note="Recheck in ten days" + +The command is re-entrant. Run it again with the same arguments and it picks +up where it stopped: the agent runs only when the task has no draft and no +admitted program. Confirming the proposed record checks is a person's step at +an interactive terminal; without one, the command stops with the draft +waiting and says so. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, Callable, Optional + +__all__ = ["add_parser", "cmd_compile_agent"] + + +def _pairs(items: Optional[list[str]], flag: str) -> dict[str, str]: + out: dict[str, str] = {} + for item in items or []: + name, sep, value = item.partition("=") + if not sep or not name: + raise SystemExit(f"{flag} takes NAME=VALUE, got {item!r}") + out[name] = value + return out + + +def add_parser(sub: argparse._SubParsersAction) -> None: + p = sub.add_parser( + "compile-agent", + help=( + "Run a computer-use agent once against --url (it writes to that app; " + "use a test copy), check the record it wrote, and compile the run " + "into a program that runs with no model calls" + ), + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "prompt", + help='the task, with {name} for each --param; or "run" to run a compiled task', + ) + p.add_argument("task_dir", nargs="?", help="with run: the compiled task directory") + p.add_argument("--param", action="append", metavar="NAME=VALUE") + p.add_argument( + "--url", + help="the app's entry URL; the agent makes real entries there, so use a test copy", + ) + p.add_argument( + "--backend", + choices=["web", "windows", "macos", "linux", "rdp", "citrix"], + default="web", + help=( + "the surface the app runs on, as for record --backend. Only web is " + "supported so far; any other is refused before anything runs" + ), + ) + p.add_argument( + "--config", + help="deployment.yaml; its backend.kind must be web (others are refused)", + ) + p.add_argument( + "--verify", + help="verify.yaml whose verifier: section reads the system of record", + ) + p.add_argument( + "--expect", + action="append", + metavar="FIELD=VALUE", + help="a field the new record must have, such as patient_id=p1", + ) + p.add_argument( + "--on-screen", + action="append", + metavar="TEXT", + help="the record's identity as the screen shows it; checked before every save", + ) + p.add_argument( + "--agent", + default="claude", + help="claude (default), or a JSON file of scripted steps (no model calls)", + ) + p.add_argument("--vision", choices=["selective", "every_turn"], default="selective") + p.add_argument("--model", default=None, help="model id (default claude-opus-5-5)") + p.add_argument("--effort", default="medium") + p.add_argument("--max-cost", type=float, default=2.0, help="USD cap for run 1") + p.add_argument("--task-id", default=None) + p.add_argument("--cache", default=".openadapt-compile-agent") + p.add_argument("--key-dir", default=None, help="JIT admission key directory") + p.add_argument("--policy", default="clinical-write", help="Flow risk policy") + p.add_argument( + "--drop-field", + action="append", + default=[], + help="a proposed match field to leave out of the checks (changes every save)", + ) + p.add_argument( + "--admit-key", + help="encrypted key of a registered admitter, or of an agent with --admission-policy", + ) + p.add_argument("--admission-policy", help="signed admission policy for --admit-key") + p.add_argument( + "--canary-param", + action="append", + metavar="NAME=VALUE", + help=( + "the value each canary saves; a canary is a real save on the record " + "--expect names, so use a value that is safe there (required to admit)" + ), + ) + p.add_argument( + "--admit-as", + help="a person who reviews, attends the canary and admits at this terminal", + ) + p.add_argument( + "--hold", + action=argparse.BooleanOptionalAction, + default=None, + help="ask at this terminal before every save in run 1 (default: on at a terminal)", + ) + p.add_argument("--show-browser", action="store_true") + p.add_argument("--json", action="store_true", help="print the result as JSON") + p.set_defaults(func=cmd_compile_agent) + + +def _interactive() -> bool: + return sys.stdin.isatty() and sys.stdout.isatty() + + +def _agent(args: argparse.Namespace) -> Any: + from openadapt_flow.compile_agent.agents import ( + DEFAULT_MODEL, + ClaudeAgent, + ScriptedAgent, + ) + + if args.agent == "claude": + return ClaudeAgent( + vision=args.vision, + model=args.model or DEFAULT_MODEL, + effort=args.effort, + max_cost_usd=args.max_cost, + ) + raw = json.loads(Path(args.agent).read_text()) + return ScriptedAgent(steps=raw["steps"], principal_id=raw.get("principal_id")) + + +def _admitter(args: argparse.Namespace) -> Any: + if not args.admit_key: + return None + from openadapt_flow.jit import authority, lifecycle + + key_dir = Path(args.key_dir) if args.key_dir else lifecycle.default_key_dir() + policy = ( + authority.SignedAdmissionPolicy.load(args.admission_policy) + if args.admission_policy + else None + ) + return authority.principal_from_key( + authority.load_encrypted_key(Path(args.admit_key)), key_dir, policy=policy + ) + + +def _terminal_confirm( + ask: Callable[[str], str], +) -> Callable[[dict[str, Any]], Optional[str]]: + def confirm(proposed: dict[str, Any]) -> Optional[str]: + print("Proposed record checks (from the records read around run 1):") + for effect in proposed["effects"]: + print(f" {effect['kind']}:") + for name, value in effect["match"].items(): + print(f" {name} = {value}") + print(f" file: {proposed['file']}") + print( + "Parameters (each run supplies a new value): " + + (", ".join(proposed["params"]) or "none") + ) + name = ask("Your name (the person confirming these checks): ").strip() + if not name: + return None + if ask("Type ok to confirm them: ").strip().lower() != "ok": + return None + return name + + return confirm + + +def _terminal_review( + ask: Callable[[str], str], +) -> Callable[[str, dict[str, Any]], bool]: + def review(stage: str, shown: dict[str, Any]) -> bool: + if stage == "review": + print("The compiled program's steps:") + for step in shown["steps"]: + print( + f" {step.get('id')}: {step.get('action')} {step.get('intent') or ''}" + ) + print(f"Parameters: {', '.join(shown['params']) or 'none'}") + prompt = "Type ok to confirm you reviewed every step: " + else: + for canary in shown["canaries"]: + print( + f"Canary: {canary['status']}, receipt {canary['receipt_outcome']}" + ) + prompt = "Type ok to admit this program: " + return ask(prompt).strip().lower() == "ok" + + return review + + +def _terminal_hold(ask: Callable[[str], str]) -> Callable[[Any], bool]: + def hold(request: Any) -> bool: + print( + f"The agent is about to press {request.control!r} with " + f"{', '.join(request.identity)} on screen." + ) + if request.screenshot: + print(f" screen: {request.screenshot}") + return ask("Allow this save? [y/N] ").strip().lower() in ("y", "yes") + + return hold + + +def _print(result: dict[str, Any], as_json: bool) -> None: + if as_json: + print(json.dumps(result, indent=2, sort_keys=True, default=str)) + return + for key, value in result.items(): + if value in (None, {}, []): + continue + if isinstance(value, dict): + value = json.dumps(value, sort_keys=True, default=str) + print(f"{key}: {value}") + + +def cmd_compile_agent( + args: argparse.Namespace, + *, + ask: Callable[[str], str] = input, + interactive: Callable[[], bool] = _interactive, +) -> int: + from openadapt_flow.compile_agent.pipeline import ( + FirstRunCompiler, + compile_first_run, + ) + + params = _pairs(args.param, "--param") + if args.prompt == "run": + if not args.task_dir: + raise SystemExit("compile-agent run needs the compiled task directory") + replay = FirstRunCompiler.load(args.task_dir).run(**params) + _print(replay.to_dict(), args.json) + return 0 if replay.verified else 1 + from openadapt_flow.compile_agent.pipeline import UnsupportedTarget, check_target + + backend = args.backend + if args.config: + from openadapt_flow.deployment import load_deployment + + backend = load_deployment(args.config, allow_unresolved=True).backend.kind + try: + check_target(args.url or "http://unset", backend) + except UnsupportedTarget as exc: + print(f"compile-agent: {exc}", file=sys.stderr) + return 2 + missing = [f for f in ("url", "verify") if not getattr(args, f)] + if missing or not args.expect or not args.on_screen: + raise SystemExit( + "compile-agent needs --url, --verify, at least one --expect FIELD=VALUE " + "and at least one --on-screen TEXT" + ) + tty = interactive() + hold = ( + _terminal_hold(ask) if (args.hold if args.hold is not None else tty) else None + ) + if hold is not None and not tty: + raise SystemExit("--hold asks a person at this terminal; there is none") + model_key = None + agent = _agent(args) + if getattr(agent, "requires_model_key", True): + from openadapt_flow.jit.keys import load_customer_model_key + + model_key = load_customer_model_key("anthropic") + result = compile_first_run( + args.prompt, + params=params, + url=args.url, + verify=args.verify, + expect=_pairs(args.expect, "--expect"), + on_screen=args.on_screen, + agent=agent, + confirm_checks=_terminal_confirm(ask) if tty else None, + drop_fields=args.drop_field, + admitter=_admitter(args), + person=args.admit_as, + review=_terminal_review(ask) if (tty and args.admit_as) else None, + canary_params=( + [_pairs(args.canary_param, "--canary-param")] if args.canary_param else None + ), + hold=hold, + task_id=args.task_id, + policy=args.policy, + cache_dir=args.cache, + key_dir=args.key_dir, + model_key=model_key, + headless=not args.show_browser, + ) + out = result.to_dict() + if result.status == "draft" and not tty: + out["next"] = ( + "a person confirms the proposed checks: run this same command again at " + "an interactive terminal" + ) + _print(out, args.json) + return ( + 0 + if result.admitted + else (3 if result.status in ("draft", "needs_person") else 1) + ) diff --git a/packages/flow/openadapt_flow/compile_agent/emit.py b/packages/flow/openadapt_flow/compile_agent/emit.py new file mode 100644 index 0000000000..12a0aabbcd --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/emit.py @@ -0,0 +1,148 @@ +"""What a compiled task is called through: an Agent Skill and an MCP tool. + +Both call :func:`openadapt_flow.compile_agent.run_compiled` on the task +directory, so every call goes through the admission check, the system-of-record +read-back and the verify-run receipt. Neither copies the bundle: a copy run +outside the task directory would skip the admission and the confirmed checks. + +The skill reuses ``emit-skill``'s step descriptions, which never carry a value +from the recording. +""" + +from __future__ import annotations + +import ast +import keyword +from pathlib import Path +from typing import TYPE_CHECKING + +from openadapt_flow.emit.skill import _recorded_literals, _slugify, _step_line +from openadapt_flow.ir import Workflow + +if TYPE_CHECKING: # pragma: no cover + from openadapt_flow.compile_agent.pipeline import FirstRunCompiler + +__all__ = ["emit_mcp", "emit_skill_for"] + +_SERVER = '''\ +"""MCP server exposing the compiled task {name!r} as one tool. + +Generated by openadapt-flow compile-agent. Requires: mcp, openadapt-flow. +Every call runs the admitted program in {task_dir}: no model calls, each save +read back from the system of record, and a verify-run receipt. +Run with: python {out_name} +""" + +from __future__ import annotations + +from mcp.server.fastmcp import FastMCP + +TASK_DIR = {task_dir!r} + +mcp = FastMCP({name!r}) + + +@mcp.tool() +def {func}({signature}) -> dict: + """{doc} + + Returns: + ``verified`` is true only when the program ended VERIFIED and the + verify-run receipt for the confirmed record checks is VERIFIED. + ``model_calls`` is 0 on a healthy run. Any other status means the run + stopped; it is in the exception inbox, and the record should be checked + before calling again. + """ + from openadapt_flow.compile_agent import run_compiled + + return run_compiled(TASK_DIR{kwargs}).to_dict() + + +if __name__ == "__main__": + mcp.run() +''' + + +def _func_name(task_id: str) -> str: + name = "".join(ch if ch.isalnum() else "_" for ch in task_id).strip("_").lower() + if not name or name[0].isdigit(): + name = f"task_{name}" + return name + + +def emit_mcp(compiler: "FirstRunCompiler", out_path: Path) -> Path: + """Write a FastMCP ``server.py`` with one tool for the task.""" + + params = sorted(compiler.task.params) + bad = [p for p in params if keyword.iskeyword(p)] + if bad: + raise ValueError(f"parameter names {bad} are Python keywords") + instruction = compiler.task.instruction.replace('"""', "'''").replace("\\", " ") + source = _SERVER.format( + name=compiler.task.task_id, + task_dir=str(compiler.dir), + out_name=out_path.name, + func=_func_name(compiler.task.task_id), + signature=", ".join(f"{p}: str" for p in params), + doc=" ".join(instruction.split()), + kwargs="".join(f", {p}={p}" for p in params), + ) + ast.parse(source) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(source, encoding="utf-8") + return out_path + + +def emit_skill_for(bundle_dir: Path, out_dir: Path, cli_line: str) -> Path: + """Write ``//SKILL.md`` for an admitted compiled task.""" + + workflow = Workflow.load(bundle_dir) + slug = _slugify(workflow.name) + skill_dir = Path(out_dir) / slug + skill_dir.mkdir(parents=True, exist_ok=True) + literals = _recorded_literals(workflow) + params = list(workflow.params) + lines: list[str] = [ + "---", + f"name: {slug}", + ( + f"description: Run the admitted '{workflow.name}' program " + f"({len(workflow.steps)} steps, no model calls) and confirm the " + "save in the system of record." + ), + "---", + "", + f"# {workflow.name}", + "", + "A computer-use agent did this task once. Its verified run was compiled " + "into this program and admitted. Running it makes no model calls.", + "", + "## What it does", + "", + ] + lines += [f"1. {_step_line(step, literals)}" for step in workflow.steps] + lines += ["", "## Parameters", ""] + if params: + lines += ["| Name | Type | Required |", "| --- | --- | --- |"] + lines += [f"| `{name}` | string | yes |" for name in params] + else: + lines.append("_This task takes no parameters._") + lines += [ + "", + "## Usage", + "", + "```bash", + cli_line, + "```", + "", + "Replace each `` with the value for this request; never reuse a " + "value from an earlier run. The command exits 0 only when the run ended " + "VERIFIED: the save was read back from the system of record and the " + "verify-run receipt for the confirmed checks is VERIFIED. Any other " + "result stops the run and puts it in the exception inbox; check the " + "record before you try again. Don't fall back to doing the task by hand " + "or with a computer-use agent.", + "", + ] + (skill_dir / "SKILL.md").write_text("\n".join(lines), encoding="utf-8") + return skill_dir diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py new file mode 100644 index 0000000000..cdfa0308d5 --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -0,0 +1,324 @@ +"""The harness a computer-use agent works in during run 1. + +The agent reads the page's structure first (:mod:`.observe`) and acts on refs. +A screenshot goes into its context only when it asks for one, or at a gate. +Every action still goes through +:class:`~openadapt_flow.jit.adapters.CaptureComputer`, so the run is recorded +and compiled exactly as any other JIT capture. + +The save gate. Before any action that commits a write (a click on a control +labelled Save, Submit, Sign, Approve, Confirm, Send or Post, or Enter in a +form field), the harness: + +1. checks that every identity string the operator named for this task (for + example ``MRN P1``) is on the screen. If one is missing, the action is not + performed and the run ends with :class:`GateRefused`: no draft is made. +2. takes a screenshot as evidence of what was on screen when the write was + committed; +3. asks the hold, when one is given (a person at a terminal, or an approval + callback). A hold that says no refuses the action the same way. + +This is the right-record check and the hold before saving from the guarded +save design. It runs in the agent's run only; the compiled program has its own +identity checks and reads every save back from the system of record. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping, Optional, Sequence + +from openadapt_flow.compile_agent import observe as _observe +from openadapt_flow.jit.adapters import CaptureComputer, Observation, translate + +__all__ = [ + "GateRefused", + "GateRequest", + "Harness", + "HarnessStats", + "SaveGate", + "StaleRef", +] + + +class GateRefused(RuntimeError): + """A write was not committed: the wrong record was on screen, or the hold + said no. Nothing in the refused action was performed.""" + + +class StaleRef(ValueError): + """The ref is not in the latest observation, or the page changed under it.""" + + +@dataclass(frozen=True) +class GateRequest: + """What the hold is asked to approve.""" + + control: str + identity: tuple[str, ...] + url: str + screenshot: Optional[Path] + + +@dataclass +class SaveGate: + """The right-record check and the hold, for every commit action. + + Args: + on_screen: Strings that must all be visible when a write is committed + (the record's identity: a name, an MRN). Required: a save with + nothing to check it against is refused. + hold: Called before the write with a :class:`GateRequest`; returns + True to let it through. None means no hold. + """ + + on_screen: Sequence[str] + hold: Optional[Callable[[GateRequest], bool]] = None + + def __post_init__(self) -> None: + self.on_screen = tuple(str(s) for s in self.on_screen if str(s).strip()) + + +_SCREENSHOT_COUNTERS = { + "requested": "screenshots_requested", + "gate": "screenshots_at_gate", + "every_turn": "screenshots_every_turn", +} + + +@dataclass +class HarnessStats: + """What the agent looked at. Screenshots sent to the model are counted by + why they were taken: the agent asked, a gate, or the every-turn baseline.""" + + observations: int = 0 + screenshots_requested: int = 0 + screenshots_at_gate: int = 0 + screenshots_every_turn: int = 0 + actions: int = 0 + gate_checks: int = 0 + #: Kept as evidence with the run; not sent to the model. + gate_evidence_screenshots: int = 0 + gate_refusals: int = 0 + + def to_dict(self) -> dict[str, int]: + return dict(self.__dict__) + + +@dataclass +class Harness: + """One run's harness. Built by the compile-agent pipeline around the capture + computer; agents call its methods.""" + + computer: CaptureComputer + gate: SaveGate + keep: Sequence[str] = () + evidence_dir: Optional[Path] = None + stats: HarnessStats = field(default_factory=HarnessStats) + #: One entry per gate decision, kept with the run. + gate_log: list[dict[str, Any]] = field(default_factory=list) + refusal: Optional[str] = None + _last: Optional[_observe.StructuralObservation] = None + + @property + def page(self) -> Any: + page = self.computer.page + if page is None: + raise RuntimeError("the compile-agent harness needs a browser page") + return page + + @property + def dialect(self) -> str: + return self.computer.dialect + + # -- observation ------------------------------------------------------------- + + def observe(self) -> _observe.StructuralObservation: + """The page's structure, scrubbed except for the task's own values.""" + + self._last = _observe.observe(self.page, keep=self.keep) + self.stats.observations += 1 + return self._last + + def screenshot(self, why: str = "requested") -> bytes: + """A screenshot for the agent's context. ``why`` is ``requested``, + ``gate`` or ``every_turn``; each is counted.""" + + from openadapt_flow.privacy import scrub_image_bytes + + counter = _SCREENSHOT_COUNTERS[why] + png = self.computer.screenshot() + setattr(self.stats, counter, getattr(self.stats, counter) + 1) + return scrub_image_bytes(png) + + # -- actions on refs (selective vision) ----------------------------------------- + + def _control(self, ref: str) -> _observe.Control: + last = self._last + if last is None or ref not in last.controls: + raise StaleRef(f"{ref!r} is not in the latest observation; observe again") + control = last.controls[ref] + at = _observe.element_at(self.page, control.x, control.y) or {} + # Same name, and the same row: a re-sorted list must not turn a ref + # into another row's control with the same label. + if ( + str(at.get("name", "")).lower() != control.name.lower() + or str(at.get("context", "")) != control.context + ): + raise StaleRef( + f"{ref!r} ({control.name!r}) is no longer at its place; observe again" + ) + return control + + def click(self, ref: str) -> Observation: + control = self._control(ref) + return self.act({"kind": "click", "x": control.x, "y": control.y}) + + def type_text(self, text: str, ref: Optional[str] = None) -> Observation: + if ref is not None: + self.click(ref) + return self.act({"kind": "type", "text": text}) + + def press(self, key: str) -> Observation: + return self.act({"kind": "press", "key": key}) + + # -- every action ------------------------------------------------------------- + + def act(self, native: Mapping[str, Any]) -> Observation: + """Perform one native action in the computer's dialect, behind the gate. + + Raises: + GateRefused: the action commits a write and the gate refused it. + UnsupportedAction: the recording can't hold the action. Nothing + in it was performed. + """ + + commit = self._commit_target(native) + if commit is not None: + self._check_gate(*commit) + obs = self.computer.execute(native) + self.stats.actions += 1 + if commit is not None: + # Let the app react before the agent looks at the result. + _settle(self.page) + return obs + + def _commit_target( + self, native: Mapping[str, Any] + ) -> Optional[tuple[str, Optional[str]]]: + """``(what commits, the text around it)`` when the action may commit a + write, else None. + + Every action that can change the app is gated unless it is plainly not + a commit: a click on a control whose label names no write, typing + without a line break, or a navigation key. A click on something the + harness can't read, Enter, Space on a control, a typed line break, and + any shortcut are all gated. + """ + + for op in translate(self.dialect, native): + if op.kind in ("click", "double_click", "right_click"): + # A click with no point lands where the pointer is. + point = (op.x, op.y) if op.x is not None else self.computer.pointer + if point is None: + continue + x, y = point + if x is None or y is None: + continue + at = _observe.element_at( + self.page, + x, + y, + backend=getattr( + getattr(self.computer, "session", None), "backend", None + ), + ) + if at is None: + return ("a click on an element the harness could not read", None) + name = str(at.get("name") or "") + # A click on a text field only focuses it ("Postal code" + # holds the word "post" but commits nothing). + if ( + at.get("interactive") + and not at.get("editable") + and _observe.commit_label(name) + ): + return (name, at.get("scope")) + elif op.kind == "type" and any(ch in (op.text or "") for ch in "\r\n"): + focused = _observe.focused_element(self.page) or {} + return ("a typed line break", focused.get("scope")) + elif op.kind == "hotkey": + focused = _observe.focused_element(self.page) or {} + shortcut = "+".join([*op.modifiers, op.key or ""]) + return (f"the shortcut {shortcut}", focused.get("scope")) + elif op.kind == "press" and op.key in ("Enter", "Space"): + focused = _observe.focused_element(self.page) or {} + if op.key == "Space" and focused.get("editable"): + continue + label = str(focused.get("name") or "the focused element") + return (f"{op.key} on {label}", focused.get("scope")) + return None + + def _check_gate(self, control: str, scope: Optional[str]) -> None: + """The right-record check: the identity must be shown, as whole words, + in the row, dialog, form or section the commit happens in.""" + + self.stats.gate_checks += 1 + missing = [s for s in self.gate.on_screen if not _observe.shows(scope or "", s)] + if not self.gate.on_screen: + missing = ["(the task names no record identity to check)"] + shot: Optional[Path] = None + png = self.computer.screenshot() + self.stats.gate_evidence_screenshots += 1 + if self.evidence_dir is not None: + self.evidence_dir.mkdir(parents=True, exist_ok=True) + shot = self.evidence_dir / f"gate-{len(self.gate_log) + 1:02d}.png" + shot.write_bytes(png) + entry: dict[str, Any] = { + "control": control, + "identity": list(self.gate.on_screen), + "missing": missing, + "url": str(getattr(self.page, "url", "")), + "screenshot": str(shot) if shot else None, + "at": time.time(), + } + if missing: + entry["decision"] = "refused_wrong_record" + self.gate_log.append(entry) + self._refuse( + f"{control!r} may commit a write, but the form, row or section it " + f"is in does not show {missing}; the right-record check refused it" + ) + if self.gate.hold is not None: + approved = bool( + self.gate.hold( + GateRequest( + control=control, + identity=tuple(self.gate.on_screen), + url=entry["url"], + screenshot=shot, + ) + ) + ) + if not approved: + entry["decision"] = "refused_by_hold" + self.gate_log.append(entry) + self._refuse(f"the hold did not approve {control!r}") + entry["decision"] = "approved_by_hold" + else: + entry["decision"] = "passed_no_hold" + self.gate_log.append(entry) + + def _refuse(self, reason: str) -> None: + self.stats.gate_refusals += 1 + self.refusal = reason + raise GateRefused(reason) + + +def _settle(page: Any, timeout_ms: int = 1500) -> None: + try: + page.wait_for_load_state("networkidle", timeout=timeout_ms) + except Exception: # noqa: BLE001 - a page that never idles is still usable + pass diff --git a/packages/flow/openadapt_flow/compile_agent/observe.py b/packages/flow/openadapt_flow/compile_agent/observe.py new file mode 100644 index 0000000000..8ce5890089 --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/observe.py @@ -0,0 +1,402 @@ +"""Structural observations of a web page, for an agent that reads before it looks. + +:func:`observe` lists what is on the page as short lines of text: headings, +text, table rows, and every visible control with a ref (``e1``, ``e2``...) the +agent acts on. Text is scrubbed with :func:`openadapt_flow.privacy.scrub_text` +before it leaves the harness, except for the values the operator put in the +task (its inputs and the record identity it names), which the agent needs to +find the right row. + +Refs are valid for one observation. Each control keeps the center of its box, +so an action on a ref becomes an ordinary click at a point and is recorded the +same way as any other click. The page itself is never modified. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Iterable, Optional, Sequence + +__all__ = [ + "Control", + "StructuralObservation", + "commit_label", + "element_at", + "observe", + "scrub_lines", + "shows", +] + +#: Controls an agent can act on. +INTERACTIVE = ( + "button, a[href], input:not([type=hidden]), textarea, select, " + "[role=button], [role=link], [role=tab], [role=checkbox], [role=radio], " + "[role=menuitem], [role=option], [contenteditable=''], [contenteditable=true]" +) + +#: Labels that commit a write. Narrower than the risk classifier's write list +#: (``openadapt_flow.risk``) on purpose: the gate checks the record identity +#: next to the control, and "Continue" or "Add" on a list page, which opens a +#: form, has none to check. The save on that form is gated. +_COMMIT_WORDS = ( + "save", + "submit", + "sign", + "approve", + "confirm", + "send", + "post", + "delete", + "remove", + "update", + "apply", + "done", + "finish", + "finaliz", + "complet", +) + +#: The container whose text must show the record's identity when a control in +#: it commits a write: the nearest row, dialog, form or section around it. +SCOPE = ( + "tr, [role=row], [role=listitem], dialog, [role=dialog], form, section, " + "article, main" +) + +_NAME_JS = """ +function oaName(el) { + var n = el.getAttribute('aria-label') || + (el.labels && el.labels.length ? el.labels[0].innerText : '') || + el.getAttribute('placeholder') || el.innerText || + (el.tagName === 'INPUT' && /^(button|submit)$/i.test(el.type) ? el.value : '') || + el.getAttribute('title') || ''; + return String(n).replace(/\\s+/g, ' ').trim().slice(0, 80); +} +function oaRow(el) { + var r = el.closest('tr, [role=row]'); + return r ? String(r.innerText).replace(/\\s+/g, ' ').trim().slice(0, 300) : ''; +} +""" + +_OBSERVE_JS = ( + "(sel) => {" + + _NAME_JS + + """ + function visible(el) { + var r = el.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return false; + if (r.bottom < 0 || r.right < 0 || r.top > innerHeight || r.left > innerWidth) + return false; + var s = getComputedStyle(el); + return s.visibility !== 'hidden' && s.display !== 'none' && s.opacity !== '0'; + } + function ownText(el) { + var t = ''; + for (var i = 0; i < el.childNodes.length; i++) { + var c = el.childNodes[i]; + if (c.nodeType === 3) t += c.textContent; + } + return t.replace(/\\s+/g, ' ').trim(); + } + var rows = Array.prototype.slice.call(document.querySelectorAll('tr')); + var out = []; + var all = document.body.querySelectorAll('*'); + for (var i = 0; i < all.length; i++) { + var el = all[i]; + var tag = el.tagName.toLowerCase(); + if (tag === 'script' || tag === 'style' || tag === 'noscript') continue; + var tr = el.closest('tr'); + var row = tr ? rows.indexOf(tr) : -1; + if (el.matches(sel)) { + if (!visible(el)) continue; + var r = el.getBoundingClientRect(); + var value = null; + if (tag === 'input' || tag === 'textarea' || tag === 'select') { + value = el.type === 'password' ? '' : String(el.value || ''); + } + var cls = (el.className && el.className.baseVal === undefined) + ? String(el.className) : ''; + out.push({ + kind: 'control', tag: tag, role: el.getAttribute('role') || '', + type: el.type || '', name: oaName(el), value: value, + focused: document.activeElement === el, + selected: /(^|\\s)(selected|active)(\\s|$)/.test(cls) || + el.getAttribute('aria-selected') === 'true' || + el.getAttribute('aria-pressed') === 'true' || !!el.checked, + disabled: !!el.disabled, + x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2), + row: row, context: oaRow(el) + }); + continue; + } + if (el.closest(sel)) continue; + var text = ownText(el); + if (!text || !visible(el)) continue; + out.push({kind: /^h[1-6]$/.test(tag) ? 'heading' : 'text', tag: tag, + text: text.slice(0, 200), row: row}); + } + return {url: location.href, title: document.title, items: out}; +} +""" +) + +_DESCRIBE_JS = """ +function oaDescribe(c, interactive, scopeSel) { + var scope = c.closest(scopeSel) || document.body; + return {tag: c.tagName.toLowerCase(), type: c.type || '', name: oaName(c), + role: c.getAttribute('role') || '', interactive: interactive, + editable: c.isContentEditable || c.tagName === 'TEXTAREA' || + (c.tagName === 'INPUT' && + !/^(button|submit|reset|checkbox|radio|image|file|range|color)$/i + .test(c.type)), + context: oaRow(c), scope: String(scope.innerText || '')}; +} +""" + +_ELEMENT_AT_JS = ( + "([x, y, sel, scopeSel]) => {" + + _NAME_JS + + _DESCRIBE_JS + + """ + var el = document.elementFromPoint(x, y); + if (!el) return null; + var c = el.closest(sel); + return oaDescribe(c || el, !!c, scopeSel); +} +""" +) + +_FOCUSED_JS = ( + "([sel, scopeSel]) => {" + + _NAME_JS + + _DESCRIBE_JS + + """ + var c = document.activeElement; + if (!c || c === document.body) return null; + return oaDescribe(c, c.matches(sel), scopeSel); +} +""" +) + + +@dataclass(frozen=True) +class Control: + """One control in an observation.""" + + ref: str + tag: str + name: str + x: int + y: int + role: str = "" + type: str = "" + #: The text of the table row the control is in, when it is in one. + context: str = "" + + @property + def kind(self) -> str: + if self.role: + return self.role + if self.tag == "input": + return f"input[{self.type or 'text'}]" + return {"a": "link"}.get(self.tag, self.tag) + + +@dataclass +class StructuralObservation: + """What the agent reads: ``text``, and the controls its refs name.""" + + url: str + title: str + text: str + controls: dict[str, Control] = field(default_factory=dict) + + def find(self, name: str, *, near: Optional[str] = None) -> Optional[Control]: + """The control named ``name`` (case-insensitive), on the line that + also shows ``near`` when given. None when none or several match.""" + + wanted = name.strip().lower() + hits: list[Control] = [] + for line in self.text.splitlines(): + if near is not None and near.lower() not in line.lower(): + continue + for ref, control in self.controls.items(): + if f"[{ref}]" in line and control.name.lower() == wanted: + hits.append(control) + unique = {c.ref: c for c in hits} + return next(iter(unique.values())) if len(unique) == 1 else None + + +def commit_label(name: str) -> bool: + """True when a control labelled ``name`` commits a write (Save, Submit).""" + + words = "".join(ch if ch.isalnum() else " " for ch in name.lower()).split() + if "sign" in words and ("in" in words or "up" in words): + return False + return any(word.startswith(stem) for word in words for stem in _COMMIT_WORDS) + + +def shows(text: str, identity: str) -> bool: + """True when ``identity`` appears in ``text`` as whole words: "MRN P1" + is not shown by "MRN P12".""" + + pattern = r"(? str: + """Scrub ``text`` line by line, leaving every ``keep`` value as it is. + + Each kept value is swapped for a placeholder the scrubber leaves alone, + and swapped back afterwards. When no scrubber is installed, or scrubbing + is off (``OPENADAPT_FLOW_SCRUB``), the text is returned unchanged. + """ + + from openadapt_flow.privacy import scrub_text, text_scrubbing_enabled + + if not text or not text_scrubbing_enabled(): + return text + kept = sorted({str(k) for k in keep if k and str(k).strip()}, key=len, reverse=True) + out: list[str] = [] + for line in text.splitlines(): + held: dict[str, str] = {} + for i, value in enumerate(kept): + if value in line: + token = f"KEEPTOKEN{i}X" + held[token] = value + line = line.replace(value, token) + line = scrub_text(line) or "" + for token, value in held.items(): + line = line.replace(token, value) + out.append(line) + return "\n".join(out) + + +def _render(raw: dict[str, Any]) -> tuple[str, dict[str, Control]]: + controls: dict[str, Control] = {} + lines: list[str] = [] + current_row: Optional[int] = None + row_parts: list[str] = [] + + def flush() -> None: + nonlocal row_parts + if row_parts: + lines.append("row: " + " | ".join(row_parts)) + row_parts = [] + + for item in raw.get("items") or []: + if item["kind"] == "control": + ref = f"e{len(controls) + 1}" + control = Control( + ref=ref, + tag=str(item["tag"]), + name=str(item["name"]), + x=int(item["x"]), + y=int(item["y"]), + role=str(item.get("role") or ""), + type=str(item.get("type") or ""), + context=str(item.get("context") or ""), + ) + controls[ref] = control + part = f'[{ref}] {control.kind} "{control.name}"' + if item.get("value") is not None: + part += f' value="{item["value"]}"' + flags = [ + flag for flag in ("focused", "selected", "disabled") if item.get(flag) + ] + if flags: + part += " (" + ", ".join(flags) + ")" + else: + part = item["text"] if item["kind"] == "text" else f"# {item['text']}" + row = int(item.get("row", -1)) + if row >= 0: + if row != current_row: + flush() + current_row = row + row_parts.append(part) + continue + flush() + current_row = None + lines.append(part) + flush() + return "\n".join(lines), controls + + +def observe(page: Any, *, keep: Sequence[str] = ()) -> StructuralObservation: + """Read ``page``'s structure: one line per text, heading, row or control.""" + + raw = page.evaluate(_OBSERVE_JS, INTERACTIVE) + body, controls = _render(raw) + header = f"page: {raw.get('title') or ''} ({raw.get('url') or ''})" + text = scrub_lines(header + "\n" + body, keep) + return StructuralObservation( + url=str(raw.get("url") or ""), + title=str(raw.get("title") or ""), + text=text, + controls=controls, + ) + + +def element_at( + page: Any, x: int, y: int, *, backend: Any = None +) -> Optional[dict[str, Any]]: + """The control (or element) at a point: tag, name, editable, scope. + + A point over an iframe is read inside the frame, through the backend's + hit-tested frame chain, so a Save button in a frame is seen as one. A + frame the backend can't map is reported as unreadable (None). + """ + + try: + at = page.evaluate(_ELEMENT_AT_JS, [int(x), int(y), INTERACTIVE, SCOPE]) + except Exception: # noqa: BLE001 - an unreadable page has no element + return None + if not at or str(at.get("tag")) not in ("iframe", "frame"): + return at + getter = getattr(backend, "_frame_point", None) + if not callable(getter): + return None + try: + point = getter(int(x), int(y)) + if point is None or not point.frame_path: + return None + return point.scope.evaluate( + _ELEMENT_AT_JS, [point.x, point.y, INTERACTIVE, SCOPE] + ) + except Exception: # noqa: BLE001 + return None + + +def focused_element(page: Any) -> Optional[dict[str, Any]]: + """The focused control, inside a frame when focus is in one.""" + + try: + got = page.evaluate(_FOCUSED_JS, [INTERACTIVE, SCOPE]) + except Exception: # noqa: BLE001 + return None + if not got or str(got.get("tag")) not in ("iframe", "frame"): + return got + # Focus is inside a frame: read the frame whose document holds it. + main = getattr(page, "main_frame", None) + for frame in getattr(page, "frames", None) or []: + if frame is main: + continue + try: + if not frame.evaluate("() => document.hasFocus()"): + continue + inner = frame.evaluate(_FOCUSED_JS, [INTERACTIVE, SCOPE]) + except Exception: # noqa: BLE001 - a detached frame holds no focus + continue + if inner and str(inner.get("tag")) not in ("iframe", "frame"): + return inner + return None + + +def page_text(page: Any) -> str: + """The page's visible text, unscrubbed, for the right-record check only.""" + + try: + return str(page.evaluate("() => document.body.innerText") or "") + except Exception: # noqa: BLE001 + return "" diff --git a/packages/flow/openadapt_flow/compile_agent/pipeline.py b/packages/flow/openadapt_flow/compile_agent/pipeline.py new file mode 100644 index 0000000000..68ae959955 --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/pipeline.py @@ -0,0 +1,1113 @@ +"""Compile a computer-use agent's first verified run into a program. + +Run 1 lets a computer-use agent do the task inside the harness +(:mod:`.harness`): structural observations first, a screenshot only when the +agent asks or at a gate, and the right-record check (plus an optional hold) +before every save. The run is recorded and compiled by ``openadapt_flow.jit``. + +Then, in order: + +1. **Record check.** The system of record is read when the run starts and + after it ends, through the ``verifier:`` section of ``verify.yaml``. Run 1 + passes only when exactly one record appeared, nothing else changed, the + record carries every field the operator expected (``expect``, such as the + patient id), and every input value landed in it. Otherwise the draft is + set aside and nothing can be admitted. +2. **Proposed checks.** The change becomes proposed record checks + (``record --verify``'s proposer). A person confirms them: they become the + program's ``verify.yaml``, and the same person confirms the parameters. +3. **Admission** under the JIT lifecycle: canaries, then a registered admitter + key, or a registered agent under a signed admission policy (self-admission + only when that policy allows it). Without either, a person admits at a + terminal. +4. **Emit.** An Agent Skill, an MCP server with one tool, and a one-line CLI + command, all calling :func:`run_compiled`. + +Every later run executes the admitted program with no model calls, reads each +save back through the verifier, and gets a ``verify-run`` receipt against the +confirmed checks. +""" + +from __future__ import annotations + +import functools +import hashlib +import json +import re +import time +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping, Optional, Sequence, Union + +from openadapt_flow.compile_agent.harness import GateRequest, Harness, SaveGate +from openadapt_flow.jit import lifecycle +from openadapt_flow.jit.compile_agent import ( + CompiledAgent, + JitResult, + TaskRequest, + WebApp, + compile_agent, +) +from openadapt_flow.jit.keys import CustomerModelKey +from openadapt_flow.jit.signature import TaskSignature + +__all__ = [ + "BACKENDS", + "UnsupportedTarget", + "CompileResult", + "FirstRunCompiler", + "RecordCheck", + "ReplayResult", + "SPEC_SCHEMA", + "check_run_record", + "compile_first_run", + "run_compiled", +] + +SPEC_SCHEMA = "openadapt.compile-agent/v1" +SPEC_FILE = "compile_agent.json" +OUT_DIR = "compiled" +PROPOSED_FILE = "verify.proposed.yaml" +CONFIRMED_FILE = "verify.yaml" +CHECKS_FILE = "record_checks.json" + +_PLACEHOLDER_RE = re.compile(r"{([A-Za-z_][A-Za-z0-9_]*)}") + +#: The surfaces ``record --backend`` drives. compile-agent runs on ``web`` only +#: so far: the JIT captures and runs through a Playwright browser, and the +#: harness reads the page's DOM. +BACKENDS = ("web", "windows", "macos", "linux", "rdp", "citrix") + + +class UnsupportedTarget(ValueError): + """The app is not a web app. compile-agent does not drive it yet, and never + falls back to a browser for it.""" + + +def check_target(url: str, backend: str = "web") -> None: + """Refuse any target that is not a web app at an http(s) URL. + + Raises: + UnsupportedTarget: a non-web backend, or a URL that is not http(s). + """ + + from urllib.parse import urlsplit + + if backend not in BACKENDS: + raise UnsupportedTarget(f"unknown backend {backend!r} (one of {BACKENDS})") + if backend != "web": + raise UnsupportedTarget( + f"compile-agent runs web apps only so far; the {backend!r} backend " + "(which record --backend drives) is not supported yet. Nothing ran." + ) + if urlsplit(url).scheme not in ("http", "https"): + raise UnsupportedTarget( + f"{url!r} is not an http(s) URL. compile-agent runs web apps only so " + "far; a desktop, RDP or Citrix target is not supported yet. Nothing ran." + ) + + +PersonOrCallback = Union[str, Callable[[dict[str, Any]], Optional[str]], None] + + +def _now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def _stamp() -> str: + return time.strftime("%Y%m%dT%H%M%S", time.gmtime()) + f"-{uuid.uuid4().hex[:6]}" + + +def _write_json(path: Path, data: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, indent=2, sort_keys=True, default=str) + "\n") + return path + + +def _slug(text: str) -> str: + words = re.findall(r"[a-z0-9]+", _PLACEHOLDER_RE.sub("", text.lower())) + return "compile-agent." + ("-".join(words[:6]) or "task") + + +# -- the run-1 record check ------------------------------------------------------ + + +@dataclass +class RecordCheck: + """Whether run 1 changed exactly the record the task asked for.""" + + passed: bool + reasons: list[str] + created: int + changed: int + removed: int + #: The new record's field names (values stay in the private snapshots). + fields: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return dict(self.__dict__) + + +def check_run_record( + before: Optional[list[dict[str, Any]]], + after: Optional[list[dict[str, Any]]], + *, + expect: Mapping[str, Any], + values: Mapping[str, str], +) -> RecordCheck: + """Run 1's change in the system of record, against what the task named. + + Passes only when one record appeared and nothing else changed, the record + has every ``expect`` field with that value, and every input value is the + value of one of its fields. + """ + + from openadapt_flow.record_sor import diff_snapshots + + if before is None or after is None: + return RecordCheck(False, ["the system of record could not be read"], 0, 0, 0) + diff = diff_snapshots(before, after) + reasons: list[str] = [] + if len(diff.created) != 1: + reasons.append(f"{len(diff.created)} records appeared; the task makes one") + if diff.changed: + reasons.append(f"{len(diff.changed)} existing records changed") + if diff.removed: + reasons.append(f"{len(diff.removed)} records disappeared") + record = diff.created[0] if len(diff.created) == 1 else {} + if record: + for name, wanted in expect.items(): + if str(record.get(name)) != str(wanted): + reasons.append( + f"the new record's {name!r} is not the one the task named " + "(wrong record)" + ) + field_values = {str(v) for v in record.values() if v is not None} + for name, value in values.items(): + if value not in field_values: + reasons.append(f"input {name!r} is not in the new record") + return RecordCheck( + passed=not reasons, + reasons=reasons, + created=len(diff.created), + changed=len(diff.changed), + removed=len(diff.removed), + fields=sorted(record), + ) + + +# -- results -------------------------------------------------------------------- + + +@dataclass +class CompileResult: + """What :func:`compile_first_run` did. + + ``status`` is ``admitted`` (later runs make no model calls), ``draft`` + (run 1 passed; checks or admission wait for a person), ``needs_person`` + (outside the agent's admission policy), or why nothing can be admitted: + ``refused_wrong_record``, ``refused_by_hold``, ``record_check_failed``, + ``canary_failed``, ``capture_failed``, ``unsupported``. + """ + + status: str + dir: Path + task_key: str + detail: Optional[str] = None + run1: dict[str, Any] = field(default_factory=dict) + record_check: Optional[dict[str, Any]] = None + proposed_checks: Optional[Path] = None + confirmed_checks: Optional[Path] = None + admission: Optional[dict[str, Any]] = None + emitted: dict[str, Any] = field(default_factory=dict) + + @property + def admitted(self) -> bool: + return self.status == "admitted" + + def to_dict(self) -> dict[str, Any]: + out = dict(self.__dict__) + for key in ("dir", "proposed_checks", "confirmed_checks"): + out[key] = str(out[key]) if out[key] is not None else None + return out + + +@dataclass +class ReplayResult: + """One run of an admitted program.""" + + status: str + verified: bool + model_calls: Optional[int] + seconds: float + run_dir: Optional[Path] + receipt_outcome: Optional[str] = None + receipt: Optional[Path] = None + halt_reason: Optional[str] = None + detail: Optional[str] = None + inbox_item: Optional[Path] = None + + def to_dict(self) -> dict[str, Any]: + out = dict(self.__dict__) + for key in ("run_dir", "receipt", "inbox_item"): + out[key] = str(out[key]) if out[key] is not None else None + return out + + +# -- the agent the JIT records ---------------------------------------------------- + + +class _HarnessedAgent: + """What ``compile_agent`` records: the inner agent, driving the harness.""" + + def __init__( + self, + inner: Any, + *, + gate: SaveGate, + keep: Sequence[str], + evidence_dir: Path, + principal_id: Optional[str], + ) -> None: + self.inner = inner + self.dialect = str(inner.dialect) + self.gate = gate + self.keep = list(keep) + self.evidence_dir = evidence_dir + self.principal_id = principal_id + self.harness: Optional[Harness] = None + self.ledger: Any = None + self.error: Optional[str] = None + + @property + def requires_model_key(self) -> bool: + return bool(getattr(self.inner, "requires_model_key", True)) + + def run(self, request: TaskRequest, computer: Any) -> None: + self.harness = Harness( + computer, self.gate, keep=self.keep, evidence_dir=self.evidence_dir + ) + try: + self.ledger = self.inner.drive(request, self.harness) + except BaseException as exc: + self.error = f"{type(exc).__name__}: {exc}" + self.ledger = getattr(self.inner, "last_ledger", None) + raise + + +# -- the pipeline ------------------------------------------------------------------- + + +class FirstRunCompiler: + """One task's compile-agent state, on disk under ``/``. + + :func:`compile_first_run` runs every step. Use the steps when a person confirms + later: :meth:`record`, :meth:`confirm_checks`, :meth:`admit`, :meth:`emit`, + then :meth:`run` for every later run. + """ + + def __init__( + self, + prompt: str, + *, + params: Sequence[str], + url: str, + verify: Union[str, Path], + expect: Mapping[str, Any], + on_screen: Sequence[str], + task_id: Optional[str] = None, + policy: str = "clinical-write", + cache_dir: Union[str, Path] = ".openadapt-compile-agent", + key_dir: Optional[Union[str, Path]] = None, + reset: Optional[Callable[[], None]] = None, + headless: bool = True, + on_warning: Optional[Callable[[str], None]] = None, + model_key: Optional[CustomerModelKey] = None, + backend: str = "web", + ) -> None: + from openadapt_flow.record_sor import load_observer + + check_target(url, backend) + + reserved = sorted(set(params) & {"reset", "task_dir"}) + if reserved: + raise ValueError(f"parameter names {reserved} are reserved") + placeholders = set(_PLACEHOLDER_RE.findall(prompt)) + if placeholders != set(params): + raise ValueError( + f"the prompt's placeholders {sorted(placeholders)} must be the " + f"parameters {sorted(params)}" + ) + if not [s for s in on_screen if str(s).strip()]: + raise ValueError( + "on_screen must name the record's identity as shown on screen " + "(for example a name and an MRN); a save with nothing to check " + "is refused" + ) + if not expect: + raise ValueError( + "expect must name at least one field of the record the task " + "writes (for example its patient id)" + ) + self.task = TaskSignature( + task_id or _slug(prompt), prompt, {name: str for name in params} + ) + self.url = url + self.verify_path = Path(verify).resolve() + self.expect = {str(k): str(v) for k, v in expect.items()} + self.on_screen = [str(s) for s in on_screen if str(s).strip()] + self.policy = policy + self.cache_dir = Path(cache_dir).resolve() + self.key_dir = Path(key_dir).resolve() if key_dir is not None else None + self.user_reset = reset + self.headless = headless + self.on_warning = on_warning + self.model_key = model_key + self.observer, self.verifier_section = load_observer(self.verify_path) + self._baseline: Optional[list[dict[str, Any]]] = None + self._pending: Optional[tuple[Any, dict[str, str], Path]] = None + self._receipt_baseline: Optional[Path] = None + + # -- layout ------------------------------------------------------------------ + + @property + def dir(self) -> Path: + return self.cache_dir / self.task.key + + @property + def out_dir(self) -> Path: + return self.dir / OUT_DIR + + def _jit(self, agent: Any = None) -> CompiledAgent: + return compile_agent( + agent, + task=self.task, + app=WebApp( + url=self.url, + record_reader=lambda: self.observer.system_of_record, + reset=self._on_reset, + headless=self.headless, + ), + verifier=self.observer.verifier, + policy=self.policy, + cache_dir=self.cache_dir, + dialect=None if agent is not None else "generic", + model_key=self.model_key, + key_dir=self.key_dir, + on_warning=self.on_warning, + ) + + def _on_reset(self) -> None: + """Called by the JIT before every capture, canary and run, just + before the browser opens: the baseline read for this run.""" + + from openadapt_flow.verify_run import capture_baseline + + if self.user_reset is not None: + self.user_reset() + self._baseline = self.observer.system_of_record + self._receipt_baseline = None + if self._pending is not None: + cfg, values, path = self._pending + capture_baseline(cfg, values, path) + self._receipt_baseline = path + + def save_spec(self) -> Path: + return _write_json( + self.dir / SPEC_FILE, + { + "schema": SPEC_SCHEMA, + "task": { + "task_id": self.task.task_id, + "instruction": self.task.instruction, + "params": sorted(self.task.params), + "key": self.task.key, + }, + "url": self.url, + "verify": str(self.verify_path), + "expect": self.expect, + "on_screen": self.on_screen, + "policy": self.policy, + "cache_dir": str(self.cache_dir), + "key_dir": str(self.key_dir) if self.key_dir else None, + "headless": self.headless, + }, + ) + + @classmethod + def load( + cls, task_dir: Union[str, Path], *, reset: Optional[Callable[[], None]] = None + ) -> "FirstRunCompiler": + spec = json.loads((Path(task_dir) / SPEC_FILE).read_text()) + if spec.get("schema") != SPEC_SCHEMA: + raise ValueError(f"{task_dir} is not a compiled agent task") + task = spec["task"] + return cls( + task["instruction"], + params=task["params"], + url=spec["url"], + verify=spec["verify"], + expect=spec["expect"], + on_screen=spec["on_screen"], + task_id=task["task_id"], + policy=spec["policy"], + cache_dir=spec["cache_dir"], + key_dir=spec.get("key_dir"), + reset=reset, + headless=bool(spec.get("headless", True)), + ) + + # -- step 1: run 1 and its record check -------------------------------------- + + def record( + self, + agent: Any, + params: Mapping[str, Any], + *, + hold: Optional[Callable[[GateRequest], bool]] = None, + principal_id: Optional[str] = None, + ) -> CompileResult: + """Run 1: the agent does the task in the harness; Flow records it. + + ``principal_id`` is the identity the recording is signed with (the + registered agent, when it will admit under a policy). Default: the + agent's own. + """ + + self.save_spec() + jit = self._jit() + values = self.task.validate(params) + if jit.admitted or jit.candidate() is not None: + raise lifecycle.AdmissionError( + "this task already has a draft or an admitted program; run 1 runs " + "only once (reject the draft to record again)" + ) + # A new run 1 makes a new draft: checks a person confirmed for an + # earlier draft must not carry over to it. + for stale in ( + PROPOSED_FILE, + CONFIRMED_FILE, + CHECKS_FILE, + "admission.json", + "emitted.json", + ): + (self.out_dir / stale).unlink(missing_ok=True) + attempt = self.dir / "attempts" / _stamp() + harnessed = _HarnessedAgent( + agent, + gate=SaveGate(self.on_screen, hold), + keep=[*self.on_screen, *values.values(), *self.expect.values()], + evidence_dir=attempt / "gates", + principal_id=principal_id or getattr(agent, "principal_id", None), + ) + jit.agent = harnessed + jit.dialect = harnessed.dialect + self._baseline = None + started = time.monotonic() + result: JitResult = jit.run(**params) + seconds = time.monotonic() - started + after = self.observer.system_of_record + harness = harnessed.harness + run1: dict[str, Any] = { + "attempt": str(attempt), + "jit_status": result.status, + "seconds": round(seconds, 3), + "agent": harnessed.ledger.to_dict() if harnessed.ledger else None, + "harness": harness.stats.to_dict() if harness else None, + "gates": harness.gate_log if harness else [], + "agent_error": harnessed.error, + } + _write_json(attempt / "run1.json", run1) + + def done(status: str, detail: Optional[str], **kw: Any) -> CompileResult: + out = CompileResult( + status=status, + dir=self.dir, + task_key=self.task.key, + detail=detail, + run1=run1, + **kw, + ) + _write_json(attempt / "result.json", out.to_dict()) + return out + + if harness is not None and harness.refusal is not None: + decision = (harness.gate_log[-1] if harness.gate_log else {}).get( + "decision", "refused_wrong_record" + ) + status = ( + "refused_by_hold" + if decision == "refused_by_hold" + else "refused_wrong_record" + ) + if jit.candidate() is not None: + # An agent that caught the refusal and went on still made a + # draft; it must never be admitted. + self._set_aside(jit, status, [harness.refusal]) + return done(status, harness.refusal) + if result.status != lifecycle_status_draft(): + return done(result.status, result.detail) + check = check_run_record( + self._baseline, after, expect=self.expect, values=values + ) + run1["record_check"] = check.to_dict() + _write_json(attempt / "run1.json", run1) + if not check.passed: + kept = self._set_aside(jit, "record_check_failed", check.reasons) + return done( + "record_check_failed", + "; ".join(check.reasons) + f" (draft set aside at {kept})", + record_check=check.to_dict(), + ) + proposed = self._propose(self._baseline or [], after or [], values) + return done( + "draft", + "run 1 passed its record check; a person confirms the proposed checks", + record_check=check.to_dict(), + proposed_checks=proposed, + ) + + def _set_aside(self, jit: CompiledAgent, status: str, reasons: list[str]) -> Path: + program = jit.program_dir + _write_json(program / "refused.json", {"status": status, "reasons": reasons}) + kept = self.dir / f"refused-{status}-{_stamp()}" + program.rename(kept) + return kept + + def _propose( + self, + before: list[dict[str, Any]], + after: list[dict[str, Any]], + values: Mapping[str, str], + ) -> Path: + from openadapt_flow.record_sor import ( + _write_private, + diff_snapshots, + propose_effects, + render_contract, + ) + + proposal = propose_effects(diff_snapshots(before, after), values) + path = self.out_dir / PROPOSED_FILE + path.parent.mkdir(parents=True, exist_ok=True) + _write_private(path, render_contract(self.verifier_section, proposal.effects)) + return path + + def proposed(self) -> dict[str, Any]: + """The proposed checks a person reviews: each effect's kind and the + fields it matches, with parameters named and literals shown.""" + + import yaml + + doc = yaml.safe_load((self.out_dir / PROPOSED_FILE).read_text()) + effects = doc.get("effects") or [] + return { + "file": str(self.out_dir / PROPOSED_FILE), + # Confirming the checks also confirms these as the parameters. + "params": sorted(self.task.params), + "effects": [ + { + "kind": e.get("kind"), + "match": { + k: ( + f"{{param: {v['param']}}}" + if "param" in v + else v.get("literal") + ) + for k, v in (e.get("match") or {}).items() + }, + } + for e in effects + ], + } + + # -- step 2: a person confirms the checks and the parameters ------------------ + + def confirm_checks( + self, *, principal_id: str, drop_fields: Sequence[str] = () + ) -> Path: + """A person confirms the proposed record checks. + + ``drop_fields`` are match fields to leave out (values that change on + every save). A field the task's ``expect`` names can't be dropped. + The result is the program's ``verify.yaml``. The same person confirms + the task's parameters, which recompiles the draft with them. + + Raises: + AdmissionError: not a person, or the identity that recorded. + ValueError: the checks would no longer pin the record or an input. + """ + + import yaml + + from openadapt_flow.record_sor import _write_private + from openadapt_flow.verify_run import parse_verify_config, sha256_tagged + + jit = self._jit() + name = lifecycle.check_principal( + principal_id, jit.program_dir, key_dir=self.key_dir + ) + pinned = sorted(set(drop_fields) & set(self.expect)) + if pinned: + raise ValueError(f"{pinned} identify the record; they can't be dropped") + proposed_path = self.out_dir / PROPOSED_FILE + doc = yaml.safe_load(proposed_path.read_text()) + effects = [] + for effect in doc.get("effects") or []: + effect = dict(effect) + effect.pop("needs_operator_confirmation", None) + effect["match"] = { + k: v + for k, v in (effect.get("match") or {}).items() + if k not in drop_fields + } + if effect.get("kind") == "record_written": + effect["probe"] = ( + "exactly one new record with " + + ", ".join( + f"{k}=" + + ( + f"parameter {v['param']!r}" + if v.get("param") + else repr(v.get("literal")) + ) + for k, v in effect["match"].items() + ) + + f" (confirmed by {name})" + ) + effects.append(effect) + written = [ + e + for e in effects + if e.get("kind") == "record_written" and e.get("count_new_only") + ] + if len(written) != 1: + raise ValueError("the checks must claim exactly one new record") + match = written[0]["match"] + for key, wanted in self.expect.items(): + if (match.get(key) or {}).get("literal") != wanted: + raise ValueError(f"the check does not pin {key!r} to {wanted!r}") + bound = {v.get("param") for v in match.values() if isinstance(v, dict)} + unbound = sorted(set(self.task.params) - bound) + if unbound: + raise ValueError(f"no check reads back input(s) {unbound}") + config = { + "schema_version": "openadapt.verify/v1", + "verifier": self.verifier_section, + "effects": effects, + } + parse_verify_config(config, source="confirmed checks") + if self.task.params: + jit.confirm_params( + principal_id=name, accept_params=sorted(self.task.params), confirm="ok" + ) + path = self.out_dir / CONFIRMED_FILE + _write_private( + path, + f"# Confirmed by {name} at {_now()} from {PROPOSED_FILE}.\n" + + yaml.safe_dump(config, sort_keys=False, allow_unicode=True), + ) + _write_json( + self.out_dir / CHECKS_FILE, + { + "confirmed_by": name, + "confirmed_at": _now(), + "proposed_sha256": hashlib.sha256( + proposed_path.read_bytes() + ).hexdigest(), + "contract_sha256": sha256_tagged(config), + "dropped_fields": sorted(drop_fields), + "params_confirmed": sorted(self.task.params), + }, + ) + lifecycle._event(jit.program_dir, "record_checks_confirmed", principal_id=name) + return path + + # -- step 3: admission ---------------------------------------------------------- + + def _confirmed_config(self) -> Any: + from openadapt_flow.verify_run import load_verify_config + + path = self.out_dir / CONFIRMED_FILE + if not path.is_file(): + raise lifecycle.AdmissionError("confirm the record checks first") + return load_verify_config(path) + + def _governed( + self, + kind: str, + values: dict[str, str], + run: Callable[[], JitResult], + ) -> tuple[JitResult, Optional[dict[str, Any]], Optional[Path]]: + """Run ``run`` with a verify-run baseline taken at its reset, then + judge the confirmed checks against the system of record.""" + + from openadapt_flow import __version__ + from openadapt_flow.verify_run import verify_run, write_result + + cfg = self._confirmed_config() + work = self.dir / "receipts" / f"{kind}-{_stamp()}" + work.mkdir(parents=True, exist_ok=True) + self._pending = (cfg, values, work / "baseline.json") + self._receipt_baseline = None + try: + result = run() + finally: + self._pending = None + if self._receipt_baseline is None: + return result, None, None + receipt = verify_run( + cfg, + params=values, + run_record={ + "executor_kind": "flow_replay", + "executor_id": self.task.key, + "executor_build": {"openadapt-flow": __version__}, + }, + authorization_ref=f"compile-agent:{self.task.key}:{kind}", + baseline_path=self._receipt_baseline, + ) + return result, receipt, write_result(receipt, work) + + def admit( + self, + *, + admitter: Any = None, + person: Optional[str] = None, + review: Optional[Callable[[str, dict[str, Any]], bool]] = None, + canary_params: Optional[Sequence[Mapping[str, Any]]] = None, + ) -> CompileResult: + """Canary the draft, then admit it. + + ``admitter`` is a registered :class:`~openadapt_flow.jit.authority.Principal`: + an admitter key, or an agent under a signed admission policy (which + needs ``min_clean_canaries`` canaries). Without one, ``person`` admits + at an interactive terminal: ``review`` shows them the steps and the + parameters (``"review"``), then the canary results (``"admit"``), and + returns True only when they confirm each. + + ``canary_params`` are the values each canary saves, one mapping per + canary (the last is reused). A canary is a real save on the record the + task names, so the operator chooses values that are safe there. + + Each canary also gets a verify-run receipt against the confirmed + checks; a canary passes only when both end VERIFIED. + """ + + jit = self._jit() + self._confirmed_config() + if admitter is None and person is None: + return self._result( + "draft", + "no admitter: pass a registered key or agent principal, or a person " + "at a terminal", + ) + rows = [dict(row) for row in canary_params or []] + if not rows: + return self._result( + "draft", + "pass canary values: each canary saves them on the record " + f"{self.expect}, so choose values that are safe to write there", + ) + if admitter is not None: + policy = getattr(admitter, "policy", None) + count = policy.policy.min_clean_canaries if policy is not None else 1 + else: + if review is None: + return self._result( + "draft", + f"{person} reviews the steps at a terminal: openadapt-flow jit " + f"review {self.dir}, then pass review= to show them here", + ) + status = jit.status() + shown = {"steps": status["steps"], "params": sorted(self.task.params)} + if not review("review", shown): + return self._result("draft", f"{person} did not confirm the review") + jit.review( + principal_id=str(person), + reviewed_steps=[s["id"] for s in status["steps"]], + confirm="ok", + ) + count = 1 + canaries: list[dict[str, Any]] = [] + for i in range(count): + params = rows[min(i, len(rows) - 1)] + values = self.task.validate(params) + try: + result, receipt, path = self._governed( + f"canary-{i + 1}", + values, + functools.partial( + jit.canary, + principal_id=admitter if admitter is not None else person, + **params, + ), + ) + except lifecycle.AdmissionError as exc: + # Outside the agent's policy: the JIT queued it for a person. + return self._result( + "needs_person", str(exc), admission={"canaries": canaries} + ) + outcome = _receipt_outcome(receipt) + canaries.append( + { + "status": result.status, + "model_calls": result.model_calls, + "receipt_outcome": outcome, + "receipt": str(path) if path else None, + "halt_reason": result.halt_reason, + } + ) + if result.status != "VERIFIED" or outcome != "VERIFIED": + return self._result( + "canary_failed", + result.halt_reason or result.detail or f"receipt {outcome}", + admission={"canaries": canaries}, + ) + if admitter is not None: + out = jit.admit_with_key(admitter) + if out["outcome"] != "admitted": + return self._result( + "needs_person", + "; ".join(out.get("reasons") or []), + admission={"canaries": canaries, **out}, + ) + manifest = out["manifest"] + else: + assert review is not None + if not review("admit", {"canaries": canaries}): + return self._result( + "draft", + f"{person} did not admit the program", + admission={"canaries": canaries}, + ) + manifest = jit.admit(principal_id=str(person), confirm="ok") + admission = { + "canaries": canaries, + "admitted_by": manifest.get("admitted_by_principal_id"), + "mode": manifest.get("admission_mode"), + "digest": manifest.get("digest"), + "policy_id": manifest.get("policy_id"), + } + _write_json(self.out_dir / "admission.json", admission) + return self._result("admitted", None, admission=admission) + + def _result(self, status: str, detail: Optional[str], **kw: Any) -> CompileResult: + confirmed = self.out_dir / CONFIRMED_FILE + return CompileResult( + status=status, + dir=self.dir, + task_key=self.task.key, + detail=detail, + proposed_checks=self.out_dir / PROPOSED_FILE, + confirmed_checks=confirmed if confirmed.is_file() else None, + **kw, + ) + + # -- step 4: emit ------------------------------------------------------------- + + def cli_line(self) -> str: + flags = " ".join( + f"--param {name}=<{name}>" for name in sorted(self.task.params) + ) + line = f"openadapt-flow compile-agent run {self.dir}" + return f"{line} {flags}".rstrip() + + def emit(self) -> dict[str, Any]: + """Write the Agent Skill, the MCP server, and the CLI line.""" + + from openadapt_flow.compile_agent.emit import emit_mcp, emit_skill_for + + jit = self._jit() + if not jit.admitted: + raise lifecycle.AdmissionError("emit an admitted program only") + cli = self.cli_line() + skill = emit_skill_for(jit.bundle_dir, self.out_dir / "skill", cli) + mcp = emit_mcp(self, self.out_dir / "mcp" / "server.py") + emitted = {"skill": str(skill), "mcp_server": str(mcp), "cli": cli} + _write_json(self.out_dir / "emitted.json", emitted) + return emitted + + # -- every later run ------------------------------------------------------------- + + def run(self, **params: Any) -> ReplayResult: + """Run the admitted program: no model calls, every save read back, + and a verify-run receipt against the confirmed checks.""" + + jit = self._jit() + values = self.task.validate(params) + started = time.monotonic() + if not jit.admitted: + return ReplayResult( + status="not_admitted", + verified=False, + model_calls=None, + seconds=0.0, + run_dir=None, + detail="this task has no admitted program; compile an agent run first", + ) + result, receipt, path = self._governed("run", values, lambda: jit.run(**params)) + outcome = _receipt_outcome(receipt) + verified = result.status == "VERIFIED" and outcome == "VERIFIED" + status = result.status + if result.status == "VERIFIED" and outcome != "VERIFIED": + status = "RECEIPT_NOT_VERIFIED" + return ReplayResult( + status=status, + verified=verified, + model_calls=result.model_calls, + seconds=round(time.monotonic() - started, 3), + run_dir=result.run_dir, + receipt_outcome=outcome, + receipt=path, + halt_reason=result.halt_reason, + detail=result.detail, + inbox_item=result.inbox_item, + ) + + +def _receipt_outcome(receipt: Optional[Mapping[str, Any]]) -> Optional[str]: + """The verify-run receipt's outcome, upper-cased like run outcomes.""" + + outcome = ((receipt or {}).get("receipt") or {}).get("outcome") + return str(outcome).upper() if outcome else None + + +def lifecycle_status_draft() -> str: + from openadapt_flow.jit.compile_agent import NEEDS_HUMAN_ADMIT + + return NEEDS_HUMAN_ADMIT + + +def _confirmer(confirm: PersonOrCallback, compiler: FirstRunCompiler) -> Optional[str]: + if confirm is None or isinstance(confirm, str): + return confirm + return confirm(compiler.proposed()) + + +def compile_first_run( + prompt: str, + *, + params: Mapping[str, Any], + url: str, + verify: Union[str, Path], + expect: Mapping[str, Any], + on_screen: Sequence[str], + agent: Any, + confirm_checks: PersonOrCallback = None, + drop_fields: Sequence[str] = (), + admitter: Any = None, + person: Optional[str] = None, + review: Optional[Callable[[str, dict[str, Any]], bool]] = None, + hold: Optional[Callable[[GateRequest], bool]] = None, + canary_params: Optional[Sequence[Mapping[str, Any]]] = None, + task_id: Optional[str] = None, + policy: str = "clinical-write", + cache_dir: Union[str, Path] = ".openadapt-compile-agent", + key_dir: Optional[Union[str, Path]] = None, + model_key: Optional[CustomerModelKey] = None, + reset: Optional[Callable[[], None]] = None, + headless: bool = True, + on_warning: Optional[Callable[[str], None]] = None, +) -> CompileResult: + """Turn an agent's first verified run of ``prompt`` into an admitted program. + + Args: + prompt: The task, with ``{name}`` for each parameter. + params: The values for run 1. + url: The app's entry URL. + verify: A ``verify.yaml`` whose ``verifier:`` section reads the system + of record (rest, sql, fhir, graphql or file). + expect: Fields the new record must have, such as ``{"patient_id": "p1"}``. + on_screen: The record's identity as the screen shows it, such as + ``["Jane Sample", "MRN P1"]``. Checked before every save. + agent: :class:`~openadapt_flow.compile_agent.agents.ClaudeAgent`, + :class:`~openadapt_flow.compile_agent.agents.ScriptedAgent`, or any + object with ``dialect`` and ``drive(request, harness)``. + confirm_checks: The person who confirms the proposed checks, or a + callback that gets them and returns that person's name (None to + leave them waiting). + drop_fields: Match fields that person leaves out of the checks. + admitter: A registered admitter key or policy agent + (:func:`~openadapt_flow.jit.authority.principal_from_key`). + person: Without an admitter, the person who reviews, attends the + canary and admits at a terminal. + review: With ``person``: shows them the steps, then the canary + results, and returns True only when they confirm each. + hold: Asked before every save in run 1; True lets it through. + canary_params: The values each canary saves, one mapping per canary + (the last is reused). A canary is a real save on the record the + task names, so choose values that are safe there. Required. + model_key: The customer's own model key, for a model-driven agent. + reset: Called before every run to reset a test app. Tests only. + + The agent is called once, only when the task has no draft and no + admitted program. + """ + + compiler = FirstRunCompiler( + prompt, + params=list(params), + url=url, + verify=verify, + expect=expect, + on_screen=on_screen, + task_id=task_id, + policy=policy, + cache_dir=cache_dir, + key_dir=key_dir, + reset=reset, + headless=headless, + on_warning=on_warning, + model_key=model_key, + ) + jit = compiler._jit() + first: Optional[CompileResult] = None + if not jit.admitted: + if jit.candidate() is None: + first = compiler.record(agent, params, hold=hold) + if first.status != "draft": + return first + if not (compiler.out_dir / CONFIRMED_FILE).is_file(): + name = _confirmer(confirm_checks, compiler) + if name is None: + return first or compiler._result( + "draft", "the proposed checks wait for a person" + ) + compiler.confirm_checks(principal_id=name, drop_fields=drop_fields) + admitted = compiler.admit( + admitter=admitter, + person=person, + review=review, + canary_params=canary_params, + ) + if first is not None: + admitted.run1 = first.run1 + admitted.record_check = first.record_check + if not admitted.admitted: + return admitted + result = admitted + else: + result = compiler._result( + "admitted", "already admitted; the agent was not called" + ) + result.emitted = compiler.emit() + _write_json(compiler.out_dir / "result.json", result.to_dict()) + return result + + +def run_compiled( + task_dir: Union[str, Path], + *, + reset: Optional[Callable[[], None]] = None, + **params: Any, +) -> ReplayResult: + """Run a compiled task's admitted program with ``params``. + + No model calls. Every save is read back through the verifier, and the run + gets a verify-run receipt against the person-confirmed checks. A run that + doesn't end VERIFIED goes to the exception inbox. + """ + + return FirstRunCompiler.load(task_dir, reset=reset).run(**params) diff --git a/packages/flow/openadapt_flow/jit/adapters.py b/packages/flow/openadapt_flow/jit/adapters.py index bb023a87f3..0d8a6dcb88 100644 --- a/packages/flow/openadapt_flow/jit/adapters.py +++ b/packages/flow/openadapt_flow/jit/adapters.py @@ -587,6 +587,29 @@ def __init__( self.log: list[dict[str, Any]] = [] self.actions = 0 + @property + def page(self) -> Any: + """The Playwright page behind the session, or None.""" + + return self._page + + @property + def pointer(self) -> Optional[tuple[int, int]]: + """Where the last click or drag left the pointer, or None.""" + + return self._pointer + + def screenshot(self) -> bytes: + """The current frame, without performing or recording an action.""" + + return self._session.screenshot() + + @property + def session(self) -> Any: + """The authoring session the actions are recorded through.""" + + return self._session + def execute(self, native: Mapping[str, Any]) -> Observation: """Perform one native action and return the new frame. diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 9e63388cdc..780523fda9 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -171,6 +171,34 @@ "path": "benchmark/comparison_artifact/comparison.json", "sha256": "c5eb58d234ecbd5b03a687745fb6afeb53d0965459b0e42779942441b19dbbd6" }, + { + "path": "benchmark/compile_agent/results/identical-inputs/runs.jsonl", + "sha256": "8bded49ba14efb9f72b58c6916a0d91380cc58e77ab9a9e5c248a35a7cd812b0" + }, + { + "path": "benchmark/compile_agent/results/pilot/replays.jsonl", + "sha256": "05104d0513de5aede95c5d06a16244e6f96ea30035870bcb70898491b34e0fc3" + }, + { + "path": "benchmark/compile_agent/results/pilot/runs.jsonl", + "sha256": "57c948a29b97e14e81d9747dd04ff230ea1605219fed6b29fbdcc28fbe01d32d" + }, + { + "path": "benchmark/compile_agent/results/pilot/summary.json", + "sha256": "65aba72eb964041d4f0717e253908901f8fe0ed3579ebbabeef1118412a26660" + }, + { + "path": "benchmark/compile_agent/results/replays.jsonl", + "sha256": "5e824e0df9ef2eeef09a8aa283f423b74d5a8b1f5273b652e516f811d0d2fdfc" + }, + { + "path": "benchmark/compile_agent/results/runs.jsonl", + "sha256": "8b1959f636d186bd1e47423b8c1b7223facb9b2a207812a1b62035d044d1d5a4" + }, + { + "path": "benchmark/compile_agent/results/summary.json", + "sha256": "6bc82419b5195c1554b4becff386b6d798b7047b08867329408eed54bbfdac11" + }, { "path": "benchmark/cosmetic_drift/results.json", "sha256": "464b6b6f2ac5850b86f7abf469b301e778c97e36d69c2af94f43b0e137fee962" diff --git a/packages/flow/tests/e2e/test_compile_agent_e2e.py b/packages/flow/tests/e2e/test_compile_agent_e2e.py new file mode 100644 index 0000000000..10ce9b9022 --- /dev/null +++ b/packages/flow/tests/e2e/test_compile_agent_e2e.py @@ -0,0 +1,422 @@ +"""compile-agent end to end on MockMed, with a fake Claude client and no network. + +The agent is the real :class:`ClaudeAgent` loop in selective-vision mode. Its +client is a fake that reads the structural listing the harness sends and +answers with tool calls, like a model would, so the loop, the harness, the +gate, the JIT capture and the admission all run for real. It asserts: + +* run 1 is recorded, passes its record check against the system of record, + the proposed checks are confirmed by a person, and the program is admitted + by a registered agent under a signed policy that allows self-admission; +* the agent saw no screenshot except the one after the save (the gate); +* the skill, the MCP server and the CLI line are emitted; +* run 2 replays with zero model calls, ends VERIFIED, and gets a VERIFIED + verify-run receipt; compiling again does not call the agent; +* an agent that opens the wrong patient is refused at the save and nothing + is written or compiled; +* the admitted program, run against an app that files the note on another + patient, stops and is not VERIFIED. +""" + +from __future__ import annotations + +import ast +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Iterator, Optional + +import pytest + +from openadapt_flow.compile_agent import ( + ClaudeAgent, + FirstRunCompiler, + compile_first_run, + run_compiled, +) +from openadapt_flow.jit.keys import CustomerModelKey +from openadapt_flow.mockmed.fault_server import serve +from openadapt_flow.tutorial import _http_json, _records + +PROMPT = ( + "Open the referral for Jane Sample, start a Triage encounter, and save " + "this note: {note}" +) +NOTE = "Synthetic follow-up in two weeks" +NEW_NOTE = "Recheck knee in ten days" +ON_SCREEN = ["Jane Sample", "MRN P1"] +EXPECT = {"patient_id": "p1", "type": "Triage"} +AGENT_ID = "agent:compile-agent-e2e" +TEST_KEY = CustomerModelKey("anthropic", "env:TEST_ONLY", "not-a-real-key") + + +@pytest.fixture(scope="module") +def mockmed() -> Iterator[str]: + base, _db, stop = serve() + yield base.rstrip("/") + stop() + + +def _reset(base: str): + return lambda: _http_json(f"{base}/api/reset", method="POST", body={}) + + +def _verify_yaml(tmp_path: Path, base: str) -> Path: + path = tmp_path / "verify.yaml" + path.write_text( + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/db\n" + " records_key: records\n" + " timeout_s: 2.0\n" + " poll_interval_s: 0.05\n" + ) + return path + + +def _policy_agent(key_dir: Path, base: str, monkeypatch: pytest.MonkeyPatch) -> Any: + """A registered agent, under a policy a person signed that lets it admit + what it recorded itself.""" + + from openadapt_flow.jit import authority + + monkeypatch.setenv(authority.KEY_PASSPHRASE_ENV, "e2e passphrase") + files = key_dir.parent / "keyfiles" + keys = {} + for role, pid, name in ( + ("policy_signer", "dana.signer", "signer"), + ("agent", AGENT_ID, "agent"), + ): + pub = authority.generate_encrypted_key(files / f"{name}.pem") + authority.register_principal( + key_dir, role=role, principal_id=pid, public_key_hex=pub + ) + keys[name] = authority.load_encrypted_key(files / f"{name}.pem") + now = datetime.now(timezone.utc) + policy = authority.AdmissionPolicy( + policy_id="mockmed-notes", + signer="dana.signer", + signed_at=(now - timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), + expires_at=(now + timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ"), + allowed_hosts=(base.split("://", 1)[1],), + allowed_actions=("click", "type"), + allowed_effects=("record_written", "field_equals"), + min_clean_canaries=2, + agent_principals=(AGENT_ID,), + allow_self_admission=True, + ) + signed = authority.sign_policy(policy, keys["signer"]) + return authority.principal_from_key(keys["agent"], key_dir, policy=signed) + + +class FakeClaude: + """Answers the harness's listings with the tool calls a model would make. + + It finds refs in the listing text only, so it fails the way a model would + if the listing stopped naming the controls. + """ + + def __init__(self, patient: str, note: str) -> None: + self.patient = patient + self.note = note + self.calls = 0 + self.messages = SimpleNamespace(create=self.create) + + @staticmethod + def _listing(messages: list[dict[str, Any]]) -> str: + last = messages[-1]["content"] + texts: list[str] = [] + for block in last: + if block.get("type") == "text": + texts.append(block["text"]) + for inner in block.get("content") or []: + if isinstance(inner, dict) and inner.get("type") == "text": + texts.append(inner["text"]) + return "\n".join(texts) + + @staticmethod + def _ref(listing: str, name: str, near: Optional[str] = None) -> str: + for line in listing.splitlines(): + if near and near not in line: + continue + found = re.search(r"\[(e\d+)\] \S+ \"" + re.escape(name) + '"', line) + if found: + return found.group(1) + raise AssertionError(f"no {name!r} in listing:\n{listing}") + + def create(self, **kw: Any) -> Any: + self.calls += 1 + assert kw["model"] == "claude-opus-5-5" + listing = self._listing(kw["messages"]) + if "Encounter saved" in listing: + tool = ("finish", {"summary": "saved"}) + elif "# Referral Tasks" in listing: + tool = ("click", {"ref": self._ref(listing, "Open", self.patient)}) + elif "# New Encounter" in listing: + if not re.search(r'"Triage" \([^)]*selected', listing): + tool = ("click", {"ref": self._ref(listing, "Triage")}) + elif f'value="{self.note}"' not in listing: + ref = self._ref(listing, "Note") + tool = ("type_text", {"text": self.note, "ref": ref}) + else: + tool = ("click", {"ref": self._ref(listing, "Save Encounter")}) + elif "New Encounter" in listing: + tool = ("click", {"ref": self._ref(listing, "New Encounter")}) + else: + raise AssertionError(f"unexpected listing:\n{listing}") + name, args = tool + return SimpleNamespace( + content=[ + { + "type": "tool_use", + "id": f"toolu_{self.calls}", + "name": name, + "input": args, + } + ], + stop_reason="tool_use", + usage=SimpleNamespace( + input_tokens=len(str(kw["messages"])) // 4, + output_tokens=30, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + ) + + +def _without_agent_args(tmp_path: Path, base: str) -> dict[str, Any]: + common = _common(tmp_path, base) + common.pop("on_warning") + return common + + +def _common(tmp_path: Path, base: str) -> dict[str, Any]: + return dict( + url=f"{base}/?fault=ok&idempotency=demo#tasks", + verify=_verify_yaml(tmp_path, base), + expect=EXPECT, + on_screen=ON_SCREEN, + task_id="mockmed.compile_agent_note", + cache_dir=tmp_path / "cache", + key_dir=tmp_path / "keys", + model_key=TEST_KEY, + reset=_reset(base), + on_warning=lambda text: None, + ) + + +def test_compile_agent_then_replay_with_no_model_calls( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + admitter = _policy_agent(tmp_path / "keys", mockmed, monkeypatch) + fake = FakeClaude("Jane Sample", NOTE) + # The registered agent drives run 1 and admits it: self-admission, which + # its signed policy allows. + agent = ClaudeAgent(client=fake, principal_id=AGENT_ID) + result = compile_first_run( + PROMPT, + params={"note": NOTE}, + agent=agent, + confirm_checks="dana.reviewer", + drop_fields=["key", "source"], + admitter=admitter, + canary_params=[{"note": "Canary test note"}], + **_common(tmp_path, mockmed), + ) + assert result.status == "admitted", result.detail + calls_after_run1 = fake.calls + assert result.run1["agent"]["model_calls"] == calls_after_run1 == 6 + harness = result.run1["harness"] + # Selective vision: no screenshot asked for; one at the save gate. + assert harness["screenshots_requested"] == 0 + assert harness["screenshots_at_gate"] == 1 + assert harness["screenshots_every_turn"] == 0 + assert harness["gate_checks"] == 1 and harness["gate_refusals"] == 0 + assert result.run1["gates"][0]["decision"] == "passed_no_hold" + assert result.record_check is not None and result.record_check["passed"] + assert result.admission is not None + assert result.admission["mode"] == "agent_policy" + assert result.admission["admitted_by"] == AGENT_ID + assert len(result.admission["canaries"]) == 2 + assert all( + c["status"] == "VERIFIED" and c["receipt_outcome"] == "VERIFIED" + for c in result.admission["canaries"] + ) + confirmed = result.confirmed_checks + assert confirmed is not None and "needs_operator_confirmation" not in ( + confirmed.read_text() + ) + # What it is called through. + skill = Path(result.emitted["skill"]) / "SKILL.md" + assert "compile-agent run" in skill.read_text() and NOTE not in skill.read_text() + server = Path(result.emitted["mcp_server"]).read_text() + ast.parse(server) + assert "run_compiled" in server and NOTE not in server + assert result.emitted["cli"].endswith("--param note=") + + # Run 2: the program, a new note, no model calls, read back and receipted. + replay = run_compiled(result.dir, reset=_reset(mockmed), note=NEW_NOTE) + assert replay.verified, replay.to_dict() + assert replay.model_calls == 0 and fake.calls == calls_after_run1 + assert replay.receipt_outcome == "VERIFIED" + rows = _records(mockmed) + assert [(r["patient_id"], r["type"], r["note"]) for r in rows] == [ + ("p1", "Triage", NEW_NOTE) + ] + + # Compiling again uses the admitted program; the agent is not called. + again = compile_first_run( + PROMPT, + params={"note": NOTE}, + agent=agent, + admitter=admitter, + **_common(tmp_path, mockmed), + ) + assert again.status == "admitted" and fake.calls == calls_after_run1 + # Run 1 never runs again on an admitted task, not even step by step. + from openadapt_flow.jit import AdmissionError + + stepwise = FirstRunCompiler( + PROMPT, params=["note"], **_without_agent_args(tmp_path, mockmed) + ) + with pytest.raises(AdmissionError, match="runs only once"): + stepwise.record(agent, {"note": NOTE}) + + # The admitted program against an app that files the note on patient p2. + common = _common(tmp_path, mockmed) + common["url"] = f"{mockmed}/?fault=wrong_record&idempotency=demo#tasks" + common.pop("model_key") + common.pop("on_warning") + wrong = FirstRunCompiler(PROMPT, params=["note"], **common).run(note=NEW_NOTE) + assert not wrong.verified and wrong.model_calls == 0 + assert wrong.status == "HALTED" and wrong.inbox_item is not None + assert fake.calls == calls_after_run1 + + +def test_wrong_patient_is_refused_at_the_save( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + admitter = _policy_agent(tmp_path / "keys", mockmed, monkeypatch) + fake = FakeClaude("Alex Testcase", NOTE) # opens the wrong chart + result = compile_first_run( + PROMPT, + params={"note": NOTE}, + agent=ClaudeAgent(client=fake), + confirm_checks="dana.reviewer", + admitter=admitter, + **_common(tmp_path, mockmed), + ) + assert result.status == "refused_wrong_record", result.detail + assert "MRN P1" in (result.detail or "") + assert result.run1["gates"][0]["decision"] == "refused_wrong_record" + assert _records(mockmed) == [] # nothing was saved + assert not (result.dir / "program").exists() # nothing was compiled + assert result.emitted == {} + + +SCRIPT = { + "principal_id": "agent:scripted-e2e", + "steps": [ + {"click": "Open", "near": "Jane Sample"}, + {"click": "New Encounter"}, + {"click": "Triage"}, + {"type": "{note}", "into": "Note"}, + {"click": "Save Encounter"}, + ], +} + + +def test_cli_scripted_agent_person_confirms_key_admits( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys +) -> None: + import json + + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent.cli import cmd_compile_agent + from openadapt_flow.jit import authority + + # The CLI has no reset hook (a real app is never reset). MockMed's demo + # idempotency key makes a second note on p1 a duplicate, so reset it + # before each run here, the way the Python API's reset= does. + original = FirstRunCompiler._on_reset + + def reset_first(self: FirstRunCompiler) -> None: + _reset(mockmed)() + original(self) + + monkeypatch.setattr(FirstRunCompiler, "_on_reset", reset_first) + monkeypatch.setenv(authority.KEY_PASSPHRASE_ENV, "e2e passphrase") + key_dir = tmp_path / "keys" + pem = tmp_path / "keyfiles" / "ci.pem" + authority.register_principal( + key_dir, + role="admitter_key", + principal_id="ci:compile-agent", + public_key_hex=authority.generate_encrypted_key(pem), + ) + script = tmp_path / "script.json" + script.write_text(json.dumps(SCRIPT)) + argv = [ + "compile-agent", + PROMPT, + "--url", + f"{mockmed}/?fault=ok&idempotency=demo#tasks", + "--verify", + str(_verify_yaml(tmp_path, mockmed)), + "--param", + f"note={NOTE}", + "--expect", + "patient_id=p1", + "--on-screen", + "Jane Sample", + "--on-screen", + "MRN P1", + "--agent", + str(script), + "--task-id", + "mockmed.compile_agent_cli", + "--cache", + str(tmp_path / "cache"), + "--key-dir", + str(key_dir), + "--admit-key", + str(pem), + "--drop-field", + "key", + "--canary-param", + "note=Canary test note", + "--json", + ] + # No terminal: run 1 passes, and the checks wait for a person. + args = build_parser().parse_args(argv) + assert cmd_compile_agent(args, interactive=lambda: False) == 3 + first = json.loads(capsys.readouterr().out) + assert first["status"] == "draft" and "next" in first + assert first["record_check"]["passed"] + + # A person at a terminal confirms; the registered key admits. The agent + # is not run again (the draft is waiting). + answers = iter(["dana.reviewer", "ok"]) + args = build_parser().parse_args([*argv, "--no-hold"]) + assert ( + cmd_compile_agent(args, ask=lambda _: next(answers), interactive=lambda: True) + == 0 + ) + out = capsys.readouterr().out + assert "Proposed record checks" in out and "patient_id = p1" in out + admitted = json.loads(out[out.index("\n{") + 1 :]) + assert admitted["status"] == "admitted", admitted["detail"] + assert admitted["admission"]["mode"] == "admitter_key" + task_dir = next((tmp_path / "cache").glob("mockmed.compile_agent_cli-*")) + emitted = json.loads((task_dir / "compiled" / "emitted.json").read_text()) + assert emitted["cli"].startswith("openadapt-flow compile-agent run ") + + run = build_parser().parse_args( + ["compile-agent", "run", str(task_dir), "--param", f"note={NEW_NOTE}", "--json"] + ) + assert cmd_compile_agent(run) == 0 + replay = json.loads(capsys.readouterr().out) + assert replay["verified"] and replay["model_calls"] == 0 + assert replay["receipt_outcome"] == "VERIFIED" diff --git a/packages/flow/tests/test_compile_agent.py b/packages/flow/tests/test_compile_agent.py new file mode 100644 index 0000000000..9ab412a0c3 --- /dev/null +++ b/packages/flow/tests/test_compile_agent.py @@ -0,0 +1,574 @@ +"""openadapt_flow.compile_agent without a browser or a model. + +The record check, the commit-label rule, scrubbing that keeps the task's own +values, the Claude loop's accounting and stops, and the emitted MCP server. +tests/e2e/test_compile_agent_e2e.py runs the whole pipeline on MockMed. +""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from openadapt_flow import privacy +from openadapt_flow.compile_agent import ( + BudgetExceeded, + ClaudeAgent, + GateRefused, + check_run_record, +) +from openadapt_flow.compile_agent.agents import AgentLedger, ModelCall +from openadapt_flow.compile_agent.observe import _render, commit_label, scrub_lines +from openadapt_flow.jit.compile_agent import TaskRequest + +BEFORE = [{"id": 1, "patient_id": "p2", "note": "old"}] + + +def _created(**fields: Any) -> list[dict[str, Any]]: + return BEFORE + [{"id": 2, **fields}] + + +def test_record_check_passes_only_for_the_named_record() -> None: + ok = check_run_record( + BEFORE, + _created(patient_id="p1", note="Follow up"), + expect={"patient_id": "p1"}, + values={"note": "Follow up"}, + ) + assert ok.passed and ok.fields == ["id", "note", "patient_id"] + + wrong = check_run_record( + BEFORE, + _created(patient_id="p2", note="Follow up"), + expect={"patient_id": "p1"}, + values={"note": "Follow up"}, + ) + assert not wrong.passed and "wrong record" in wrong.reasons[0] + + lost = check_run_record( + BEFORE, + _created(patient_id="p1", note=""), + expect={"patient_id": "p1"}, + values={"note": "Follow up"}, + ) + assert lost.reasons == ["input 'note' is not in the new record"] + + nothing = check_run_record(BEFORE, BEFORE, expect={"patient_id": "p1"}, values={}) + assert not nothing.passed and nothing.created == 0 + + two = check_run_record( + [], + [{"id": 1, "patient_id": "p1"}, {"id": 2, "patient_id": "p1"}], + expect={"patient_id": "p1"}, + values={}, + ) + assert not two.passed + + edited = check_run_record( + BEFORE, + [ + {"id": 1, "patient_id": "p2", "note": "changed"}, + {"id": 2, "patient_id": "p1"}, + ], + expect={"patient_id": "p1"}, + values={}, + ) + assert edited.reasons == ["1 existing records changed"] + + unread = check_run_record(None, BEFORE, expect={"patient_id": "p1"}, values={}) + assert not unread.passed + + +@pytest.mark.parametrize( + ("label", "commits"), + [ + ("Save Encounter", True), + ("Submit", True), + ("Sign note", True), + ("Approve claim", True), + ("Sign In", False), + ("Open", False), + ("New Encounter", False), + ("Continue to tasks", False), + ("Saved searches", True), + ], +) +def test_commit_labels(label: str, commits: bool) -> None: + assert commit_label(label) is commits + + +def test_listing_groups_rows_and_names_refs() -> None: + raw = { + "items": [ + {"kind": "heading", "text": "Referral Tasks", "row": -1}, + {"kind": "text", "text": "Jane Sample", "row": 1}, + {"kind": "text", "text": "High", "row": 1}, + { + "kind": "control", + "tag": "button", + "name": "Open", + "x": 10, + "y": 20, + "row": 1, + "selected": False, + }, + { + "kind": "control", + "tag": "textarea", + "name": "Note", + "value": "", + "x": 5, + "y": 6, + "row": -1, + "focused": True, + }, + ] + } + text, controls = _render(raw) + assert text.splitlines() == [ + "# Referral Tasks", + 'row: Jane Sample | High | [e1] button "Open"', + '[e2] textarea "Note" value="" (focused)', + ] + assert (controls["e1"].x, controls["e1"].y) == (10, 20) + + +class _Scrubber: + def scrub_text(self, text: str, is_separated: bool = False) -> str: + return re.sub(r"[A-Z][a-z]+ [A-Z][a-z]+", "", text) + + +def test_scrubbing_keeps_the_values_the_operator_named(monkeypatch) -> None: + monkeypatch.setenv("OPENADAPT_FLOW_SCRUB", "on") + privacy.set_text_scrubber(_Scrubber()) + try: + text = "row: Jane Sample | Knee pain\nrow: Alex Testcase | Cardiology" + assert scrub_lines(text, keep=["Jane Sample"]) == ( + "row: Jane Sample | Knee pain\nrow: | Cardiology" + ) + finally: + privacy.reset_scrubbers() + + +# -- the Claude loop ------------------------------------------------------------- + + +class _Harness: + """Stands in for the browser harness: a fixed listing and counters.""" + + def __init__(self, refuse: bool = False) -> None: + self.refuse = refuse + self.clicks: list[str] = [] + self.stats = SimpleNamespace(gate_checks=0, screenshots_requested=0) + + def observe(self) -> Any: + return SimpleNamespace(text='page: x\n[e1] button "Save"') + + def screenshot(self, why: str = "requested") -> bytes: + self.stats.screenshots_requested += 1 + return b"\x89PNG fake" + + def click(self, ref: str) -> None: + if ref == "e9": + from openadapt_flow.compile_agent.harness import StaleRef + + raise StaleRef("e9 is not in the latest observation") + if self.refuse: + raise GateRefused("wrong record") + self.clicks.append(ref) + self.stats.gate_checks += 1 + + +class _Client: + def __init__(self, script: list[tuple[str, dict[str, Any]]], tokens: int = 100): + self.script = list(script) + self.tokens = tokens + self.requests: list[dict[str, Any]] = [] + self.messages = SimpleNamespace(create=self.create) + + def create(self, **kw: Any) -> Any: + self.requests.append(kw) + name, args = self.script.pop(0) + return SimpleNamespace( + content=[ + { + "type": "tool_use", + "id": f"t{len(self.requests)}", + "name": name, + "input": args, + } + ], + stop_reason="tool_use", + usage=SimpleNamespace( + input_tokens=self.tokens, + output_tokens=10, + cache_creation_input_tokens=0, + cache_read_input_tokens=50, + ), + ) + + +REQUEST = TaskRequest(task_id="t", instruction="Save it", inputs={}) + + +def test_claude_loop_counts_tokens_and_the_gate_screenshot() -> None: + client = _Client( + [ + ("click", {"ref": "e9"}), + ("click", {"ref": "e1"}), + ("finish", {"summary": "done"}), + ] + ) + harness = _Harness() + ledger = ClaudeAgent(client=client).drive(REQUEST, harness) # type: ignore[arg-type] + assert ledger.finished and ledger.summary == "done" + assert ledger.model_calls == 3 and ledger.tool_errors == 1 # the stale ref + assert harness.clicks == ["e1"] + # The save's result carried a screenshot: the verification gate. + messages = client.requests[2]["messages"] # the loop appends to one list + third = messages[4]["content"][0]["content"] + assert [b["type"] for b in third] == ["text", "image"] + assert ledger.to_dict()["cache_read_input_tokens"] == 150 + assert client.requests[0]["extra_body"]["output_config"] == {"effort": "medium"} + assert client.requests[0]["tools"][0]["name"] == "observe" + + +def test_every_turn_mode_uses_the_computer_toolset() -> None: + agent = ClaudeAgent( + vision="every_turn", client=_Client([("finish", {"summary": "x"})]) + ) + assert agent.dialect == "anthropic" + harness = _Harness() + agent.drive(REQUEST, harness) # type: ignore[arg-type] + tools = agent.client.requests[0]["tools"] + assert tools[0]["type"] == "computer_toolset_20260801" and "name" not in tools[0] + + +def test_a_refused_save_ends_the_loop() -> None: + client = _Client([("click", {"ref": "e1"})]) + with pytest.raises(GateRefused): + ClaudeAgent(client=client).drive(REQUEST, _Harness(refuse=True)) # type: ignore[arg-type] + + +def test_the_cost_cap_ends_the_loop() -> None: + client = _Client([("observe", {})] * 5, tokens=400_000) + with pytest.raises(BudgetExceeded): + ClaudeAgent(client=client, max_cost_usd=1.0).drive(REQUEST, _Harness()) # type: ignore[arg-type] + assert len(client.requests) == 1 + + +def test_ledger_prices_opus() -> None: + ledger = AgentLedger(agent="claude", model="claude-opus-5-5") + ledger.calls.append( + ModelCall(1, 1_000_000, 100_000, 0, 1_000_000, 1.0, "tool_use", 0) + ) + assert ledger.cost_usd() == pytest.approx(4.0 + 2.0 + 0.2) + + +def test_the_emitted_mcp_server_parses(tmp_path: Path) -> None: + from openadapt_flow.compile_agent.emit import emit_mcp + from openadapt_flow.jit.signature import TaskSignature + + fake = SimpleNamespace( + task=TaskSignature("demo.add-note", 'Add """{note}"""', {"note": str}), + dir=tmp_path / "task", + ) + path = emit_mcp(fake, tmp_path / "server.py") # type: ignore[arg-type] + tree = ast.parse(path.read_text()) + funcs = [n.name for n in tree.body if isinstance(n, ast.FunctionDef)] + assert funcs == ["demo_add_note"] + assert "note: str" in path.read_text() + + +# -- the save gate ---------------------------------------------------------------- + + +class _Page: + url = "http://app.test/#encounter" + + def __init__(self, screen: str, element: Any = "save", focused: Any = None): + self.screen = screen + self.element = element + self.focused = focused + + def _describe(self, name: str, **kw: Any) -> dict[str, Any]: + return { + "tag": "button", + "type": "submit", + "name": name, + "role": "", + "interactive": True, + "editable": False, + "context": "", + "scope": self.screen, + **kw, + } + + def evaluate(self, script: str, arg: Any = None) -> Any: + if "elementFromPoint" in script: + if self.element is None: + raise RuntimeError("navigating") + if self.element == "save": + return self._describe("Save") + return self._describe(**self.element) + if "activeElement" in script: + return self._describe(**self.focused) if self.focused else None + return None + + +class _Computer: + dialect = "anthropic" + pointer = (40, 50) + + def __init__(self, screen: str, **page: Any) -> None: + self.page = _Page(screen, **page) + self.executed: list[dict[str, Any]] = [] + + def screenshot(self) -> bytes: + return b"png" + + def execute(self, native: dict[str, Any]) -> Any: + self.executed.append(native) + return SimpleNamespace(png=b"png", notes=[]) + + +def _refused(computer: _Computer, native: dict[str, Any], tmp_path: Path) -> Any: + from openadapt_flow.compile_agent.harness import Harness, SaveGate + + harness = Harness(computer, SaveGate(["MRN P1"]), evidence_dir=tmp_path) # type: ignore[arg-type] + with pytest.raises(GateRefused, match="MRN P1"): + harness.act(native) + assert computer.executed == [] + return harness + + +@pytest.mark.parametrize( + "native", [{"action": "left_click"}, {"action": "left_click", "coordinate": [1, 2]}] +) +def test_a_save_on_the_wrong_record_is_never_performed( + native: dict, tmp_path: Path +) -> None: + harness = _refused(_Computer("Alex Testcase — MRN P2"), native, tmp_path) + assert harness.gate_log[0]["decision"] == "refused_wrong_record" + assert (tmp_path / "gate-01.png").is_file() + + +def test_the_identity_must_be_whole_words(tmp_path: Path) -> None: + _refused( + _Computer("Sam Specimen — MRN P12"), + {"action": "left_click", "coordinate": [1, 2]}, + tmp_path, + ) + + +BUTTON = {"name": "Save note", "tag": "div", "role": "button"} +TEXT = {"name": "Note", "tag": "textarea", "editable": True} + + +@pytest.mark.parametrize( + ("page", "native"), + [ + ({"focused": TEXT}, {"action": "type", "text": "done\n"}), + ({"focused": {"name": "Open"}}, {"action": "key", "text": "space"}), + ({"focused": TEXT}, {"action": "key", "text": "ctrl+Return"}), + ({"focused": TEXT}, {"action": "key", "text": "ctrl+s"}), + ({"focused": TEXT}, {"action": "key", "text": "Tab Return"}), + ({"focused": TEXT}, {"action": "key", "text": "Return"}), + ({"element": BUTTON}, {"action": "left_click", "coordinate": [1, 2]}), + ({"element": None}, {"action": "left_click", "coordinate": [1, 2]}), + ( + {"element": {"name": "Delete encounter"}}, + {"action": "double_click", "coordinate": [1, 2]}, + ), + ], +) +def test_every_way_to_commit_is_gated(page: dict, native: dict, tmp_path: Path) -> None: + _refused(_Computer("Alex Testcase — MRN P2", **page), native, tmp_path) + + +@pytest.mark.parametrize( + ("page", "native"), + [ + ({"element": {"name": "Open"}}, {"action": "left_click", "coordinate": [1, 2]}), + ({"focused": TEXT}, {"action": "type", "text": "no line break"}), + ({"focused": TEXT}, {"action": "key", "text": "space"}), + ({"focused": TEXT}, {"action": "key", "text": "Tab"}), + ], +) +def test_plain_actions_are_not_gated(page: dict, native: dict) -> None: + from openadapt_flow.compile_agent.harness import Harness, SaveGate + + computer = _Computer("Alex Testcase — MRN P2", **page) + harness = Harness(computer, SaveGate(["MRN P1"])) # type: ignore[arg-type] + harness.act(native) + assert len(computer.executed) == 1 and harness.stats.gate_checks == 0 + + +def test_the_hold_decides_after_the_identity_check(tmp_path: Path) -> None: + from openadapt_flow.compile_agent.harness import Harness, SaveGate + + asked: list[Any] = [] + computer = _Computer("Jane Sample — MRN P1") + deny = Harness( + computer, # type: ignore[arg-type] + SaveGate(["MRN P1"], hold=lambda req: asked.append(req) or False), + ) + with pytest.raises(GateRefused, match="hold"): + deny.act({"action": "left_click", "coordinate": [1, 2]}) + assert computer.executed == [] and asked[0].control == "Save" + + allow = Harness(computer, SaveGate(["MRN P1"], hold=lambda req: True)) # type: ignore[arg-type] + allow.act({"action": "left_click", "coordinate": [1, 2]}) + assert len(computer.executed) == 1 + assert allow.gate_log[-1]["decision"] == "approved_by_hold" + + +def test_a_ref_is_stale_when_its_row_changed() -> None: + from openadapt_flow.compile_agent.harness import Harness, SaveGate, StaleRef + from openadapt_flow.compile_agent.observe import Control, StructuralObservation + + computer = _Computer( + "x", element={"name": "Save", "context": "Alex Testcase | Save"} + ) + harness = Harness(computer, SaveGate(["MRN P1"])) # type: ignore[arg-type] + harness._last = StructuralObservation( + url="", + title="", + text="", + controls={ + "e1": Control("e1", "button", "Save", 1, 2, context="Jane Sample | Save") + }, + ) + with pytest.raises(StaleRef, match="no longer at its place"): + harness.click("e1") + assert computer.executed == [] + + +# -- targets ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("backend", ["windows", "macos", "linux", "rdp", "citrix"]) +def test_a_non_web_backend_is_refused_before_anything_runs( + backend: str, tmp_path: Path, capsys +) -> None: + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent import FirstRunCompiler + from openadapt_flow.compile_agent.cli import cmd_compile_agent + from openadapt_flow.compile_agent.pipeline import UnsupportedTarget + + args = build_parser().parse_args( + ["compile-agent", "Save {note}", "--backend", backend, "--url", "http://a"] + ) + assert cmd_compile_agent(args) == 2 + assert f"the {backend!r} backend" in capsys.readouterr().err + with pytest.raises(UnsupportedTarget, match="web apps only"): + FirstRunCompiler( + "Save {note}", + params=["note"], + url="http://a", + verify=tmp_path / "missing.yaml", # never read: refused first + expect={"patient_id": "p1"}, + on_screen=["MRN P1"], + backend=backend, + ) + + +def test_a_deployment_naming_another_backend_is_refused(tmp_path: Path, capsys) -> None: + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent.cli import cmd_compile_agent + + config = tmp_path / "deployment.yaml" + config.write_text("backend:\n kind: citrix\n") + args = build_parser().parse_args( + ["compile-agent", "Save {note}", "--url", "http://a", "--config", str(config)] + ) + assert cmd_compile_agent(args) == 2 + assert "'citrix' backend" in capsys.readouterr().err + + +def test_a_non_http_url_is_refused(capsys) -> None: + from openadapt_flow.__main__ import build_parser + from openadapt_flow.compile_agent.cli import cmd_compile_agent + + args = build_parser().parse_args( + ["compile-agent", "Save {note}", "--url", "rdp://host"] + ) + assert cmd_compile_agent(args) == 2 + assert "not an http(s) URL" in capsys.readouterr().err + + +# -- frames and text fields ----------------------------------------------------------- + + +class _FramePage(_Page): + """A top page whose point and focus land on an iframe.""" + + def __init__(self, inner: Any) -> None: + super().__init__("outer page with no identity") + self.inner = inner + self.main_frame = object() + self.frames = [self.main_frame, inner] + + def evaluate(self, script: str, arg: Any = None) -> Any: + if "elementFromPoint" in script or "activeElement" in script: + return self._describe("frame", tag="iframe") + return None + + +def test_a_save_inside_an_iframe_is_read_in_the_frame() -> None: + from openadapt_flow.compile_agent.observe import element_at + + inner = _Page("Alex Testcase — MRN P2") + backend = SimpleNamespace( + _frame_point=lambda x, y: SimpleNamespace( + frame_path=("iframe#chart",), scope=inner, x=x - 10, y=y - 10 + ) + ) + page = _FramePage(inner) + assert element_at(page, 50, 60, backend=backend)["name"] == "Save" + # A frame the backend can't map is unreadable, so a click there is gated. + assert element_at(page, 50, 60) is None + + +def test_a_click_in_an_iframe_is_gated(tmp_path: Path) -> None: + computer = _Computer("x") + inner = _Page("Alex Testcase — MRN P2") + computer.page = _FramePage(inner) + computer.session = SimpleNamespace( + backend=SimpleNamespace( + _frame_point=lambda x, y: SimpleNamespace( + frame_path=("iframe",), scope=inner, x=x, y=y + ) + ) + ) + _refused(computer, {"action": "left_click", "coordinate": [5, 5]}, tmp_path) + + +def test_focus_inside_an_iframe_is_followed() -> None: + from openadapt_flow.compile_agent.observe import focused_element + + class _Frame: + def evaluate(self, script: str, arg: Any = None) -> Any: + if "hasFocus" in script: + return True + return {"tag": "input", "name": "Note", "scope": "Jane Sample — MRN P1"} + + page = _FramePage(_Frame()) + assert focused_element(page)["scope"] == "Jane Sample — MRN P1" + + +def test_a_click_on_a_text_field_is_not_a_commit() -> None: + from openadapt_flow.compile_agent.harness import Harness, SaveGate + + computer = _Computer( + "Alex Testcase — MRN P2", + element={"name": "Postal code", "tag": "input", "editable": True}, + ) + harness = Harness(computer, SaveGate(["MRN P1"])) # type: ignore[arg-type] + harness.act({"action": "left_click", "coordinate": [1, 2]}) + assert len(computer.executed) == 1 and harness.stats.gate_checks == 0