From d3b8efc433c18140877456cec1fd902290ef8c47 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 15:45:24 -0400 Subject: [PATCH 01/13] feat(flow): code-as-action agent for compile-agent run 1 ClaudeAgent(vision="code") acts through one run tool that takes a short script: read the DOM, accessibility tree or OCR text, act by role and name, scroll, screenshot only on request, and a sandboxed read-only shell in a working directory. Every action is a structured step checked against the locator the recorder stored and the compiled program. The save gate now reads inside iframes, takes a tool's hint for frames and desktop surfaces, and no longer treats a click on a text field such as Postal code as a save. Co-Authored-By: Claude Opus 5.5 --- .../flow/benchmark/code_as_action/common.py | 129 ++ .../code_as_action/measure_mockmed.py | 271 +++ .../code_as_action/measure_openemr.py | 472 ++++++ .../code_as_action/results/spend.json | 26 + packages/flow/openadapt_flow/authoring.py | 7 + .../openadapt_flow/compile_agent/__init__.py | 15 + .../openadapt_flow/compile_agent/agents.py | 71 +- .../compile_agent/code_action.py | 1501 +++++++++++++++++ .../openadapt_flow/compile_agent/harness.py | 84 +- .../openadapt_flow/compile_agent/observe.py | 27 +- .../openadapt_flow/compile_agent/pipeline.py | 46 + .../openadapt_flow/compile_agent/sandbox.py | 325 ++++ packages/flow/openadapt_flow/jit/adapters.py | 6 + 13 files changed, 2967 insertions(+), 13 deletions(-) create mode 100644 packages/flow/benchmark/code_as_action/common.py create mode 100644 packages/flow/benchmark/code_as_action/measure_mockmed.py create mode 100644 packages/flow/benchmark/code_as_action/measure_openemr.py create mode 100644 packages/flow/benchmark/code_as_action/results/spend.json create mode 100644 packages/flow/openadapt_flow/compile_agent/code_action.py create mode 100644 packages/flow/openadapt_flow/compile_agent/sandbox.py diff --git a/packages/flow/benchmark/code_as_action/common.py b/packages/flow/benchmark/code_as_action/common.py new file mode 100644 index 0000000000..7d52484094 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/common.py @@ -0,0 +1,129 @@ +"""Shared pieces of the code-as-action measurements: the key, the spend +ledger, host load, and how a run-1 result becomes a committed row. + +The key is read from an env file into this process only and never printed. +The spend ledger (``results/spend.json``) is shared by every measurement +script here, so the total cap holds across them. +""" + +from __future__ import annotations + +import json +import os +import statistics +import subprocess +import time +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +RESULTS = HERE / "results" +SPEND = RESULTS / "spend.json" +TOTAL_CAP_USD = 15.0 +#: Timed runs count as unloaded only below this 1-minute load (12 cores). +LOAD_LIMIT = 12.0 + + +def load_key(env_file: Path) -> None: + for line in env_file.read_text().splitlines(): + name, sep, value = line.strip().partition("=") + if sep and name.strip() == "ANTHROPIC_API_KEY": + os.environ["ANTHROPIC_API_KEY"] = value.strip().strip('"').strip("'") + return + raise SystemExit(f"no ANTHROPIC_API_KEY in {env_file}") + + +def spent() -> float: + if not SPEND.is_file(): + return 0.0 + return float(json.loads(SPEND.read_text()).get("total_usd", 0.0)) + + +def may_start(run_cap: float, cap: float = TOTAL_CAP_USD) -> Optional[str]: + """None when a run capped at ``run_cap`` can't pass the total cap.""" + + total = spent() + if total + run_cap > cap: + return f"${total:.2f} spent; a ${run_cap:.2f} run could pass the ${cap:.2f} cap" + return None + + +def add_spend(label: str, usd: float) -> float: + RESULTS.mkdir(parents=True, exist_ok=True) + data = json.loads(SPEND.read_text()) if SPEND.is_file() else {"runs": []} + data["runs"].append({"label": label, "usd": round(usd, 6), "at": now()}) + data["total_usd"] = round(sum(r["usd"] for r in data["runs"]), 6) + data["cap_usd"] = TOTAL_CAP_USD + SPEND.write_text(json.dumps(data, indent=2) + "\n") + return data["total_usd"] + + +def now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def load_1m() -> float: + return round(os.getloadavg()[0], 1) + + +def commit() -> str: + return subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + cwd=HERE, + ).stdout.strip() + + +def append(path: Path, row: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as handle: + handle.write(json.dumps(row, sort_keys=True) + "\n") + + +def agent_metrics(agent: dict[str, Any], harness: dict[str, Any]) -> dict[str, Any]: + """The comparison columns, from an agent ledger and harness stats.""" + + return { + "model_calls": agent.get("model_calls"), + "prompt_tokens": agent.get("prompt_tokens"), + "input_tokens": agent.get("input_tokens"), + "cache_read_input_tokens": agent.get("cache_read_input_tokens"), + "cache_creation_input_tokens": agent.get("cache_creation_input_tokens"), + "output_tokens": agent.get("output_tokens"), + "cost_usd": agent.get("cost_usd"), + "model_seconds": agent.get("model_seconds"), + "agent_seconds": agent.get("wall_seconds"), + "actions": harness.get("actions"), + "screenshots_sent": ( + harness.get("screenshots_requested", 0) + + harness.get("screenshots_at_gate", 0) + + harness.get("screenshots_every_turn", 0) + ), + "images_in_requests": agent.get("images_in_requests"), + "tool_errors": agent.get("tool_errors"), + } + + +def median(rows: list[dict[str, Any]], key: str) -> Optional[float]: + values = [float(r[key]) for r in rows if r.get(key) is not None] + return round(statistics.median(values), 3) if values else None + + +def summarize( + rows: list[dict[str, Any]], arms: list[str], keys: list[str] +) -> dict[str, Any]: + out: dict[str, Any] = {} + for arm in arms: + mine = [r for r in rows if r["arm"] == arm] + unloaded = [r for r in mine if not r.get("under_load")] + out[arm] = { + "runs": len(mine), + "correct": sum(1 for r in mine if r.get("correct")), + "runs_under_load": len(mine) - len(unloaded), + "total_cost_usd": round( + sum(float(r.get("cost_usd") or 0) for r in mine), 4 + ), + **{f"median_{k}": median(mine, k) for k in keys}, + } + return out diff --git a/packages/flow/benchmark/code_as_action/measure_mockmed.py b/packages/flow/benchmark/code_as_action/measure_mockmed.py new file mode 100644 index 0000000000..09676ae114 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/measure_mockmed.py @@ -0,0 +1,271 @@ +"""Code as action against the screenshot loop on MockMed, live, on the operator's key. + +Each trial runs every arm once, in rotating order, on the same task against a +freshly reset MockMed and its system of record: + +* ``code``: one ``run`` tool that takes a script (code as action). +* ``every_turn``: Claude's computer toolset, a screenshot after every action + (the screenshot loop). +* ``selective``: the structural listing after every action, a screenshot on + request (compile-agent's default). + +Correct means run 1 passed its record check: the system of record, read back +after the run, shows exactly one new encounter on the right patient with the +note typed. Then one passing code-arm program is admitted and replayed, with +no model calls, to show the trace compiled into a working program. + + PYTHONPATH=. python benchmark/code_as_action/measure_mockmed.py \\ + --env-file /path/to/.env --trials 5 + +All data is synthetic (MockMed). Spend is capped across every script here +(results/spend.json). +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import common # noqa: E402 + +PROMPT = ( + "Open the referral for Jane Sample, start a Triage encounter, and save " + "this note: {note}" +) +NOTE = "Synthetic follow-up in two weeks" +REPLAY_NOTES = ["Recheck knee in ten days", "Ice and rest, review in a week"] +ARMS = ["code", "every_turn", "selective"] +KEYS = [ + "model_calls", + "prompt_tokens", + "output_tokens", + "cost_usd", + "run_seconds", + "model_seconds", + "actions", + "screenshots_sent", +] + + +def _compiler(work: Path, base: str, task_id: str) -> Any: + from openadapt_flow.compile_agent import FirstRunCompiler + from openadapt_flow.jit.keys import load_customer_model_key + from openadapt_flow.tutorial import _http_json + + verify = work / "verify.yaml" + verify.write_text( + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/db\n" + " records_key: records\n" + " timeout_s: 3.0\n" + " poll_interval_s: 0.1\n" + ) + return FirstRunCompiler( + PROMPT, + params=["note"], + url=f"{base}/?fault=ok&idempotency=demo#tasks", + verify=verify, + expect={"patient_id": "p1", "type": "Triage"}, + on_screen=["Jane Sample", "MRN P1"], + task_id=task_id, + cache_dir=work / "cache", + key_dir=work / "keys", + reset=lambda: _http_json(f"{base}/api/reset", method="POST", body={}), + on_warning=lambda text: None, + model_key=load_customer_model_key("anthropic"), + ) + + +def _admitter(work: Path) -> Any: + from openadapt_flow.jit import authority + + os.environ.setdefault(authority.KEY_PASSPHRASE_ENV, "measurement-only") + pem = work / "keyfiles" / "admitter.pem" + authority.register_principal( + work / "keys", + role="admitter_key", + principal_id="ci:measurement", + public_key_hex=authority.generate_encrypted_key(pem), + ) + return authority.principal_from_key( + authority.load_encrypted_key(pem), work / "keys" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--env-file", type=Path, required=True) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--arms", default=",".join(ARMS)) + parser.add_argument("--model", default="claude-sonnet-5-5") + parser.add_argument("--effort", default="medium") + parser.add_argument("--max-run-usd", type=float, default=0.75) + parser.add_argument("--replays", type=int, default=2) + parser.add_argument("--out", type=Path, default=common.RESULTS / "mockmed") + parser.add_argument("--label", default="") + args = parser.parse_args() + common.load_key(args.env_file) + arms = [a for a in args.arms.split(",") if a] + + from openadapt_flow.compile_agent import ClaudeAgent + from openadapt_flow.mockmed.fault_server import serve + + runs_path = args.out / "runs.jsonl" + rows: list[dict[str, Any]] = [] + work_root = Path(tempfile.mkdtemp(prefix="code-action-mockmed-")) + base, _db, stop = serve() + base = base.rstrip("/") + passing: list[Any] = [] + stopped = None + try: + for trial in range(1, args.trials + 1): + shift = (trial - 1) % len(arms) + order = arms[shift:] + arms[:shift] + for arm in order: + stopped = common.may_start(args.max_run_usd) + if stopped: + break + work = work_root / f"t{trial}-{arm}" + work.mkdir(parents=True) + compiler = _compiler(work, base, f"mockmed.code-action-{arm}-t{trial}") + agent = ClaudeAgent( + vision=arm, + model=args.model, + effort=args.effort, + max_cost_usd=args.max_run_usd, + ) + note = f"{NOTE}, trial {trial}" + load_before = common.load_1m() + started = time.time() + try: + result = compiler.record(agent, {"note": note}) + status, detail, run1 = result.status, result.detail, result.run1 + except Exception as exc: # noqa: BLE001 - a failed run is a row + status, detail = "error", f"{type(exc).__name__}: {exc}" + ledger = getattr(agent, "last_ledger", None) + run1 = {"agent": ledger.to_dict() if ledger else {}} + wall = time.time() - started + load_after = common.load_1m() + agent_row = run1.get("agent") or {} + metrics = common.agent_metrics(agent_row, run1.get("harness") or {}) + cost = float(metrics.get("cost_usd") or 0.0) + total = common.add_spend(f"mockmed {arm} t{trial}", cost) + row = { + "app": "mockmed", + "label": args.label or None, + "trial": trial, + "arm": arm, + "model": args.model, + "order": order.index(arm), + "started_at": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime(started) + ), + "status": status, + "correct": status == "draft", + "record_check": run1.get("record_check"), + "detail": detail if status != "draft" else None, + "run_seconds": round(float(run1.get("seconds") or wall), 3), + "host_load_1m": [load_before, load_after], + "under_load": max(load_before, load_after) >= common.LOAD_LIMIT, + **metrics, + "code_action": run1.get("code_action"), + "gates": [ + { + k: v + for k, v in g.items() + if k not in ("screenshot", "url", "at") + } + for g in run1.get("gates") or [] + ], + "agent_summary": agent_row.get("summary"), + "calls": agent_row.get("calls"), + } + rows.append(row) + common.append(runs_path, row) + print( + f"t{trial} {arm}: {status}; {metrics['model_calls']} calls, " + f"{metrics['actions']} actions, {metrics['prompt_tokens']} prompt " + f"tokens, ${cost:.4f}, {row['run_seconds']} s, load " + f"{load_before}->{load_after} (total ${total:.2f})", + flush=True, + ) + if status == "draft" and arm == "code": + passing.append(compiler) + if stopped: + print(f"stopping: {stopped}", flush=True) + break + replays: list[dict[str, Any]] = [] + if passing and args.replays: + compiler = passing[0] + compiler.confirm_checks( + principal_id="measurement.operator", drop_fields=["key", "source"] + ) + admitted = compiler.admit( + admitter=_admitter(compiler.dir.parent.parent), + canary_params=[{"note": "Canary test note 1"}], + ) + print(f"admit code: {admitted.status} {admitted.detail or ''}", flush=True) + if admitted.admitted: + for i, note in enumerate(REPLAY_NOTES[: args.replays], start=1): + replay = compiler.run(note=note) + row = { + "app": "mockmed", + "program_from": "code", + "replay": i, + "status": replay.status, + "verified": replay.verified, + "model_calls": replay.model_calls, + "seconds": replay.seconds, + "receipt_outcome": replay.receipt_outcome, + "host_load_1m": common.load_1m(), + } + replays.append(row) + common.append(args.out / "replays.jsonl", row) + print(f"replay {i}: {row}", flush=True) + finally: + stop() + every = [json.loads(line) for line in runs_path.read_text().splitlines() if line] + summary = { + "arms": common.summarize(every, arms, KEYS), + "environment": { + "model": args.model, + "effort": args.effort, + "app": "MockMed fault server, ?fault=ok&idempotency=demo, 1280x800", + "engine_commit": common.commit(), + "python": sys.version.split()[0], + "platform": platform.platform(), + "measured_at": common.now(), + "load_limit_1m": common.LOAD_LIMIT, + "spend_total_usd": common.spent(), + }, + } + replay_path = args.out / "replays.jsonl" + if replay_path.is_file(): + rep = [ + json.loads(line) for line in replay_path.read_text().splitlines() if line + ] + summary["replays"] = { + "runs": len(rep), + "verified": sum(1 for r in rep if r["verified"]), + "model_calls_total": sum(r["model_calls"] or 0 for r in rep), + } + (args.out / "summary.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n" + ) + print(json.dumps(summary["arms"], indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py new file mode 100644 index 0000000000..0ddea92221 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -0,0 +1,472 @@ +"""Code as action against the screenshot loop on the local OpenEMR referral task. + +The task is the referral benchmark's (``benchmark/openemr_referral_timing``): +register a referred, made-up patient in OpenEMR from 12 structured fields, +without creating a duplicate. The same eight referrals are used, including r6, +whose name matches an existing chart with a different birth date (the agent +must stop, not save). + +Both arms run in the same harness: a fresh browser, sign-in and the blank New +Patient form off the clock (as in the referral study), then every action goes +through the compile-agent harness and is recorded by the JIT capture: + +* ``code``: ``ClaudeAgent(vision="code")``, one ``run`` tool. +* ``every_turn``: ``ClaudeAgent(vision="every_turn")``, Claude's computer + toolset with a screenshot after every action (the screenshot loop). + +The test setup reads the result back on its own, by SQL on the fixture +database: correct means exactly one new chart whose 12 fields match (or, for +r6, no new chart). For the code arm the recording is also compiled and the +compiled program is checked to carry the locator of every element the agent +acted on. + + OPENEMR_FIXTURE_STATE=... OPENEMR_FIXTURE_LOCK=... PYTHONPATH=. \\ + python benchmark/code_as_action/measure_openemr.py --env-file .env \\ + --reset --referrals r1,r2,r6 --arms code,every_turn + +Made-up patients only (example.invalid emails, 555-01xx phones). +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import platform +import shutil +import sys +import tempfile +import time +import types +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +FLOW = HERE.parent.parent +sys.path.insert(0, str(HERE)) +# ``benchmark`` here is a namespace package; an installed regular package of +# the same name would win the import, so bind this one first. +_pkg = types.ModuleType("benchmark") +_pkg.__path__ = [str(FLOW / "benchmark")] # type: ignore[attr-defined] +sys.modules["benchmark"] = _pkg + +import common # noqa: E402 + +INSTRUCTION = ( + "Register this referred patient in the EMR. Do not create a duplicate chart. " + "If the EMR shows a possible existing patient, stop and report instead of saving." +) + +#: The referral study's eight made-up referrals (structured fields only). +REFERRALS: dict[str, dict[str, str]] = { + "r1": dict( + title="Ms.", + fname="Nadia", + lname="Brennan", + dob="1971-04-19", + sex="Female", + street="27 WILLOW BEND ROAD", + phone_home="555-0121", + email="nadia.brennan@example.invalid", + ), + "r2": dict( + title="Mr.", + fname="Marcus", + lname="Feld", + dob="1958-09-02", + sex="Male", + street="114 QUARRY HILL DRIVE", + phone_home="555-0124", + email="marcus.feld@example.invalid", + ), + "r3": dict( + title="Mrs.", + fname="Ines", + lname="Carvalho", + dob="1949-12-11", + sex="Female", + street="9 LANTERN COURT", + phone_home="555-0131", + email="ines.carvalho@example.invalid", + ), + "r4": dict( + title="Mr.", + fname="Theo", + lname="Lindqvist", + dob="1983-07-26", + sex="Male", + street="63 FERRY STREET", + phone_home="555-0136", + email="theo.lindqvist@example.invalid", + ), + "r5": dict( + title="Ms.", + fname="Grace", + lname="Adeyemi", + dob="1990-02-08", + sex="Female", + street="402 CEDAR ROW", + phone_home="555-0145", + email="grace.adeyemi@example.invalid", + ), + "r6": dict( + title="Mrs.", + fname="Rosa", + lname="Delgado", + dob="1958-03-22", + sex="Female", + street="7 MILL ROAD", + phone_home="555-0188", + email="rosa.delgado@example.invalid", + ), + "r7": dict( + title="Mr.", + fname="Arlo", + lname="Whitcombe", + dob="1966-05-30", + sex="Male", + street="88 BEACON TERRACE", + phone_home="555-0152", + email="arlo.whitcombe@example.invalid", + ), + "r8": dict( + title="Ms.", + fname="Priya", + lname="Raman", + dob="1975-10-14", + sex="Female", + street="15 STATION AVENUE", + phone_home="555-0167", + email="priya.raman@example.invalid", + ), +} +MISMATCH = {"r6"} +EXISTING = dict( + title="Mrs.", + fname="Rosa", + lname="Delgado", + dob="1958-03-27", + sex="Female", + street="311 BIRCH COURT", + phone_home="555-0119", + email="r.delgado.prior@example.invalid", +) +COMMON_FIELDS = dict( + city="EXAMPLETON", state="MA", state_label="Massachusetts", postal_code="02139" +) + +SQL_FIELDS = ( + "id", + "pid", + "title", + "fname", + "lname", + "DOB", + "sex", + "street", + "city", + "state", + "postal_code", + "phone_home", + "email", + "country_code", +) +KEYS = [ + "model_calls", + "prompt_tokens", + "output_tokens", + "cost_usd", + "run_seconds", + "model_seconds", + "actions", + "screenshots_sent", +] + + +def spec_for(key: str) -> Any: + from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + + return SyntheticPatientSpec(**REFERRALS[key], **COMMON_FIELDS) + + +def task_prompt(spec: Any) -> str: + fields = json.dumps(spec.params(), indent=2) + return ( + "You are operating OpenEMR, an electronic medical record, in a web browser. It is a " + "local test copy and every patient in it is made up. You are already signed in, and " + "the New Patient form is open.\n\n" + f"{INSTRUCTION}\n\n" + "Referred patient (structured fields):\n" + f"{fields}\n\n" + "When you have finished, or if you stop, reply with a short report: what you did, " + "whether a chart was saved, and anything that did not match." + ) + + +def _q(value: str) -> str: + assert "'" not in value and "\\" not in value + return "'" + value + "'" + + +def sql_patients(fixture: Any, where: str) -> list[dict[str, str]]: + out = fixture._db( + "SELECT " + + ",".join(SQL_FIELDS) + + " FROM openemr.patient_data WHERE " + + where + + " ORDER BY id" + ).decode() + return [ + {k: ("" if v == "NULL" else v) for k, v in zip(SQL_FIELDS, line.split("\t"))} + for line in out.splitlines() + ] + + +def _demo() -> Any: + spec = importlib.util.spec_from_file_location( + "openemr_local_demo", FLOW / "scripts" / "openemr_local_demo.py" + ) + module = importlib.util.module_from_spec(spec) # type: ignore[arg-type] + sys.modules["openemr_local_demo"] = module + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +def run_one( + fixture: Any, demo: Any, arm: str, key: str, args: Any, work: Path +) -> dict[str, Any]: + from openadapt_flow.authoring import AuthoringSession + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.compile_agent import ClaudeAgent + from openadapt_flow.compile_agent.harness import Harness, SaveGate + from openadapt_flow.jit.adapters import CaptureComputer + from openadapt_flow.jit.compile_agent import TaskRequest + from openadapt_flow.jit.keys import load_customer_model_key + + spec = spec_for(key) + agent = ClaudeAgent( + vision=arm, + model=args.model, + effort=args.effort, + max_cost_usd=args.max_run_usd, + max_turns=args.max_turns, + ) + before_all = sql_patients(fixture, "1=1") + backend, close = PlaywrightBackend.launch(fixture.ui_base_url, headless=True) + error: Optional[str] = None + ledger: Any = None + harness: Any = None + program: Optional[dict[str, Any]] = None + started = time.time() + load_before = load_after = common.load_1m() + wall = 0.0 + try: + demo._preauthenticate_browser(backend, fixture) + backend.page.wait_for_function( + "() => !document.querySelector('.modal-backdrop')", timeout=30_000 + ) + session = AuthoringSession( + backend, work / "recording", backend_kind="web", app_url=fixture.ui_base_url + ) + computer = CaptureComputer(session, dialect=agent.dialect, page=backend.page) + session.start_record() + values = [str(v) for v in spec.params().values()] + # A new patient's identity is typed, not printed: the gate checks that + # the commit happens in the new-patient confirm dialog. + harness = Harness( + computer, + SaveGate(["Create New Patient"]), + keep=values, + evidence_dir=work / "gates", + ) + request = TaskRequest( + task_id=f"openemr.referral.{key}", + instruction=task_prompt(spec), + inputs=spec.params(), + model_key=load_customer_model_key("anthropic"), + ) + load_before = common.load_1m() + started = time.time() + try: + ledger = agent.drive(request, harness) + except Exception as exc: # noqa: BLE001 - a failed run is a row + error = f"{type(exc).__name__}: {exc}"[:500] + ledger = getattr(agent, "last_ledger", None) + wall = time.time() - started + load_after = common.load_1m() + trace = getattr(agent, "last_trace", None) + if trace is not None and computer.actions: + try: + from openadapt_flow.compile_agent.code_action import ( + check_trace_against_ir, + ) + from openadapt_flow.ir import Workflow + + session.compile( + work / "bundle", name=f"openemr-referral-{key}", distill=True + ) + program = check_trace_against_ir( + trace.to_dict(), Workflow.load(work / "bundle") + ) + except Exception as exc: # noqa: BLE001 - reported with the row + program = { + "passed": False, + "error": f"{type(exc).__name__}: {exc}"[:300], + } + finally: + close() + time.sleep(1.0) + after_all = sql_patients(fixture, "1=1") + before_ids = {r["id"] for r in before_all} + new_rows = [r for r in after_all if r["id"] not in before_ids] + fields = spec.fields + match = None + if len(new_rows) == 1: + match = {k: new_rows[0].get(k) == v for k, v in fields.items()} + if key in MISMATCH: + correct = not new_rows + else: + correct = match is not None and all(match.values()) + agent_row = ledger.to_dict() if ledger is not None else {} + metrics = common.agent_metrics( + agent_row, harness.stats.to_dict() if harness else {} + ) + trace = getattr(agent, "last_trace", None) + return { + "app": "openemr-referral", + "arm": arm, + "referral": key, + "mismatch_case": key in MISMATCH, + "model": args.model, + "started_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(started)), + "correct": correct, + "new_charts": len(new_rows), + "fields_matched": (sum(match.values()) if match else 0), + "fields_total": len(fields), + "field_mismatches": sorted(k for k, ok in (match or {}).items() if not ok), + "error": error, + "run_seconds": round(wall, 3), + "host_load_1m": [load_before, load_after], + "under_load": max(load_before, load_after) >= common.LOAD_LIMIT, + **metrics, + "gates": [ + { + k: v + for k, v in g.items() + if k not in ("screenshot", "url", "at", "identity") + } + for g in (harness.gate_log if harness else []) + ], + "code_action": trace.summary() if trace is not None else None, + "program": program, + "agent_summary": agent_row.get("summary"), + "calls": agent_row.get("calls"), + "scripts": trace.log if trace is not None else None, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--env-file", type=Path, required=True) + parser.add_argument("--arms", default="code,every_turn") + parser.add_argument("--referrals", default="r1,r2,r3,r4,r5,r6,r7,r8") + parser.add_argument("--rounds", type=int, default=1) + parser.add_argument( + "--reset", + action="store_true", + help="reset the fixture and seed the existing chart first", + ) + parser.add_argument("--model", default="claude-sonnet-5-5") + parser.add_argument("--effort", default="medium") + parser.add_argument("--max-run-usd", type=float, default=0.75) + parser.add_argument("--max-turns", type=int, default=60) + parser.add_argument("--out", type=Path, default=common.RESULTS / "openemr") + args = parser.parse_args() + common.load_key(args.env_file) + arms = [a for a in args.arms.split(",") if a] + keys = [k for k in args.referrals.split(",") if k] + + from benchmark.openemr_local import qualify_hooks + from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + + fixture = qualify_hooks.fixture() + demo = _demo() + meta: dict[str, Any] = {"started_at": common.now()} + if args.reset: + meta["baseline_sha256"] = fixture.reset() + existing = SyntheticPatientSpec(**EXISTING, **COMMON_FIELDS) + meta["seeded_existing_chart_status"] = ( + fixture.token_session("actor") + .post( + fixture.api_base_url + "/apis/default/api/patient", + json=dict(existing.fields), + ) + .status_code + ) + work_root = Path(tempfile.mkdtemp(prefix="code-action-openemr-")) + rows_path = args.out / "runs.jsonl" + try: + for round_ in range(1, args.rounds + 1): + for i, key in enumerate(keys): + # Alternate which arm goes first, so drift hits both. + order = arms if (i + round_) % 2 == 0 else list(reversed(arms)) + for arm in order: + if key not in MISMATCH and sql_patients( + fixture, f"email={_q(REFERRALS[key]['email'])}" + ): + # The other arm already registered this referral; a + # second save would be a duplicate. Remove nothing: + # skip it and say so. + print(f"{key} {arm}: skipped (chart exists)", flush=True) + continue + stopped = common.may_start(args.max_run_usd) + if stopped: + print(f"stopping: {stopped}", flush=True) + raise StopIteration + work = work_root / f"{round_}-{key}-{arm}" + work.mkdir(parents=True) + row = run_one(fixture, demo, arm, key, args, work) + row["round"] = round_ + total = common.add_spend( + f"openemr {arm} {key}", float(row["cost_usd"] or 0) + ) + common.append(rows_path, row) + print( + f"{key} {arm}: correct={row['correct']} new={row['new_charts']} " + f"fields={row['fields_matched']}/{row['fields_total']} " + f"{row['model_calls']} calls {row['actions']} actions " + f"{row['prompt_tokens']} prompt tokens ${float(row['cost_usd'] or 0):.4f} " + f"{row['run_seconds']} s load {row['host_load_1m']} err={row['error']} " + f"(total ${total:.2f})", + flush=True, + ) + shutil.rmtree(work, ignore_errors=True) + except StopIteration: + pass + finally: + shutil.rmtree(work_root, ignore_errors=True) + every = [json.loads(line) for line in rows_path.read_text().splitlines() if line] + summary = { + "arms": common.summarize(every, arms, KEYS), + "environment": { + "model": args.model, + "effort": args.effort, + "app": "OpenEMR local fixture (pinned image), New Patient form, 1280x800, headless", + "engine_commit": common.commit(), + "python": sys.version.split()[0], + "platform": platform.platform(), + "measured_at": common.now(), + "load_limit_1m": common.LOAD_LIMIT, + "spend_total_usd": common.spent(), + **meta, + }, + } + (args.out / "summary.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n" + ) + print(json.dumps(summary["arms"], indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/code_as_action/results/spend.json b/packages/flow/benchmark/code_as_action/results/spend.json new file mode 100644 index 0000000000..76bf84e950 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/spend.json @@ -0,0 +1,26 @@ +{ + "runs": [ + { + "label": "mockmed code t1", + "usd": 0.010642, + "at": "2026-10-18T19:24:34Z" + }, + { + "label": "mockmed code t1", + "usd": 0.010804, + "at": "2026-10-18T19:25:21Z" + }, + { + "label": "mockmed code t1", + "usd": 0.012239, + "at": "2026-10-18T19:25:56Z" + }, + { + "label": "openemr code r1", + "usd": 0.047661, + "at": "2026-10-18T19:41:04Z" + } + ], + "total_usd": 0.081346, + "cap_usd": 15.0 +} diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index 67a2926797..75e2dd6d8a 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -533,6 +533,13 @@ def __init__( def backend_kind(self) -> str: return self._kind + @property + def backend(self) -> Any: + """The live backend the recorder drives. Callers read it; actions go + through this session so they are recorded.""" + + return self._backend + @property def screen_only(self) -> bool: """True for a remote session: pixels and keys, no DOM or UIA.""" diff --git a/packages/flow/openadapt_flow/compile_agent/__init__.py b/packages/flow/openadapt_flow/compile_agent/__init__.py index ebcf6f593d..0660a878ff 100644 --- a/packages/flow/openadapt_flow/compile_agent/__init__.py +++ b/packages/flow/openadapt_flow/compile_agent/__init__.py @@ -6,6 +6,9 @@ record shows exactly the record the task named, the run is compiled, a person confirms the proposed record checks, and the program is admitted under the JIT lifecycle. Every later run makes no model calls. See docs/COMPILE_AGENT.md. + +``ClaudeAgent(vision="code")`` acts through code instead: one terminal-style +tool, structured steps, a sandboxed shell. See docs/CODE_AS_ACTION.md. """ from openadapt_flow.compile_agent.agents import ( @@ -14,6 +17,13 @@ ClaudeAgent, ScriptedAgent, ) +from openadapt_flow.compile_agent.code_action import ( + CodeActionTool, + LeftTheApp, + ScriptedCodeAgent, + ScriptError, + check_trace_against_ir, +) from openadapt_flow.compile_agent.harness import ( GateRefused, GateRequest, @@ -34,16 +44,21 @@ "AgentLedger", "BudgetExceeded", "ClaudeAgent", + "CodeActionTool", "CompileResult", "FirstRunCompiler", "GateRefused", "GateRequest", "Harness", + "LeftTheApp", "RecordCheck", "ReplayResult", "SaveGate", + "ScriptError", "ScriptedAgent", + "ScriptedCodeAgent", "check_run_record", + "check_trace_against_ir", "compile_first_run", "run_compiled", ] diff --git a/packages/flow/openadapt_flow/compile_agent/agents.py b/packages/flow/openadapt_flow/compile_agent/agents.py index f160fd1025..4c15f9a268 100644 --- a/packages/flow/openadapt_flow/compile_agent/agents.py +++ b/packages/flow/openadapt_flow/compile_agent/agents.py @@ -11,6 +11,11 @@ ``every_turn`` The baseline: Claude's computer toolset (``computer_toolset_20260801``), with a screenshot returned after every action. +``code`` + Code as action (:mod:`.code_action`): one ``run`` tool that takes a short + script. The model queries the structure, acts by role and name, reads + back, keeps notes in a sandboxed working directory, and asks for a + screenshot only when it wants one. Several actions fit in one call. Both modes perform every action through the same harness and capture computer, so both are recorded and compiled the same way. Each model call's token usage @@ -63,7 +68,7 @@ }, } -VISION_MODES = ("selective", "every_turn") +VISION_MODES = ("selective", "every_turn", "code") class BudgetExceeded(RuntimeError): @@ -220,6 +225,21 @@ def drive(self, request: TaskRequest, harness: Harness) -> AgentLedger: given, and nothing else. When the task is done, or cannot be done, call \ finish with a one-line summary.""" +_CODE_SYSTEM = """\ +You operate an application for the user through a terminal-style tool, run. \ +Each call takes a short script, one command per line, and returns what each \ +line printed. Read the app's structure with tree, find, text and read; act on \ +elements by role and name with click, fill, select and press; check results \ +by reading again. Put several commands in one script when you know what comes \ +next: refs change after every action, so later lines in a script name their \ +targets by role and name (click role=button name="Save"). Filter listings \ +with grep= to keep them short. Call screenshot only \ +when the text is not enough. Before any save, the harness checks that the \ +right record is on screen; a save on the wrong record is refused and ends the \ +run. Do exactly the task, with the values given, and nothing else. After the \ +save, read the screen to confirm it. When the task is done, or cannot be \ +done, call finish with a one-line summary.""" + _FINISH_TOOL = { "name": "finish", "description": "End the task. Call it once, when the task is done or cannot be done.", @@ -297,6 +317,26 @@ def drive(self, request: TaskRequest, harness: Harness) -> AgentLedger: ] +def _run_tool_spec() -> dict[str, Any]: + from openadapt_flow.compile_agent.code_action import COMMANDS + + return { + "name": "run", + "description": "Run a short script against the app.\n" + COMMANDS, + "input_schema": { + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "Commands, one per line.", + } + }, + "required": ["script"], + "additionalProperties": False, + }, + } + + def _block_dict(block: Any) -> dict[str, Any]: if isinstance(block, dict): return block @@ -386,7 +426,7 @@ def __post_init__(self) -> None: @property def dialect(self) -> str: - return "generic" if self.vision == "selective" else "anthropic" + return "anthropic" if self.vision == "every_turn" else "generic" def _client(self, request: TaskRequest) -> Any: if self.client is not None: @@ -405,7 +445,26 @@ def drive(self, request: TaskRequest, harness: Harness) -> AgentLedger: #: Kept so a run that raises still reports what it spent. self.last_ledger = ledger selective = self.vision == "selective" - if selective: + self._tool: Any = None + if self.vision == "code": + from openadapt_flow.compile_agent.code_action import ( + CodeActionTool, + _watch, + ) + + self._tool = CodeActionTool(harness, watch=_watch(request, harness)) + self.last_trace = self._tool.trace + system = _CODE_SYSTEM + tools = [_run_tool_spec(), _FINISH_TOOL] + first = [ + {"type": "text", "text": f"Task: {request.instruction}"}, + { + "type": "text", + "text": "The harness ran `tree` for you. Its output:\n\n" + + self._tool.run("tree").text, + }, + ] + elif selective: system = _SELECTIVE_SYSTEM tools: list[dict[str, Any]] = list(_SELECTIVE_TOOLS) first: list[dict[str, Any]] = [ @@ -511,7 +570,11 @@ def _run_tool( if toolset: result["toolset_name"] = toolset try: - if selective: + if self._tool is not None: + if name != "run": + raise LookupError(f"unknown tool {name!r}; use run or finish") + result["content"] = self._tool.run(str(args.get("script", ""))).blocks() + elif selective: result["content"] = self._selective(name, args, harness) else: result["content"] = self._computer(name, args, harness) diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py new file mode 100644 index 0000000000..e33c384615 --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -0,0 +1,1501 @@ +"""Code as action: the run-1 agent works through one terminal-style tool. + +The agent gets a single tool, ``run``, that takes a short script: one command +per line. It reads the app's structure (a web page's DOM, a desktop app's +accessibility tree, or OCR text on a pixel-only screen), acts on elements by +role and name, asks for a screenshot only when it wants one, and keeps notes +in a working directory with a read-only shell. One model call can read, act +and check several times, so a task takes fewer calls and far fewer tokens than +a screenshot after every click. + +This follows the 2026 harness results: CUAWright (a terminal as the only +action interface, state kept on disk, vision on demand), OpenGCUA (GUI, +accessibility and shell actions composed in one program) and ANIS (commands +with observable effects, read back after a write). Unlike those harnesses the +script is not general code: every line is either a read or one recordable +action, so each action goes through the save gate and is recorded as a step. + +Commands (see :data:`COMMANDS`):: + + tree [grep=TEXT] [max=N] the structure, one line per row or control + find [role=R] [name=TEXT] [near=TEXT] controls that match, with @refs + text [grep=TEXT] the visible text + read TARGET an element's value and row + click TARGET + fill TARGET value=TEXT types into an empty field + select TARGET option=TEXT picks an option + press KEY Enter, Tab, Escape, ... + scroll down|up [px=N] at the pointer, recorded like any action + screenshot an image of the screen, only when asked + sh COMMAND read-only shell in the working directory + save FILE the previous command's output, to FILE + wait ms=N + +A TARGET is a ref from the latest listing (``@3``) or a description +(``role=button name="Open" near="Jane Sample"``; on a pixel-only screen, +``text="Save"``). A description must match exactly one element. + +Every action becomes a structured step in :class:`ActionTrace`: the element's +identity (role, name, the row it is in), the action and the value, the +recorder's event index, and the structural locator the recorder captured for +it. :func:`check_trace_against_ir` then checks that the compiled program +carries that locator on its step. The trace's reads are evidence for the +record check: which of the task's identity strings and values the agent read, +and whether it read its inputs back after the save. + +Safety is the harness's: the right-record check and the hold run before every +action that may commit a write (:class:`~.harness.Harness`), the shell can't +reach a network or write outside its directory (:mod:`.sandbox`), and a web +action that leaves the app's origin ends the run. +""" + +from __future__ import annotations + +import re +import shlex +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, ClassVar, Iterable, Mapping, Optional, Sequence +from urllib.parse import urlsplit + +from openadapt_flow.compile_agent import observe as _observe +from openadapt_flow.compile_agent.harness import ( + CommitHint, + Harness, +) +from openadapt_flow.compile_agent.sandbox import Sandbox, ShellRefused +from openadapt_flow.jit.adapters import UnsupportedAction + +__all__ = [ + "COMMANDS", + "ActionTrace", + "CodeActionTool", + "Element", + "LeftTheApp", + "ScriptError", + "ScriptedCodeAgent", + "ToolOutput", + "check_trace_against_ir", +] + +COMMANDS = """\ +One command per line; lines run in order and stop at the first error. + tree [grep=TEXT] [max=N] the structure: headings, text, rows, controls with @refs + find [role=R] [name=TEXT] [near=TEXT] controls that match (name is exact, near is in the same row) + text [grep=TEXT] visible text lines + read TARGET an element's value and the row it is in + click TARGET + fill TARGET value=TEXT click an empty field and type TEXT + select TARGET option=TEXT choose an option in a list or drop-down + press KEY Enter, Tab, Escape, ArrowDown, ... + scroll down|up [px=N] scroll where the pointer is (click inside the area first) + screenshot an image of the screen (costly; only when the text is not enough) + sh COMMAND read-only shell in your working directory: grep, head, wc, sort, ... joined with | + save FILE write the previous command's output to FILE in the working directory + wait ms=N pause up to 2000 ms +TARGET is @N from the latest tree or find, or role=R name="..." [near="..."]; +on a screen with no structure, text="..." (from OCR). Quote values with spaces. +Refs change after every action. After a write is committed the script stops, +so you can check the result.""" + +#: Roles the agent may name, mapped to the roles a listing uses. +_ROLE_ALIASES = { + "field": "textbox", + "input": "textbox", + "textarea": "textbox", + "text field": "textbox", + "dropdown": "combobox", + "select": "combobox", + "listbox": "combobox", + "a": "link", +} + +_MAX_WAIT_MS = 2000 +_MAX_LINES = 40 + + +class ScriptError(ValueError): + """A line could not run. Nothing in it was performed; the agent sees why.""" + + +class LeftTheApp(RuntimeError): + """An action took the browser to another origin. The run ends: the agent + may act on the target app only.""" + + +def _norm(text: Any) -> str: + return " ".join(str(text or "").split()).lower() + + +@dataclass +class Element: + """One element the agent can name: from the DOM, a tree, or OCR.""" + + ref: str + role: str + name: str + x: int + y: int + w: int = 0 + h: int = 0 + value: Optional[str] = None + #: The text of the table row the element is in. + context: str = "" + #: The text of the row, dialog, form or section around it (the gate's scope). + scope: str = "" + tag: str = "" + options: tuple[str, ...] = () + focused: bool = False + disabled: bool = False + checked: bool = False + frame: int = 0 + source: str = "dom" + automation_id: Optional[str] = None + #: A desktop tree node the authoring session remembers. + node_id: Optional[str] = None + + def identity(self) -> dict[str, Any]: + out: dict[str, Any] = {"role": self.role, "name": self.name} + if self.context: + out["row"] = self.context + if self.automation_id: + out["automation_id"] = self.automation_id + out["source"] = self.source + if self.frame: + out["frame"] = self.frame + return out + + def line(self) -> str: + part = f'@{self.ref} {self.role} "{self.name}"' + if self.value is not None: + part += f' value="{self.value}"' + if self.options: + shown = ", ".join(f'"{o}"' for o in self.options[:12]) + more = f", +{len(self.options) - 12} more" if len(self.options) > 12 else "" + part += f" options=[{shown}{more}]" + flags = [ + name + for name, on in ( + ("focused", self.focused), + ("checked", self.checked), + ("disabled", self.disabled), + ) + if on + ] + if flags: + part += " (" + ", ".join(flags) + ")" + return part + + +# -- surfaces ------------------------------------------------------------------------ + +_DOM_JS = ( + "([sel, scopeSel]) => {\nconst fieldLabel = " + + "%(field_label)s" + + r"""; + const clean = (s) => String(s || '').replace(/\s+/g, ' ').trim(); + function nameOf(el) { + let n = clean(el.getAttribute('aria-label')); + if (!n) n = clean(fieldLabel(el)); + if (!n && el.tagName === 'INPUT' && /^(button|submit|reset)$/i.test(el.type)) + n = clean(el.value); + if (!n) n = clean(el.textContent).slice(0, 120); + if (!n) n = clean(el.getAttribute('title')); + return n.slice(0, 120); + } + function roleOf(el) { + const r = el.getAttribute('role'); + if (r) return r; + const tag = el.tagName.toLowerCase(); + if (tag === 'a') return 'link'; + if (tag === 'button') return 'button'; + if (tag === 'select') return 'combobox'; + if (tag === 'textarea') return 'textbox'; + if (tag === 'input') { + const t = (el.type || 'text').toLowerCase(); + if (t === 'checkbox' || t === 'radio') return t; + if (/^(button|submit|reset|image)$/.test(t)) return 'button'; + return 'textbox'; + } + if (el.isContentEditable) return 'textbox'; + return tag; + } + function visible(el) { + const r = el.getBoundingClientRect(); + if (r.width <= 0 || r.height <= 0) return false; + if (r.bottom < 0 || r.right < 0 || r.top > innerHeight || r.left > innerWidth) + return false; + const s = getComputedStyle(el); + return s.visibility !== 'hidden' && s.display !== 'none' && s.opacity !== '0'; + } + function ownText(el) { + let t = ''; + for (const c of el.childNodes) if (c.nodeType === 3) t += c.textContent; + return clean(t); + } + // A row with many controls is page layout, not a record row. + const rows = Array.from(document.querySelectorAll('tr, [role=row]')) + .filter(r => r.querySelectorAll(sel).length <= 8); + const rowOf = (el) => { + let r = el.closest('tr, [role=row]'); + while (r && rows.indexOf(r) < 0) r = r.parentElement && r.parentElement.closest('tr, [role=row]'); + return r; + }; + const out = []; + const scopeText = new Map(); + for (const el of document.body ? document.body.querySelectorAll('*') : []) { + const tag = el.tagName.toLowerCase(); + if (tag === 'script' || tag === 'style' || tag === 'noscript') continue; + const tr = rowOf(el); + const row = tr ? rows.indexOf(tr) : -1; + if (el.matches(sel)) { + if (!visible(el)) { + const b = el.getBoundingClientRect(), st = getComputedStyle(el); + if (b.width > 0 && b.height > 0 && st.visibility !== 'hidden' && + st.display !== 'none' && (b.top > innerHeight || b.bottom < 0)) + out.push({kind: 'offscreen', where: b.top > innerHeight ? 'below' : 'above', + name: nameOf(el), row: -1}); + continue; + } + const r = el.getBoundingClientRect(); + let value = null, options = []; + if (tag === 'input' || tag === 'textarea') + value = el.type === 'password' ? '' : String(el.value || ''); + if (tag === 'select') { + options = Array.from(el.options).map(o => clean(o.textContent)).slice(0, 80); + const chosen = el.selectedOptions && el.selectedOptions[0]; + value = chosen ? clean(chosen.textContent) : ''; + } + const scope = el.closest(scopeSel) || document.body; + if (!scopeText.has(scope)) { + // The gate's scope: the text around the control, and the values in + // its fields (a new record's identity is typed, not printed). + const values = Array.from(scope.querySelectorAll('input, textarea, select')) + .filter(f => f.type !== 'password' && f.type !== 'hidden') + .map(f => f.tagName === 'SELECT' + ? clean(f.selectedOptions && f.selectedOptions[0] ? + f.selectedOptions[0].textContent : '') + : clean(f.value)) + .filter(Boolean); + scopeText.set(scope, (clean(scope.innerText) + ' ' + values.join(' ')).slice(0, 6000)); + } + out.push({kind: 'control', tag: tag, role: roleOf(el), name: nameOf(el), + value: value, options: options, + focused: document.activeElement === el, + checked: !!el.checked || el.getAttribute('aria-checked') === 'true' || + el.getAttribute('aria-selected') === 'true', + disabled: !!el.disabled || el.getAttribute('aria-disabled') === 'true', + x: r.left, y: r.top, w: r.width, h: r.height, row: row, + context: tr ? clean(tr.innerText).slice(0, 300) : '', + scope: scopeText.get(scope)}); + continue; + } + if (el.closest(sel)) continue; + const text = ownText(el); + if (!text || !visible(el)) continue; + out.push({kind: /^h[1-6]$/.test(tag) ? 'heading' : 'text', text: text.slice(0, 200), + row: row}); + } + return {url: location.href, title: document.title, items: out}; +}""" +) + +_HIT_JS = ( + "([px, py, sel]) => {\nconst fieldLabel = " + + "%(field_label)s" + + r"""; + const clean = (s) => String(s || '').replace(/\s+/g, ' ').trim(); + const hit = document.elementFromPoint(px, py); + if (!hit) return null; + const el = hit.closest(sel); + if (!el) return {name: '', context: ''}; + let n = clean(el.getAttribute('aria-label')); + if (!n) n = clean(fieldLabel(el)); + if (!n && el.tagName === 'INPUT' && /^(button|submit|reset)$/i.test(el.type)) + n = clean(el.value); + if (!n) n = clean(el.textContent).slice(0, 120); + if (!n) n = clean(el.getAttribute('title')); + let tr = el.closest('tr, [role=row]'); + while (tr && tr.querySelectorAll(sel).length > 8) + tr = tr.parentElement && tr.parentElement.closest('tr, [role=row]'); + let value = null; + if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') + value = el.type === 'password' ? '' : String(el.value || ''); + if (el.tagName === 'SELECT') { + const chosen = el.selectedOptions && el.selectedOptions[0]; + value = chosen ? clean(chosen.textContent) : ''; + } + return {name: n.slice(0, 120), context: tr ? clean(tr.innerText).slice(0, 300) : '', + value: value}; +}""" +) + +_FOCUS_JS = r"""([sel, scopeSel]) => { + const clean = (s) => String(s || '').replace(/\s+/g, ' ').trim(); + let el = document.activeElement; + if (!el || el === document.body) return null; + if (el.tagName === 'IFRAME' || el.tagName === 'FRAME') return {frame: true}; + const scope = el.closest(scopeSel) || document.body; + return {name: clean(el.getAttribute('aria-label') || el.getAttribute('name') || + el.innerText || el.value).slice(0, 120), + scope: clean(scope.innerText).slice(0, 4000)}; +}""" + + +def _field_label_js() -> str: + from openadapt_flow.backends.playwright_backend import _FIELD_LABEL_JS + + return _FIELD_LABEL_JS + + +class DomSurface: + """A web page, every frame included. Click points are top-level pixels.""" + + kind = "dom" + + def __init__(self, page: Any, backend: Any = None) -> None: + self.page = page + self.backend = backend + label = _field_label_js() + self._dom_js = _DOM_JS.replace("%(field_label)s", label) + self._hit_js = _HIT_JS.replace("%(field_label)s", label) + + def _frames(self) -> list[tuple[int, Any, float, float]]: + """``(index, frame, x offset, y offset)`` for the page and each visible + frame in it.""" + + out: list[tuple[int, Any, float, float]] = [(0, self.page.main_frame, 0.0, 0.0)] + for index, frame in enumerate(self.page.frames): + if frame is self.page.main_frame: + continue + try: + box = frame.frame_element().bounding_box() + except Exception: # noqa: BLE001 - a detached frame is not listed + continue + if not box or box["width"] <= 0 or box["height"] <= 0: + continue + out.append((index, frame, float(box["x"]), float(box["y"]))) + return out + + def read(self) -> tuple[str, list[dict[str, Any]], list[Element]]: + """``(header, items, elements)`` across every frame.""" + + items: list[dict[str, Any]] = [] + elements: list[Element] = [] + header = "" + for index, frame, ox, oy in self._frames(): + try: + raw = frame.evaluate( + self._dom_js, [_observe.INTERACTIVE, _observe.SCOPE] + ) + except Exception: # noqa: BLE001 - a frame that navigates away is skipped + continue + if index == 0: + header = f"page: {raw.get('title') or ''} ({raw.get('url') or ''})" + elif raw.get("items"): + items.append( + { + "kind": "heading", + "text": f"frame {index}: {raw.get('title') or ''}", + "row": -1, + "frame": index, + } + ) + hidden: dict[str, list[str]] = {} + for item in raw.get("items") or []: + item = dict(item, frame=index) + if item["kind"] == "offscreen": + hidden.setdefault(str(item["where"]), []).append(str(item["name"])) + continue + if item["kind"] == "control": + w, h = float(item["w"]), float(item["h"]) + element = Element( + ref=str(len(elements) + 1), + role=str(item.get("role") or item.get("tag") or ""), + name=str(item.get("name") or ""), + x=int(ox + float(item["x"]) + w / 2), + y=int(oy + float(item["y"]) + h / 2), + w=int(w), + h=int(h), + value=item.get("value"), + context=str(item.get("context") or ""), + scope=str(item.get("scope") or ""), + tag=str(item.get("tag") or ""), + options=tuple(str(o) for o in item.get("options") or ()), + focused=bool(item.get("focused")), + disabled=bool(item.get("disabled")), + checked=bool(item.get("checked")), + frame=index, + source="dom", + ) + elements.append(element) + item["ref"] = element.ref + items.append(item) + for where, names in sorted(hidden.items()): + shown = ", ".join(f'"{n}"' for n in names[:8] if n) + more = f" and {len(names) - 8} more" if len(names) > 8 else "" + items.append( + { + "kind": "text", + "row": -1, + "frame": index, + "text": f"({len(names)} controls {where} the visible area" + + (f": {shown}{more}" if shown else "") + + f"; scroll {'down' if where == 'below' else 'up'} to reach them)", + } + ) + return header, items, elements + + def origin(self) -> Optional[str]: + parts = urlsplit(str(getattr(self.page, "url", "") or "")) + return f"{parts.scheme}://{parts.netloc}" if parts.scheme else None + + def at(self, element: Element) -> Optional[dict[str, Any]]: + """What a click at the element's point would hit, read in its frame.""" + + point = None + getter = getattr(self.backend, "_frame_point", None) + if callable(getter): + try: + point = getter(element.x, element.y) + except Exception: # noqa: BLE001 + point = None + try: + if point is not None: + return point.scope.evaluate( + self._hit_js, [point.x, point.y, _observe.INTERACTIVE] + ) + if element.frame: + return None + return self.page.evaluate( + self._hit_js, [element.x, element.y, _observe.INTERACTIVE] + ) + except Exception: # noqa: BLE001 + return None + + def focused(self) -> Optional[dict[str, Any]]: + try: + got = self.page.evaluate(_FOCUS_JS, [_observe.INTERACTIVE, _observe.SCOPE]) + if got and got.get("frame"): + for _index, frame, _ox, _oy in self._frames()[1:]: + inner = frame.evaluate( + _FOCUS_JS, [_observe.INTERACTIVE, _observe.SCOPE] + ) + if inner and not inner.get("frame"): + return inner + return None + return got + except Exception: # noqa: BLE001 + return None + + def screen_text(self) -> str: + texts: list[str] = [] + for _index, frame, _ox, _oy in self._frames(): + try: + texts.append(str(frame.evaluate("() => document.body.innerText") or "")) + except Exception: # noqa: BLE001 + continue + return "\n".join(texts) + + def locator_unique(self, locator: Any) -> Optional[bool]: + resolve = getattr(self.backend, "_locator_with_scope", None) + if not callable(resolve): + return None + try: + got = resolve(locator) + return None if got is None else got[1].count() == 1 + except Exception: # noqa: BLE001 + return None + + +class TreeSurface: + """A desktop app with an accessibility layer (UIA, AX, AT-SPI). + + It lists what :meth:`AuthoringSession.observe` projects (the same tree the + MCP recording tools read) and finds an exact role and name with the + backend's ``locate_structural``. Text comes from OCR. + """ + + kind = "tree" + + def __init__(self, session: Any, backend: Any) -> None: + self.session = session + self.backend = backend + + def read(self) -> tuple[str, list[dict[str, Any]], list[Element]]: + try: + payload = self.session.observe() + except Exception: # noqa: BLE001 - no tree is an ordinary miss + payload = {"tree": []} + elements: list[Element] = [] + items: list[dict[str, Any]] = [] + nodes = getattr(self.session, "_nodes", {}) + for node in payload.get("tree") or []: + pixels = nodes.get(node.get("node_id")) + if pixels is None: + continue + element = Element( + ref=str(len(elements) + 1), + role=str(node.get("role") or "unknown"), + name=str(node.get("name") or ""), + x=int(pixels.x + pixels.w / 2), + y=int(pixels.y + pixels.h / 2), + w=int(pixels.w), + h=int(pixels.h), + focused=bool(node.get("focused")), + disabled=not bool(node.get("enabled", True)), + source="tree", + automation_id=node.get("automation_id"), + node_id=node.get("node_id"), + ) + elements.append(element) + items.append({"kind": "control", "ref": element.ref, "row": -1}) + header = f"window: {self.session.backend_kind} app" + if not elements: + header += " (no accessibility listing; text is from OCR)" + return header, items, elements + + def locate(self, role: str, name: str) -> Optional[Element]: + from openadapt_flow.ir import StructuralLocator + + locate = getattr(self.backend, "locate_structural", None) + if not callable(locate): + return None + try: + handle = locate(StructuralLocator(role=role, name=name)) + except Exception as exc: # noqa: BLE001 - ambiguity is the agent's to fix + raise ScriptError( + f"{role} {name!r} is not unique on screen: {exc}" + ) from exc + if handle is None: + return None + x, y = int(handle.point[0]), int(handle.point[1]) + return Element(ref="", role=role, name=name, x=x, y=y, source="tree") + + def origin(self) -> Optional[str]: + return None + + def at(self, element: Element) -> Optional[dict[str, Any]]: + getter = getattr(self.backend, "structural_locator_at", None) + if not callable(getter): + return None + try: + loc = getter(element.x, element.y) + except Exception: # noqa: BLE001 + return None + return None if loc is None else {"name": loc.name or "", "context": ""} + + def focused(self) -> Optional[dict[str, Any]]: + return None + + def screen_text(self) -> str: + return _ocr_text(self.session.screenshot()) + + def locator_unique(self, locator: Any) -> Optional[bool]: + locate = getattr(self.backend, "locate_structural", None) + if not callable(locate): + return None + try: + return locate(locator) is not None + except Exception: # noqa: BLE001 - refused as ambiguous + return False + + +class PixelSurface: + """An RDP or Citrix window: pixels only. Elements are OCR text lines.""" + + kind = "pixels" + + def __init__( + self, session: Any, ocr: Optional[Callable[[bytes], list[Any]]] = None + ): + self.session = session + self._ocr = ocr + + def _lines(self) -> list[Any]: + png = self.session.screenshot() + if self._ocr is not None: + return list(self._ocr(png)) + from openadapt_flow.vision.ocr import ocr + + return list(ocr(png)) + + def read(self) -> tuple[str, list[dict[str, Any]], list[Element]]: + elements: list[Element] = [] + items: list[dict[str, Any]] = [] + lines = self._lines() + everything = "\n".join(str(line.text) for line in lines) + for line in lines: + x, y, w, h = (int(v) for v in line.region) + element = Element( + ref=str(len(elements) + 1), + role="text", + name=str(line.text), + x=x + w // 2, + y=y + h // 2, + w=w, + h=h, + scope=everything, + source="ocr", + ) + elements.append(element) + items.append({"kind": "control", "ref": element.ref, "row": -1}) + return "screen: pixels only (text is from OCR)", items, elements + + def origin(self) -> Optional[str]: + return None + + def at(self, element: Element) -> Optional[dict[str, Any]]: + return None + + def focused(self) -> Optional[dict[str, Any]]: + return None + + def screen_text(self) -> str: + return "\n".join(str(line.text) for line in self._lines()) + + def locator_unique(self, locator: Any) -> Optional[bool]: + return None + + +def _ocr_text(png: bytes) -> str: + try: + from openadapt_flow.vision.ocr import ocr + + return "\n".join(str(line.text) for line in ocr(png)) + except Exception: # noqa: BLE001 - no OCR engine installed + return "" + + +def surface_for(harness: Harness) -> Any: + """The surface the agent reads: the DOM, a desktop tree, or pixels.""" + + session = getattr(harness.computer, "session", None) + backend = getattr(session, "backend", None) + if harness.has_page: + return DomSurface(harness.page, backend) + if getattr(session, "screen_only", False): + return PixelSurface(session) + return TreeSurface(session, backend) + + +# -- the trace -------------------------------------------------------------------------- + + +@dataclass +class ActionTrace: + """What the agent did, as structured steps, and what it read.""" + + surface: str + steps: list[dict[str, Any]] = field(default_factory=list) + reads: list[dict[str, Any]] = field(default_factory=list) + #: The task's identity strings and values: which ones each read showed. + watch: dict[str, str] = field(default_factory=dict) + scripts: int = 0 + #: Every script and what it printed (scrubbed, cut to 3,000 characters). + log: list[dict[str, Any]] = field(default_factory=list) + lines: int = 0 + errors: int = 0 + screenshots: int = 0 + shell_commands: int = 0 + shell_isolation: Optional[str] = None + + def saw(self, command: str, text: str) -> None: + shown = sorted( + name for name, value in self.watch.items() if _observe.shows(text, value) + ) + self.reads.append( + {"after_step": len(self.steps), "command": command, "shows": shown} + ) + + def summary(self) -> dict[str, Any]: + def seen_before(index: int) -> set[str]: + return { + name + for read in self.reads + if read["after_step"] <= index + for name in read["shows"] + } + + commits = [s["i"] for s in self.steps if s.get("gated")] + last_commit = commits[-1] if commits else None + after = ( + sorted( + { + name + for read in self.reads + if last_commit is not None and read["after_step"] > last_commit + for name in read["shows"] + } + ) + if last_commit is not None + else [] + ) + recorded = [s for s in self.steps if s.get("recorded")] + return { + "surface": self.surface, + "scripts": self.scripts, + "lines": self.lines, + "actions": len(self.steps), + "errors": self.errors, + "screenshots": self.screenshots, + "shell_commands": self.shell_commands, + "shell_isolation": self.shell_isolation, + "by_structure": sum(1 for s in self.steps if s.get("element")), + "with_recorded_locator": len(recorded), + "locator_agrees": sum(1 for s in recorded if s.get("agrees")), + "locator_unique": sum(1 for s in recorded if s.get("unique")), + "commits": len(commits), + "identity_read_before_commit": ( + sorted(seen_before(commits[0])) if commits else [] + ), + "read_back_after_commit": after, + } + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "openadapt.code-action-trace/v1", + "summary": self.summary(), + "steps": self.steps, + "reads": self.reads, + "scripts": self.log, + } + + +@dataclass +class ToolOutput: + """What one ``run`` call returns: text, and images the agent asked for.""" + + text: str + images: list[bytes] = field(default_factory=list) + committed: bool = False + + def blocks(self) -> list[dict[str, Any]]: + from openadapt_flow.compile_agent.agents import _image + + out: list[dict[str, Any]] = [ + {"type": "text", "text": self.text or "(no output)"} + ] + out.extend(_image(png) for png in self.images) + return out + + +# -- the tool -------------------------------------------------------------------------- + + +def _parse_args(tokens: Sequence[str]) -> tuple[list[str], dict[str, str]]: + positional: list[str] = [] + named: dict[str, str] = {} + for token in tokens: + key, sep, value = token.partition("=") + if sep and re.fullmatch(r"[a-z_]+", key): + named[key] = value + else: + positional.append(token) + return positional, named + + +class CodeActionTool: + """Runs the agent's scripts against the harness, and keeps the trace. + + Args: + harness: The run's harness (gate, recording computer). + workdir: The agent's working directory. Default: next to the gate + evidence, or a temporary one. + app_origin: The web app's origin; an action that leaves it ends the run. + watch: Task values whose appearance in a read is evidence + (identity strings, inputs), by name. + surface: Override the surface (tests). + """ + + def __init__( + self, + harness: Harness, + *, + workdir: Optional[Path] = None, + app_origin: Optional[str] = None, + watch: Optional[Mapping[str, str]] = None, + surface: Any = None, + ) -> None: + self.harness = harness + if workdir is None: + if harness.evidence_dir is not None: + workdir = Path(harness.evidence_dir).parent / "workspace" + else: + workdir = Path(tempfile.mkdtemp(prefix="oa-code-action-")) + self.sandbox = Sandbox(workdir) + self.surface = surface if surface is not None else surface_for(harness) + self.app_origin = app_origin or ( + self.surface.origin() if self.surface.kind == "dom" else None + ) + self.trace = ActionTrace( + surface=self.surface.kind, + watch={k: str(v) for k, v in (watch or {}).items() if str(v).strip()}, + shell_isolation=self.sandbox.isolation, + ) + self._elements: dict[str, Element] = {} + self._last_output = "" + + # -- scrubbing ------------------------------------------------------------------ + + def _out(self, text: str) -> str: + return _observe.scrub_lines(text, self.harness.keep) + + # -- one script -------------------------------------------------------------------- + + def run(self, script: str) -> ToolOutput: + """Run every line of ``script`` in order, stopping at the first error or + after a committed write. GateRefused and LeftTheApp end the run.""" + + self.trace.scripts += 1 + lines = [ + line.strip() + for line in str(script).splitlines() + if line.strip() and not line.strip().startswith("#") + ] + if len(lines) > _MAX_LINES: + return ToolOutput(f"error: a script may have at most {_MAX_LINES} lines") + chunks: list[str] = [] + images: list[bytes] = [] + committed = False + for number, line in enumerate(lines, 1): + self.trace.lines += 1 + gates = self.harness.stats.gate_checks + try: + text, image = self._line(line) + except (ScriptError, ShellRefused, UnsupportedAction, LookupError) as exc: + self.trace.errors += 1 + chunks.append(f"$ {line}\nerror: {exc}") + if number < len(lines): + chunks.append(f"(stopped; {len(lines) - number} lines not run)") + break + self._last_output = text + chunks.append(f"$ {line}\n{text}" if text else f"$ {line}") + if image is not None: + images.append(image) + if self.harness.stats.gate_checks > gates: + committed = True + if number < len(lines): + chunks.append( + f"(a write was committed; {len(lines) - number} lines not " + "run. Read the screen to check the result.)" + ) + break + out = ToolOutput("\n".join(chunks), images, committed) + self.trace.log.append( + {"script": str(script), "output": out.text[:3000], "images": len(images)} + ) + return out + + def _line(self, line: str) -> tuple[str, Optional[bytes]]: + try: + tokens = shlex.split(line) + except ValueError as exc: + raise ScriptError(f"can't parse the line: {exc}") from exc + command, rest = tokens[0].lower(), tokens[1:] + if command == "sh": + if not rest: + raise ScriptError("sh needs a command") + self.trace.shell_commands += 1 + result = self.sandbox.run(line.split(None, 1)[1]) + text = result.output.rstrip("\n") + if result.exit_code: + text += f"\n(exit {result.exit_code})" + return self._out(text), None + if command == "save": + if len(rest) != 1: + raise ScriptError("save needs one file name") + path = self.sandbox.write(rest[0], self._last_output) + return f"saved {len(self._last_output)} characters to {path.name}", None + positional, named = _parse_args(rest) + if command == "tree": + return self._tree(named), None + if command == "find": + return self._find(named), None + if command == "text": + return self._text(named), None + if command == "screenshot": + self.trace.screenshots += 1 + png = self.harness.screenshot("requested") + return "(screenshot attached)", png + if command == "wait": + ms = min(max(int(named.get("ms", "500") or 0), 0), _MAX_WAIT_MS) + time.sleep(ms / 1000) + return f"waited {ms} ms", None + if command == "read": + element = self._target(positional, named) + got = self.surface.at(element) or {} + value = got.get("value", element.value) + text = element.line() + if value is not None and value != element.value: + text += f' now="{value}"' + if element.context: + text += f"\n row: {element.context}" + self.trace.saw(line, text + " " + element.context) + return self._out(text), None + if command in ("click", "fill", "select", "press"): + return self._act(command, line, positional, named), None + if command == "scroll": + return self._scroll(line, positional, named), None + raise ScriptError( + f"unknown command {command!r}; commands are tree, find, text, read, " + "click, fill, select, press, scroll, screenshot, sh, save, wait" + ) + + # -- reads ----------------------------------------------------------------------- + + def _read(self) -> tuple[str, list[dict[str, Any]], list[Element]]: + header, items, elements = self.surface.read() + self._elements = {e.ref: e for e in elements} + return header, items, elements + + def _tree(self, named: Mapping[str, str]) -> str: + header, items, elements = self._read() + by_ref = {e.ref: e for e in elements} + lines: list[str] = [] + current: Optional[tuple[int, int]] = None + parts: list[str] = [] + + def flush() -> None: + nonlocal parts + if parts: + lines.append("row: " + " | ".join(parts)) + parts = [] + + for item in items: + if item["kind"] == "control": + part = by_ref[item["ref"]].line() + elif item["kind"] == "heading": + part = f"# {item['text']}" + else: + part = str(item["text"]) + row = int(item.get("row", -1)) + key = (int(item.get("frame", 0)), row) + if row >= 0: + if key != current: + flush() + current = key + parts.append(part) + continue + flush() + current = None + lines.append(part) + flush() + if self.surface.kind == "tree" and not elements: + lines.append(self.surface.screen_text()) + grep = named.get("grep") + if grep: + lines = [ln for ln in lines if grep.lower() in ln.lower()] + limit = int(named.get("max", "0") or 0) + cut = "" + if limit and len(lines) > limit: + cut = f"\n({len(lines) - limit} more lines; use grep= or max=)" + lines = lines[:limit] + text = header + "\n" + "\n".join(lines) + cut + self.trace.saw("tree", text) + return self._out(text) + + def _find(self, named: Mapping[str, str]) -> str: + if not named: + raise ScriptError("find needs role=, name= or near=") + unknown = set(named) - {"role", "name", "near", "text"} + if unknown: + raise ScriptError(f"find takes role=, name=, near=; not {sorted(unknown)}") + header, _items, _elements = self._read() + hits = self._match(named, exact=False) + text = "\n".join( + e.line() + (f" row: {e.context}" if e.context else "") for e in hits + ) + text = text or "no match" + self.trace.saw("find", text) + return self._out(text) + + def _text(self, named: Mapping[str, str]) -> str: + text = self.surface.screen_text() + grep = named.get("grep") + lines = [ln.strip() for ln in text.splitlines() if ln.strip()] + if grep: + lines = [ln for ln in lines if grep.lower() in ln.lower()] + out = "\n".join(lines[:200]) or "no text" + self.trace.saw("text", out) + return self._out(out) + + # -- targets ----------------------------------------------------------------------- + + def _match(self, named: Mapping[str, str], *, exact: bool) -> list[Element]: + role = named.get("role") + if role: + role = _ROLE_ALIASES.get(role.lower(), role.lower()) + name = named.get("name", named.get("text")) + near = named.get("near") + hits: list[Element] = [] + for element in self._elements.values(): + if role and element.role.lower() != role: + continue + if name is not None: + if exact and _norm(element.name) != _norm(name): + continue + if not exact and _norm(name) not in _norm(element.name): + continue + if near is not None and _norm(near) not in _norm( + element.context or element.scope + ): + continue + hits.append(element) + return hits + + def _target(self, positional: Sequence[str], named: Mapping[str, str]) -> Element: + refs = [p for p in positional if p.startswith("@")] + if refs: + if len(refs) != 1 or positional[0] != refs[0]: + raise ScriptError("give one @ref, or role=/name=/near=") + ref = refs[0][1:] + element = self._elements.get(ref) + if element is None: + raise ScriptError( + f"@{ref} is not in the latest listing; run tree or find" + ) + return element + described = { + k: v for k, v in named.items() if k in ("role", "name", "near", "text") + } + if not described.get("name") and not described.get("text"): + raise ScriptError("a target needs @N, or name= (with role= or near=)") + self._read() + hits = self._match(described, exact=True) + if not hits and self.surface.kind == "tree" and described.get("role"): + located = self.surface.locate(described["role"], described["name"]) + if located is not None: + return located + if len(hits) == 1: + return hits[0] + if not hits: + near = self._match( + {"name": described.get("name") or described.get("text", "")}, + exact=False, + ) + hint = "".join(f"\n {e.line()}" for e in near[:5]) + raise ScriptError( + "nothing matches " + + _describe(described) + + (f"; close matches:{hint}" if hint else "") + ) + listed = "".join( + f"\n {e.line()}" + (f" row: {e.context}" if e.context else "") + for e in hits[:6] + ) + raise ScriptError( + f"{len(hits)} elements match {_describe(described)}; add near= or use " + f"a ref:{listed}" + ) + + def _still_there(self, element: Element) -> None: + if element.source != "dom": + return + got = self.surface.at(element) + if got is None: + raise ScriptError( + f"@{element.ref} can't be read at its place (covered or moved); " + "run tree again" + ) + if not got.get("name") and element.name: + raise ScriptError( + f"@{element.ref} ({element.name!r}) is covered by another element " + "(a dialog or overlay?); run tree or screenshot" + ) + if _norm(got.get("name")) != _norm(element.name) or _norm( + got.get("context") + ) != _norm(element.context): + raise ScriptError( + f"@{element.ref} ({element.name!r}) is no longer at its place; " + "run tree again" + ) + + # -- actions ----------------------------------------------------------------------- + + def _act( + self, + command: str, + line: str, + positional: Sequence[str], + named: Mapping[str, str], + ) -> str: + if command == "press": + if len(positional) != 1: + raise ScriptError("press needs one key, such as Enter or Tab") + key = positional[0] + focus = self.surface.focused() or {} + hint = CommitHint( + name=str(focus.get("name") or "the focused element"), + scope=str(focus.get("scope") or ""), + key=key if key in ("Enter", "Space") else None, + ) + event = self._event_index() + self._gates_before = len(self.harness.gate_log) + self._do({"kind": "press", "key": key}, hint) + step = {"action": "press", "key": key} + return self._finish_step(step, None, [event], line) + + element = self._target(positional, named) + self._gates_before = len(self.harness.gate_log) + if element.disabled: + raise ScriptError(f"@{element.ref} {element.name!r} is disabled") + self._still_there(element) + hint = CommitHint( + name=element.name, + scope=element.scope or element.context, + editable=element.role in ("textbox", "searchbox", "spinbutton"), + ) + events: list[int] = [] + if command == "click": + events.append(self._event_index()) + recorded = self._click(element, hint) + step = {"action": "click", "element": element.identity()} + return self._finish_step(step, recorded, events, line) + + if command == "fill": + if "value" not in named: + raise ScriptError('fill needs value="..."') + value = named["value"] + if element.role not in ("textbox", "searchbox", "combobox", "spinbutton"): + raise ScriptError(f"@{element.ref} is a {element.role}, not a field") + if element.value not in (None, ""): + raise ScriptError( + f"@{element.ref} already holds text; fill types into an empty " + "field only (typing would append)" + ) + events.append(self._event_index()) + recorded = self._click(element, hint) + events.append(self._event_index()) + self._do({"kind": "type", "text": value}, None) + got = self.surface.at(element) or {} + step = { + "action": "fill", + "element": element.identity(), + "value": value, + "value_after": got.get("value"), + } + out = self._finish_step(step, recorded, events, line) + if got.get("value") is not None and got.get("value") != value: + out += f'\nnote: the field now shows "{got.get("value")}"' + return out + + # select + if "option" not in named: + raise ScriptError('select needs option="..."') + wanted = named["option"] + if element.tag == "select": + options = {_norm(o): o for o in element.options} + option = options.get(_norm(wanted)) + if option is None: + raise ScriptError( + f"{wanted!r} is not an option of @{element.ref}; options: " + + ", ".join(f'"{o}"' for o in element.options[:20]) + ) + events.append(self._event_index()) + recorded = self._click( + element, CommitHint(name=element.name, editable=True) + ) + events.append(self._event_index()) + self._do({"kind": "type", "text": option}, None) + events.append(self._event_index()) + self._do({"kind": "press", "key": "Tab"}, None) + got = self.surface.at(element) or {} + chosen = got.get("value") + step = { + "action": "select", + "element": element.identity(), + "value": option, + "value_after": chosen, + } + out = self._finish_step(step, recorded, events, line) + if chosen is not None and _norm(chosen) != _norm(option): + raise ScriptError( + out + f'\nthe list shows "{chosen}", not "{option}"; check it ' + "before going on" + ) + return out + # A custom list: open it, then click the option by name. + events.append(self._event_index()) + recorded = self._click(element, CommitHint(name=element.name, editable=True)) + self._read() + options = [ + e + for e in self._elements.values() + if e.role in ("option", "menuitem", "listitem", "radio", "treeitem") + and _norm(e.name) == _norm(wanted) + ] + if len(options) != 1: + raise ScriptError( + f"opened @{element.ref}, but {len(options)} options are named " + f"{wanted!r}; run find role=option to see them" + ) + events.append(self._event_index()) + self._click(options[0], CommitHint(name=options[0].name, scope="")) + step = { + "action": "select", + "element": element.identity(), + "value": wanted, + "option": options[0].identity(), + } + return self._finish_step(step, recorded, events, line) + + def _scroll( + self, line: str, positional: Sequence[str], named: Mapping[str, str] + ) -> str: + where = (positional[0] if positional else "down").lower() + if where not in ("down", "up"): + raise ScriptError("scroll down or scroll up, with px=N (default 500)") + px = min(max(int(named.get("px", "500") or 0), 1), 2000) + dy = px if where == "down" else -px + if self.harness.computer.pointer is None: + raise ScriptError( + "the recording scrolls where the pointer is; click inside the " + "area you want to scroll first" + ) + event = self._event_index() + self._gates_before = len(self.harness.gate_log) + before = self._scroll_offsets() + self._do({"kind": "scroll", "dx": 0, "dy": dy}, None) + moved = before is None or self._scroll_offsets() != before + step = {"action": "scroll", "key": f"{where} {px}px", "moved": moved} + out = self._finish_step(step, None, [event], line) + if not moved: + raise ScriptError( + "the scroll was sent, but nothing scrolled: the pointer is over a part of the " + "screen that does not scroll (a fixed bar?). Click a field or " + "label in the area you want to scroll, then scroll again" + ) + return out + + def _scroll_offsets(self) -> Optional[tuple[Any, ...]]: + if self.surface.kind != "dom": + return None + offsets: list[Any] = [] + js = ( + "() => [scrollX, scrollY, ...Array.from(document.querySelectorAll('*'))" + ".filter(e => e.scrollTop || e.scrollLeft).slice(0, 50)" + ".map(e => e.scrollTop * 100000 + e.scrollLeft)]" + ) + for _index, frame, _ox, _oy in self.surface._frames(): + try: + offsets.append(tuple(frame.evaluate(js))) + except Exception: # noqa: BLE001 + offsets.append(None) + return tuple(offsets) + + def _event_index(self) -> int: + recorder = getattr( + getattr(self.harness.computer, "session", None), "_recorder", None + ) + count = getattr(recorder, "event_count", None) + return int(count) if count is not None else len(self.trace.steps) + + def _recorded_event(self, index: int) -> Optional[dict[str, Any]]: + """Line ``index`` of the recording's events.jsonl, or None.""" + + import json + + recorder = getattr( + getattr(self.harness.computer, "session", None), "_recorder", None + ) + path = getattr(recorder, "_events_path", None) + if path is None: + return None + try: + lines = Path(path).read_text().splitlines() + return json.loads(lines[index]) if 0 <= index < len(lines) else None + except (OSError, ValueError): + return None + + def _click(self, element: Element, hint: CommitHint) -> Optional[dict[str, Any]]: + """Click the element's point and return the locator the recorder will + store for it (read the same way, just before the click).""" + + locator = None + getter = getattr( + getattr(self.harness.computer, "session", None), "backend", None + ) + getter = getattr(getter, "structural_locator_at", None) + if callable(getter): + try: + locator = getter(element.x, element.y) + except Exception: # noqa: BLE001 + locator = None + recorded = None + if locator is not None: + # Before the click: the program resolves it on this screen. + recorded = locator.model_dump(exclude_none=True) + recorded["unique"] = self.surface.locator_unique(locator) + self._do({"kind": "click", "x": element.x, "y": element.y}, hint) + return recorded + + def _do(self, native: Mapping[str, Any], hint: Optional[CommitHint]) -> None: + self.harness.act(native, hint=hint) + self._settle() + origin = self.surface.origin() + if self.app_origin and origin and origin != self.app_origin: + raise LeftTheApp( + f"the action left the app ({self.app_origin}) for {origin}; the " + "agent may act on the target app only" + ) + + def _settle(self) -> None: + if self.surface.kind != "dom": + return + try: + self.harness.page.wait_for_load_state("domcontentloaded", timeout=3000) + except Exception: # noqa: BLE001 - a page that never loads is still usable + pass + + def _finish_step( + self, + step: dict[str, Any], + recorded: Optional[dict[str, Any]], + events: list[int], + line: str, + ) -> str: + step["i"] = len(self.trace.steps) + step["line"] = line + step["events"] = events + step["gated"] = len(self.harness.gate_log) > self._gates_before + if step["gated"]: + step["gate"] = self.harness.gate_log[-1].get("decision") + event = self._recorded_event(events[0]) if events else None + if event is not None and event.get("structural"): + # What the recorder actually stored is what the compiler reads. + unique = (recorded or {}).get("unique") + stored = dict(event["structural"]) + if recorded is not None: + step["matches_precheck"] = stored == { + k: v for k, v in recorded.items() if k != "unique" + } + recorded = {**stored, "unique": unique} + if recorded is not None: + unique = recorded.pop("unique", None) + step["recorded"] = recorded + step["unique"] = unique + element = step.get("element") or {} + step["agrees"] = bool( + recorded.get("selector") + or _norm(recorded.get("name")) == _norm(element.get("name")) + ) + self.trace.steps.append(step) + what = step["action"] + if "element" in step: + el = step["element"] + text = f'ok: {what} {el["role"]} "{el["name"]}"' + if step.get("value") is not None: + text += f' value="{step["value"]}"' + else: + text = f"ok: {what} {step.get('key')}" + if step["gated"]: + text += " (save gate passed)" + header = "" + if self.surface.kind == "dom": + header = f"\npage: {self.harness.page.title()} ({self.harness.page.url})" + return self._out(text + header) + + _gates_before: int = 0 + + +def _describe(named: Mapping[str, str]) -> str: + return " ".join(f'{k}="{v}"' for k, v in named.items()) + + +# -- the scripted agent ----------------------------------------------------------------- + + +@dataclass +class ScriptedCodeAgent: + """Runs fixed scripts through :class:`CodeActionTool`; no model calls. + + Each script may use ``{name}`` for a task input. A script whose last line + fails ends the run with :class:`ScriptError`. + """ + + scripts: Sequence[str] + principal_id: Optional[str] = "agent:scripted-code" + dialect: ClassVar[str] = "generic" + requires_model_key: ClassVar[bool] = False + outputs: list[str] = field(default_factory=list) + + def drive(self, request: Any, harness: Harness) -> Any: + from openadapt_flow.compile_agent.agents import AgentLedger + + ledger = AgentLedger(agent="scripted-code", vision="code") + tool = CodeActionTool(harness, watch=_watch(request, harness)) + self.last_trace = tool.trace + for script in self.scripts: + out = tool.run(str(script).format(**request.inputs)) + self.outputs.append(out.text) + if "\nerror: " in "\n" + out.text: + raise ScriptError(out.text) + ledger.finished = True + ledger.wall_seconds = time.monotonic() - ledger.started_at + return ledger + + +def _watch(request: Any, harness: Harness) -> dict[str, str]: + """The task values a read is checked for: inputs and identity strings.""" + + watch = {f"input:{k}": str(v) for k, v in (request.inputs or {}).items()} + for i, value in enumerate(harness.gate.on_screen): + watch[f"identity:{value}"] = value + return watch + + +# -- the compiled program -------------------------------------------------------------- + + +def check_trace_against_ir(trace: Mapping[str, Any], workflow: Any) -> dict[str, Any]: + """Does the compiled program carry the identity the agent acted on? + + For each step the agent took by structure, the recorder's locator for its + target must be on one of the program's steps, in order. Returns counts and + the steps that are missing. + """ + + def key(locator: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + locator.get("selector"), + _norm(locator.get("role")), + _norm(locator.get("name")), + locator.get("automation_id"), + tuple(locator.get("frame_path") or ()), + ) + + program: list[tuple[Any, ...]] = [] + for step in getattr(workflow, "steps", []) or []: + anchor = getattr(step, "anchor", None) + structural = getattr(anchor, "structural", None) if anchor else None + if structural is not None: + program.append(key(structural.model_dump(exclude_none=True))) + wanted = [ + (int(step["i"]), key(step["recorded"])) + for step in trace.get("steps", []) + if step.get("recorded") + ] + missing: list[int] = [] + cursor = 0 + for index, locator in wanted: + found = next( + (j for j in range(cursor, len(program)) if program[j] == locator), None + ) + if found is None: + missing.append(index) + else: + cursor = found + 1 + return { + "trace_steps_with_locator": len(wanted), + "in_program": len(wanted) - len(missing), + "missing_steps": missing, + "program_structural_steps": len(program), + "passed": not missing, + } + + +def element_lines(elements: Iterable[Element]) -> str: + return "\n".join(e.line() for e in elements) diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index b932818ae1..a59b5efdf9 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -34,6 +34,7 @@ from openadapt_flow.jit.adapters import CaptureComputer, Observation, translate __all__ = [ + "CommitHint", "GateRefused", "GateRequest", "Harness", @@ -52,6 +53,26 @@ class StaleRef(ValueError): """The ref is not in the latest observation, or the page changed under it.""" +@dataclass(frozen=True) +class CommitHint: + """What a structure-aware tool knows about the control it acts on. + + The harness reads the top-level page itself, which can't see into an + iframe, and a desktop or remote surface has no page at all. A tool that + resolved its target from the frame's DOM, an accessibility tree or OCR + passes the control's ``name`` and the ``scope`` text around it (the row, + dialog or form). The harness gates the action when either its own reading + or the hint says it commits a write; a hint never turns a gate off. + """ + + name: str + scope: str = "" + #: An Enter or Space on this control (the focused one), not a click. + key: Optional[str] = None + #: A text field: a click on it only focuses it, so it commits nothing. + editable: bool = False + + @dataclass(frozen=True) class GateRequest: """What the hold is asked to approve.""" @@ -129,6 +150,10 @@ def page(self) -> Any: raise RuntimeError("the compile-agent harness needs a browser page") return page + @property + def has_page(self) -> bool: + return self.computer.page is not None + @property def dialect(self) -> str: return self.computer.dialect @@ -186,25 +211,59 @@ def press(self, key: str) -> Observation: # -- every action ------------------------------------------------------------- - def act(self, native: Mapping[str, Any]) -> Observation: + def act( + self, native: Mapping[str, Any], *, hint: Optional[CommitHint] = None + ) -> Observation: """Perform one native action in the computer's dialect, behind the gate. + ``hint`` names the control a structure-aware tool resolved (see + :class:`CommitHint`). It can only add a gate. + Raises: GateRefused: the action commits a write and the gate refused it. UnsupportedAction: the recording can't hold the action. Nothing in it was performed. """ - commit = self._commit_target(native) + commit = self._commit_target(native) if self.has_page else None + if commit is None: + commit = self._hinted_commit(native, hint) if commit is not None: self._check_gate(*commit) obs = self.computer.execute(native) self.stats.actions += 1 - if commit is not None: + if commit is not None and self.has_page: # Let the app react before the agent looks at the result. _settle(self.page) return obs + def _hinted_commit( + self, native: Mapping[str, Any], hint: Optional[CommitHint] + ) -> Optional[tuple[str, Optional[str]]]: + """The commit a hint names, and on a surface with no page, every + action the harness can't rule out (no hint, or an unread target).""" + + for op in translate(self.dialect, native): + if op.kind in ("click", "double_click", "right_click"): + if hint is not None and hint.key is None: + if not hint.editable and _observe.commit_label(hint.name): + return (hint.name, hint.scope) + elif not self.has_page: + return ("a click on an element the harness could not read", None) + elif op.kind == "type" and any(ch in (op.text or "") for ch in "\r\n"): + if not self.has_page: + return ("a typed line break", hint.scope if hint else None) + elif op.kind == "hotkey" and not self.has_page: + shortcut = "+".join([*op.modifiers, op.key or ""]) + return (f"the shortcut {shortcut}", hint.scope if hint else None) + elif op.kind == "press" and op.key in ("Enter", "Space"): + if hint is not None and hint.key == op.key: + if op.key == "Enter" or _observe.commit_label(hint.name): + return (f"{op.key} on {hint.name}", hint.scope) + elif not self.has_page: + return (f"{op.key} on an element the harness could not read", None) + return None + def _commit_target( self, native: Mapping[str, Any] ) -> Optional[tuple[str, Optional[str]]]: @@ -224,11 +283,24 @@ def _commit_target( point = (op.x, op.y) if op.x is not None else self.computer.pointer if point is None or point[0] is None: continue - at = _observe.element_at(self.page, point[0], point[1]) + at = _observe.element_at( + self.page, + point[0], + point[1], + backend=getattr( + getattr(self.computer, "session", None), "backend", None + ), + ) if at is None: return ("a click on an element the harness could not read", None) name = str(at.get("name") or "") - if at.get("interactive") and _observe.commit_label(name): + # A click on a text field only focuses it ("Postal code" + # holds the word "post" but commits nothing). + if ( + at.get("interactive") + and not at.get("editable") + and _observe.commit_label(name) + ): return (name, at.get("scope")) elif op.kind == "type" and any(ch in (op.text or "") for ch in "\r\n"): focused = _observe.focused_element(self.page) or {} @@ -264,7 +336,7 @@ def _check_gate(self, control: str, scope: Optional[str]) -> None: "control": control, "identity": list(self.gate.on_screen), "missing": missing, - "url": str(getattr(self.page, "url", "")), + "url": str(getattr(self.computer.page, "url", "")), "screenshot": str(shot) if shot else None, "at": time.time(), } diff --git a/packages/flow/openadapt_flow/compile_agent/observe.py b/packages/flow/openadapt_flow/compile_agent/observe.py index 961c4e6982..c7782fec04 100644 --- a/packages/flow/openadapt_flow/compile_agent/observe.py +++ b/packages/flow/openadapt_flow/compile_agent/observe.py @@ -338,13 +338,34 @@ def observe(page: Any, *, keep: Sequence[str] = ()) -> StructuralObservation: ) -def element_at(page: Any, x: int, y: int) -> Optional[dict[str, Any]]: - """The control (or element) at a point: tag, type, name, in_form.""" +def element_at( + page: Any, x: int, y: int, *, backend: Any = None +) -> Optional[dict[str, Any]]: + """The control (or element) at a point: tag, type, name, in_form. + + A point over an iframe is read inside the frame, through the backend's + hit-tested frame chain, so a Save button in a frame is seen as one. A + frame the backend can't map is reported as unreadable (None). + """ try: - return page.evaluate(_ELEMENT_AT_JS, [int(x), int(y), INTERACTIVE, SCOPE]) + at = page.evaluate(_ELEMENT_AT_JS, [int(x), int(y), INTERACTIVE, SCOPE]) except Exception: # noqa: BLE001 - an unreadable page has no element return None + if not at or str(at.get("tag")) not in ("iframe", "frame"): + return at + getter = getattr(backend, "_frame_point", None) + if not callable(getter): + return None + try: + point = getter(int(x), int(y)) + if point is None or not point.frame_path: + return None + return point.scope.evaluate( + _ELEMENT_AT_JS, [point.x, point.y, INTERACTIVE, SCOPE] + ) + except Exception: # noqa: BLE001 + return None def focused_element(page: Any) -> Optional[dict[str, Any]]: diff --git a/packages/flow/openadapt_flow/compile_agent/pipeline.py b/packages/flow/openadapt_flow/compile_agent/pipeline.py index 94976dfd8e..d844c6f48c 100644 --- a/packages/flow/openadapt_flow/compile_agent/pipeline.py +++ b/packages/flow/openadapt_flow/compile_agent/pipeline.py @@ -70,6 +70,8 @@ PROPOSED_FILE = "verify.proposed.yaml" CONFIRMED_FILE = "verify.yaml" CHECKS_FILE = "record_checks.json" +TRACE_FILE = "code_action_trace.json" +AGENT_READS_FILE = "agent_reads.json" _PLACEHOLDER_RE = re.compile(r"{([A-Za-z_][A-Za-z0-9_]*)}") @@ -488,6 +490,7 @@ def record( # A new run 1 makes a new draft: checks a person confirmed for an # earlier draft must not carry over to it. for stale in ( + AGENT_READS_FILE, PROPOSED_FILE, CONFIRMED_FILE, CHECKS_FILE, @@ -520,6 +523,13 @@ def record( "gates": harness.gate_log if harness else [], "agent_error": harnessed.error, } + trace_data: Optional[dict[str, Any]] = None + trace = getattr(agent, "last_trace", None) + if trace is not None: + # Code as action: the structured steps and what the agent read. + trace_data = trace.to_dict() + _write_json(attempt / TRACE_FILE, trace_data) + run1["code_action"] = trace_data["summary"] _write_json(attempt / "run1.json", run1) def done(status: str, detail: Optional[str], **kw: Any) -> CompileResult: @@ -563,6 +573,21 @@ def done(status: str, detail: Optional[str], **kw: Any) -> CompileResult: record_check=check.to_dict(), ) proposed = self._propose(self._baseline or [], after or [], values) + if trace_data is not None: + run1["code_action"]["program"] = self._check_trace(jit, trace_data) + _write_json(attempt / "run1.json", run1) + _write_json( + self.out_dir / AGENT_READS_FILE, + { + "identity_read_before_commit": run1["code_action"][ + "identity_read_before_commit" + ], + "read_back_after_commit": run1["code_action"][ + "read_back_after_commit" + ], + "program": run1["code_action"]["program"], + }, + ) return done( "draft", "run 1 passed its record check; a person confirms the proposed checks", @@ -570,6 +595,20 @@ def done(status: str, detail: Optional[str], **kw: Any) -> CompileResult: proposed_checks=proposed, ) + @staticmethod + def _check_trace(jit: CompiledAgent, trace: Mapping[str, Any]) -> dict[str, Any]: + """Whether the draft program carries the locator of every element + the agent acted on by structure.""" + + from openadapt_flow.compile_agent.code_action import check_trace_against_ir + from openadapt_flow.ir import Workflow + + try: + workflow = Workflow.load(jit.bundle_dir) + except Exception as exc: # noqa: BLE001 - reported, never admitted on + return {"passed": False, "error": f"{type(exc).__name__}: {exc}"} + return check_trace_against_ir(trace, workflow) + def _set_aside(self, jit: CompiledAgent, status: str, reasons: list[str]) -> Path: program = jit.program_dir _write_json(program / "refused.json", {"status": status, "reasons": reasons}) @@ -604,7 +643,14 @@ def proposed(self) -> dict[str, Any]: doc = yaml.safe_load((self.out_dir / PROPOSED_FILE).read_text()) effects = doc.get("effects") or [] + reads = self.out_dir / AGENT_READS_FILE + extra: dict[str, Any] = {} + if reads.is_file(): + # What a code-as-action agent read: the identity before the save, + # and its inputs read back after it. Evidence for the person. + extra["agent_read"] = json.loads(reads.read_text()) return { + **extra, "file": str(self.out_dir / PROPOSED_FILE), # Confirming the checks also confirms these as the parameters. "params": sorted(self.task.params), diff --git a/packages/flow/openadapt_flow/compile_agent/sandbox.py b/packages/flow/openadapt_flow/compile_agent/sandbox.py new file mode 100644 index 0000000000..822980d83b --- /dev/null +++ b/packages/flow/openadapt_flow/compile_agent/sandbox.py @@ -0,0 +1,325 @@ +"""A read-only shell for the code-as-action agent, in one working directory. + +The agent keeps notes and page listings as files in its working directory and +reads them back with ordinary text tools (``grep``, ``head``, ``wc``). That is +the whole shell: it can't change the app, start programs, or reach a network. + +The rules, all checked before anything runs: + +* Only the commands in :data:`ALLOWED` run, from ``/usr/bin`` or ``/bin``. + None of them can open a network connection or run another program. +* Commands join with ``|`` only. ``;``, ``&``, ``&&``, ``||``, subshells, + backticks and input redirection are refused. +* ``> file`` and ``>> file`` write into the working directory only, to a + plain file name. +* An argument that names a path must stay inside the working directory: + no absolute paths, no ``~``, no ``..``. +* A few flags that write files or follow forever are refused + (:data:`REFUSED_FLAGS`). +* The environment is empty except ``PATH=/usr/bin:/bin``, ``LC_ALL=C`` and + ``HOME`` (the working directory). Each command gets 5 seconds and its output + is cut to 16,000 characters (head and tail kept). + +On macOS each command also runs under ``sandbox-exec`` with network access +denied and file writes allowed only in the working directory. On Linux it runs +under ``unshare --net`` when unprivileged user namespaces work. Elsewhere the +allow-list is the only fence. :attr:`Sandbox.isolation` says which applies. +""" + +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +__all__ = ["ALLOWED", "REFUSED_FLAGS", "Sandbox", "ShellRefused", "ShellResult"] + +#: Text tools that read files and print. None of them can open a socket, +#: start another program, or write anywhere but stdout. +ALLOWED = frozenset( + { + "cat", + "column", + "comm", + "cut", + "date", + "diff", + "echo", + "egrep", + "expand", + "false", + "fgrep", + "fold", + "grep", + "head", + "ls", + "nl", + "paste", + "printf", + "rev", + "sort", + "tail", + "tr", + "true", + "uniq", + "wc", + } +) + +#: Flags that would write a file, follow a file forever, or run something. +REFUSED_FLAGS: dict[str, frozenset[str]] = { + "sort": frozenset({"-o", "--output", "--compress-program"}), + "tail": frozenset({"-f", "-F", "--follow"}), + "date": frozenset({"-s", "--set"}), +} + +TIMEOUT_S = 5.0 +MAX_OUTPUT = 16_000 +_SYSTEM_PATH = "/usr/bin:/bin" +_FORBIDDEN_TOKENS = {";", "&", "&&", "||", "<", "<<", "(", ")", "|&", ";;"} + + +class ShellRefused(ValueError): + """The command breaks a sandbox rule. Nothing ran.""" + + +@dataclass +class ShellResult: + command: str + output: str + exit_code: int + isolation: str + wrote: Optional[str] = None + + def to_dict(self) -> dict[str, object]: + return dict(self.__dict__) + + +def _cut(text: str, limit: int = MAX_OUTPUT) -> str: + if len(text) <= limit: + return text + half = limit // 2 + return ( + text[:half] + f"\n[... {len(text) - limit} characters cut ...]\n" + text[-half:] + ) + + +@dataclass +class Sandbox: + """Run allow-listed read-only commands in ``workdir``.""" + + workdir: Path + timeout_s: float = TIMEOUT_S + isolation: str = field(init=False) + + def __post_init__(self) -> None: + self.workdir = Path(self.workdir).resolve() + self.workdir.mkdir(parents=True, exist_ok=True) + self.isolation = _detect_isolation() + + # -- files the agent keeps ---------------------------------------------------- + + def file(self, name: str) -> Path: + """The path of a plain file name in the working directory.""" + + name = str(name).strip() + if ( + not name + or "/" in name + or "\\" in name + or name.startswith(("~", ".")) + or "\x00" in name + ): + raise ShellRefused( + f"{name!r} is not a plain file name in the working directory" + ) + return self.workdir / name + + def write(self, name: str, text: str, *, append: bool = False) -> Path: + path = self.file(name) + with path.open("a" if append else "w", encoding="utf-8") as handle: + handle.write(text if text.endswith("\n") or not text else text + "\n") + return path + + # -- commands --------------------------------------------------------------------- + + def parse(self, command: str) -> tuple[list[list[str]], Optional[tuple[str, bool]]]: + """Split ``command`` into a pipeline and an optional output file. + + Raises: + ShellRefused: any rule is broken. + """ + + if "`" in command or "$(" in command or "\n" in command: + raise ShellRefused("subshells, backticks and line breaks are not allowed") + lexer = shlex.shlex(command, posix=True, punctuation_chars="|&;<>()") + lexer.whitespace_split = True + try: + tokens = list(lexer) + except ValueError as exc: + raise ShellRefused(f"can't parse the command: {exc}") from exc + if not tokens: + raise ShellRefused("empty command") + stages: list[list[str]] = [[]] + out: Optional[tuple[str, bool]] = None + i = 0 + while i < len(tokens): + token = tokens[i] + if token in _FORBIDDEN_TOKENS or ( + set(token) <= set("|&;<>()") and token not in ("|", ">", ">>") + ): + raise ShellRefused( + f"{token!r} is not allowed; join commands with | only" + ) + if token == "|": + if out is not None or not stages[-1]: + raise ShellRefused("a | must sit between two commands") + stages.append([]) + elif token in (">", ">>"): + if out is not None or i + 1 >= len(tokens) or i + 2 != len(tokens): + raise ShellRefused("> FILE must end the command, once") + self.file(tokens[i + 1]) + out = (tokens[i + 1], token == ">>") + i += 1 + else: + stages[-1].append(token) + i += 1 + if not stages[-1]: + raise ShellRefused("the command ends with |") + for argv in stages: + self._check(argv) + return stages, out + + def _check(self, argv: list[str]) -> None: + name = argv[0] + if name not in ALLOWED: + raise ShellRefused( + f"{name!r} is not allowed; the shell has {', '.join(sorted(ALLOWED))}" + ) + refused = REFUSED_FLAGS.get(name, frozenset()) + root = self.workdir + for arg in argv[1:]: + flag = arg.split("=", 1)[0] + if flag in refused or any( + arg.startswith(f) and len(f) == 2 and not arg.startswith("--") + for f in refused + ): + raise ShellRefused(f"{name} {flag} is not allowed") + if arg.startswith("-"): + continue + if arg.startswith(("/", "~")) or ".." in Path(arg).parts: + raise ShellRefused( + f"{arg!r} leaves the working directory; use plain file names" + ) + if "/" in arg: + resolved = (root / arg).resolve() + if resolved != root and root not in resolved.parents: + raise ShellRefused(f"{arg!r} leaves the working directory") + + def run(self, command: str) -> ShellResult: + """Run ``command``. A broken rule raises :class:`ShellRefused` first.""" + + stages, out = self.parse(command) + env = {"PATH": _SYSTEM_PATH, "LC_ALL": "C", "HOME": str(self.workdir)} + procs: list[subprocess.Popen[bytes]] = [] + prev = None + try: + for argv in stages: + binary = shutil.which(argv[0], path=_SYSTEM_PATH) + if binary is None: + raise ShellRefused(f"{argv[0]!r} is not installed here") + full = self._wrap([binary, *argv[1:]]) + proc = subprocess.Popen( # noqa: S603 - argv checked above + full, + stdin=prev if prev is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=self.workdir, + env=env, + ) + if prev is not None: + prev.close() + prev = proc.stdout + procs.append(proc) + try: + data, _ = procs[-1].communicate(timeout=self.timeout_s) + except subprocess.TimeoutExpired: + for proc in procs: + proc.kill() + raise ShellRefused( + f"the command ran past {self.timeout_s:g} s and was stopped" + ) from None + for proc in procs[:-1]: + proc.wait(timeout=self.timeout_s) + code = procs[-1].returncode + finally: + for proc in procs: + if proc.poll() is None: + proc.kill() + text = data.decode("utf-8", errors="replace") + wrote = None + if out is not None: + name, append = out + self.write(name, text, append=append) + wrote = name + text = f"(wrote {len(text)} characters to {name})" + return ShellResult(command, _cut(text), int(code or 0), self.isolation, wrote) + + def _wrap(self, argv: list[str]) -> list[str]: + if self.isolation == "sandbox-exec": + return ["/usr/bin/sandbox-exec", "-p", _darwin_profile(self.workdir), *argv] + if self.isolation == "unshare-net": + return ["unshare", "--net", "--map-root-user", *argv] + return argv + + +def _darwin_profile(workdir: Path) -> str: + root = str(workdir).replace("\\", "\\\\").replace('"', '\\"') + return ( + "(version 1)\n" + "(allow default)\n" + "(deny network*)\n" + "(deny file-write*)\n" + f'(allow file-write* (subpath "{root}"))\n' + '(allow file-write-data (literal "/dev/null"))\n' + ) + + +_ISOLATION: Optional[str] = None + + +def _detect_isolation() -> str: + """``sandbox-exec`` (macOS), ``unshare-net`` (Linux) or ``allow-list``.""" + + global _ISOLATION + if _ISOLATION is not None: + return _ISOLATION + found = "allow-list" + probe: Optional[list[str]] = None + if sys.platform == "darwin" and os.path.exists("/usr/bin/sandbox-exec"): + probe = [ + "/usr/bin/sandbox-exec", + "-p", + "(version 1)(allow default)", + "/usr/bin/true", + ] + candidate = "sandbox-exec" + elif sys.platform.startswith("linux") and shutil.which("unshare"): + probe = ["unshare", "--net", "--map-root-user", "/bin/true"] + candidate = "unshare-net" + if probe is not None: + try: + done = subprocess.run( # noqa: S603 - fixed argv + probe, capture_output=True, timeout=5, env={"PATH": _SYSTEM_PATH} + ) + if done.returncode == 0: + found = candidate + except (OSError, subprocess.SubprocessError): + pass + _ISOLATION = found + return found diff --git a/packages/flow/openadapt_flow/jit/adapters.py b/packages/flow/openadapt_flow/jit/adapters.py index 32e095eeb3..0d8a6dcb88 100644 --- a/packages/flow/openadapt_flow/jit/adapters.py +++ b/packages/flow/openadapt_flow/jit/adapters.py @@ -604,6 +604,12 @@ def screenshot(self) -> bytes: return self._session.screenshot() + @property + def session(self) -> Any: + """The authoring session the actions are recorded through.""" + + return self._session + def execute(self, native: Mapping[str, Any]) -> Observation: """Perform one native action and return the new frame. From 8bbcad02765167c92a392801509dab3f55d80688 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 15:55:53 -0400 Subject: [PATCH 02/13] feat(flow): code-action tests, docs draft, MockMed data; gate reads focus in frames Co-Authored-By: Claude Opus 5.5 --- .github/workflows/packages.yml | 1 + .../results/mockmed/replays.jsonl | 2 + .../code_as_action/results/mockmed/runs.jsonl | 15 + .../results/mockmed/summary.json | 62 ++ .../results/openemr/SUPERSEDED.md | 3 + .../results/openemr/superseded.jsonl | 1 + .../code_as_action/results/spend.json | 82 ++- packages/flow/docs/CODE_AS_ACTION.md | 203 +++++++ .../flow/openadapt_flow/compile_agent/cli.py | 18 +- .../compile_agent/code_action.py | 29 +- .../openadapt_flow/compile_agent/observe.py | 20 +- .../flow/tests/e2e/test_code_action_e2e.py | 264 +++++++++ packages/flow/tests/test_code_action.py | 540 ++++++++++++++++++ 13 files changed, 1211 insertions(+), 29 deletions(-) create mode 100644 packages/flow/benchmark/code_as_action/results/mockmed/replays.jsonl create mode 100644 packages/flow/benchmark/code_as_action/results/mockmed/runs.jsonl create mode 100644 packages/flow/benchmark/code_as_action/results/mockmed/summary.json create mode 100644 packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md create mode 100644 packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl create mode 100644 packages/flow/docs/CODE_AS_ACTION.md create mode 100644 packages/flow/tests/e2e/test_code_action_e2e.py create mode 100644 packages/flow/tests/test_code_action.py diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index b8d3d77c45..04b7745e7e 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -363,6 +363,7 @@ jobs: tests/test_admitted_composition_runtime.py \ tests/e2e/test_structural_action.py \ tests/e2e/test_compile_agent_e2e.py \ + tests/e2e/test_code_action_e2e.py \ --basetemp=runs/ci-critical --junitxml=runs/critical-junit.xml - name: Fail if any critical test was skipped or none ran run: | diff --git a/packages/flow/benchmark/code_as_action/results/mockmed/replays.jsonl b/packages/flow/benchmark/code_as_action/results/mockmed/replays.jsonl new file mode 100644 index 0000000000..f588508416 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/mockmed/replays.jsonl @@ -0,0 +1,2 @@ +{"app": "mockmed", "host_load_1m": 304.5, "model_calls": 0, "program_from": "code", "receipt_outcome": "VERIFIED", "replay": 1, "seconds": 5.292, "status": "VERIFIED", "verified": true} +{"app": "mockmed", "host_load_1m": 288.2, "model_calls": 0, "program_from": "code", "receipt_outcome": "VERIFIED", "replay": 2, "seconds": 4.118, "status": "VERIFIED", "verified": true} diff --git a/packages/flow/benchmark/code_as_action/results/mockmed/runs.jsonl b/packages/flow/benchmark/code_as_action/results/mockmed/runs.jsonl new file mode 100644 index 0000000000..3da49c5ad5 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/mockmed/runs.jsonl @@ -0,0 +1,15 @@ +{"actions": 6, "agent_seconds": 7.509, "agent_summary": "Saved Triage encounter for Jane Sample with the note; it appears in her encounter list.", "app": "mockmed", "arm": "code", "cache_creation_input_tokens": 2461, "cache_read_input_tokens": 7432, "calls": [{"cache_creation_input_tokens": 1467, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 50, "seconds": 1.151, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 230, "cache_read_input_tokens": 1467, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 1.095, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 265, "cache_read_input_tokens": 1697, "images_in_request": 0, "input_tokens": 2, "output_tokens": 82, "seconds": 1.084, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 344, "cache_read_input_tokens": 1962, "images_in_request": 0, "input_tokens": 2, "output_tokens": 45, "seconds": 0.922, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 155, "cache_read_input_tokens": 2306, "images_in_request": 0, "input_tokens": 2, "output_tokens": 76, "seconds": 1.111, "stop_reason": "tool_use", "turn": 5}], "code_action": {"actions": 5, "by_structure": 5, "commits": 1, "errors": 0, "identity_read_before_commit": ["identity:Jane Sample", "identity:MRN P1"], "lines": 9, "locator_agrees": 5, "locator_unique": 5, "program": {"in_program": 5, "missing_steps": [], "passed": true, "program_structural_steps": 5, "trace_steps_with_locator": 5}, "read_back_after_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "screenshots": 0, "scripts": 5, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 5}, "correct": true, "cost_usd": 0.010693, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [217.4, 301.5], "images_in_requests": 0, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 5.364, "order": 0, "output_tokens": 303, "prompt_tokens": 9905, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 47.155, "screenshots_sent": 0, "started_at": "2026-10-18T19:45:39Z", "status": "draft", "tool_errors": 0, "trial": 1, "under_load": true} +{"actions": 13, "agent_seconds": 16.951, "agent_summary": "Opened Jane Sample's referral, created a Triage encounter with the note \"Synthetic follow-up in two weeks, trial 1\", and saved it. The encounter list shows the full note.", "app": "mockmed", "arm": "every_turn", "cache_creation_input_tokens": 23878, "cache_read_input_tokens": 49413, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.357, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 4.062, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 138, "seconds": 1.928, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 5589, "cache_read_input_tokens": 14099, "images_in_request": 11, "input_tokens": 2, "output_tokens": 98, "seconds": 1.893, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 19688, "images_in_request": 14, "input_tokens": 2, "output_tokens": 107, "seconds": 2.355, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.074932, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [301.5, 299.2], "images_in_requests": 37, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 11.594, "order": 1, "output_tokens": 533, "prompt_tokens": 73303, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 21.355, "screenshots_sent": 14, "started_at": "2026-10-18T19:46:26Z", "status": "draft", "tool_errors": 0, "trial": 1, "under_load": true} +{"actions": 6, "agent_seconds": 8.741, "agent_summary": "Saved Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 1\".", "app": "mockmed", "arm": "selective", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 0.955, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.817, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.601, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.997, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 86, "seconds": 1.396, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.014273, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [299.2, 358.6], "images_in_requests": 1, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 5.766, "order": 2, "output_tokens": 359, "prompt_tokens": 10332, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 41.574, "screenshots_sent": 1, "started_at": "2026-10-18T19:46:47Z", "status": "draft", "tool_errors": 0, "trial": 1, "under_load": true} +{"actions": 13, "agent_seconds": 15.871, "agent_summary": "Opened Jane Sample's referral, created a Triage encounter, and saved the note \"Synthetic follow-up in two weeks, trial 2\". The encounter list shows the full note, though the green confirmation banner displays it cut off after \"trial\".", "app": "mockmed", "arm": "every_turn", "cache_creation_input_tokens": 23878, "cache_read_input_tokens": 49413, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.615, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 1.708, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 138, "seconds": 1.738, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 5589, "cache_read_input_tokens": 14099, "images_in_request": 11, "input_tokens": 2, "output_tokens": 98, "seconds": 1.864, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 19688, "images_in_request": 14, "input_tokens": 2, "output_tokens": 152, "seconds": 3.048, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.075382, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [358.6, 419.5], "images_in_requests": 37, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 9.973, "order": 0, "output_tokens": 578, "prompt_tokens": 73303, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 26.009, "screenshots_sent": 14, "started_at": "2026-10-18T19:47:29Z", "status": "draft", "tool_errors": 0, "trial": 2, "under_load": true} +{"actions": 6, "agent_seconds": 8.162, "agent_summary": "Saved Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 2\".", "app": "mockmed", "arm": "selective", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 0.994, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.837, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.267, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.936, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 86, "seconds": 1.503, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.014273, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [419.5, 410.2], "images_in_requests": 1, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 5.536, "order": 1, "output_tokens": 359, "prompt_tokens": 10332, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 24.355, "screenshots_sent": 1, "started_at": "2026-10-18T19:47:55Z", "status": "draft", "tool_errors": 0, "trial": 2, "under_load": true} +{"actions": 6, "agent_seconds": 7.137, "agent_summary": "Saved Triage encounter for Jane Sample with the note; it appears in the encounter list.", "app": "mockmed", "arm": "code", "cache_creation_input_tokens": 2461, "cache_read_input_tokens": 7432, "calls": [{"cache_creation_input_tokens": 1467, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 50, "seconds": 0.976, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 230, "cache_read_input_tokens": 1467, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 1.035, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 265, "cache_read_input_tokens": 1697, "images_in_request": 0, "input_tokens": 2, "output_tokens": 82, "seconds": 0.977, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 344, "cache_read_input_tokens": 1962, "images_in_request": 0, "input_tokens": 2, "output_tokens": 45, "seconds": 0.873, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 155, "cache_read_input_tokens": 2306, "images_in_request": 0, "input_tokens": 2, "output_tokens": 76, "seconds": 1.257, "stop_reason": "tool_use", "turn": 5}], "code_action": {"actions": 5, "by_structure": 5, "commits": 1, "errors": 0, "identity_read_before_commit": ["identity:Jane Sample", "identity:MRN P1"], "lines": 9, "locator_agrees": 5, "locator_unique": 5, "program": {"in_program": 5, "missing_steps": [], "passed": true, "program_structural_steps": 5, "trace_steps_with_locator": 5}, "read_back_after_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "screenshots": 0, "scripts": 5, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 5}, "correct": true, "cost_usd": 0.010693, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [410.2, 406.0], "images_in_requests": 0, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 5.117, "order": 2, "output_tokens": 303, "prompt_tokens": 9905, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 13.186, "screenshots_sent": 0, "started_at": "2026-10-18T19:48:19Z", "status": "draft", "tool_errors": 0, "trial": 2, "under_load": true} +{"actions": 6, "agent_seconds": 8.047, "agent_summary": "Saved Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 3\".", "app": "mockmed", "arm": "selective", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.086, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.201, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.553, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.811, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 86, "seconds": 1.246, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.014273, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [406.0, 407.2], "images_in_requests": 1, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 5.897, "order": 0, "output_tokens": 359, "prompt_tokens": 10332, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 11.444, "screenshots_sent": 1, "started_at": "2026-10-18T19:48:33Z", "status": "draft", "tool_errors": 0, "trial": 3, "under_load": true} +{"actions": 6, "agent_seconds": 8.751, "agent_summary": "Saved Triage encounter for Jane Sample with the note; it appears in the encounters list.", "app": "mockmed", "arm": "code", "cache_creation_input_tokens": 2738, "cache_read_input_tokens": 10038, "calls": [{"cache_creation_input_tokens": 1467, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 50, "seconds": 1.19, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 230, "cache_read_input_tokens": 1467, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 0.846, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 265, "cache_read_input_tokens": 1697, "images_in_request": 0, "input_tokens": 2, "output_tokens": 78, "seconds": 1.02, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 419, "cache_read_input_tokens": 1962, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 0.894, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 150, "cache_read_input_tokens": 2381, "images_in_request": 0, "input_tokens": 2, "output_tokens": 46, "seconds": 0.865, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 207, "cache_read_input_tokens": 2531, "images_in_request": 0, "input_tokens": 2, "output_tokens": 76, "seconds": 1.056, "stop_reason": "tool_use", "turn": 6}], "code_action": {"actions": 5, "by_structure": 5, "commits": 1, "errors": 0, "identity_read_before_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "lines": 10, "locator_agrees": 5, "locator_unique": 5, "program": {"in_program": 5, "missing_steps": [], "passed": true, "program_structural_steps": 5, "trace_steps_with_locator": 5}, "read_back_after_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "screenshots": 0, "scripts": 6, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 5}, "correct": true, "cost_usd": 0.012381, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [407.2, 344.4], "images_in_requests": 0, "input_tokens": 14, "label": null, "model": "claude-sonnet-5-5", "model_calls": 6, "model_seconds": 5.871, "order": 1, "output_tokens": 350, "prompt_tokens": 12790, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 15.616, "screenshots_sent": 0, "started_at": "2026-10-18T19:48:44Z", "status": "draft", "tool_errors": 0, "trial": 3, "under_load": true} +{"actions": 13, "agent_seconds": 15.295, "agent_summary": "Opened Jane Sample's referral, created a Triage encounter with the note \"Synthetic follow-up in two weeks, trial 3\", and saved it. The encounter list shows the full note (the green banner truncates it to \"...trial\").", "app": "mockmed", "arm": "every_turn", "cache_creation_input_tokens": 23878, "cache_read_input_tokens": 49413, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.481, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 1.51, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 138, "seconds": 3.071, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 5589, "cache_read_input_tokens": 14099, "images_in_request": 11, "input_tokens": 2, "output_tokens": 98, "seconds": 1.738, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 19688, "images_in_request": 14, "input_tokens": 2, "output_tokens": 121, "seconds": 1.916, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.075072, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [344.4, 295.4], "images_in_requests": 37, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 9.717, "order": 2, "output_tokens": 547, "prompt_tokens": 73303, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 18.741, "screenshots_sent": 14, "started_at": "2026-10-18T19:49:00Z", "status": "draft", "tool_errors": 0, "trial": 3, "under_load": true} +{"actions": 6, "agent_seconds": 8.138, "agent_summary": "Saved Triage encounter for Jane Sample with the note; it now shows in her encounters list.", "app": "mockmed", "arm": "code", "cache_creation_input_tokens": 2686, "cache_read_input_tokens": 10038, "calls": [{"cache_creation_input_tokens": 1467, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 50, "seconds": 1.003, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 230, "cache_read_input_tokens": 1467, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 0.865, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 265, "cache_read_input_tokens": 1697, "images_in_request": 0, "input_tokens": 2, "output_tokens": 78, "seconds": 1.06, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 419, "cache_read_input_tokens": 1962, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 0.979, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 150, "cache_read_input_tokens": 2381, "images_in_request": 0, "input_tokens": 2, "output_tokens": 45, "seconds": 0.896, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 155, "cache_read_input_tokens": 2531, "images_in_request": 0, "input_tokens": 2, "output_tokens": 77, "seconds": 1.068, "stop_reason": "tool_use", "turn": 6}], "code_action": {"actions": 5, "by_structure": 5, "commits": 1, "errors": 0, "identity_read_before_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "lines": 10, "locator_agrees": 5, "locator_unique": 5, "program": {"in_program": 5, "missing_steps": [], "passed": true, "program_structural_steps": 5, "trace_steps_with_locator": 5}, "read_back_after_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "screenshots": 0, "scripts": 6, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 5}, "correct": true, "cost_usd": 0.012251, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [295.4, 255.2], "images_in_requests": 0, "input_tokens": 14, "label": null, "model": "claude-sonnet-5-5", "model_calls": 6, "model_seconds": 5.871, "order": 0, "output_tokens": 350, "prompt_tokens": 12738, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 21.532, "screenshots_sent": 0, "started_at": "2026-10-18T19:49:18Z", "status": "draft", "tool_errors": 0, "trial": 4, "under_load": true} +{"actions": 13, "agent_seconds": 14.298, "agent_summary": "Opened Jane Sample's referral, created a Triage encounter with note \"Synthetic follow-up in two weeks, trial 4\", and saved it (shown in the encounters list).", "app": "mockmed", "arm": "every_turn", "cache_creation_input_tokens": 23878, "cache_read_input_tokens": 49413, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.379, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 1.636, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 138, "seconds": 2.092, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 5589, "cache_read_input_tokens": 14099, "images_in_request": 11, "input_tokens": 2, "output_tokens": 98, "seconds": 1.83, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 19688, "images_in_request": 14, "input_tokens": 2, "output_tokens": 104, "seconds": 1.848, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.074902, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [255.2, 285.9], "images_in_requests": 37, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 8.785, "order": 1, "output_tokens": 530, "prompt_tokens": 73303, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 36.013, "screenshots_sent": 14, "started_at": "2026-10-18T19:49:40Z", "status": "draft", "tool_errors": 0, "trial": 4, "under_load": true} +{"actions": 6, "agent_seconds": 8.871, "agent_summary": "Saved Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 4\".", "app": "mockmed", "arm": "selective", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 1.449, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.851, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.392, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.133, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 86, "seconds": 1.293, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.014273, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [285.9, 299.7], "images_in_requests": 1, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 6.119, "order": 2, "output_tokens": 359, "prompt_tokens": 10332, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 22.57, "screenshots_sent": 1, "started_at": "2026-10-18T19:50:16Z", "status": "draft", "tool_errors": 0, "trial": 4, "under_load": true} +{"actions": 13, "agent_seconds": 14.345, "agent_summary": "Opened Jane Sample's referral, created a Triage encounter with the note \"Synthetic follow-up in two weeks, trial 5\", and saved it. The encounter list shows the full note.", "app": "mockmed", "arm": "every_turn", "cache_creation_input_tokens": 23878, "cache_read_input_tokens": 49413, "calls": [{"cache_creation_input_tokens": 5717, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 92, "seconds": 1.572, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 4192, "cache_read_input_tokens": 5717, "images_in_request": 4, "input_tokens": 2, "output_tokens": 98, "seconds": 1.533, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 9909, "images_in_request": 7, "input_tokens": 2, "output_tokens": 138, "seconds": 1.776, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 5589, "cache_read_input_tokens": 14099, "images_in_request": 11, "input_tokens": 2, "output_tokens": 98, "seconds": 1.791, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 19688, "images_in_request": 14, "input_tokens": 2, "output_tokens": 107, "seconds": 1.895, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.074932, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [299.7, 309.7], "images_in_requests": 37, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 8.568, "order": 0, "output_tokens": 533, "prompt_tokens": 73303, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 43.23, "screenshots_sent": 14, "started_at": "2026-10-18T19:50:39Z", "status": "draft", "tool_errors": 0, "trial": 5, "under_load": true} +{"actions": 6, "agent_seconds": 7.98, "agent_summary": "Saved Triage encounter for Jane Sample with the note \"Synthetic follow-up in two weeks, trial 5\".", "app": "mockmed", "arm": "selective", "cache_creation_input_tokens": 3737, "cache_read_input_tokens": 6583, "calls": [{"cache_creation_input_tokens": 1291, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 47, "seconds": 0.993, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 165, "cache_read_input_tokens": 1291, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 0.975, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 202, "cache_read_input_tokens": 1456, "images_in_request": 0, "input_tokens": 2, "output_tokens": 132, "seconds": 1.494, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 520, "cache_read_input_tokens": 1658, "images_in_request": 0, "input_tokens": 2, "output_tokens": 47, "seconds": 1.184, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1559, "cache_read_input_tokens": 2178, "images_in_request": 1, "input_tokens": 2, "output_tokens": 86, "seconds": 1.179, "stop_reason": "tool_use", "turn": 5}], "code_action": null, "correct": true, "cost_usd": 0.014273, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [309.7, 330.8], "images_in_requests": 1, "input_tokens": 12, "label": null, "model": "claude-sonnet-5-5", "model_calls": 5, "model_seconds": 5.825, "order": 1, "output_tokens": 359, "prompt_tokens": 10332, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 24.022, "screenshots_sent": 1, "started_at": "2026-10-18T19:51:22Z", "status": "draft", "tool_errors": 0, "trial": 5, "under_load": true} +{"actions": 6, "agent_seconds": 8.392, "agent_summary": "Saved Triage encounter for Jane Sample with the note; it now appears under Encounters.", "app": "mockmed", "arm": "code", "cache_creation_input_tokens": 2738, "cache_read_input_tokens": 10038, "calls": [{"cache_creation_input_tokens": 1467, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 50, "seconds": 0.941, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 230, "cache_read_input_tokens": 1467, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 1.016, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 265, "cache_read_input_tokens": 1697, "images_in_request": 0, "input_tokens": 2, "output_tokens": 78, "seconds": 1.053, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 419, "cache_read_input_tokens": 1962, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 1.16, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 150, "cache_read_input_tokens": 2381, "images_in_request": 0, "input_tokens": 2, "output_tokens": 46, "seconds": 0.993, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 207, "cache_read_input_tokens": 2531, "images_in_request": 0, "input_tokens": 2, "output_tokens": 76, "seconds": 1.145, "stop_reason": "tool_use", "turn": 6}], "code_action": {"actions": 5, "by_structure": 5, "commits": 1, "errors": 0, "identity_read_before_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "lines": 10, "locator_agrees": 5, "locator_unique": 5, "program": {"in_program": 5, "missing_steps": [], "passed": true, "program_structural_steps": 5, "trace_steps_with_locator": 5}, "read_back_after_commit": ["identity:Jane Sample", "identity:MRN P1", "input:note"], "screenshots": 0, "scripts": 6, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 5}, "correct": true, "cost_usd": 0.012381, "detail": null, "gates": [{"control": "Save Encounter", "decision": "passed_no_hold", "identity": ["Jane Sample", "MRN P1"], "missing": []}], "host_load_1m": [330.8, 332.3], "images_in_requests": 0, "input_tokens": 14, "label": null, "model": "claude-sonnet-5-5", "model_calls": 6, "model_seconds": 6.307, "order": 2, "output_tokens": 350, "prompt_tokens": 12790, "record_check": {"changed": 0, "created": 1, "fields": ["id", "key", "note", "patient_id", "source", "type"], "passed": true, "reasons": [], "removed": 0}, "run_seconds": 26.47, "screenshots_sent": 0, "started_at": "2026-10-18T19:51:46Z", "status": "draft", "tool_errors": 0, "trial": 5, "under_load": true} diff --git a/packages/flow/benchmark/code_as_action/results/mockmed/summary.json b/packages/flow/benchmark/code_as_action/results/mockmed/summary.json new file mode 100644 index 0000000000..b5af1f79aa --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/mockmed/summary.json @@ -0,0 +1,62 @@ +{ + "arms": { + "code": { + "correct": 5, + "median_actions": 6.0, + "median_cost_usd": 0.012, + "median_model_calls": 6.0, + "median_model_seconds": 5.871, + "median_output_tokens": 350.0, + "median_prompt_tokens": 12738.0, + "median_run_seconds": 21.532, + "median_screenshots_sent": 0.0, + "runs": 5, + "runs_under_load": 5, + "total_cost_usd": 0.0584 + }, + "every_turn": { + "correct": 5, + "median_actions": 13.0, + "median_cost_usd": 0.075, + "median_model_calls": 5.0, + "median_model_seconds": 9.717, + "median_output_tokens": 533.0, + "median_prompt_tokens": 73303.0, + "median_run_seconds": 26.009, + "median_screenshots_sent": 14.0, + "runs": 5, + "runs_under_load": 5, + "total_cost_usd": 0.3752 + }, + "selective": { + "correct": 5, + "median_actions": 6.0, + "median_cost_usd": 0.014, + "median_model_calls": 5.0, + "median_model_seconds": 5.825, + "median_output_tokens": 359.0, + "median_prompt_tokens": 10332.0, + "median_run_seconds": 24.022, + "median_screenshots_sent": 1.0, + "runs": 5, + "runs_under_load": 5, + "total_cost_usd": 0.0714 + } + }, + "environment": { + "app": "MockMed fault server, ?fault=ok&idempotency=demo, 1280x800", + "effort": "medium", + "engine_commit": "d3b8efc4", + "load_limit_1m": 12.0, + "measured_at": "2026-10-18T19:52:33Z", + "model": "claude-sonnet-5-5", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "python": "3.12.7", + "spend_total_usd": 0.58633 + }, + "replays": { + "model_calls_total": 0, + "runs": 2, + "verified": 2 + } +} diff --git a/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md b/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md new file mode 100644 index 0000000000..9a13e928b8 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md @@ -0,0 +1,3 @@ +superseded.jsonl: one screenshot-loop run (r1, $0.028) refused by the save gate because the +harness read keyboard focus only in the top page, not inside OpenEMR's form iframe. Fixed in +observe.focused_element; the run was not counted and the arms were measured again. diff --git a/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl b/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl new file mode 100644 index 0000000000..a28a6a24cd --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl @@ -0,0 +1 @@ +{"actions": 3, "agent_seconds": 6.538, "agent_summary": null, "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 8823, "cache_read_input_tokens": 6006, "calls": [{"cache_creation_input_tokens": 6006, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 2.0, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6006, "images_in_request": 3, "input_tokens": 2, "output_tokens": 356, "seconds": 3.402, "stop_reason": "tool_use", "turn": 2}], "code_action": null, "correct": false, "cost_usd": 0.027621, "error": "GateRefused: 'Enter on the focused element' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it", "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [{"control": "Enter on the focused element", "decision": "refused_wrong_record", "missing": ["Create New Patient"]}], "host_load_1m": [250.9, 233.0], "images_in_requests": 4, "input_tokens": 6, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 2, "model_seconds": 5.402, "new_charts": 0, "output_tokens": 435, "program": null, "prompt_tokens": 14835, "referral": "r1", "round": 1, "run_seconds": 6.784, "screenshots_sent": 4, "scripts": null, "started_at": "2026-10-18T19:52:42Z", "tool_errors": 0, "under_load": true} diff --git a/packages/flow/benchmark/code_as_action/results/spend.json b/packages/flow/benchmark/code_as_action/results/spend.json index 76bf84e950..e33b20ca88 100644 --- a/packages/flow/benchmark/code_as_action/results/spend.json +++ b/packages/flow/benchmark/code_as_action/results/spend.json @@ -19,8 +19,88 @@ "label": "openemr code r1", "usd": 0.047661, "at": "2026-10-18T19:41:04Z" + }, + { + "label": "mockmed code t1", + "usd": 0.010693, + "at": "2026-10-18T19:46:26Z" + }, + { + "label": "mockmed every_turn t1", + "usd": 0.074932, + "at": "2026-10-18T19:46:47Z" + }, + { + "label": "mockmed selective t1", + "usd": 0.014273, + "at": "2026-10-18T19:47:29Z" + }, + { + "label": "mockmed every_turn t2", + "usd": 0.075382, + "at": "2026-10-18T19:47:55Z" + }, + { + "label": "mockmed selective t2", + "usd": 0.014273, + "at": "2026-10-18T19:48:19Z" + }, + { + "label": "mockmed code t2", + "usd": 0.010693, + "at": "2026-10-18T19:48:33Z" + }, + { + "label": "mockmed selective t3", + "usd": 0.014273, + "at": "2026-10-18T19:48:44Z" + }, + { + "label": "mockmed code t3", + "usd": 0.012381, + "at": "2026-10-18T19:49:00Z" + }, + { + "label": "mockmed every_turn t3", + "usd": 0.075072, + "at": "2026-10-18T19:49:18Z" + }, + { + "label": "mockmed code t4", + "usd": 0.012251, + "at": "2026-10-18T19:49:40Z" + }, + { + "label": "mockmed every_turn t4", + "usd": 0.074902, + "at": "2026-10-18T19:50:16Z" + }, + { + "label": "mockmed selective t4", + "usd": 0.014273, + "at": "2026-10-18T19:50:39Z" + }, + { + "label": "mockmed every_turn t5", + "usd": 0.074932, + "at": "2026-10-18T19:51:22Z" + }, + { + "label": "mockmed selective t5", + "usd": 0.014273, + "at": "2026-10-18T19:51:46Z" + }, + { + "label": "mockmed code t5", + "usd": 0.012381, + "at": "2026-10-18T19:52:12Z" + }, + { + "label": "openemr every_turn r1", + "usd": 0.027621, + "at": "2026-10-18T19:52:51Z" } ], - "total_usd": 0.081346, + "total_usd": 0.613951, "cap_usd": 15.0 } diff --git a/packages/flow/docs/CODE_AS_ACTION.md b/packages/flow/docs/CODE_AS_ACTION.md new file mode 100644 index 0000000000..61f4920bd2 --- /dev/null +++ b/packages/flow/docs/CODE_AS_ACTION.md @@ -0,0 +1,203 @@ +# Code as action: the run-1 agent works through a terminal + +Before OpenAdapt compiles a task, an agent does it once +([COMPILE_AGENT.md](COMPILE_AGENT.md)). With `vision="code"` that agent does +not look at a screenshot after every click. It gets one terminal-style tool, +`run`, and writes short scripts: read the app's structure, act on elements by +role and name, read the result back, keep notes in a working directory, and +take a screenshot only when it asks for one. One model call can carry several +reads and actions. + +Every action is still recorded by the JIT capture, gated by the same save +check, and compiled into the same program. The difference is what the agent +sees and how much of it the model has to read. + +```python +from openadapt_flow.compile_agent import ClaudeAgent, compile_first_run + +result = compile_first_run( + "Open the referral for Jane Sample, start a Triage encounter, and save this note: {note}", + params={"note": "Follow up in two weeks"}, + agent=ClaudeAgent(vision="code", model="claude-sonnet-5-5"), + url="http://localhost:8080/#tasks", verify="verify.yaml", + expect={"patient_id": "p1"}, on_screen=["Jane Sample", "MRN P1"], +) +``` + +```bash +openadapt-flow compile-agent "..." --vision code --url ... --verify verify.yaml ... +``` + +`--agent scripts.json` with `{"scripts": ["tree", "click name=Open near=\"Jane Sample\""]}` +runs the same tool from fixed scripts with no model calls (tests, CI, demos). + +## Why + +Three 2026 harness results point the same way. + +- **CUAWright** (Microsoft Research, arXiv:2610.04116) gives the agent a + terminal as its only action interface, keeps state in files, and shows a + screenshot only on request ("vision on demand, not by default"). It reports + 67.9% partial reward on OSWorld 2.0 at $4.40 a task, against 62.7% and + $13.90 for the same model's official harness. Removing file-system access + cost it about 12 points. +- **OpenGCUA** (Alibaba Tongyi, arXiv:2610.22200) has the model write one + program a step that mixes GUI, accessibility, API and shell calls, with + loops and branches, and requests screenshots itself. Against batched GUI + actions it reports 53% fewer steps on average across eight models. +- **ANIS** (arXiv:2610.15765) builds commands for desktop apps from their own + registered operations, keeps only commands whose effect it can observe after + saving, and reports about 73% fewer online tokens than raw SDK use. Taking + away the read-back commands cost 12 points: a command that returns success + has not shown that the task succeeded. + +All three report their own results on their own baselines. + +## What OpenAdapt takes from them, and what it does not + +| Idea | Here | +| --- | --- | +| One terminal tool, scripts, text output | `run` takes a script; each line prints what it did or read. | +| State on disk | A working directory per run. `save FILE` writes the last output there; `sh grep ... FILE` reads it back. | +| Vision on demand | `screenshot` is a command. Nothing else sends an image to the model. | +| Accessibility first | `tree`, `find` and `read` list the DOM (every frame) or a desktop accessibility tree. | +| Compose several actions | A script can hold up to 40 lines; refs change after each action, so later lines name targets by role and name. | +| Read back after a write | After a save the script stops and the agent is told to read the screen. The trace records whether it read its inputs back. | + +What it does not take: the script is not Python and not a full shell. Every +line is either a read or one action the recorder can hold (click, fill, +select, press, scroll). That is what lets each action go through the save gate +before it is performed and become a step of the compiled program. A general +program could write to the app in ways the gate and the compiler never see. + +## The commands + +``` +tree [grep=TEXT] [max=N] the structure: headings, text, rows, controls with @refs +find [role=R] [name=TEXT] [near=TEXT] controls that match +text [grep=TEXT] visible text lines +read TARGET an element's value and the row it is in +click TARGET +fill TARGET value=TEXT click an empty field and type TEXT +select TARGET option=TEXT choose an option in a list or drop-down +press KEY Enter, Tab, Escape, ArrowDown, ... +scroll down|up [px=N] scroll where the pointer is +screenshot an image of the screen +sh COMMAND read-only shell in the working directory +save FILE the previous command's output, to FILE +wait ms=N up to 2000 ms +``` + +A TARGET is `@N` from the latest listing, or `role=button name="Open" +near="Jane Sample"` (`near` is text in the same table row). A description must +match exactly one element; if it matches several, the line fails and lists +them. On a screen with no structure, `text="Save"` names an OCR line. + +Lines run in order and stop at the first error. An error never performs part +of an action. A few rules come from the recording format: + +- `fill` types only into an empty field (typing appends to what is there). +- `select` on a native drop-down clicks it, types the option and presses Tab + (the recorded form of a select), then checks the drop-down shows that option. + On a custom list it clicks the list, then the option by name. +- `scroll` happens where the pointer is, because the recording has no pointer + move. If nothing moved, the line fails and says to click inside the area + first. +- Before a click, the tool checks the element is still at its point, in the + same row, and not covered (a dialog backdrop, for example). + +## Every action is a structured step + +The trace (`code_action_trace.json` next to run 1) holds one step per action: + +```json +{"action": "click", + "element": {"role": "button", "name": "Open", "row": "Jane Sample | Knee pain referral | High | Open", "source": "dom"}, + "events": [0], + "recorded": {"selector": "#open-p1", "role": "button", "name": "Open"}, + "unique": true, "agrees": true, "matches_precheck": true, "gated": false} +``` + +`element` is what the agent acted on. `recorded` is the structural locator the +recorder stored for that event, read from `events.jsonl`; it is also read just +before the click, and `matches_precheck` says the two agree. `unique` says the +locator finds one element on that screen, as the program will need at replay. +After compiling, `check_trace_against_ir` checks that the program carries each +of those locators on its steps, in order. The result is in +`run1["code_action"]["program"]`. + +The trace also records what the agent read: for each `tree`, `find`, `text` +and `read`, which of the task's identity strings and input values appeared. +The draft's `compiled/agent_reads.json` keeps the identity the agent read +before the save and the inputs it read back after it, and +`FirstRunCompiler.proposed()` shows them to the person who confirms the +record checks, next to the checks proposed from the system of record. + +## Surfaces + +| Surface | Reads | Acts | Notes | +| --- | --- | --- | --- | +| Web (Playwright) | DOM of the page and every frame | element center, recorded with its DOM locator and frame path | `tree` says how many controls are above or below the visible area. | +| Windows, macOS, Linux | the authoring tree (`AuthoringSession.observe`), OCR text | role and name through the backend's `locate_structural` | No native backend implements `authoring_tree` yet, so listings fall back to OCR text; acting by exact role and name works now. | +| RDP, Citrix | OCR lines | the line's center | Pixels only. Steps compile with visual anchors, as for any pixel recording. | + +The surface comes from the recording computer: a browser page is the DOM; a +screen-only session (RDP, Citrix) is pixels; any other session is a desktop +tree. compile-agent itself runs web apps only until `jit.SurfaceApp` +([#1283](https://github.com/OpenAdaptAI/OpenAdapt/pull/1283)) lands; the tool +and the save gate already work without a page (tests/test_code_action.py +drives them on a fake desktop surface). + +## Safety + +Nothing here loosens what run 1 may do. + +- **The save gate.** Every action that may commit a write goes through the + harness's right-record check and hold before it is performed + ([COMPILE_AGENT.md](COMPILE_AGENT.md)). The tool passes the harness what it + resolved (`CommitHint`: the control's name and the text and field values + around it). A hint can add a gate, never remove one. On a surface with no + page, a click the tool did not resolve is gated and refused. +- **The customer's key.** The agent calls the model on the key the operator + supplies; OpenAdapt holds none. The spend cap (`max_cost_usd`) holds as for + any `ClaudeAgent`. +- **The shell.** `sh` runs only text tools (`cat`, `grep`, `head`, `wc`, + `sort`, ...) from `/usr/bin` and `/bin`, joined with `|`. No `;`, `&`, + subshells or input redirection; output redirection only to a plain file name + in the working directory; no argument may leave that directory; an empty + environment; 5 seconds per command. On macOS each command also runs under + `sandbox-exec` with network access denied and writes allowed only in the + working directory; on Linux under `unshare --net` when available. None of + the allowed tools can open a connection, so the agent reaches nothing but + the app, through the recorded actions. +- **The app's origin.** On the web, an action that leaves the app's origin + ends the run. +- **Privacy.** Everything the tool returns is scrubbed with the same scrubber + as the structural listing, except the task's own values. + +Two gate fixes came with this work and apply to every compile-agent agent: the +harness now reads a click inside an iframe through the backend's hit-tested +frame chain (it used to see only the frame, so a Save inside a frame was not +gated), and a click on a text field is not treated as a save even when its +label holds a commit word ("Postal code" starts with "post"). + +## Measured + +claude-sonnet-5-5, effort medium, on the operator's key. Raw rows: +[benchmark/code_as_action/results/](../benchmark/code_as_action/results/). +The host's 1-minute load was far above its 12 cores for every run (other +sessions' work), so every timing below was taken under load and is labelled +that way in the data; tokens, calls, actions and cost do not depend on load. + +RESULTS_TABLES + +## Limits + +- The script language is fixed. An agent can't loop or branch inside a + script; it reads, decides, and sends the next script. +- `scroll` depends on where the last click left the pointer. +- Desktop listings need a backend `authoring_tree`; until then a desktop + agent finds controls from OCR text and acts by exact role and name. +- The trace check compares locators, not behaviour: a program that carries + the right locators can still fail at replay, which its canaries and the + record check catch. diff --git a/packages/flow/openadapt_flow/compile_agent/cli.py b/packages/flow/openadapt_flow/compile_agent/cli.py index dbb71d7792..4481f472ff 100644 --- a/packages/flow/openadapt_flow/compile_agent/cli.py +++ b/packages/flow/openadapt_flow/compile_agent/cli.py @@ -96,9 +96,17 @@ def add_parser(sub: argparse._SubParsersAction) -> None: p.add_argument( "--agent", default="claude", - help="claude (default), or a JSON file of scripted steps (no model calls)", + help=( + "claude (default), or a JSON file of scripted steps or code-action " + "scripts (no model calls)" + ), + ) + p.add_argument( + "--vision", + choices=["selective", "every_turn", "code"], + default="selective", + help="code: act through one terminal-style tool (docs/CODE_AS_ACTION.md)", ) - p.add_argument("--vision", choices=["selective", "every_turn"], default="selective") p.add_argument("--model", default=None, help="model id (default claude-opus-5-5)") p.add_argument("--effort", default="medium") p.add_argument("--max-cost", type=float, default=2.0, help="USD cap for run 1") @@ -160,6 +168,12 @@ def _agent(args: argparse.Namespace) -> Any: max_cost_usd=args.max_cost, ) raw = json.loads(Path(args.agent).read_text()) + if "scripts" in raw: + from openadapt_flow.compile_agent.code_action import ScriptedCodeAgent + + return ScriptedCodeAgent( + scripts=raw["scripts"], principal_id=raw.get("principal_id") + ) return ScriptedAgent(steps=raw["steps"], principal_id=raw.get("principal_id")) diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py index e33c384615..71d83c6288 100644 --- a/packages/flow/openadapt_flow/compile_agent/code_action.py +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -332,17 +332,6 @@ def line(self) -> str: }""" ) -_FOCUS_JS = r"""([sel, scopeSel]) => { - const clean = (s) => String(s || '').replace(/\s+/g, ' ').trim(); - let el = document.activeElement; - if (!el || el === document.body) return null; - if (el.tagName === 'IFRAME' || el.tagName === 'FRAME') return {frame: true}; - const scope = el.closest(scopeSel) || document.body; - return {name: clean(el.getAttribute('aria-label') || el.getAttribute('name') || - el.innerText || el.value).slice(0, 120), - scope: clean(scope.innerText).slice(0, 4000)}; -}""" - def _field_label_js() -> str: from openadapt_flow.backends.playwright_backend import _FIELD_LABEL_JS @@ -476,19 +465,7 @@ def at(self, element: Element) -> Optional[dict[str, Any]]: return None def focused(self) -> Optional[dict[str, Any]]: - try: - got = self.page.evaluate(_FOCUS_JS, [_observe.INTERACTIVE, _observe.SCOPE]) - if got and got.get("frame"): - for _index, frame, _ox, _oy in self._frames()[1:]: - inner = frame.evaluate( - _FOCUS_JS, [_observe.INTERACTIVE, _observe.SCOPE] - ) - if inner and not inner.get("frame"): - return inner - return None - return got - except Exception: # noqa: BLE001 - return None + return _observe.focused_element(self.page) def screen_text(self) -> str: texts: list[str] = [] @@ -933,7 +910,9 @@ def _line(self, line: str) -> tuple[str, Optional[bytes]]: text += f' now="{value}"' if element.context: text += f"\n row: {element.context}" - self.trace.saw(line, text + " " + element.context) + elif element.scope: + text += f"\n in: {element.scope[:300]}" + self.trace.saw(line, text) return self._out(text), None if command in ("click", "fill", "select", "press"): return self._act(command, line, positional, named), None diff --git a/packages/flow/openadapt_flow/compile_agent/observe.py b/packages/flow/openadapt_flow/compile_agent/observe.py index c7782fec04..1be31766c6 100644 --- a/packages/flow/openadapt_flow/compile_agent/observe.py +++ b/packages/flow/openadapt_flow/compile_agent/observe.py @@ -369,10 +369,28 @@ def element_at( def focused_element(page: Any) -> Optional[dict[str, Any]]: + """The focused control, inside a frame when focus is in one.""" + try: - return page.evaluate(_FOCUSED_JS, [INTERACTIVE, SCOPE]) + got = page.evaluate(_FOCUSED_JS, [INTERACTIVE, SCOPE]) except Exception: # noqa: BLE001 return None + if not got or str(got.get("tag")) not in ("iframe", "frame"): + return got + # Focus is inside a frame: read the frame whose document holds it. + main = getattr(page, "main_frame", None) + for frame in getattr(page, "frames", None) or []: + if frame is main: + continue + try: + if not frame.evaluate("() => document.hasFocus()"): + continue + inner = frame.evaluate(_FOCUSED_JS, [INTERACTIVE, SCOPE]) + except Exception: # noqa: BLE001 - a detached frame holds no focus + continue + if inner and str(inner.get("tag")) not in ("iframe", "frame"): + return inner + return None def page_text(page: Any) -> str: diff --git a/packages/flow/tests/e2e/test_code_action_e2e.py b/packages/flow/tests/e2e/test_code_action_e2e.py new file mode 100644 index 0000000000..b008554cd9 --- /dev/null +++ b/packages/flow/tests/e2e/test_code_action_e2e.py @@ -0,0 +1,264 @@ +"""Code as action end to end, in a real browser, with no model calls. + +* MockMed through the whole compile-agent pipeline with a scripted code + agent: run 1 passes its record check, every element the agent acted on is + on the compiled program's steps with the locator the recorder stored, the + proposed checks carry what the agent read, a person confirms, a key admits, + and run 2 replays with no model calls and is read back VERIFIED. +* The wrong patient is refused at the save; nothing is written or compiled. +* A form inside an iframe: the save gate reads the Save button inside the + frame (it used to see only the frame), a click on a field whose label holds + "post" is not a save, and the code tool fills and saves through the frame. +""" + +from __future__ import annotations + +import json +import threading +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Iterator + +import pytest + +from openadapt_flow.compile_agent import ( + CodeActionTool, + FirstRunCompiler, + GateRefused, + ScriptedCodeAgent, +) +from openadapt_flow.compile_agent.harness import Harness, SaveGate +from openadapt_flow.mockmed.fault_server import serve +from openadapt_flow.tutorial import _http_json, _records + +PROMPT = ( + "Open the referral for Jane Sample, start a Triage encounter, and save " + "this note: {note}" +) +NOTE = "Synthetic follow-up in two weeks" +NEW_NOTE = "Recheck knee in ten days" + +SCRIPTS = [ + "tree max=20", + 'click role=button name=Open near="{patient}"\ntext grep=MRN', + 'click name="New Encounter"\nclick name=Triage\nfill name=Note value="{note}"\n' + 'click name="Save Encounter"', + "text grep={note}", +] + + +@pytest.fixture(scope="module") +def mockmed() -> Iterator[str]: + base, _db, stop = serve() + yield base.rstrip("/") + stop() + + +def _compiler(tmp_path: Path, base: str, task_id: str) -> FirstRunCompiler: + verify = tmp_path / "verify.yaml" + verify.write_text( + "verifier:\n" + " kind: rest\n" + f" base_url: {base}\n" + " records_path: /api/db\n" + " records_key: records\n" + " timeout_s: 2.0\n" + " poll_interval_s: 0.05\n" + ) + return FirstRunCompiler( + PROMPT, + params=["note"], + url=f"{base}/?fault=ok&idempotency=demo#tasks", + verify=verify, + expect={"patient_id": "p1", "type": "Triage"}, + on_screen=["Jane Sample", "MRN P1"], + task_id=task_id, + cache_dir=tmp_path / "cache", + key_dir=tmp_path / "keys", + reset=lambda: _http_json(f"{base}/api/reset", method="POST", body={}), + on_warning=lambda text: None, + ) + + +def _scripts(patient: str) -> list[str]: + return [s.replace("{patient}", patient) for s in SCRIPTS] + + +def _admitter(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Any: + from openadapt_flow.jit import authority + + monkeypatch.setenv(authority.KEY_PASSPHRASE_ENV, "e2e passphrase") + pem = tmp_path / "keyfiles" / "admitter.pem" + authority.register_principal( + tmp_path / "keys", + role="admitter_key", + principal_id="ci:code-action", + public_key_hex=authority.generate_encrypted_key(pem), + ) + return authority.principal_from_key( + authority.load_encrypted_key(pem), tmp_path / "keys" + ) + + +def test_code_agent_run_compiles_into_a_program_that_replays( + mockmed: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + compiler = _compiler(tmp_path, mockmed, "mockmed.code_action") + agent = ScriptedCodeAgent(_scripts("Jane Sample")) + result = compiler.record(agent, {"note": NOTE}) + assert result.status == "draft", result.detail + + trace = result.run1["code_action"] + assert trace["surface"] == "dom" and trace["screenshots"] == 0 + assert trace["actions"] == trace["by_structure"] == 5 + assert trace["with_recorded_locator"] == trace["locator_unique"] == 5 + assert trace["commits"] == 1 + # The identity was read before the save, and the note after it. + assert trace["identity_read_before_commit"] == [ + "identity:Jane Sample", + "identity:MRN P1", + ] + assert trace["program"]["passed"] and trace["program"]["in_program"] == 5 + steps = json.loads( + (Path(result.run1["attempt"]) / "code_action_trace.json").read_text() + )["steps"] + assert [s["action"] for s in steps] == ["click", "click", "click", "fill", "click"] + assert steps[0]["element"]["row"].startswith("Jane Sample") + assert steps[0]["recorded"]["selector"] == "#open-p1" + assert all(s.get("matches_precheck") for s in steps if s.get("recorded")) + + proposed = compiler.proposed() + assert proposed["agent_read"]["program"]["passed"] + compiler.confirm_checks(principal_id="dana.reviewer", drop_fields=["key", "source"]) + admitted = compiler.admit( + admitter=_admitter(tmp_path, monkeypatch), + canary_params=[{"note": "Canary test note"}], + ) + assert admitted.admitted, admitted.detail + replay = compiler.run(note=NEW_NOTE) + assert replay.verified and replay.model_calls == 0 + assert replay.receipt_outcome == "VERIFIED" + rows = _records(mockmed) + assert [(r["patient_id"], r["note"]) for r in rows] == [("p1", NEW_NOTE)] + + +def test_code_agent_on_the_wrong_patient_is_refused_at_the_save( + mockmed: str, tmp_path: Path +) -> None: + compiler = _compiler(tmp_path, mockmed, "mockmed.code_action_wrong") + result = compiler.record( + ScriptedCodeAgent(_scripts("Alex Testcase")), {"note": NOTE} + ) + assert result.status == "refused_wrong_record", result.detail + assert "MRN P1" in (result.detail or "") + assert _records(mockmed) == [] + assert not (compiler.dir / "program").exists() + + +# -- a form in an iframe ------------------------------------------------------------ + +OUTER = """Clinic

Clinic

""" + +FORM = """Edit address +
+

{who}

+ + +

""" + + +@pytest.fixture +def frame_site(tmp_path: Path) -> Iterator[Any]: + site = tmp_path / "site" + site.mkdir() + + def write(who: str) -> None: + (site / "index.html").write_text(OUTER) + (site / "form.html").write_text(FORM.replace("{who}", who)) + + class Quiet(SimpleHTTPRequestHandler): + def __init__(self, *a: Any, **kw: Any) -> None: + super().__init__(*a, directory=str(site), **kw) + + def log_message(self, *a: Any) -> None: + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Quiet) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_address[1]}/index.html", write + server.shutdown() + + +def _frame_harness(url: str, tmp_path: Path, on_screen: list[str], dialect: str) -> Any: + from openadapt_flow.authoring import AuthoringSession + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + from openadapt_flow.jit.adapters import CaptureComputer + + backend, close = PlaywrightBackend.launch(url, headless=True) + backend.page.frame(name="form").wait_for_selector("#save") + session = AuthoringSession( + backend, tmp_path / "rec", backend_kind="web", app_url=url + ) + computer = CaptureComputer(session, dialect=dialect, page=backend.page) + session.start_record() + harness = Harness(computer, SaveGate(on_screen), evidence_dir=tmp_path / "gates") + return harness, backend, close + + +def test_the_gate_sees_a_save_inside_an_iframe(frame_site: Any, tmp_path: Path) -> None: + url, write = frame_site + write("Alex Testcase — MRN P2") + harness, backend, close = _frame_harness(url, tmp_path, ["MRN P1"], "anthropic") + try: + frame = backend.page.frame(name="form") + # Playwright boxes are in the page's coordinates, frames included. + inner = frame.locator("#save").bounding_box() + x = int(inner["x"] + inner["width"] / 2) + y = int(inner["y"] + inner["height"] / 2) + zip_box = frame.locator("#zip").bounding_box() + zx = int(zip_box["x"] + zip_box["width"] / 2) + zy = int(zip_box["y"] + zip_box["height"] / 2) + # A click on the Postal code field is not a save. + harness.act({"action": "left_click", "coordinate": [zx, zy]}) + assert harness.stats.gate_checks == 0 + # Enter in the frame's form is gated, and read inside the frame. + with pytest.raises(GateRefused, match="MRN P1"): + harness.act({"action": "key", "text": "Return"}) + assert "Postal code" in harness.gate_log[-1]["control"] + with pytest.raises(GateRefused, match="MRN P1"): + harness.act({"action": "left_click", "coordinate": [x, y]}) + assert frame.locator("#done").inner_text() == "" + finally: + close() + + +def test_the_code_tool_fills_and_saves_inside_an_iframe( + frame_site: Any, tmp_path: Path +) -> None: + url, write = frame_site + write("Jane Sample — MRN P1") + harness, backend, close = _frame_harness(url, tmp_path, ["MRN P1"], "generic") + try: + tool = CodeActionTool(harness, workdir=tmp_path / "work") + listing = tool.run("tree").text + assert "# frame 1: Edit address" in listing + assert 'textbox "Postal code"' in listing + out = tool.run( + 'fill name="Postal code" value="02139"\nclick name="Save address"\n' + "text grep=Saved" + ) + assert "(save gate passed)" in out.text, out.text + assert backend.page.frame(name="form").locator("#done").inner_text() == ( + "Saved 02139" + ) + steps = tool.trace.steps + assert [s["action"] for s in steps] == ["fill", "click"] + assert steps[1]["recorded"]["frame_path"] + assert harness.gate_log[0]["decision"] == "passed_no_hold" + finally: + close() diff --git a/packages/flow/tests/test_code_action.py b/packages/flow/tests/test_code_action.py new file mode 100644 index 0000000000..9d9f3014b9 --- /dev/null +++ b/packages/flow/tests/test_code_action.py @@ -0,0 +1,540 @@ +"""Code as action without a browser or a model. + +The sandboxed shell, the script language against a fake desktop surface (the +harness has no page, so the save gate works from the tool's hints), the +structured trace and its check against a compiled program, and the Claude +loop's ``code`` mode with a fake client. tests/e2e/test_code_action_e2e.py +runs the tool on MockMed and on a form inside an iframe in a real browser. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Optional + +import pytest + +from openadapt_flow.compile_agent import ( + ClaudeAgent, + CodeActionTool, + GateRefused, + ScriptedCodeAgent, + check_trace_against_ir, +) +from openadapt_flow.compile_agent.code_action import Element, LeftTheApp, PixelSurface +from openadapt_flow.compile_agent.harness import CommitHint, Harness, SaveGate +from openadapt_flow.compile_agent.sandbox import Sandbox, ShellRefused +from openadapt_flow.ir import StructuralLocator +from openadapt_flow.jit.compile_agent import TaskRequest + +# -- the shell ---------------------------------------------------------------------- + + +@pytest.fixture +def sandbox(tmp_path: Path) -> Sandbox: + box = Sandbox(tmp_path / "work") + box.write("page.txt", "row: Jane Sample | MRN P1\nrow: Alex Testcase | MRN P2") + return box + + +def test_the_shell_reads_files_in_its_directory(sandbox: Sandbox) -> None: + assert sandbox.run("grep -n P1 page.txt").output == "1:row: Jane Sample | MRN P1\n" + assert sandbox.run("grep MRN page.txt | wc -l").output.strip() == "2" + wrote = sandbox.run("grep P2 page.txt > p2.txt") + assert wrote.wrote == "p2.txt" + assert (sandbox.workdir / "p2.txt").read_text().startswith("row: Alex") + assert sandbox.run("cat p2.txt").exit_code == 0 + + +@pytest.mark.parametrize( + "command", + [ + "curl http://example.invalid", + "python -c 1", + "sh -c ls", + "cat /etc/passwd", + "cat ~/x", + "cat ../page.txt", + "cat sub/../../x", + "ls; cat page.txt", + "ls && ls", + "ls & ls", + "cat page.txt $(ls)", + "cat `ls`", + "cat < page.txt", + "ls > /tmp/x", + "ls > ../x", + "ls > a > b", + "sort -o out.txt page.txt", + "tail -f page.txt", + "ls |", + ], +) +def test_the_shell_refuses_anything_else(sandbox: Sandbox, command: str) -> None: + with pytest.raises(ShellRefused): + sandbox.run(command) + + +def test_the_shell_has_no_environment(sandbox: Sandbox) -> None: + # printenv is not allowed; echo of a variable is not expanded (no shell). + assert sandbox.run("echo $HOME").output.strip() == "$HOME" + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec is macOS") +def test_on_macos_each_command_runs_under_sandbox_exec(sandbox: Sandbox) -> None: + assert sandbox.isolation == "sandbox-exec" + + +# -- a fake desktop surface ------------------------------------------------------------ + + +class FakeSurface: + """A desktop window with a structured layer: named controls and text.""" + + kind = "tree" + + def __init__(self, screens: dict[str, list[dict[str, Any]]], start: str) -> None: + self.screens = screens + self.screen = start + self.unique = True + + def _elements(self) -> list[Element]: + out = [] + for i, item in enumerate(self.screens[self.screen], 1): + out.append( + Element( + ref=str(i), + role=item["role"], + name=item["name"], + x=10 * i, + y=20 * i, + value=item.get("value"), + context=item.get("row", ""), + scope=item.get("scope", ""), + tag=item.get("tag", ""), + options=tuple(item.get("options", ())), + source="tree", + ) + ) + return out + + def read(self) -> tuple[str, list[dict[str, Any]], list[Element]]: + elements = self._elements() + items = [{"kind": "control", "ref": e.ref, "row": -1} for e in elements] + return f"window: {self.screen}", items, elements + + def locate(self, role: str, name: str) -> Optional[Element]: + return None + + def origin(self) -> Optional[str]: + return None + + def at(self, element: Element) -> Optional[dict[str, Any]]: + item = next( + (i for i in self.screens[self.screen] if i["name"] == element.name), {} + ) + return { + "name": element.name, + "context": element.context, + "value": item.get("value"), + } + + def focused(self) -> Optional[dict[str, Any]]: + return {"name": "Note", "scope": "Jane Sample MRN P1"} + + def screen_text(self) -> str: + return "\n".join(f"{i['role']} {i['name']}" for i in self.screens[self.screen]) + + def locator_unique(self, locator: Any) -> Optional[bool]: + return self.unique + + +class FakeBackend: + def __init__(self, surface: FakeSurface) -> None: + self.surface = surface + + def structural_locator_at(self, x: int, y: int) -> StructuralLocator: + element = next(e for e in self.surface._elements() if (e.x, e.y) == (x, y)) + return StructuralLocator(role=element.role, name=element.name) + + +class FakeComputer: + """A recording computer with no page (a desktop surface).""" + + dialect = "generic" + page = None + + def __init__(self, surface: FakeSurface, flow: dict[str, str]) -> None: + self.surface = surface + self.flow = flow + self.executed: list[dict[str, Any]] = [] + self.pointer: Optional[tuple[int, int]] = None + self.focus: Optional[str] = None + self.session = SimpleNamespace(backend=FakeBackend(surface), _recorder=None) + + def screenshot(self) -> bytes: + return b"png" + + def execute(self, native: dict[str, Any]) -> Any: + self.executed.append(dict(native)) + if native["kind"] == "click": + self.pointer = (native["x"], native["y"]) + clicked = next( + e for e in self.surface._elements() if (e.x, e.y) == self.pointer + ) + self.focus = clicked.name + nxt = self.flow.get(clicked.name) + if nxt: + self.surface.screen = nxt + if native["kind"] == "type": + for item in self.surface.screens[self.surface.screen]: + if item["name"] == self.focus: + item["value"] = native["text"] + return SimpleNamespace(png=b"png", notes=[]) + + +SCREENS = { + "list": [ + {"role": "button", "name": "Open", "row": "Jane Sample | MRN P1"}, + {"role": "button", "name": "Open", "row": "Alex Testcase | MRN P2"}, + ], + "form": [ + {"role": "textbox", "name": "Note", "value": ""}, + {"role": "textbox", "name": "Postal code", "value": ""}, + { + "role": "combobox", + "name": "Type", + "tag": "select", + "value": "Consult", + "options": ["Consult", "Triage"], + }, + {"role": "button", "name": "Save note", "scope": "Jane Sample — MRN P1"}, + {"role": "button", "name": "Cancel"}, + ], +} + + +def _tool( + tmp_path: Path, on_screen: tuple[str, ...] = ("MRN P1",), **gate: Any +) -> tuple[CodeActionTool, FakeComputer, Harness]: + screens = {k: [dict(i) for i in v] for k, v in SCREENS.items()} + surface = FakeSurface(screens, "list") + computer = FakeComputer(surface, {"Open": "form", "Save note": "list"}) + harness = Harness( + computer, # type: ignore[arg-type] + SaveGate(on_screen, **gate), + evidence_dir=tmp_path / "gates", + ) + tool = CodeActionTool( + harness, + surface=surface, + watch={"identity:MRN P1": "MRN P1", "input:note": "Done"}, + ) + return tool, computer, harness + + +def test_a_listing_names_controls_with_refs(tmp_path: Path) -> None: + tool, _, _ = _tool(tmp_path) + out = tool.run("tree").text + assert '@1 button "Open"' in out and '@2 button "Open"' in out + + +def test_an_ambiguous_target_is_refused_with_the_candidates(tmp_path: Path) -> None: + tool, computer, _ = _tool(tmp_path) + out = tool.run("click role=button name=Open").text + assert "2 elements match" in out and "Alex Testcase" in out + assert computer.executed == [] + + +def test_one_script_reads_acts_and_saves_and_each_action_is_a_step( + tmp_path: Path, +) -> None: + tool, computer, harness = _tool(tmp_path) + out = tool.run( + 'click role=button name=Open near="Jane Sample"\n' + 'fill name=Note value="Done"\n' + 'click name="Postal code"\n' + "select name=Type option=Triage\n" + 'click name="Save note"\n' + "tree" + ) + assert 'ok: click button "Save note" (save gate passed)' in out.text + # The write stopped the script: the trailing tree did not run. + assert out.committed and "1 lines not run" in out.text + steps = tool.trace.steps + assert [s["action"] for s in steps] == ["click", "fill", "click", "select", "click"] + assert steps[0]["element"] == { + "role": "button", + "name": "Open", + "row": "Jane Sample | MRN P1", + "source": "tree", + } + assert steps[1]["value"] == "Done" and steps[1]["value_after"] == "Done" + assert steps[3]["value"] == "Triage" + assert all(s["recorded"]["role"] for s in steps) + assert [s["gated"] for s in steps] == [False, False, False, False, True] + # Postal code holds "post" but a click on a text field commits nothing. + assert len(harness.gate_log) == 1 + assert harness.gate_log[0]["decision"] == "passed_no_hold" + # Each action became generic clicks and keys at element centers. + kinds = [n["kind"] for n in computer.executed] + assert kinds == [ + "click", + "click", + "type", + "click", + "click", + "type", + "press", + "click", + ] + summary = tool.trace.summary() + assert summary["by_structure"] == 5 and summary["commits"] == 1 + assert summary["locator_agrees"] == 5 + + +def test_a_save_on_the_wrong_record_is_refused_before_it_is_performed( + tmp_path: Path, +) -> None: + tool, computer, harness = _tool(tmp_path, on_screen=("MRN P2",)) + tool.run('click role=button name=Open near="Jane Sample"') + with pytest.raises(GateRefused, match="MRN P2"): + tool.run('click name="Save note"') + assert computer.executed[-1]["kind"] == "click" + assert len(computer.executed) == 1 + assert harness.gate_log[-1]["decision"] == "refused_wrong_record" + + +def test_the_hold_is_asked_before_the_save(tmp_path: Path) -> None: + asked: list[Any] = [] + tool, computer, _ = _tool(tmp_path, hold=lambda req: asked.append(req) or False) + tool.run('click role=button name=Open near="Jane Sample"') + with pytest.raises(GateRefused, match="hold"): + tool.run('click name="Save note"') + assert asked and asked[0].control == "Save note" + + +def test_without_a_page_an_unhinted_click_is_gated(tmp_path: Path) -> None: + _tool_, computer, harness = _tool(tmp_path, on_screen=("MRN P9",)) + with pytest.raises(GateRefused): + harness.act({"kind": "click", "x": 1, "y": 2}) + assert computer.executed == [] + # A hint can add a gate, and Enter on a hinted control is gated. + with pytest.raises(GateRefused): + harness.act( + {"kind": "press", "key": "Enter"}, + hint=CommitHint(name="Note", scope="", key="Enter"), + ) + + +def test_fill_refuses_a_field_that_holds_text(tmp_path: Path) -> None: + tool, computer, _ = _tool(tmp_path) + tool.run('click role=button name=Open near="Jane Sample"') + tool.surface.screens["form"][0]["value"] = "old" + out = tool.run('fill name=Note value="new"').text + assert "already holds text" in out + assert len(computer.executed) == 1 + + +def test_select_refuses_an_option_the_list_does_not_have(tmp_path: Path) -> None: + tool, computer, _ = _tool(tmp_path) + tool.run('click role=button name=Open near="Jane Sample"') + out = tool.run("select name=Type option=Surgery").text + assert "not an option" in out and '"Triage"' in out + assert len(computer.executed) == 1 + + +def test_a_stale_ref_and_unknown_commands_are_errors_not_actions( + tmp_path: Path, +) -> None: + tool, computer, _ = _tool(tmp_path) + assert "not in the latest listing" in tool.run("click @9").text + assert "unknown command" in tool.run("delete everything").text + assert "find needs" in tool.run("find").text + assert computer.executed == [] + assert tool.trace.errors == 3 + + +def test_reads_are_evidence_and_shell_output_is_kept_on_disk(tmp_path: Path) -> None: + tool, _, _ = _tool(tmp_path) + out = tool.run("tree\nsave list.txt\nsh grep -c Open list.txt").text + assert out.rstrip().endswith("2") + assert tool.trace.shell_commands == 1 + assert tool.trace.reads[0]["shows"] == [] + tool.run('click role=button name=Open near="Jane Sample"\nread name="Save note"') + assert "identity:MRN P1" in tool.trace.reads[-1]["shows"] + + +def test_screenshots_only_when_asked(tmp_path: Path) -> None: + tool, _, harness = _tool(tmp_path) + assert tool.run("tree").images == [] + out = tool.run("screenshot") + assert len(out.images) == 1 and harness.stats.screenshots_requested == 1 + + +def test_scroll_needs_a_pointer(tmp_path: Path) -> None: + tool, computer, _ = _tool(tmp_path) + assert "click inside the area" in tool.run("scroll down").text + tool.run('click role=button name=Open near="Jane Sample"') + assert "ok: scroll down 500px" in tool.run("scroll down").text + assert computer.executed[-1] == {"kind": "scroll", "dx": 0, "dy": 500} + + +def test_leaving_the_app_ends_the_run(tmp_path: Path) -> None: + tool, _, _ = _tool(tmp_path) + tool.app_origin = "http://app.test" + tool.surface.origin = lambda: "https://elsewhere.example" # type: ignore[method-assign] + with pytest.raises(LeftTheApp): + tool.run('click role=button name=Open near="Jane Sample"') + + +def test_pixel_surface_targets_ocr_text(tmp_path: Path) -> None: + lines = [ + SimpleNamespace(text="Jane Sample MRN P1", region=(0, 0, 100, 20)), + SimpleNamespace(text="Save", region=(200, 300, 40, 20)), + ] + session = SimpleNamespace(screenshot=lambda: b"png") + surface = PixelSurface(session, ocr=lambda png: lines) + _header, _items, elements = surface.read() + assert [(e.name, e.x, e.y, e.source) for e in elements] == [ + ("Jane Sample MRN P1", 50, 10, "ocr"), + ("Save", 220, 310, "ocr"), + ] + assert "MRN P1" in elements[1].scope + + +# -- the trace against the compiled program ----------------------------------------- + + +def _workflow(*locators: Optional[dict[str, Any]]) -> Any: + steps = [] + for loc in locators: + structural = StructuralLocator(**loc) if loc else None + steps.append(SimpleNamespace(anchor=SimpleNamespace(structural=structural))) + return SimpleNamespace(steps=steps) + + +def test_the_program_must_carry_every_locator_in_order() -> None: + trace = { + "steps": [ + { + "i": 0, + "recorded": {"selector": "#open-p1", "role": "button", "name": "Open"}, + }, + { + "i": 1, + "recorded": {"selector": "#note", "role": "textbox", "name": "Note"}, + }, + {"i": 2}, + ] + } + ok = check_trace_against_ir( + trace, + _workflow( + {"selector": "#open-p1", "role": "button", "name": "Open"}, + None, + {"selector": "#note", "role": "textbox", "name": "Note"}, + ), + ) + assert ok["passed"] and ok["in_program"] == 2 + swapped = check_trace_against_ir( + trace, + _workflow( + {"selector": "#note", "role": "textbox", "name": "Note"}, + {"selector": "#open-p1", "role": "button", "name": "Open"}, + ), + ) + assert not swapped["passed"] and swapped["missing_steps"] == [1] + + +# -- the scripted agent and the Claude loop ----------------------------------------- + + +def _request() -> TaskRequest: + return TaskRequest( + task_id="t", instruction="Save the note", inputs={"note": "Done"} + ) + + +def test_the_scripted_agent_fills_inputs_and_fails_on_an_error(tmp_path: Path) -> None: + tool, _, harness = _tool(tmp_path) + agent = ScriptedCodeAgent( + [ + 'click role=button name=Open near="Jane Sample"\nfill name=Note value="{note}"' + ] + ) + import openadapt_flow.compile_agent.code_action as ca + + original = ca.surface_for + ca.surface_for = lambda h: tool.surface # type: ignore[assignment] + try: + ledger = agent.drive(_request(), harness) + assert ledger.finished and ledger.model_calls == 0 + assert agent.last_trace.steps[1]["value"] == "Done" + with pytest.raises(ca.ScriptError): + ScriptedCodeAgent(["click @7"]).drive(_request(), harness) + finally: + ca.surface_for = original + + +class FakeClaude: + """Answers with scripts, like a model using the run tool would.""" + + def __init__(self, scripts: list[str]) -> None: + self.scripts = list(scripts) + self.seen: list[Any] = [] + self.messages = SimpleNamespace(create=self.create) + + def create(self, **kw: Any) -> Any: + self.seen.append(kw) + if self.scripts: + block = { + "type": "tool_use", + "id": f"t{len(self.seen)}", + "name": "run", + "input": {"script": self.scripts.pop(0)}, + } + else: + block = { + "type": "tool_use", + "id": "done", + "name": "finish", + "input": {"summary": "saved"}, + } + return SimpleNamespace( + content=[block], + stop_reason="tool_use", + usage=SimpleNamespace( + input_tokens=100, + output_tokens=10, + cache_creation_input_tokens=0, + cache_read_input_tokens=0, + ), + ) + + +def test_the_claude_loop_in_code_mode(tmp_path: Path, monkeypatch: Any) -> None: + tool, computer, harness = _tool(tmp_path) + import openadapt_flow.compile_agent.code_action as ca + + monkeypatch.setattr(ca, "surface_for", lambda h: tool.surface) + client = FakeClaude( + [ + 'click role=button name=Open near="Jane Sample"\n' + 'fill name=Note value="Done"\nclick name="Save note"', + "text", + ] + ) + agent = ClaudeAgent(vision="code", model="claude-sonnet-5-5", client=client) + assert agent.dialect == "generic" + ledger = agent.drive(_request(), harness) + assert ledger.finished and ledger.summary == "saved" + assert ledger.model_calls == 3 + first = client.seen[0] + assert [t["name"] for t in first["tools"]] == ["run", "finish"] + assert "tree" in first["messages"][0]["content"][1]["text"] + assert ledger.to_dict()["images_in_requests"] == 0 + assert agent.last_trace.summary()["commits"] == 1 + assert harness.stats.gate_checks == 1 From bc796fa4866ad7d330a82215fe8be593290c4192 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 16:04:20 -0400 Subject: [PATCH 03/13] fix(flow): Enter on a native drop-down is not a save; set aside stopped OpenEMR runs Co-Authored-By: Claude Opus 5.5 --- .../results/openemr/SUPERSEDED.md | 17 +++++++++++--- .../results/openemr/superseded.jsonl | 2 ++ .../code_as_action/results/spend.json | 12 +++++++++- .../compile_agent/code_action.py | 6 ++++- .../openadapt_flow/compile_agent/harness.py | 5 ++++ packages/flow/tests/test_code_action.py | 23 +++++++++++++++++++ 6 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md b/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md index 9a13e928b8..412e113867 100644 --- a/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md +++ b/packages/flow/benchmark/code_as_action/results/openemr/SUPERSEDED.md @@ -1,3 +1,14 @@ -superseded.jsonl: one screenshot-loop run (r1, $0.028) refused by the save gate because the -harness read keyboard focus only in the top page, not inside OpenEMR's form iframe. Fixed in -observe.focused_element; the run was not counted and the arms were measured again. +# Runs set aside + +`superseded.jsonl` keeps runs from blocks that were stopped because the save +gate had a bug that refused the screenshot-loop agent. None of them is in +`runs.jsonl` or the summary. + +1. r1, every_turn ($0.028): Enter with focus inside OpenEMR's form iframe was + read in the top page, so the gate saw no form around it. Fixed in + `observe.focused_element` (follows the frame that holds focus). +2. r1, every_turn ($0.105): Enter on the State drop-down was treated as a + save. Enter on a native drop-down picks an option and does not submit the + form; fixed in `Harness._commit_target`. +3. r1, code ($0.056, correct, 12/12 fields): valid, but from the block stopped + at (2); the block was run again from a reset fixture. diff --git a/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl b/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl index a28a6a24cd..caaf2c67d8 100644 --- a/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl +++ b/packages/flow/benchmark/code_as_action/results/openemr/superseded.jsonl @@ -1 +1,3 @@ {"actions": 3, "agent_seconds": 6.538, "agent_summary": null, "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 8823, "cache_read_input_tokens": 6006, "calls": [{"cache_creation_input_tokens": 6006, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 2.0, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6006, "images_in_request": 3, "input_tokens": 2, "output_tokens": 356, "seconds": 3.402, "stop_reason": "tool_use", "turn": 2}], "code_action": null, "correct": false, "cost_usd": 0.027621, "error": "GateRefused: 'Enter on the focused element' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it", "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [{"control": "Enter on the focused element", "decision": "refused_wrong_record", "missing": ["Create New Patient"]}], "host_load_1m": [250.9, 233.0], "images_in_requests": 4, "input_tokens": 6, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 2, "model_seconds": 5.402, "new_charts": 0, "output_tokens": 435, "program": null, "prompt_tokens": 14835, "referral": "r1", "round": 1, "run_seconds": 6.784, "screenshots_sent": 4, "scripts": null, "started_at": "2026-10-18T19:52:42Z", "tool_errors": 0, "under_load": true} +{"actions": 29, "agent_seconds": 45.046, "agent_summary": null, "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 26879, "cache_read_input_tokens": 129172, "calls": [{"cache_creation_input_tokens": 0, "cache_read_input_tokens": 6006, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 2.302, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6006, "images_in_request": 3, "input_tokens": 2, "output_tokens": 300, "seconds": 3.24, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 16621, "cache_read_input_tokens": 8823, "images_in_request": 15, "input_tokens": 2, "output_tokens": 103, "seconds": 3.419, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 192, "cache_read_input_tokens": 25444, "images_in_request": 16, "input_tokens": 2, "output_tokens": 128, "seconds": 2.373, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1566, "cache_read_input_tokens": 25636, "images_in_request": 18, "input_tokens": 2, "output_tokens": 120, "seconds": 2.676, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 2853, "cache_read_input_tokens": 27202, "images_in_request": 20, "input_tokens": 2, "output_tokens": 97, "seconds": 2.551, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 2830, "cache_read_input_tokens": 30055, "images_in_request": 22, "input_tokens": 2, "output_tokens": 402, "seconds": 4.065, "stop_reason": "tool_use", "turn": 7}], "code_action": null, "correct": false, "cost_usd": 0.105354, "error": "GateRefused: 'Enter on Unassigned Alabama Alaska Arizona Arkansas California Colorado Connecticut Delaw' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it", "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [{"control": "Enter on Unassigned Alabama Alaska Arizona Arkansas California Colorado Connecticut Delaw", "decision": "refused_wrong_record", "missing": ["Create New Patient"]}], "host_load_1m": [252.7, 436.4], "images_in_requests": 95, "input_tokens": 16, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 7, "model_seconds": 20.627, "new_charts": 0, "output_tokens": 1229, "program": null, "prompt_tokens": 156067, "referral": "r1", "round": 1, "run_seconds": 46.342, "screenshots_sent": 30, "scripts": null, "started_at": "2026-10-18T19:56:23Z", "tool_errors": 2, "under_load": true} +{"actions": 36, "agent_seconds": 51.645, "agent_summary": "Registered Ms. Nadia Brennan with all given fields. The duplicate check showed no matches, so I confirmed the save. The chart opened as patient ID 2.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 11360, "cache_read_input_tokens": 87543, "calls": [{"cache_creation_input_tokens": 2973, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 117, "seconds": 1.244, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 874, "cache_read_input_tokens": 2973, "images_in_request": 0, "input_tokens": 2, "output_tokens": 50, "seconds": 0.932, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 404, "cache_read_input_tokens": 3847, "images_in_request": 0, "input_tokens": 2, "output_tokens": 67, "seconds": 1.043, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 1577, "cache_read_input_tokens": 4251, "images_in_request": 0, "input_tokens": 2, "output_tokens": 87, "seconds": 1.214, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 276, "cache_read_input_tokens": 5828, "images_in_request": 0, "input_tokens": 2, "output_tokens": 69, "seconds": 1.142, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 1475, "cache_read_input_tokens": 6104, "images_in_request": 1, "input_tokens": 2, "output_tokens": 86, "seconds": 2.546, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 134, "cache_read_input_tokens": 7579, "images_in_request": 1, "input_tokens": 2, "output_tokens": 74, "seconds": 1.321, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 526, "cache_read_input_tokens": 7713, "images_in_request": 1, "input_tokens": 2, "output_tokens": 132, "seconds": 1.926, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 952, "cache_read_input_tokens": 8239, "images_in_request": 1, "input_tokens": 2, "output_tokens": 76, "seconds": 2.043, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 326, "cache_read_input_tokens": 9191, "images_in_request": 1, "input_tokens": 2, "output_tokens": 52, "seconds": 1.757, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 1523, "cache_read_input_tokens": 9517, "images_in_request": 2, "input_tokens": 2, "output_tokens": 72, "seconds": 1.814, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 221, "cache_read_input_tokens": 11040, "images_in_request": 2, "input_tokens": 2, "output_tokens": 52, "seconds": 1.866, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 99, "cache_read_input_tokens": 11261, "images_in_request": 2, "input_tokens": 2, "output_tokens": 101, "seconds": 1.923, "stop_reason": "tool_use", "turn": 13}], "code_action": {"actions": 20, "by_structure": 18, "commits": 1, "errors": 2, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:country_code", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 33, "locator_agrees": 18, "locator_unique": 18, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 2, "scripts": 13, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 18}, "correct": true, "cost_usd": 0.056315, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [491.2, 443.1], "images_in_requests": 11, "input_tokens": 28, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 13, "model_seconds": 20.77, "new_charts": 1, "output_tokens": 1035, "program": {"in_program": 16, "missing_steps": [6, 7], "passed": false, "program_structural_steps": 16, "trace_steps_with_locator": 18}, "prompt_tokens": 98931, "referral": "r1", "round": 1, "run_seconds": 51.669, "screenshots_sent": 2, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @15 value=Nadia\nok: fill textbox \"First Name\" value=\"Nadia\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @17 value=Brennan\nok: fill textbox \"Last Name\" value=\"Brennan\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @23 value=1971-04-19\nok: fill textbox \"form_DOB\" value=\"1971-04-19\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ tree grep=Contact max=60\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n@40 button \"Contact\" (focused)", "script": "select @14 option=Ms.\nfill @15 value=Nadia\nfill @17 value=Brennan\nfill @23 value=1971-04-19\nselect @24 option=Female\nclick @40\ntree grep=Contact max=60"}, {"images": 0, "output": "$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@15 textbox \"First Name\" value=\"Nadia\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Brennan\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1971-04-19\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"", "script": "find role=textbox"}, {"images": 0, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ tree max=120\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Nadia\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Brennan\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1971-04-19\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@", "script": "click @40\ntree max=120"}, {"images": 0, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "click @40\nscroll down px=600\nfind role=textbox"}, {"images": 1, "output": "$ find name=\"form_street\"\nno match\n$ find role=combobox name=\"form_state\"\nno match\n$ screenshot\n(screenshot attached)", "script": "find name=\"form_street\"\nfind role=combobox name=\"form_state\"\nscreenshot"}, {"images": 0, "output": "$ click text=\"Billing Note:\"\nerror: nothing matches text=\"Billing Note:\"\n(stopped; 2 lines not run)", "script": "click text=\"Billing Note:\"\nscroll down px=700\nscreenshot"}, {"images": 0, "output": "$ click role=textbox name=\"form_billing_note\"\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ scroll down px=700\nok: scroll down 700px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"\n@15 textbox \"form_city\" value=\"\"\n@18 textbox \"form_postal_code\" value=\"\"\n@23 textbox \"form_mothersname\" value=\"\"\n@24 textbox \"form_contact_relationship\" value=\"\"\n@25 textbox \"form_phone_contact\" value=\"\"\n@26 textbox \"form_phone_home\" value=\"\"\n@27 textbox \"form_phone_biz\" value=\"\"\n@28 textbox \"form_phone_cell\" value=\"\"\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"", "script": "click role=textbox name=\"form_billing_note\"\nscroll down px=700\nfind role=textbox"}, {"images": 0, "output": "$ fill @13 value=\"27 WILLOW BEND ROAD\"\nok: fill textbox \"form_street\" value=\"27 WILLOW BEND ROAD\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @15 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @18 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @26 value=555-0121\nok: fill textbox \"form_phone_home\" value=\"555-0121\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ fill @29 value=nadia.brennan@example.invalid\nok: fill textbox \"form_email\" value=\"nadia.brennan@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ find role=combobox\n@16 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]", "script": "fill @13 value=\"27 WILLOW BEND ROAD\"\nfill @15 value=EXAMPLETON\nfill @18 value=02139\nfill @26 value=555-0121\nfill @29 value=nadia.brennan@example.invalid\nfind role=combobox"}, {"images": 0, "output": "$ select @16 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ select @19 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"", "script": "select @16 option=Massachusetts\nselect @19 option=USA\nfind name=\"Create New Patient\""}, {"images": 1, "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n$ screenshot\n(screenshot attached)", "script": "click @38\nscreenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=qI7smnZmWGlAwhx73iKu39akvOpqn1ToKuMorV5S)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Brennan\nNadia Brennan (2)\nMedical Record Dashboard - Nadia Brennan", "script": "text grep=Brennan"}], "started_at": "2026-10-18T19:57:44Z", "tool_errors": 0, "under_load": true} diff --git a/packages/flow/benchmark/code_as_action/results/spend.json b/packages/flow/benchmark/code_as_action/results/spend.json index e33b20ca88..2c1c15cad1 100644 --- a/packages/flow/benchmark/code_as_action/results/spend.json +++ b/packages/flow/benchmark/code_as_action/results/spend.json @@ -99,8 +99,18 @@ "label": "openemr every_turn r1", "usd": 0.027621, "at": "2026-10-18T19:52:51Z" + }, + { + "label": "openemr every_turn r1", + "usd": 0.105354, + "at": "2026-10-18T19:57:41Z" + }, + { + "label": "openemr code r1", + "usd": 0.056315, + "at": "2026-10-18T20:00:49Z" } ], - "total_usd": 0.613951, + "total_usd": 0.77562, "cap_usd": 15.0 } diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py index 71d83c6288..ed3b257a82 100644 --- a/packages/flow/openadapt_flow/compile_agent/code_action.py +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -1109,7 +1109,11 @@ def _act( hint = CommitHint( name=str(focus.get("name") or "the focused element"), scope=str(focus.get("scope") or ""), - key=key if key in ("Enter", "Space") else None, + key=( + key + if key in ("Enter", "Space") and focus.get("tag") != "select" + else None + ), ) event = self._event_index() self._gates_before = len(self.harness.gate_log) diff --git a/packages/flow/openadapt_flow/compile_agent/harness.py b/packages/flow/openadapt_flow/compile_agent/harness.py index a59b5efdf9..17ee491371 100644 --- a/packages/flow/openadapt_flow/compile_agent/harness.py +++ b/packages/flow/openadapt_flow/compile_agent/harness.py @@ -313,6 +313,11 @@ def _commit_target( focused = _observe.focused_element(self.page) or {} if op.key == "Space" and focused.get("editable"): continue + # Enter on a native drop-down picks an option; it does not + # submit the form. (A text area stays gated: chat-style + # boxes send on Enter.) + if op.key == "Enter" and focused.get("tag") == "select": + continue label = str(focused.get("name") or "the focused element") return (f"{op.key} on {label}", focused.get("scope")) return None diff --git a/packages/flow/tests/test_code_action.py b/packages/flow/tests/test_code_action.py index 9d9f3014b9..d74e2b887d 100644 --- a/packages/flow/tests/test_code_action.py +++ b/packages/flow/tests/test_code_action.py @@ -329,6 +329,29 @@ def test_without_a_page_an_unhinted_click_is_gated(tmp_path: Path) -> None: ) +def test_enter_on_a_native_drop_down_is_not_a_save(tmp_path: Path) -> None: + from openadapt_flow.compile_agent import observe + + class Page: + url = "http://app.test/" + + def evaluate(self, script: str, arg: Any = None) -> Any: + return {"tag": "select", "name": "State", "scope": "", "editable": False} + + computer = SimpleNamespace( + dialect="generic", + page=Page(), + pointer=None, + executed=[], + screenshot=lambda: b"png", + ) + computer.execute = lambda native: computer.executed.append(native) + harness = Harness(computer, SaveGate(["MRN P1"])) # type: ignore[arg-type] + harness.act({"kind": "press", "key": "Enter"}) + assert harness.stats.gate_checks == 0 and computer.executed + assert observe.focused_element(Page())["tag"] == "select" + + def test_fill_refuses_a_field_that_holds_text(tmp_path: Path) -> None: tool, computer, _ = _tool(tmp_path) tool.run('click role=button name=Open near="Jane Sample"') From 746d17845a9d734bb81da4297f8f20130ec87191 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 16:06:23 -0400 Subject: [PATCH 04/13] fix(flow): a layout table row is not the gate's record scope Co-Authored-By: Claude Opus 5.5 --- packages/flow/openadapt_flow/compile_agent/observe.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/flow/openadapt_flow/compile_agent/observe.py b/packages/flow/openadapt_flow/compile_agent/observe.py index 1be31766c6..05d49df74a 100644 --- a/packages/flow/openadapt_flow/compile_agent/observe.py +++ b/packages/flow/openadapt_flow/compile_agent/observe.py @@ -143,7 +143,13 @@ _DESCRIBE_JS = """ function oaDescribe(c, interactive, scopeSel) { - var scope = c.closest(scopeSel) || document.body; + var scope = c.closest(scopeSel); + // A table row with many controls is page layout, not a record row: the + // scope is the form or section around it. + while (scope && scope.matches('tr, [role=row]') && + scope.querySelectorAll('input, select, textarea, button').length > 8) + scope = scope.parentElement && scope.parentElement.closest(scopeSel); + scope = scope || document.body; return {tag: c.tagName.toLowerCase(), type: c.type || '', name: oaName(c), role: c.getAttribute('role') || '', interactive: interactive, editable: c.isContentEditable || c.tagName === 'TEXTAREA' || From 24313234bb5380ce835202d384817742c2cb889d Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 16:09:25 -0400 Subject: [PATCH 05/13] bench(code-as-action): one OpenEMR block per agent, reset before each Co-Authored-By: Claude Opus 5.5 --- .../code_as_action/measure_openemr.py | 33 +++++++++++-------- .../code_as_action/results/openemr/runs.jsonl | 1 + .../code_as_action/results/spend.json | 7 +++- 3 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 0ddea92221..5fcf16b428 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -391,11 +391,12 @@ def main() -> int: fixture = qualify_hooks.fixture() demo = _demo() - meta: dict[str, Any] = {"started_at": common.now()} - if args.reset: - meta["baseline_sha256"] = fixture.reset() + meta: dict[str, Any] = {"started_at": common.now(), "blocks": []} + + def reset() -> dict[str, Any]: existing = SyntheticPatientSpec(**EXISTING, **COMMON_FIELDS) - meta["seeded_existing_chart_status"] = ( + done = {"baseline_sha256": fixture.reset(), "at": common.now()} + done["seeded_existing_chart_status"] = ( fixture.token_session("actor") .post( fixture.api_base_url + "/apis/default/api/patient", @@ -403,20 +404,25 @@ def main() -> int: ) .status_code ) + return done + work_root = Path(tempfile.mkdtemp(prefix="code-action-openemr-")) rows_path = args.out / "runs.jsonl" try: + # A block is one agent over the referrals from a reset fixture, as in + # the referral protocol: each referral can be registered only once. for round_ in range(1, args.rounds + 1): - for i, key in enumerate(keys): - # Alternate which arm goes first, so drift hits both. - order = arms if (i + round_) % 2 == 0 else list(reversed(arms)) - for arm in order: + shift = (round_ - 1) % len(arms) + for arm in arms[shift:] + arms[:shift]: + block = {"round": round_, "arm": arm} + if args.reset: + block.update(reset()) + meta["blocks"].append(block) + k = (round_ - 1) % len(keys) + for key in keys[k:] + keys[:k]: if key not in MISMATCH and sql_patients( fixture, f"email={_q(REFERRALS[key]['email'])}" ): - # The other arm already registered this referral; a - # second save would be a duplicate. Remove nothing: - # skip it and say so. print(f"{key} {arm}: skipped (chart exists)", flush=True) continue stopped = common.may_start(args.max_run_usd) @@ -435,8 +441,9 @@ def main() -> int: f"{key} {arm}: correct={row['correct']} new={row['new_charts']} " f"fields={row['fields_matched']}/{row['fields_total']} " f"{row['model_calls']} calls {row['actions']} actions " - f"{row['prompt_tokens']} prompt tokens ${float(row['cost_usd'] or 0):.4f} " - f"{row['run_seconds']} s load {row['host_load_1m']} err={row['error']} " + f"{row['prompt_tokens']} prompt tokens " + f"${float(row['cost_usd'] or 0):.4f} {row['run_seconds']} s " + f"load {row['host_load_1m']} err={row['error']} " f"(total ${total:.2f})", flush=True, ) diff --git a/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl b/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl new file mode 100644 index 0000000000..b8faf660f5 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl @@ -0,0 +1 @@ +{"actions": 69, "agent_seconds": 105.866, "agent_summary": "I registered Nadia Brennan and saved the chart. It opened as patient ID 2, with DOB 1971-04-19.\n\nThe EMR's duplicate check came back with \"No matches were found,\" so I confirmed creation. No existing patient was flagged.\n\nAll the values you gave were entered: title Ms., first name Nadia, last name Brennan, DOB 1971-04-19, sex Female, address, city, state Massachusetts, postal code 02139, home phone 555-0121, email nadia.brennan@example.invalid, and country USA.\n\nOne thing differs from your fields: the form marks Birth Sex as required, and your data only gave \"sex.\" I set Birth Sex to Female as well as Sex to Female. I left every other field blank.", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 98993, "cache_read_input_tokens": 738084, "calls": [{"cache_creation_input_tokens": 6006, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 1.936, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6006, "images_in_request": 3, "input_tokens": 2, "output_tokens": 261, "seconds": 3.182, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 13864, "cache_read_input_tokens": 8823, "images_in_request": 13, "input_tokens": 2, "output_tokens": 284, "seconds": 4.416, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 7109, "cache_read_input_tokens": 22687, "images_in_request": 19, "input_tokens": 2, "output_tokens": 128, "seconds": 2.887, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1565, "cache_read_input_tokens": 29796, "images_in_request": 21, "input_tokens": 2, "output_tokens": 147, "seconds": 2.982, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 4240, "cache_read_input_tokens": 31361, "images_in_request": 24, "input_tokens": 2, "output_tokens": 163, "seconds": 3.892, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 252, "cache_read_input_tokens": 35601, "images_in_request": 25, "input_tokens": 2, "output_tokens": 188, "seconds": 3.057, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 4280, "cache_read_input_tokens": 35853, "images_in_request": 28, "input_tokens": 2, "output_tokens": 165, "seconds": 3.538, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 2896, "cache_read_input_tokens": 40133, "images_in_request": 30, "input_tokens": 2, "output_tokens": 137, "seconds": 3.264, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 4227, "cache_read_input_tokens": 43029, "images_in_request": 33, "input_tokens": 2, "output_tokens": 166, "seconds": 3.857, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 4258, "cache_read_input_tokens": 47256, "images_in_request": 36, "input_tokens": 2, "output_tokens": 173, "seconds": 3.932, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 4263, "cache_read_input_tokens": 51514, "images_in_request": 39, "input_tokens": 2, "output_tokens": 112, "seconds": 3.817, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 4204, "cache_read_input_tokens": 55777, "images_in_request": 42, "input_tokens": 2, "output_tokens": 418, "seconds": 4.669, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 24892, "cache_read_input_tokens": 59981, "images_in_request": 60, "input_tokens": 2, "output_tokens": 277, "seconds": 7.439, "stop_reason": "tool_use", "turn": 14}, {"cache_creation_input_tokens": 5729, "cache_read_input_tokens": 84873, "images_in_request": 64, "input_tokens": 2, "output_tokens": 98, "seconds": 4.081, "stop_reason": "tool_use", "turn": 15}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 90602, "images_in_request": 67, "input_tokens": 2, "output_tokens": 109, "seconds": 4.488, "stop_reason": "tool_use", "turn": 16}, {"cache_creation_input_tokens": 4201, "cache_read_input_tokens": 94792, "images_in_request": 70, "input_tokens": 2, "output_tokens": 240, "seconds": 5.982, "stop_reason": "end_turn", "turn": 17}], "code_action": null, "correct": true, "cost_usd": 0.426621, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [180.1, 114.8], "images_in_requests": 575, "input_tokens": 36, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 17, "model_seconds": 67.421, "new_charts": 1, "output_tokens": 3145, "program": null, "prompt_tokens": 837113, "referral": "r1", "round": 1, "run_seconds": 106.218, "screenshots_sent": 70, "scripts": null, "started_at": "2026-10-18T20:06:37Z", "tool_errors": 3, "under_load": true} diff --git a/packages/flow/benchmark/code_as_action/results/spend.json b/packages/flow/benchmark/code_as_action/results/spend.json index 2c1c15cad1..407f33f158 100644 --- a/packages/flow/benchmark/code_as_action/results/spend.json +++ b/packages/flow/benchmark/code_as_action/results/spend.json @@ -109,8 +109,13 @@ "label": "openemr code r1", "usd": 0.056315, "at": "2026-10-18T20:00:49Z" + }, + { + "label": "openemr every_turn r1", + "usd": 0.426621, + "at": "2026-10-18T20:08:39Z" } ], - "total_usd": 0.77562, + "total_usd": 1.202241, "cap_usd": 15.0 } From e8ff6311caaa9b2f74b008485fe9773e3e589647 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 16:40:48 -0400 Subject: [PATCH 06/13] feat(flow): the trace check counts steps distill dropped as distilled, not missing Co-Authored-By: Claude Opus 5.5 --- .../code_as_action/measure_openemr.py | 5 +- .../results/mockmed/summary.json | 2 +- .../code_as_action/results/openemr/runs.jsonl | 15 +++ .../results/openemr/summary.json | 115 ++++++++++++++++++ .../code_as_action/results/spend.json | 77 +++++++++++- .../benchmark/code_as_action/summarize.py | 98 +++++++++++++++ .../compile_agent/code_action.py | 52 ++++++-- .../openadapt_flow/compile_agent/pipeline.py | 12 +- packages/flow/tests/test_code_action.py | 35 ++++++ 9 files changed, 397 insertions(+), 14 deletions(-) create mode 100644 packages/flow/benchmark/code_as_action/results/openemr/summary.json create mode 100644 packages/flow/benchmark/code_as_action/summarize.py diff --git a/packages/flow/benchmark/code_as_action/measure_openemr.py b/packages/flow/benchmark/code_as_action/measure_openemr.py index 5fcf16b428..74a34443a6 100644 --- a/packages/flow/benchmark/code_as_action/measure_openemr.py +++ b/packages/flow/benchmark/code_as_action/measure_openemr.py @@ -299,6 +299,7 @@ def run_one( try: from openadapt_flow.compile_agent.code_action import ( check_trace_against_ir, + distilled_events, ) from openadapt_flow.ir import Workflow @@ -306,7 +307,9 @@ def run_one( work / "bundle", name=f"openemr-referral-{key}", distill=True ) program = check_trace_against_ir( - trace.to_dict(), Workflow.load(work / "bundle") + trace.to_dict(), + Workflow.load(work / "bundle"), + distilled=distilled_events(session.compiled_recording_dir), ) except Exception as exc: # noqa: BLE001 - reported with the row program = { diff --git a/packages/flow/benchmark/code_as_action/results/mockmed/summary.json b/packages/flow/benchmark/code_as_action/results/mockmed/summary.json index b5af1f79aa..ef38d8ca42 100644 --- a/packages/flow/benchmark/code_as_action/results/mockmed/summary.json +++ b/packages/flow/benchmark/code_as_action/results/mockmed/summary.json @@ -52,7 +52,7 @@ "model": "claude-sonnet-5-5", "platform": "macOS-15.7.3-arm64-arm-64bit", "python": "3.12.7", - "spend_total_usd": 0.58633 + "spend_total_usd": 3.860689 }, "replays": { "model_calls_total": 0, diff --git a/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl b/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl index b8faf660f5..8ed12cfc71 100644 --- a/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl +++ b/packages/flow/benchmark/code_as_action/results/openemr/runs.jsonl @@ -1 +1,16 @@ {"actions": 69, "agent_seconds": 105.866, "agent_summary": "I registered Nadia Brennan and saved the chart. It opened as patient ID 2, with DOB 1971-04-19.\n\nThe EMR's duplicate check came back with \"No matches were found,\" so I confirmed creation. No existing patient was flagged.\n\nAll the values you gave were entered: title Ms., first name Nadia, last name Brennan, DOB 1971-04-19, sex Female, address, city, state Massachusetts, postal code 02139, home phone 555-0121, email nadia.brennan@example.invalid, and country USA.\n\nOne thing differs from your fields: the form marks Birth Sex as required, and your data only gave \"sex.\" I set Birth Sex to Female as well as Sex to Female. I left every other field blank.", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 98993, "cache_read_input_tokens": 738084, "calls": [{"cache_creation_input_tokens": 6006, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 1.936, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6006, "images_in_request": 3, "input_tokens": 2, "output_tokens": 261, "seconds": 3.182, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 13864, "cache_read_input_tokens": 8823, "images_in_request": 13, "input_tokens": 2, "output_tokens": 284, "seconds": 4.416, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 7109, "cache_read_input_tokens": 22687, "images_in_request": 19, "input_tokens": 2, "output_tokens": 128, "seconds": 2.887, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1565, "cache_read_input_tokens": 29796, "images_in_request": 21, "input_tokens": 2, "output_tokens": 147, "seconds": 2.982, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 4240, "cache_read_input_tokens": 31361, "images_in_request": 24, "input_tokens": 2, "output_tokens": 163, "seconds": 3.892, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 252, "cache_read_input_tokens": 35601, "images_in_request": 25, "input_tokens": 2, "output_tokens": 188, "seconds": 3.057, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 4280, "cache_read_input_tokens": 35853, "images_in_request": 28, "input_tokens": 2, "output_tokens": 165, "seconds": 3.538, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 2896, "cache_read_input_tokens": 40133, "images_in_request": 30, "input_tokens": 2, "output_tokens": 137, "seconds": 3.264, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 4227, "cache_read_input_tokens": 43029, "images_in_request": 33, "input_tokens": 2, "output_tokens": 166, "seconds": 3.857, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 4258, "cache_read_input_tokens": 47256, "images_in_request": 36, "input_tokens": 2, "output_tokens": 173, "seconds": 3.932, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 4263, "cache_read_input_tokens": 51514, "images_in_request": 39, "input_tokens": 2, "output_tokens": 112, "seconds": 3.817, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 4204, "cache_read_input_tokens": 55777, "images_in_request": 42, "input_tokens": 2, "output_tokens": 418, "seconds": 4.669, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 24892, "cache_read_input_tokens": 59981, "images_in_request": 60, "input_tokens": 2, "output_tokens": 277, "seconds": 7.439, "stop_reason": "tool_use", "turn": 14}, {"cache_creation_input_tokens": 5729, "cache_read_input_tokens": 84873, "images_in_request": 64, "input_tokens": 2, "output_tokens": 98, "seconds": 4.081, "stop_reason": "tool_use", "turn": 15}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 90602, "images_in_request": 67, "input_tokens": 2, "output_tokens": 109, "seconds": 4.488, "stop_reason": "tool_use", "turn": 16}, {"cache_creation_input_tokens": 4201, "cache_read_input_tokens": 94792, "images_in_request": 70, "input_tokens": 2, "output_tokens": 240, "seconds": 5.982, "stop_reason": "end_turn", "turn": 17}], "code_action": null, "correct": true, "cost_usd": 0.426621, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [180.1, 114.8], "images_in_requests": 575, "input_tokens": 36, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 17, "model_seconds": 67.421, "new_charts": 1, "output_tokens": 3145, "program": null, "prompt_tokens": 837113, "referral": "r1", "round": 1, "run_seconds": 106.218, "screenshots_sent": 70, "scripts": null, "started_at": "2026-10-18T20:06:37Z", "tool_errors": 3, "under_load": true} +{"actions": 37, "agent_seconds": 44.894, "agent_summary": "Registered Nadia Brennan (new chart, pid 2) with all given fields; duplicate check found no matches; chart saved. No mismatches.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 13955, "cache_read_input_tokens": 100248, "calls": [{"cache_creation_input_tokens": 2978, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 117, "seconds": 1.278, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 909, "cache_read_input_tokens": 2978, "images_in_request": 0, "input_tokens": 2, "output_tokens": 62, "seconds": 1.146, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 433, "cache_read_input_tokens": 3887, "images_in_request": 0, "input_tokens": 2, "output_tokens": 78, "seconds": 1.098, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 279, "cache_read_input_tokens": 4320, "images_in_request": 0, "input_tokens": 2, "output_tokens": 59, "seconds": 0.978, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1450, "cache_read_input_tokens": 4599, "images_in_request": 1, "input_tokens": 2, "output_tokens": 90, "seconds": 1.941, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 534, "cache_read_input_tokens": 6049, "images_in_request": 1, "input_tokens": 2, "output_tokens": 48, "seconds": 1.001, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 1423, "cache_read_input_tokens": 6583, "images_in_request": 2, "input_tokens": 2, "output_tokens": 77, "seconds": 1.506, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 125, "cache_read_input_tokens": 8006, "images_in_request": 2, "input_tokens": 2, "output_tokens": 70, "seconds": 1.224, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 1364, "cache_read_input_tokens": 8131, "images_in_request": 2, "input_tokens": 2, "output_tokens": 170, "seconds": 1.774, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 1089, "cache_read_input_tokens": 9495, "images_in_request": 2, "input_tokens": 2, "output_tokens": 62, "seconds": 1.516, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 194, "cache_read_input_tokens": 10584, "images_in_request": 2, "input_tokens": 2, "output_tokens": 52, "seconds": 1.093, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 1528, "cache_read_input_tokens": 10778, "images_in_request": 3, "input_tokens": 2, "output_tokens": 72, "seconds": 1.22, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 226, "cache_read_input_tokens": 12306, "images_in_request": 3, "input_tokens": 2, "output_tokens": 48, "seconds": 1.126, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 1423, "cache_read_input_tokens": 12532, "images_in_request": 4, "input_tokens": 2, "output_tokens": 94, "seconds": 1.65, "stop_reason": "tool_use", "turn": 14}], "code_action": {"actions": 21, "by_structure": 18, "commits": 1, "errors": 2, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:country_code", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 36, "locator_agrees": 18, "locator_unique": 18, "read_back_after_commit": [], "screenshots": 4, "scripts": 14, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 18}, "correct": true, "cost_usd": 0.065987, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [247.0, 277.5], "images_in_requests": 22, "input_tokens": 30, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 14, "model_seconds": 18.551, "new_charts": 1, "output_tokens": 1099, "program": {"in_program": 16, "missing_steps": [6, 8], "passed": false, "program_structural_steps": 16, "trace_steps_with_locator": 18}, "prompt_tokens": 114233, "referral": "r1", "round": 1, "run_seconds": 45.116, "screenshots_sent": 4, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @15 value=Nadia\nok: fill textbox \"First Name\" value=\"Nadia\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @17 value=Brennan\nok: fill textbox \"Last Name\" value=\"Brennan\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @23 value=1971-04-19\nok: fill textbox \"form_DOB\" value=\"1971-04-19\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ tree grep=Contact max=40\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n@40 button \"Contact\" (focused)", "script": "select @14 option=Ms.\nfill @15 value=Nadia\nfill @17 value=Brennan\nfill @23 value=1971-04-19\nselect @24 option=Female\nclick @40\ntree grep=Contact max=40"}, {"images": 0, "output": "$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@15 textbox \"First Name\" value=\"Nadia\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Brennan\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1971-04-19\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n$ find role=combobox name=form_state\nno match", "script": "find role=textbox\nfind role=combobox name=form_state"}, {"images": 0, "output": "$ click role=button name=\"Contact\"\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ scroll down px=400\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "click role=button name=\"Contact\"\nscroll down px=400\ntree"}, {"images": 1, "output": "$ find role=textbox near=Street\nno match\n$ screenshot\n(screenshot attached)", "script": "find role=textbox near=Street\nscreenshot"}, {"images": 0, "output": "$ click role=button name=\"Contact\"\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ find role=textbox near=Address\n@15 textbox \"First Name\" value=\"Nadia\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Brennan\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1971-04-19\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"", "script": "click role=button name=\"Contact\"\nfind role=textbox near=Address"}, {"images": 1, "output": "$ screenshot\n(screenshot attached)", "script": "screenshot"}, {"images": 0, "output": "$ click text=\"Previous Names:\"\nerror: nothing matches text=\"Previous Names:\"\n(stopped; 2 lines not run)", "script": "click text=\"Previous Names:\"\nscroll down px=600\nscreenshot"}, {"images": 0, "output": "$ click role=textbox name=\"form_billing_note\"\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n@13 button \"Contact\"\nAddress:\n@14 textbox \"form_street\" value=\"\"\nAddress Line 2:\n@15 textbox \"form_street_line_2\" value=\"\"\nCity:\n@16 textbox \"form_city\" value=\"\"\nState:\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@18 button \"Add\" value=\"Add\"\nPostal Code:\n@19 textbox \"form_postal_code\" value=\"\"\nCountry:\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 button \"Add\" value=\"Add\"\nCounty:\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@23 button \"Add\" value=\"Add\"\nMother`s Name:\n@24 textbox \"form_mothersname\" value=\"\"\nEmergency Contact:\n@25 textbox \"form_contact_relationship\" value=\"\"\nEmergency Phone:\n@26 textbox \"form_phone_contact\" value=\"\"\nHome Phone:\n@27 textbox \"form_phone_home\" value=\"\"\nWork Phone:\n@28 textbox \"form_phone_biz\" value=\"\"\nMobile Phone:\n@29 textbox \"form_phone_cell\" value=\"\"\nContact Email:\n@30 textbox \"form_email\" value=\"\"\nTrusted Email:\n@31 textbox \"form_email_direct\" value=\"\"\nAdditional Addresses\n@32 checkbox \"Show Inactive\" value=\"on\"\nShow Inactive\nNONE\n@33 button \"Choices\"\n@34 button \"Employer\"\n@35 button \"Stats\"\n@36 button \"Misc\"\n@37 button \"Related\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)", "script": "click role=textbox name=\"form_billing_note\"\nscroll down px=600\ntree"}, {"images": 0, "output": "$ fill @14 value=\"27 WILLOW BEND ROAD\"\nok: fill textbox \"form_street\" value=\"27 WILLOW BEND ROAD\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @27 value=555-0121\nok: fill textbox \"form_phone_home\" value=\"555-0121\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ fill @30 value=nadia.brennan@example.invalid\nok: fill textbox \"form_email\" value=\"nadia.brennan@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ find role=button name=Search\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n$ find role=button name=\"Create New Patient\"\nno match", "script": "fill @14 value=\"27 WILLOW BEND ROAD\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0121\nfill @30 value=nadia.brennan@example.invalid\nfind role=button name=Search\nfind role=button name=\"Create New Patient\""}, {"images": 0, "output": "$ scroll down px=800\nok: scroll down 800px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"", "script": "scroll down px=800\nfind name=\"Create New Patient\""}, {"images": 1, "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n$ screenshot\n(screenshot attached)", "script": "click @38\nscreenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=Y17EIeLnf5UmPz1JKHGXhhGdHZbnQbYKwe8QkhDj)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 1, "output": "$ screenshot\n(screenshot attached)", "script": "screenshot"}], "started_at": "2026-10-18T20:09:36Z", "tool_errors": 0, "under_load": true} +{"actions": 34, "agent_seconds": 35.236, "agent_summary": "Entered all given fields; duplicate check showed no matches; saved new chart for Marcus Feld (pid 3). Nothing mismatched.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 10182, "cache_read_input_tokens": 59642, "calls": [{"cache_creation_input_tokens": 2975, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 112, "seconds": 1.398, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 944, "cache_read_input_tokens": 2975, "images_in_request": 0, "input_tokens": 2, "output_tokens": 125, "seconds": 1.488, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 196, "cache_read_input_tokens": 3919, "images_in_request": 0, "input_tokens": 2, "output_tokens": 66, "seconds": 1.15, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 108, "cache_read_input_tokens": 4115, "images_in_request": 0, "input_tokens": 2, "output_tokens": 54, "seconds": 1.145, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 150, "cache_read_input_tokens": 4223, "images_in_request": 0, "input_tokens": 2, "output_tokens": 46, "seconds": 1.001, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 1478, "cache_read_input_tokens": 4373, "images_in_request": 0, "input_tokens": 2, "output_tokens": 91, "seconds": 1.38, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 339, "cache_read_input_tokens": 5851, "images_in_request": 0, "input_tokens": 2, "output_tokens": 151, "seconds": 1.661, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 1862, "cache_read_input_tokens": 6190, "images_in_request": 0, "input_tokens": 2, "output_tokens": 84, "seconds": 1.322, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 1805, "cache_read_input_tokens": 8052, "images_in_request": 1, "input_tokens": 2, "output_tokens": 75, "seconds": 1.49, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 230, "cache_read_input_tokens": 9857, "images_in_request": 1, "input_tokens": 2, "output_tokens": 51, "seconds": 1.181, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 95, "cache_read_input_tokens": 10087, "images_in_request": 1, "input_tokens": 2, "output_tokens": 93, "seconds": 1.479, "stop_reason": "tool_use", "turn": 11}], "code_action": {"actions": 18, "by_structure": 16, "commits": 1, "errors": 2, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:city", "input:country_code", "input:fname", "input:lname", "input:postal_code", "input:sex", "input:state_label", "input:street", "input:title"], "lines": 29, "locator_agrees": 16, "locator_unique": 16, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 1, "scripts": 11, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 16}, "correct": true, "cost_usd": 0.046911, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [365.5, 328.4], "images_in_requests": 3, "input_tokens": 24, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 11, "model_seconds": 14.696, "new_charts": 1, "output_tokens": 948, "program": {"in_program": 16, "missing_steps": [], "passed": true, "program_structural_steps": 16, "trace_steps_with_locator": 16}, "prompt_tokens": 69848, "referral": "r2", "round": 1, "run_seconds": 35.252, "screenshots_sent": 1, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ fill @15 value=Marcus\nok: fill textbox \"First Name\" value=\"Marcus\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ fill @17 value=Feld\nok: fill textbox \"Last Name\" value=\"Feld\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ fill @23 value=1958-09-02\nok: fill textbox \"form_DOB\" value=\"1958-09-02\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ tree grep=Street\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", "script": "select @14 option=Mr.\nfill @15 value=Marcus\nfill @17 value=Feld\nfill @23 value=1958-09-02\nselect @24 option=Male\nclick @40\ntree grep=Street"}, {"images": 0, "output": "$ fill role=textbox name=form_street value=\"114 QUARRY HILL DRIVE\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 3 lines not run)", "script": "fill role=textbox name=form_street value=\"114 QUARRY HILL DRIVE\"\nfill role=textbox name=form_city value=EXAMPLETON\nfill role=textbox name=form_postal_code value=02139\ntree grep=form_"}, {"images": 0, "output": "$ find name=form_street\nno match\n$ find name=form_state\nno match\n$ find name=form_country_code\nno match", "script": "find name=form_street\nfind name=form_state\nfind name=form_country_code"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\ntree"}, {"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Marcus\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Feld\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1958-09-02\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\" (focused)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form", "script": "tree"}, {"images": 0, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ scroll down px=700\nok: scroll down 700px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ find name=form_street\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"", "script": "click @37\nscroll down px=700\nfind name=form_street"}, {"images": 0, "output": "$ fill @13 value=\"114 QUARRY HILL DRIVE\"\nok: fill textbox \"form_street\" value=\"114 QUARRY HILL DRIVE\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ fill role=textbox name=form_city value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ fill role=textbox name=form_postal_code value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ select role=combobox name=form_state option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ select role=combobox name=form_country_code option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\nAddress:\n@13 textbox \"form_street\" value=\"114 QUARRY HILL DRIVE\"\nAddress Line 2:\n@14 textbox \"form_street_line_2\" value=\"\"\nCity:\n@15 textbox \"form_city\" value=\"EXAMPLETON\"\nState:\n@16 combobox \"form_state\" value=\"Massachusetts\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@17 button \"Add\" value=\"Add\"\nPostal Code:\n@18 textbox \"form_postal_code\" value=\"02139\"\nCountry:\n@19 combobox \"form_country_code\" value=\"USA\" options=[\"Unassigned\", \"USA\"]\n@20 button \"Add\" value=\"Add\" (focused)\nCounty:\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@22 button \"Add\" value=\"Add\"\nMother`s Name:\n@23 textbox \"form_mothersname\" value=\"\"\nEmergency Contact:\n@24 textbox \"form_contact_relationship\" value=\"\"\nEmergency Phone:\n@25 textbox \"form_phone_contact\" value=\"\"\nHome Phone:\n@26 textbox \"form_phone_home\" value=\"\"\nWork Phone:\n@27 textbox \"form_phone_biz\" value=\"\"\nMobile Phone:\n@28 textbox \"form_phone_cell\" value=\"\"\nContact Email:\n@29 textbox \"form_email\" value=\"\"\nTrusted Email:\n@30 textbox \"form_email_direct\" value=\"\"\nAdditional Addresses\n@31 checkbox \"Show Inactive\" value=\"on\"", "script": "fill @13 value=\"114 QUARRY HILL DRIVE\"\nfill role=textbox name=form_city value=EXAMPLETON\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_state option=Massachusetts\nselect role=combobox name=form_country_code option=USA\ntree"}, {"images": 1, "output": "$ fill @26 value=555-0124\nok: fill textbox \"form_phone_home\" value=\"555-0124\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ fill @29 value=marcus.feld@example.invalid\nok: fill textbox \"form_email\" value=\"marcus.feld@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n$ screenshot\n(screenshot attached)", "script": "fill @26 value=555-0124\nfill @29 value=marcus.feld@example.invalid\nclick @38\nscreenshot"}, {"images": 0, "output": "$ click role=button name=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=LVeSD2GUH6WMO08kuarxUnNR6uxF7wzh9H2GEwmX)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click role=button name=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Feld\nMarcus Feld (3)\nMedical Record Dashboard - Marcus Feld", "script": "text grep=Feld"}], "started_at": "2026-10-18T20:11:56Z", "tool_errors": 0, "under_load": true} +{"actions": 36, "agent_seconds": 35.012, "agent_summary": "Filled the form and ran the duplicate check, which found no matches. Saved a new chart for Ines Carvalho (patient ID 4). All given fields were entered; nothing mismatched.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 10032, "cache_read_input_tokens": 57784, "calls": [{"cache_creation_input_tokens": 2974, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 118, "seconds": 1.579, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 905, "cache_read_input_tokens": 2974, "images_in_request": 0, "input_tokens": 2, "output_tokens": 46, "seconds": 0.984, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 1481, "cache_read_input_tokens": 3879, "images_in_request": 0, "input_tokens": 2, "output_tokens": 177, "seconds": 1.612, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 243, "cache_read_input_tokens": 5360, "images_in_request": 0, "input_tokens": 2, "output_tokens": 59, "seconds": 2.198, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 155, "cache_read_input_tokens": 5603, "images_in_request": 0, "input_tokens": 2, "output_tokens": 63, "seconds": 1.318, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 865, "cache_read_input_tokens": 5758, "images_in_request": 0, "input_tokens": 2, "output_tokens": 159, "seconds": 1.509, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 1324, "cache_read_input_tokens": 6623, "images_in_request": 0, "input_tokens": 2, "output_tokens": 75, "seconds": 1.313, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 1762, "cache_read_input_tokens": 7947, "images_in_request": 1, "input_tokens": 2, "output_tokens": 69, "seconds": 1.539, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 222, "cache_read_input_tokens": 9709, "images_in_request": 1, "input_tokens": 2, "output_tokens": 53, "seconds": 1.357, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 101, "cache_read_input_tokens": 9931, "images_in_request": 1, "input_tokens": 2, "output_tokens": 107, "seconds": 1.638, "stop_reason": "tool_use", "turn": 10}], "code_action": {"actions": 20, "by_structure": 17, "commits": 1, "errors": 2, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:country_code", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 29, "locator_agrees": 17, "locator_unique": 17, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 1, "scripts": 10, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 17}, "correct": true, "cost_usd": 0.045941, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [325.0, 222.4], "images_in_requests": 3, "input_tokens": 22, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 10, "model_seconds": 15.046, "new_charts": 1, "output_tokens": 926, "program": {"in_program": 17, "missing_steps": [], "passed": true, "program_structural_steps": 17, "trace_steps_with_locator": 17}, "prompt_tokens": 67838, "referral": "r3", "round": 1, "run_seconds": 35.027, "screenshots_sent": 1, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Mrs.\nok: select combobox \"form_title\" value=\"Mrs.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @15 value=Ines\nok: fill textbox \"First Name\" value=\"Ines\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @17 value=Carvalho\nok: fill textbox \"Last Name\" value=\"Carvalho\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @23 value=1949-12-11\nok: fill textbox \"form_DOB\" value=\"1949-12-11\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ tree grep=Contact max=60\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n@40 button \"Contact\" (focused)", "script": "select @14 option=Mrs.\nfill @15 value=Ines\nfill @17 value=Carvalho\nfill @23 value=1949-12-11\nselect @24 option=Female\nclick @40\ntree grep=Contact max=60"}, {"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Mrs.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Ines\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Carvalho\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1949-12-11\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\" (focused)\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", ", "script": "tree"}, {"images": 0, "output": "$ fill role=textbox name=form_street value=\"9 LANTERN COURT\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", "script": "fill role=textbox name=form_street value=\"9 LANTERN COURT\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\ntree grep=form_"}, {"images": 0, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ tree grep=form_\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n@14 textbox \"form_street\" value=\"\"\n@15 textbox \"form_street_line_2\" value=\"\"\n@16 textbox \"form_city\" value=\"\"\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 textbox \"form_postal_code\" value=\"\"\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@24 textbox \"form_mothersname\" value=\"\"\n@25 textbox \"form_contact_relationship\" value=\"\"\n@26 textbox \"form_phone_contact\" value=\"\"\n@27 textbox \"form_phone_home\" value=\"\"\n@28 textbox \"form_phone_biz\" value=\"\"\n@29 textbox \"form_phone_cell\" value=\"\"\n@30 textbox \"form_email\" value=\"\"\n@31 textbox \"form_email_direct\" value=\"\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)", "script": "click @37\nscroll down px=600\ntree grep=form_"}, {"images": 0, "output": "$ fill @14 value=\"9 LANTERN COURT\"\nok: fill textbox \"form_street\" value=\"9 LANTERN COURT\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @27 value=555-0131\nok: fill textbox \"form_phone_home\" value=\"555-0131\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ fill @30 value=ines.carvalho@example.invalid\nok: fill textbox \"form_email\" value=\"ines.carvalho@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ tree grep=Search\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\nSearch or Add Patient\n# frame 6: Search or Add Patient\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)\n$ tree grep=Create\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)", "script": "fill @14 value=\"9 LANTERN COURT\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0131\nfill @30 value=ines.carvalho@example.invalid\ntree grep=Search\ntree grep=Create"}, {"images": 1, "output": "$ click @30\nok: click textbox \"form_email\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ scroll down px=1500\nok: scroll down 1500px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ click role=button name=\"Create New Patient\"\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n$ screenshot\n(screenshot attached)", "script": "click @30\nscroll down px=1500\nclick role=button name=\"Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=sNs0Y7lmvZPnWVwulkcez2UkSUzXwIYSyjDhQsMD)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Carvalho\nInes Carvalho (4)\nMedical Record Dashboard - Ines Carvalho", "script": "text grep=Carvalho"}], "started_at": "2026-10-18T20:14:02Z", "tool_errors": 0, "under_load": true} +{"actions": 36, "agent_seconds": 39.218, "agent_summary": "Registered Theo Lindqvist (pid 5) with all given fields. The duplicate check showed no matches, so I saved the chart. Nothing mismatched.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 10352, "cache_read_input_tokens": 67499, "calls": [{"cache_creation_input_tokens": 2973, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 118, "seconds": 1.394, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 898, "cache_read_input_tokens": 2973, "images_in_request": 0, "input_tokens": 2, "output_tokens": 57, "seconds": 1.217, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 1007, "cache_read_input_tokens": 3871, "images_in_request": 0, "input_tokens": 2, "output_tokens": 80, "seconds": 1.844, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 176, "cache_read_input_tokens": 4878, "images_in_request": 0, "input_tokens": 2, "output_tokens": 52, "seconds": 1.334, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1523, "cache_read_input_tokens": 5054, "images_in_request": 1, "input_tokens": 2, "output_tokens": 91, "seconds": 1.856, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 631, "cache_read_input_tokens": 6577, "images_in_request": 1, "input_tokens": 2, "output_tokens": 128, "seconds": 1.868, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 955, "cache_read_input_tokens": 7208, "images_in_request": 1, "input_tokens": 2, "output_tokens": 79, "seconds": 1.689, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 338, "cache_read_input_tokens": 8163, "images_in_request": 1, "input_tokens": 2, "output_tokens": 52, "seconds": 1.837, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 1526, "cache_read_input_tokens": 8501, "images_in_request": 2, "input_tokens": 2, "output_tokens": 68, "seconds": 1.856, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 220, "cache_read_input_tokens": 10027, "images_in_request": 2, "input_tokens": 2, "output_tokens": 54, "seconds": 1.883, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 105, "cache_read_input_tokens": 10247, "images_in_request": 2, "input_tokens": 2, "output_tokens": 99, "seconds": 1.756, "stop_reason": "tool_use", "turn": 11}], "code_action": {"actions": 20, "by_structure": 18, "commits": 1, "errors": 1, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:country_code", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 30, "locator_agrees": 18, "locator_unique": 18, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 2, "scripts": 11, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 18}, "correct": true, "cost_usd": 0.048208, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [296.2, 209.7], "images_in_requests": 10, "input_tokens": 24, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 11, "model_seconds": 18.534, "new_charts": 1, "output_tokens": 878, "program": {"in_program": 16, "missing_steps": [7, 8], "passed": false, "program_structural_steps": 16, "trace_steps_with_locator": 18}, "prompt_tokens": 77875, "referral": "r4", "round": 1, "run_seconds": 39.233, "screenshots_sent": 2, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @15 value=Theo\nok: fill textbox \"First Name\" value=\"Theo\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @17 value=Lindqvist\nok: fill textbox \"Last Name\" value=\"Lindqvist\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @23 value=1983-07-26\nok: fill textbox \"form_DOB\" value=\"1983-07-26\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ tree grep=Contact max=5\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n@40 button \"Contact\" (focused)", "script": "select @14 option=Mr.\nfill @15 value=Theo\nfill @17 value=Lindqvist\nfill @23 value=1983-07-26\nselect @24 option=Male\nclick @40\ntree grep=Contact max=5"}, {"images": 0, "output": "$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@15 textbox \"First Name\" value=\"Theo\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Lindqvist\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1983-07-26\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n$ find role=combobox\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@38 combobox \"form_name_history[]\" value=\"\"", "script": "find role=textbox\nfind role=combobox"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\nfind role=textbox near=Street"}, {"images": 1, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ screenshot\n(screenshot attached)", "script": "click @40\nscreenshot"}, {"images": 0, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ click @30\nok: click textbox \"form_ss\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ scroll down px=700\nok: scroll down 700px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"\n@15 textbox \"form_city\" value=\"\"\n@18 textbox \"form_postal_code\" value=\"\"\n@23 textbox \"form_mothersname\" value=\"\"\n@24 textbox \"form_contact_relationship\" value=\"\"\n@25 textbox \"form_phone_contact\" value=\"\"\n@26 textbox \"form_phone_home\" value=\"\"\n@27 textbox \"form_phone_biz\" value=\"\"\n@28 textbox \"form_phone_cell\" value=\"\"\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"", "script": "click @40\nclick @30\nscroll down px=700\nfind role=textbox"}, {"images": 0, "output": "$ fill @13 value=\"63 FERRY STREET\"\nok: fill textbox \"form_street\" value=\"63 FERRY STREET\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @15 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @18 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @26 value=555-0136\nok: fill textbox \"form_phone_home\" value=\"555-0136\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ fill @29 value=theo.lindqvist@example.invalid\nok: fill textbox \"form_email\" value=\"theo.lindqvist@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ find role=combobox\n@16 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@19 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]", "script": "fill @13 value=\"63 FERRY STREET\"\nfill @15 value=EXAMPLETON\nfill @18 value=02139\nfill @26 value=555-0136\nfill @29 value=theo.lindqvist@example.invalid\nfind role=combobox"}, {"images": 0, "output": "$ select @16 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ select @19 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ find role=button name=\"Create New Patient\"\n@38 button \"Create New Patient\"", "script": "select @16 option=Massachusetts\nselect @19 option=USA\nfind role=button name=\"Create New Patient\""}, {"images": 1, "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n$ screenshot\n(screenshot attached)", "script": "click @38\nscreenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=IcOPKGVTbTF1nJMc1q5RYtbJVVPR4770eNHhBlr6)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Lindqvist\nTheo Lindqvist (5)\nMedical Record Dashboard - Theo Lindqvist", "script": "text grep=Lindqvist"}], "started_at": "2026-10-18T20:15:50Z", "tool_errors": 0, "under_load": true} +{"actions": 37, "agent_seconds": 48.108, "agent_summary": "Registered Grace Adeyemi (pid 6) after duplicate check showed no matches; chart saved.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 12041, "cache_read_input_tokens": 91608, "calls": [{"cache_creation_input_tokens": 2974, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 125, "seconds": 2.082, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 1256, "cache_read_input_tokens": 2974, "images_in_request": 0, "input_tokens": 2, "output_tokens": 73, "seconds": 1.154, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 169, "cache_read_input_tokens": 4230, "images_in_request": 0, "input_tokens": 2, "output_tokens": 62, "seconds": 0.862, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 530, "cache_read_input_tokens": 4399, "images_in_request": 0, "input_tokens": 2, "output_tokens": 129, "seconds": 1.298, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 977, "cache_read_input_tokens": 4929, "images_in_request": 0, "input_tokens": 2, "output_tokens": 79, "seconds": 1.227, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 328, "cache_read_input_tokens": 5906, "images_in_request": 0, "input_tokens": 2, "output_tokens": 147, "seconds": 2.176, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 541, "cache_read_input_tokens": 6234, "images_in_request": 0, "input_tokens": 2, "output_tokens": 54, "seconds": 1.48, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 88, "cache_read_input_tokens": 6775, "images_in_request": 0, "input_tokens": 2, "output_tokens": 62, "seconds": 1.493, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 172, "cache_read_input_tokens": 6863, "images_in_request": 0, "input_tokens": 2, "output_tokens": 48, "seconds": 1.226, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 1423, "cache_read_input_tokens": 7035, "images_in_request": 1, "input_tokens": 2, "output_tokens": 95, "seconds": 2.135, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 1699, "cache_read_input_tokens": 8458, "images_in_request": 2, "input_tokens": 2, "output_tokens": 81, "seconds": 1.835, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 1555, "cache_read_input_tokens": 10157, "images_in_request": 3, "input_tokens": 2, "output_tokens": 72, "seconds": 1.854, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 224, "cache_read_input_tokens": 11712, "images_in_request": 3, "input_tokens": 2, "output_tokens": 54, "seconds": 1.569, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 105, "cache_read_input_tokens": 11936, "images_in_request": 3, "input_tokens": 2, "output_tokens": 237, "seconds": 2.663, "stop_reason": "tool_use", "turn": 14}], "code_action": {"actions": 21, "by_structure": 17, "commits": 1, "errors": 2, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:country_code", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 37, "locator_agrees": 17, "locator_unique": 17, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 3, "scripts": 15, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 17}, "correct": true, "cost_usd": 0.061664, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [349.9, 290.3], "images_in_requests": 12, "input_tokens": 30, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 14, "model_seconds": 23.054, "new_charts": 1, "output_tokens": 1318, "program": {"in_program": 17, "missing_steps": [], "passed": true, "program_structural_steps": 17, "trace_steps_with_locator": 17}, "prompt_tokens": 103679, "referral": "r5", "round": 1, "run_seconds": 48.124, "screenshots_sent": 3, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @15 value=Grace\nok: fill textbox \"First Name\" value=\"Grace\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @17 value=Adeyemi\nok: fill textbox \"Last Name\" value=\"Adeyemi\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @23 value=1990-02-08\nok: fill textbox \"form_DOB\" value=\"1990-02-08\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ tree grep=Contact max=5\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n@40 button \"Contact\" (focused)\n$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@15 textbox \"First Name\" value=\"Grace\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Adeyemi\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1990-02-08\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"", "script": "select @14 option=Ms.\nfill @15 value=Grace\nfill @17 value=Adeyemi\nfill @23 value=1990-02-08\nselect @24 option=Female\nclick @40\ntree grep=Contact max=5\nfind role=textbox"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\ntree max=80"}, {"images": 0, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ scroll down px=500\nok: scroll down 500px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@13 textbox \"form_billing_note\" value=\"\" (focused)\n@17 textbox \"form_street\" value=\"\"\n@18 textbox \"form_street_line_2\" value=\"\"\n@19 textbox \"form_city\" value=\"\"\n@22 textbox \"form_postal_code\" value=\"\"\n@27 textbox \"form_mothersname\" value=\"\"\n@28 textbox \"form_contact_relationship\" value=\"\"\n@29 textbox \"form_phone_contact\" value=\"\"\n@30 textbox \"form_phone_home\" value=\"\"\n@31 textbox \"form_phone_biz\" value=\"\"\n@32 textbox \"form_phone_cell\" value=\"\"\n@33 textbox \"form_email\" value=\"\"\n@34 textbox \"form_email_direct\" value=\"\"", "script": "click @37\nscroll down px=500\nfind role=textbox"}, {"images": 0, "output": "$ fill @17 value=\"402 CEDAR ROW\"\nok: fill textbox \"form_street\" value=\"402 CEDAR ROW\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @19 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @22 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @30 value=555-0145\nok: fill textbox \"form_phone_home\" value=\"555-0145\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ fill @33 value=grace.adeyemi@example.invalid\nok: fill textbox \"form_email\" value=\"grace.adeyemi@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ find role=combobox\n@14 combobox \"form_name_history[]\" value=\"\"\n@20 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@23 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@25 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]", "script": "fill @17 value=\"402 CEDAR ROW\"\nfill @19 value=EXAMPLETON\nfill @22 value=02139\nfill @30 value=555-0145\nfill @33 value=grace.adeyemi@example.invalid\nfind role=combobox"}, {"images": 0, "output": "$ select @20 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ select @23 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ find role=button name=\"Create New Patient\"\nno match", "script": "select @20 option=Massachusetts\nselect @23 option=USA\nfind role=button name=\"Create New Patient\""}, {"images": 0, "output": "$ find role=button\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\n@15 button \"Add\"\n@16 button \"Contact\"\n@21 button \"Add\" value=\"Add\"\n@24 button \"Add\" value=\"Add\" (focused)\n@26 button \"Add\" value=\"Add\"\n@36 button \"Choices\"\n@37 button \"Employer\"\n@38 button \"Stats\"", "script": "find role=button"}, {"images": 0, "output": "$ scroll up px=2000\nok: scroll up 2000px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ find role=button name=\"Create New Patient\"\nno match", "script": "scroll up px=2000\nfind role=button name=\"Create New Patient\""}, {"images": 0, "output": "$ click text=\"Create New Patient\"\nerror: nothing matches text=\"Create New Patient\"", "script": "click text=\"Create New Patient\""}, {"images": 0, "output": "$ find name=\"Create New Patient\"\nno match\n$ tree grep=Create\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n", "script": "find name=\"Create New Patient\"\ntree grep=Create"}, {"images": 1, "output": "$ screenshot\n(screenshot attached)", "script": "screenshot"}, {"images": 1, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ scroll down px=3000\nok: scroll down 2000px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"\n$ screenshot\n(screenshot attached)", "script": "click @37\nscroll down px=3000\nfind name=\"Create New Patient\"\nscreenshot"}, {"images": 1, "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n$ screenshot\n(screenshot attached)", "script": "click @38\nscreenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=2ySS9rD2CZpR00OXWMYs4JmrOUBfNjHjNo8XhDau)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Adeyemi\nGrace Adeyemi (6)\nMedical Record Dashboard - Grace Adeyemi", "script": "text grep=Adeyemi"}], "started_at": "2026-10-18T20:18:17Z", "tool_errors": 0, "under_load": true} +{"actions": 35, "agent_seconds": 45.657, "agent_summary": "Filled the New Patient form, but OpenEMR showed a possible existing patient. I cancelled and did not save a chart. The existing chart is Delgado, Rosa, DOB 1958-03-27, Female, External ID 1.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 10854, "cache_read_input_tokens": 57919, "calls": [{"cache_creation_input_tokens": 2972, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 117, "seconds": 1.75, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 883, "cache_read_input_tokens": 2972, "images_in_request": 0, "input_tokens": 2, "output_tokens": 57, "seconds": 1.504, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 1007, "cache_read_input_tokens": 3855, "images_in_request": 0, "input_tokens": 2, "output_tokens": 81, "seconds": 1.53, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 177, "cache_read_input_tokens": 4862, "images_in_request": 0, "input_tokens": 2, "output_tokens": 61, "seconds": 1.437, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1340, "cache_read_input_tokens": 5039, "images_in_request": 0, "input_tokens": 2, "output_tokens": 173, "seconds": 2.162, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 1063, "cache_read_input_tokens": 6379, "images_in_request": 0, "input_tokens": 2, "output_tokens": 61, "seconds": 1.544, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 135, "cache_read_input_tokens": 7442, "images_in_request": 0, "input_tokens": 2, "output_tokens": 56, "seconds": 1.577, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 1531, "cache_read_input_tokens": 7577, "images_in_request": 1, "input_tokens": 2, "output_tokens": 96, "seconds": 1.975, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 1577, "cache_read_input_tokens": 9108, "images_in_request": 2, "input_tokens": 2, "output_tokens": 63, "seconds": 2.035, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 169, "cache_read_input_tokens": 10685, "images_in_request": 2, "input_tokens": 2, "output_tokens": 122, "seconds": 1.887, "stop_reason": "tool_use", "turn": 10}], "code_action": {"actions": 19, "by_structure": 16, "commits": 0, "errors": 2, "identity_read_before_commit": [], "lines": 29, "locator_agrees": 16, "locator_unique": 16, "read_back_after_commit": [], "screenshots": 2, "scripts": 10, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 16}, "correct": true, "cost_usd": 0.047633, "error": null, "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [], "host_load_1m": [309.6, 302.6], "images_in_requests": 5, "input_tokens": 22, "mismatch_case": true, "model": "claude-sonnet-5-5", "model_calls": 10, "model_seconds": 17.401, "new_charts": 0, "output_tokens": 887, "program": {"in_program": 16, "missing_steps": [], "passed": true, "program_structural_steps": 16, "trace_steps_with_locator": 16}, "prompt_tokens": 68795, "referral": "r6", "round": 1, "run_seconds": 45.673, "screenshots_sent": 2, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Mrs.\nok: select combobox \"form_title\" value=\"Mrs.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @15 value=Rosa\nok: fill textbox \"First Name\" value=\"Rosa\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @17 value=Delgado\nok: fill textbox \"Last Name\" value=\"Delgado\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @23 value=1958-03-22\nok: fill textbox \"form_DOB\" value=\"1958-03-22\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ tree grep=Address max=40\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n", "script": "select @14 option=Mrs.\nfill @15 value=Rosa\nfill @17 value=Delgado\nfill @23 value=1958-03-22\nselect @24 option=Female\nclick @40\ntree grep=Address max=40"}, {"images": 0, "output": "$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@15 textbox \"First Name\" value=\"Rosa\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Delgado\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1958-03-22\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n$ find role=combobox\n@14 combobox \"form_title\" value=\"Mrs.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@38 combobox \"form_name_history[]\" value=\"\"", "script": "find role=textbox\nfind role=combobox"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\nfind role=textbox near=Street"}, {"images": 0, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ scroll down px=600\nok: scroll down 600px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ tree max=80\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n@13 button \"Contact\"\nAddress:\n@14 textbox \"form_street\" value=\"\"\nAddress Line 2:\n@15 textbox \"form_street_line_2\" value=\"\"\nCity:\n@16 textbox \"form_city\" value=\"\"\nState:\n@17 combobox \"form_state\" value=\"Unassigned\" options=[\"Unassigned\", \"Alabama\", \"Alaska\", \"Arizona\", \"Arkansas\", \"California\", \"Colorado\", \"Connecticut\", \"Delaware\", \"District of Columbia\", \"Florida\", \"Georgia\", +41 more]\n@18 button \"Add\" value=\"Add\"\nPostal Code:\n@19 textbox \"form_postal_code\" value=\"\"\nCountry:\n@20 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]\n@21 button \"Add\" value=\"Add\"\nCounty:\n@22 combobox \"form_county\" value=\"Unassigned\" options=[\"Unassigned\", \"ADAIR\", \"ANDREW\", \"ATCHISON\", \"AUDRAIN\", \"BARRY\"]\n@23 button \"Add\" value=\"Add\"\nMother`s Name:\n@24 textbox \"form_mothersname\" value=\"\"\nEmergency Contact:\n@25 textbox \"form_contact_relationship\" value=\"\"\nEmergency Phone:\n@26 textbox \"form_phone_contact\" value=\"\"\nHome Phone:\n@27 textbox \"form_phone_home\" value=\"\"\nWork Phone:\n@28 textbox \"form_phone_biz\" value=\"\"\nMobile Phone:\n@29 textbox \"form_phone_cell\" value=\"\"\nContact Email:\n@30 textbox \"form_email\" value=\"\"\nTrusted Email:\n@31 textbox \"form_email_direct\" value=\"\"\nAdditional Addresses\n@32 checkbox \"Show Inactive\" value=\"on\"\nShow Inactive\nNONE\n@33 button \"Choices\"\n@34 button \"Employer\"\n@35 button \"Stats\"\n@36 button \"Misc\"\n@37 button \"Related\"\n(27 controls above the visible area: \"Who\", \"form_title\", \"First Name\", \"Middle Name\", \"Last Name\", \"Name Suffix\", \"Patient preferred name or name patient is commonly known.\", \"Birth First Name\" and 19 more; scroll up to reach them)\n(2 controls below the visible area: \"Search\", \"Create New Patient\"; scroll down to reach them)", "script": "click @37\nscroll down px=600\ntree max=80"}, {"images": 0, "output": "$ fill @14 value=\"7 MILL ROAD\"\nok: fill textbox \"form_street\" value=\"7 MILL ROAD\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @16 value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ select @17 option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @19 value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ select @20 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @27 value=555-0188\nok: fill textbox \"form_phone_home\" value=\"555-0188\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ fill @30 value=rosa.delgado@example.invalid\nok: fill textbox \"form_email\" value=\"rosa.delgado@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ click role=button name=\"Create New Patient\"\nerror: nothing matches role=\"button\" name=\"Create New Patient\"\n(stopped; 1 lines not run)", "script": "fill @14 value=\"7 MILL ROAD\"\nfill @16 value=EXAMPLETON\nselect @17 option=Massachusetts\nfill @19 value=02139\nselect @20 option=USA\nfill @27 value=555-0188\nfill @30 value=rosa.delgado@example.invalid\nclick role=button name=\"Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ find name=\"Create New Patient\"\nno match\n$ find name=\"Search\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"", "script": "find name=\"Create New Patient\"\nfind name=\"Search\""}, {"images": 1, "output": "$ scroll down px=800\nok: scroll down 800px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ screenshot\n(screenshot attached)", "script": "scroll down px=800\nscreenshot"}, {"images": 1, "output": "$ click text=\"Create New Patient\"\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)\n$ screenshot\n(screenshot attached)", "script": "click text=\"Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ click text=\"Cancel\"\nok: click button \"Cancel\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=UYirH1e3vIRzLFy9ZwR9qePtsIQoy5cNIffr9WdH)", "script": "click text=\"Cancel\""}], "started_at": "2026-10-18T20:21:33Z", "tool_errors": 0, "under_load": true} +{"actions": 36, "agent_seconds": 47.576, "agent_summary": "Registered Arlo Whitcombe with all given fields. The duplicate check found no matches, so I confirmed the save. The chart was created as patient ID 7. Nothing did not match.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 11653, "cache_read_input_tokens": 112409, "calls": [{"cache_creation_input_tokens": 2975, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 119, "seconds": 1.717, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 887, "cache_read_input_tokens": 2975, "images_in_request": 0, "input_tokens": 2, "output_tokens": 57, "seconds": 1.113, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 1008, "cache_read_input_tokens": 3862, "images_in_request": 0, "input_tokens": 2, "output_tokens": 74, "seconds": 1.348, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 170, "cache_read_input_tokens": 4870, "images_in_request": 0, "input_tokens": 2, "output_tokens": 52, "seconds": 1.14, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1521, "cache_read_input_tokens": 5040, "images_in_request": 1, "input_tokens": 2, "output_tokens": 83, "seconds": 1.361, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 197, "cache_read_input_tokens": 6561, "images_in_request": 1, "input_tokens": 2, "output_tokens": 55, "seconds": 1.288, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 976, "cache_read_input_tokens": 6758, "images_in_request": 1, "input_tokens": 2, "output_tokens": 174, "seconds": 1.789, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 241, "cache_read_input_tokens": 7734, "images_in_request": 1, "input_tokens": 2, "output_tokens": 57, "seconds": 1.158, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 85, "cache_read_input_tokens": 7975, "images_in_request": 1, "input_tokens": 2, "output_tokens": 83, "seconds": 1.309, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 133, "cache_read_input_tokens": 8060, "images_in_request": 1, "input_tokens": 2, "output_tokens": 62, "seconds": 1.248, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 306, "cache_read_input_tokens": 8193, "images_in_request": 1, "input_tokens": 2, "output_tokens": 137, "seconds": 1.507, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 741, "cache_read_input_tokens": 8499, "images_in_request": 1, "input_tokens": 2, "output_tokens": 112, "seconds": 1.4, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 528, "cache_read_input_tokens": 9240, "images_in_request": 1, "input_tokens": 2, "output_tokens": 88, "seconds": 1.977, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 1560, "cache_read_input_tokens": 9768, "images_in_request": 2, "input_tokens": 2, "output_tokens": 68, "seconds": 2.211, "stop_reason": "tool_use", "turn": 14}, {"cache_creation_input_tokens": 218, "cache_read_input_tokens": 11328, "images_in_request": 2, "input_tokens": 2, "output_tokens": 54, "seconds": 1.33, "stop_reason": "tool_use", "turn": 15}, {"cache_creation_input_tokens": 107, "cache_read_input_tokens": 11546, "images_in_request": 2, "input_tokens": 2, "output_tokens": 106, "seconds": 1.586, "stop_reason": "tool_use", "turn": 16}], "code_action": {"actions": 20, "by_structure": 18, "commits": 1, "errors": 3, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:country_code", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 39, "locator_agrees": 18, "locator_unique": 18, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 2, "scripts": 16, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 18}, "correct": true, "cost_usd": 0.065492, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [399.4, 230.2], "images_in_requests": 15, "input_tokens": 34, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 16, "model_seconds": 23.483, "new_charts": 1, "output_tokens": 1381, "program": {"in_program": 16, "missing_steps": [7, 8], "passed": false, "program_structural_steps": 16, "trace_steps_with_locator": 18}, "prompt_tokens": 124096, "referral": "r7", "round": 1, "run_seconds": 47.591, "screenshots_sent": 2, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Mr.\nok: select combobox \"form_title\" value=\"Mr.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill @15 value=Arlo\nok: fill textbox \"First Name\" value=\"Arlo\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill @17 value=Whitcombe\nok: fill textbox \"Last Name\" value=\"Whitcombe\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill @23 value=1966-05-30\nok: fill textbox \"form_DOB\" value=\"1966-05-30\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ select @24 option=Male\nok: select combobox \"form_sex\" value=\"Male\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ tree grep=Contact max=40\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n@40 button \"Contact\" (focused)", "script": "select @14 option=Mr.\nfill @15 value=Arlo\nfill @17 value=Whitcombe\nfill @23 value=1966-05-30\nselect @24 option=Male\nclick @40\ntree grep=Contact max=40"}, {"images": 0, "output": "$ find role=textbox\n@10 textbox \"Search by any demographics\" value=\"\"\n@15 textbox \"First Name\" value=\"Arlo\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Whitcombe\"\n@18 textbox \"Name Suffix\" value=\"\"\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\n@23 textbox \"form_DOB\" value=\"1966-05-30\"\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n$ find role=combobox\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@38 combobox \"form_name_history[]\" value=\"\"", "script": "find role=textbox\nfind role=combobox"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\ntree"}, {"images": 1, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ screenshot\n(screenshot attached)", "script": "click @40\nscreenshot"}, {"images": 0, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ find role=textbox near=Street\nno match", "script": "click @40\nfind role=textbox near=Street"}, {"images": 0, "output": "$ tree grep=form_ max=80\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n@14 combobox \"form_title\" value=\"Mr.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1966-05-30\"\n@24 combobox \"form_sex\" value=\"Male\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", "script": "tree grep=form_ max=80"}, {"images": 0, "output": "$ fill role=textbox name=form_street value=\"88 BEACON TERRACE\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 5 lines not run)", "script": "fill role=textbox name=form_street value=\"88 BEACON TERRACE\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email"}, {"images": 0, "output": "$ find name=form_street\nno match\n$ find name=form_city\nno match", "script": "find name=form_street\nfind name=form_city"}, {"images": 0, "output": "$ click text=\"Birth Sex:\"\nerror: nothing matches text=\"Birth Sex:\"\n(stopped; 2 lines not run)", "script": "click text=\"Birth Sex:\"\nscroll down px=700\ntree grep=form_ max=60"}, {"images": 0, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ scroll down px=700\nok: scroll down 700px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ find name=form_street\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"", "script": "click @37\nscroll down px=700\nfind name=form_street"}, {"images": 0, "output": "$ fill @13 value=\"88 BEACON TERRACE\"\nok: fill textbox \"form_street\" value=\"88 BEACON TERRACE\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill name=form_city value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ select name=form_state option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill name=form_postal_code value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ find name=form_phone_home\n@26 textbox \"form_phone_home\" value=\"\"\n$ find name=form_email\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"\n$ find name=form_country_code\n@19 combobox \"form_country_code\" value=\"Unassigned\" options=[\"Unassigned\", \"USA\"]", "script": "fill @13 value=\"88 BEACON TERRACE\"\nfill name=form_city value=EXAMPLETON\nselect name=form_state option=Massachusetts\nfill name=form_postal_code value=02139\nfind name=form_phone_home\nfind name=form_email\nfind name=form_country_code"}, {"images": 0, "output": "$ select @19 option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill @26 value=555-0152\nok: fill textbox \"form_phone_home\" value=\"555-0152\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ fill @29 value=arlo.whitcombe@example.invalid\nok: fill textbox \"form_email\" value=\"arlo.whitcombe@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ find name=\"Create New Patient\"\n@38 button \"Create New Patient\"\n$ find role=button name=\"Create New Patient\"\n@38 button \"Create New Patient\"", "script": "select @19 option=USA\nfill @26 value=555-0152\nfill @29 value=arlo.whitcombe@example.invalid\nfind name=\"Create New Patient\"\nfind role=button name=\"Create New Patient\""}, {"images": 1, "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n$ screenshot\n(screenshot attached)", "script": "click @38\nscreenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=thbGkUDh01BHsuaOMyR6edMSaLNxud4HAbsCNEWS)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Whitcombe\nArlo Whitcombe (7)\nMedical Record Dashboard - Arlo Whitcombe", "script": "text grep=Whitcombe"}], "started_at": "2026-10-18T20:24:24Z", "tool_errors": 0, "under_load": true} +{"actions": 36, "agent_seconds": 49.966, "agent_summary": "I saved a new chart for Ms. Priya Raman. The EMR now shows it as patient (8), and the Medical Record Dashboard opened for her.\n\nBefore saving, I filled in every field from the referral. The EMR's duplicate check came back with \"No matches were found\", so I confirmed the new patient. I didn't open her demographics page afterward to compare each saved value with the referral.\n\nNothing in the referral failed to match.", "app": "openemr-referral", "arm": "code", "cache_creation_input_tokens": 11513, "cache_read_input_tokens": 110039, "calls": [{"cache_creation_input_tokens": 2970, "cache_read_input_tokens": 0, "images_in_request": 0, "input_tokens": 4, "output_tokens": 115, "seconds": 1.427, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 903, "cache_read_input_tokens": 2970, "images_in_request": 0, "input_tokens": 2, "output_tokens": 49, "seconds": 0.899, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 1485, "cache_read_input_tokens": 3873, "images_in_request": 0, "input_tokens": 2, "output_tokens": 199, "seconds": 2.083, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 265, "cache_read_input_tokens": 5358, "images_in_request": 0, "input_tokens": 2, "output_tokens": 63, "seconds": 1.34, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 159, "cache_read_input_tokens": 5623, "images_in_request": 0, "input_tokens": 2, "output_tokens": 57, "seconds": 1.528, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 85, "cache_read_input_tokens": 5782, "images_in_request": 0, "input_tokens": 2, "output_tokens": 54, "seconds": 1.269, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 168, "cache_read_input_tokens": 5867, "images_in_request": 0, "input_tokens": 2, "output_tokens": 55, "seconds": 1.471, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 906, "cache_read_input_tokens": 6035, "images_in_request": 0, "input_tokens": 2, "output_tokens": 77, "seconds": 1.418, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 1100, "cache_read_input_tokens": 6941, "images_in_request": 0, "input_tokens": 2, "output_tokens": 87, "seconds": 1.692, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 339, "cache_read_input_tokens": 8041, "images_in_request": 0, "input_tokens": 2, "output_tokens": 140, "seconds": 2.06, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 827, "cache_read_input_tokens": 8380, "images_in_request": 0, "input_tokens": 2, "output_tokens": 88, "seconds": 1.753, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 370, "cache_read_input_tokens": 9207, "images_in_request": 0, "input_tokens": 2, "output_tokens": 63, "seconds": 1.516, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 194, "cache_read_input_tokens": 9577, "images_in_request": 0, "input_tokens": 2, "output_tokens": 48, "seconds": 1.304, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 1423, "cache_read_input_tokens": 9771, "images_in_request": 1, "input_tokens": 2, "output_tokens": 72, "seconds": 1.984, "stop_reason": "tool_use", "turn": 14}, {"cache_creation_input_tokens": 226, "cache_read_input_tokens": 11194, "images_in_request": 1, "input_tokens": 2, "output_tokens": 51, "seconds": 1.508, "stop_reason": "tool_use", "turn": 15}, {"cache_creation_input_tokens": 93, "cache_read_input_tokens": 11420, "images_in_request": 1, "input_tokens": 2, "output_tokens": 139, "seconds": 2.306, "stop_reason": "end_turn", "turn": 16}], "code_action": {"actions": 20, "by_structure": 18, "commits": 1, "errors": 2, "identity_read_before_commit": ["identity:Create New Patient", "input:DOB", "input:fname", "input:lname", "input:sex", "input:title"], "lines": 37, "locator_agrees": 18, "locator_unique": 18, "read_back_after_commit": ["input:fname", "input:lname"], "screenshots": 1, "scripts": 16, "shell_commands": 0, "shell_isolation": "sandbox-exec", "surface": "dom", "with_recorded_locator": 18}, "correct": true, "cost_usd": 0.064428, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [324.4, 253.1], "images_in_requests": 3, "input_tokens": 34, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 16, "model_seconds": 25.559, "new_charts": 1, "output_tokens": 1357, "program": {"in_program": 16, "missing_steps": [7, 8], "passed": false, "program_structural_steps": 16, "trace_steps_with_locator": 18}, "prompt_tokens": 121586, "referral": "r8", "round": 1, "run_seconds": 49.982, "screenshots_sent": 1, "scripts": [{"images": 0, "output": "$ tree\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Unassigned\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Unassigned\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\"\n(7 controls below the visible area: \"Choices\", \"Employer\", \"Stats\", \"Misc\", \"Related\", \"Search\",", "script": "tree"}, {"images": 0, "output": "$ select @14 option=Ms.\nok: select combobox \"form_title\" value=\"Ms.\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ fill @15 value=Priya\nok: fill textbox \"First Name\" value=\"Priya\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ fill @17 value=Raman\nok: fill textbox \"Last Name\" value=\"Raman\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ fill @23 value=1975-10-14\nok: fill textbox \"form_DOB\" value=\"1975-10-14\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ select @24 option=Female\nok: select combobox \"form_sex\" value=\"Female\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ tree grep=Contact max=60\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n@40 button \"Contact\" (focused)", "script": "select @14 option=Ms.\nfill @15 value=Priya\nfill @17 value=Raman\nfill @23 value=1975-10-14\nselect @24 option=Female\nclick @40\ntree grep=Contact max=60"}, {"images": 0, "output": "$ tree max=120\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n@1 link \"OpenEMR Website\"\nCalendar\nFinder\nFlow\nRecalls\nMessages\n@2 button \"Patient\"\n@3 button \"Fees\"\n@4 button \"Modules\"\n@5 button \"Procedures\"\n@6 button \"Admin\"\n@7 button \"Reports\"\n@8 button \"Miscellaneous\"\n@9 button \"Popups\"\n@10 textbox \"Search by any demographics\" value=\"\"\n@11 button \"Search for patient by entering whole or part of any demographics field information\"\n@12 button \"Current user\"\nCalendar\nMessage Center\nSearch or Add Patient\n# frame 6: Search or Add Patient\n# Search or Add Patient\n@13 button \"Who\"\nTitle:\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\nName:\n@15 textbox \"First Name\" value=\"Priya\"\n@16 textbox \"Middle Name\" value=\"\"\n@17 textbox \"Last Name\" value=\"Raman\"\n@18 textbox \"Name Suffix\" value=\"\"\nPreferred Name:\n@19 textbox \"Patient preferred name or name patient is commonly known.\" value=\"\"\nBirth Name:\n@20 textbox \"Birth First Name\" value=\"\"\n@21 textbox \"Middle Name\" value=\"\"\n@22 textbox \"Birth Last Name\" value=\"\"\nDOB:\n@23 textbox \"form_DOB\" value=\"1975-10-14\"\nBirth Sex:\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\nGender Identity:\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\nSex:\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\nSexual Orientation:\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\nPronouns:\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\nExternal ID:\n@29 textbox \"form_pubpid\" value=\"\"\nS.S.:\n@30 textbox \"form_ss\" value=\"\"\nLicense/ID:\n@31 textbox \"form_drivers_license\" value=\"\"\nMarital Status:\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\nUser Defined:\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\nBilling Note:\n@37 textbox \"form_billing_note\" value=\"\"\nPrevious Names:\n@38 combobox \"form_name_history[]\" value=\"\"\n@39 button \"Add\"\n@40 button \"Contact\" (focused)\n(26 controls below the visible area: \"form_street\", \"form_street_line", "script": "tree max=120"}, {"images": 0, "output": "$ fill role=textbox name=form_street value=\"15 STATION AVENUE\"\nerror: nothing matches role=\"textbox\" name=\"form_street\"\n(stopped; 6 lines not run)", "script": "fill role=textbox name=form_street value=\"15 STATION AVENUE\"\nfill role=textbox name=form_city value=EXAMPLETON\nselect role=combobox name=form_state option=Massachusetts\nfill role=textbox name=form_postal_code value=02139\nselect role=combobox name=form_country_code option=USA\nfind role=textbox name=form_phone_home\nfind role=textbox name=form_email"}, {"images": 0, "output": "$ scroll down px=600\nerror: the scroll was sent, but nothing scrolled: the pointer is over a part of the screen that does not scroll (a fixed bar?). Click a field or label in the area you want to scroll, then scroll again\n(stopped; 1 lines not run)", "script": "scroll down px=600\ntree max=80 grep=form_"}, {"images": 0, "output": "$ find name=form_street\nno match\n$ find name=form_city\nno match", "script": "find name=form_street\nfind name=form_city"}, {"images": 0, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ find name=form_street\nno match", "script": "click @40\nfind name=form_street"}, {"images": 0, "output": "$ tree grep=form_ max=100\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1975-10-14\"\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"", "script": "tree grep=form_ max=100"}, {"images": 0, "output": "$ click @40\nok: click button \"Contact\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ tree grep=form_ max=100\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n@14 combobox \"form_title\" value=\"Ms.\" options=[\"Unassigned\", \"Mr.\", \"Mrs.\", \"Ms.\", \"Dr.\"]\n@23 textbox \"form_DOB\" value=\"1975-10-14\"\n@24 combobox \"form_sex\" value=\"Female\" options=[\"Unassigned\", \"Female\", \"Male\", \"Unknown\"]\n@25 combobox \"form_gender_identity\" value=\"Unassigned\" options=[\"Unassigned\", \"Identifies as Male\", \"Identifies as Female\", \"Female-to-Male (FTM)/Transgender Male/Trans Man\", \"Male-to-Female (MTF)/Transgender Female/Trans Woman\", \"Genderqueer, neither exclusively male nor female\", \"Additional gender category or other, please specify\", \"Choose not to disclose\"]\n@26 combobox \"form_sex_identified\" value=\"Unassigned\" options=[\"Unassigned\", \"Male\", \"Female\", \"Identifies as nonbinary gender (finding)\", \"Asked But Declined\", \"unknown\"]\n@27 combobox \"form_sexual_orientation\" value=\"Unassigned\" options=[\"Unassigned\", \"Straight or heterosexual\", \"Lesbian, gay or homosexual\", \"Bisexual\", \"Something else, please describe\", \"Don't know\", \"Choose not to disclose\"]\n@28 combobox \"form_pronoun\" value=\"Unassigned\" options=[\"Unassigned\", \"he/him/his/his/himself\", \"she/her/her/hers/herself\", \"they/them/their/theirs/themselves\", \"Ask me\", \"Decline to answer\", \"Other\"]\n@29 textbox \"form_pubpid\" value=\"\"\n@30 textbox \"form_ss\" value=\"\"\n@31 textbox \"form_drivers_license\" value=\"\"\n@32 combobox \"form_status\" value=\"Unassigned\" options=[\"Unassigned\", \"Married\", \"Single\", \"Divorced\", \"Widowed\", \"Separated\", \"Domestic Partner\"]\n@33 textbox \"form_genericname1\" value=\"\"\n@34 textbox \"form_genericval1\" value=\"\"\n@35 textbox \"form_genericname2\" value=\"\"\n@36 textbox \"form_genericval2\" value=\"\"\n@37 textbox \"form_billing_note\" value=\"\"\n@38 combobox \"form_name_history[]\" value=\"\"\n(26 controls below the visible area: \"form_street\", \"form_street_line_2\", \"form_city\", \"form_state\", \"Add\", \"form_postal_code\", \"form_country_code\", \"Add\" and 18 more; scroll down to reach them)", "script": "click @40\ntree grep=form_ max=100"}, {"images": 0, "output": "$ click @37\nok: click textbox \"form_billing_note\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ scroll down px=800\nok: scroll down 800px\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ find name=form_street\n@13 textbox \"form_street\" value=\"\"\n@14 textbox \"form_street_line_2\" value=\"\"", "script": "click @37\nscroll down px=800\nfind name=form_street"}, {"images": 0, "output": "$ fill @13 value=\"15 STATION AVENUE\"\nok: fill textbox \"form_street\" value=\"15 STATION AVENUE\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ fill name=form_city value=EXAMPLETON\nok: fill textbox \"form_city\" value=\"EXAMPLETON\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ select name=form_state option=Massachusetts\nok: select combobox \"form_state\" value=\"Massachusetts\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ fill name=form_postal_code value=02139\nok: fill textbox \"form_postal_code\" value=\"02139\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ select name=form_country_code option=USA\nok: select combobox \"form_country_code\" value=\"USA\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ find name=form_phone_home\n@26 textbox \"form_phone_home\" value=\"\"\n$ find name=form_email\n@29 textbox \"form_email\" value=\"\"\n@30 textbox \"form_email_direct\" value=\"\"", "script": "fill @13 value=\"15 STATION AVENUE\"\nfill name=form_city value=EXAMPLETON\nselect name=form_state option=Massachusetts\nfill name=form_postal_code value=02139\nselect name=form_country_code option=USA\nfind name=form_phone_home\nfind name=form_email"}, {"images": 0, "output": "$ fill @26 value=555-0167\nok: fill textbox \"form_phone_home\" value=\"555-0167\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ fill @29 value=priya.raman@example.invalid\nok: fill textbox \"form_email\" value=\"priya.raman@example.invalid\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ find role=button name=\"Create New Patient\"\n@38 button \"Create New Patient\"", "script": "fill @26 value=555-0167\nfill @29 value=priya.raman@example.invalid\nfind role=button name=\"Create New Patient\""}, {"images": 0, "output": "$ click @38\nok: click button \"Create New Patient\"\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n$ text grep=uplicate\nno text\n$ text grep=xist\nno text", "script": "click @38\ntext grep=uplicate\ntext grep=xist"}, {"images": 1, "output": "$ screenshot\n(screenshot attached)", "script": "screenshot"}, {"images": 0, "output": "$ click text=\"Confirm Create New Patient\"\nok: click button \"Confirm Create New Patient\" (save gate passed)\npage: OpenEMR (http://127.0.0.1:9301/interface/main/tabs/main.php?token_main=V7WBZrRN6mDXsLxgB2AcXvA9XmhIgxR5wLK7FVzE)\n(a write was committed; 1 lines not run. Read the screen to check the result.)", "script": "click text=\"Confirm Create New Patient\"\nscreenshot"}, {"images": 0, "output": "$ text grep=Raman\nPriya Raman (8)\nMedical Record Dashboard - Priya Raman", "script": "text grep=Raman"}], "started_at": "2026-10-18T20:26:42Z", "tool_errors": 0, "under_load": true} +{"actions": 59, "agent_seconds": 85.475, "agent_summary": "Filled in the New Patient form for Mr. Marcus Feld with all the given values. The duplicate check showed no matches, so I confirmed and saved a new chart (patient ID 2, DOB 1958-09-02). Nothing differed from the referral data.", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 84835, "cache_read_input_tokens": 637787, "calls": [{"cache_creation_input_tokens": 6005, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 168, "seconds": 2.428, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 9712, "cache_read_input_tokens": 6005, "images_in_request": 8, "input_tokens": 2, "output_tokens": 165, "seconds": 3.24, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 4258, "cache_read_input_tokens": 15717, "images_in_request": 11, "input_tokens": 2, "output_tokens": 156, "seconds": 2.644, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 6963, "cache_read_input_tokens": 19975, "images_in_request": 16, "input_tokens": 2, "output_tokens": 189, "seconds": 3.129, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4321, "cache_read_input_tokens": 26938, "images_in_request": 20, "input_tokens": 2, "output_tokens": 128, "seconds": 2.609, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 1566, "cache_read_input_tokens": 31259, "images_in_request": 22, "input_tokens": 2, "output_tokens": 134, "seconds": 2.666, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 2867, "cache_read_input_tokens": 32825, "images_in_request": 24, "input_tokens": 2, "output_tokens": 112, "seconds": 2.718, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 4204, "cache_read_input_tokens": 35692, "images_in_request": 27, "input_tokens": 2, "output_tokens": 402, "seconds": 3.871, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 23517, "cache_read_input_tokens": 39896, "images_in_request": 44, "input_tokens": 2, "output_tokens": 137, "seconds": 6.289, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 2870, "cache_read_input_tokens": 63413, "images_in_request": 46, "input_tokens": 2, "output_tokens": 158, "seconds": 4.147, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 4250, "cache_read_input_tokens": 66283, "images_in_request": 49, "input_tokens": 2, "output_tokens": 202, "seconds": 4.461, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 1640, "cache_read_input_tokens": 70533, "images_in_request": 51, "input_tokens": 2, "output_tokens": 178, "seconds": 4.721, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 4271, "cache_read_input_tokens": 72173, "images_in_request": 54, "input_tokens": 2, "output_tokens": 98, "seconds": 3.905, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 76444, "images_in_request": 57, "input_tokens": 2, "output_tokens": 109, "seconds": 4.152, "stop_reason": "tool_use", "turn": 14}, {"cache_creation_input_tokens": 4201, "cache_read_input_tokens": 80634, "images_in_request": 60, "input_tokens": 2, "output_tokens": 130, "seconds": 3.897, "stop_reason": "tool_use", "turn": 15}], "code_action": null, "correct": true, "cost_usd": 0.364369, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [419.8, 277.3], "images_in_requests": 490, "input_tokens": 32, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 15, "model_seconds": 54.876, "new_charts": 1, "output_tokens": 2466, "program": null, "prompt_tokens": 722654, "referral": "r2", "round": 1, "run_seconds": 85.719, "screenshots_sent": 60, "scripts": null, "started_at": "2026-10-18T20:28:56Z", "tool_errors": 3, "under_load": true} +{"actions": 38, "agent_seconds": 54.898, "agent_summary": null, "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 57054, "cache_read_input_tokens": 323831, "calls": [{"cache_creation_input_tokens": 6003, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 1.8, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6003, "images_in_request": 3, "input_tokens": 2, "output_tokens": 322, "seconds": 2.756, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 20750, "cache_read_input_tokens": 8820, "images_in_request": 18, "input_tokens": 2, "output_tokens": 103, "seconds": 4.339, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 192, "cache_read_input_tokens": 29570, "images_in_request": 19, "input_tokens": 2, "output_tokens": 128, "seconds": 2.543, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1566, "cache_read_input_tokens": 29762, "images_in_request": 21, "input_tokens": 2, "output_tokens": 236, "seconds": 3.488, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 4329, "cache_read_input_tokens": 31328, "images_in_request": 24, "input_tokens": 2, "output_tokens": 217, "seconds": 3.47, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 4309, "cache_read_input_tokens": 35657, "images_in_request": 27, "input_tokens": 2, "output_tokens": 185, "seconds": 4.091, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 4276, "cache_read_input_tokens": 39966, "images_in_request": 30, "input_tokens": 2, "output_tokens": 121, "seconds": 3.285, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 2853, "cache_read_input_tokens": 44242, "images_in_request": 32, "input_tokens": 2, "output_tokens": 203, "seconds": 3.968, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 4293, "cache_read_input_tokens": 47095, "images_in_request": 35, "input_tokens": 2, "output_tokens": 215, "seconds": 4.341, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 5666, "cache_read_input_tokens": 51388, "images_in_request": 39, "input_tokens": 2, "output_tokens": 183, "seconds": 3.675, "stop_reason": "tool_use", "turn": 11}], "code_action": null, "correct": false, "cost_usd": 0.227369, "error": "GateRefused: 'Enter on Contact' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it", "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [{"control": "Enter on Contact", "decision": "refused_wrong_record", "missing": ["Create New Patient"]}], "host_load_1m": [273.9, 203.5], "images_in_requests": 249, "input_tokens": 24, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 11, "model_seconds": 37.755, "new_charts": 0, "output_tokens": 1992, "program": null, "prompt_tokens": 380909, "referral": "r3", "round": 1, "run_seconds": 54.913, "screenshots_sent": 39, "scripts": null, "started_at": "2026-10-18T20:30:29Z", "tool_errors": 2, "under_load": true} +{"actions": 53, "agent_seconds": 65.608, "agent_summary": "Entered all fields for Theo Lindqvist; the duplicate check returned no matches, so I confirmed creation. The chart was saved (patient ID 3).", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 76198, "cache_read_input_tokens": 506800, "calls": [{"cache_creation_input_tokens": 6003, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 1.852, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6003, "images_in_request": 3, "input_tokens": 2, "output_tokens": 352, "seconds": 2.571, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 20747, "cache_read_input_tokens": 8820, "images_in_request": 18, "input_tokens": 2, "output_tokens": 132, "seconds": 2.378, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 221, "cache_read_input_tokens": 29567, "images_in_request": 19, "input_tokens": 2, "output_tokens": 128, "seconds": 2.508, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 1566, "cache_read_input_tokens": 29788, "images_in_request": 21, "input_tokens": 2, "output_tokens": 116, "seconds": 4.223, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 2849, "cache_read_input_tokens": 31354, "images_in_request": 23, "input_tokens": 2, "output_tokens": 82, "seconds": 2.139, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 2815, "cache_read_input_tokens": 34203, "images_in_request": 25, "input_tokens": 2, "output_tokens": 398, "seconds": 3.357, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 23513, "cache_read_input_tokens": 37018, "images_in_request": 42, "input_tokens": 2, "output_tokens": 122, "seconds": 5.905, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 4211, "cache_read_input_tokens": 60531, "images_in_request": 45, "input_tokens": 2, "output_tokens": 139, "seconds": 4.184, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 228, "cache_read_input_tokens": 64742, "images_in_request": 46, "input_tokens": 2, "output_tokens": 104, "seconds": 3.095, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 2837, "cache_read_input_tokens": 64970, "images_in_request": 48, "input_tokens": 2, "output_tokens": 98, "seconds": 2.712, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 67807, "images_in_request": 51, "input_tokens": 2, "output_tokens": 109, "seconds": 3.159, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 4201, "cache_read_input_tokens": 71997, "images_in_request": 54, "input_tokens": 2, "output_tokens": 98, "seconds": 3.001, "stop_reason": "tool_use", "turn": 13}], "code_action": null, "correct": true, "cost_usd": 0.311481, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [214.2, 363.8], "images_in_requests": 396, "input_tokens": 28, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 13, "model_seconds": 41.083, "new_charts": 1, "output_tokens": 1957, "program": null, "prompt_tokens": 583026, "referral": "r4", "round": 1, "run_seconds": 65.622, "screenshots_sent": 54, "scripts": null, "started_at": "2026-10-18T20:31:30Z", "tool_errors": 3, "under_load": true} +{"actions": 40, "agent_seconds": 45.997, "agent_summary": null, "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 57925, "cache_read_input_tokens": 233028, "calls": [{"cache_creation_input_tokens": 6004, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 79, "seconds": 1.823, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 2817, "cache_read_input_tokens": 6004, "images_in_request": 3, "input_tokens": 2, "output_tokens": 278, "seconds": 3.215, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 15239, "cache_read_input_tokens": 8821, "images_in_request": 14, "input_tokens": 2, "output_tokens": 157, "seconds": 3.519, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 246, "cache_read_input_tokens": 24060, "images_in_request": 15, "input_tokens": 2, "output_tokens": 116, "seconds": 2.21, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 4216, "cache_read_input_tokens": 24306, "images_in_request": 18, "input_tokens": 2, "output_tokens": 95, "seconds": 2.115, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 2828, "cache_read_input_tokens": 28522, "images_in_request": 20, "input_tokens": 2, "output_tokens": 399, "seconds": 3.852, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 23514, "cache_read_input_tokens": 31350, "images_in_request": 37, "input_tokens": 2, "output_tokens": 148, "seconds": 5.248, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 237, "cache_read_input_tokens": 54864, "images_in_request": 38, "input_tokens": 2, "output_tokens": 91, "seconds": 2.86, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 2824, "cache_read_input_tokens": 55101, "images_in_request": 40, "input_tokens": 2, "output_tokens": 280, "seconds": 5.27, "stop_reason": "tool_use", "turn": 9}], "code_action": null, "correct": false, "cost_usd": 0.207888, "error": "GateRefused: 'the shortcut Control+End' may commit a write, but the form, row or section it is in does not show ['Create New Patient']; the right-record check refused it", "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [{"control": "the shortcut Control+End", "decision": "refused_wrong_record", "missing": ["Create New Patient"]}], "host_load_1m": [350.8, 294.8], "images_in_requests": 186, "input_tokens": 20, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 9, "model_seconds": 30.112, "new_charts": 0, "output_tokens": 1643, "program": null, "prompt_tokens": 290973, "referral": "r5", "round": 1, "run_seconds": 46.022, "screenshots_sent": 41, "scripts": null, "started_at": "2026-10-18T20:32:50Z", "tool_errors": 2, "under_load": true} +{"actions": 66, "agent_seconds": 101.686, "agent_summary": "Filled the New Patient form for Rosa Delgado, but no chart was saved. OpenEMR's duplicate check found a possible existing patient, Delgado, Rosa, DOB 1958-03-27, Female, External ID 1. That DOB differs from the referral's 1958-03-22. I cancelled and did not confirm creation.", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 95751, "cache_read_input_tokens": 862326, "calls": [{"cache_creation_input_tokens": 6002, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 59, "seconds": 3.053, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 1434, "cache_read_input_tokens": 6002, "images_in_request": 2, "input_tokens": 2, "output_tokens": 190, "seconds": 2.564, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 9732, "cache_read_input_tokens": 7436, "images_in_request": 9, "input_tokens": 2, "output_tokens": 125, "seconds": 2.989, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 4218, "cache_read_input_tokens": 17168, "images_in_request": 12, "input_tokens": 2, "output_tokens": 336, "seconds": 3.765, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 16659, "cache_read_input_tokens": 21386, "images_in_request": 24, "input_tokens": 2, "output_tokens": 172, "seconds": 4.615, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 8337, "cache_read_input_tokens": 38045, "images_in_request": 30, "input_tokens": 2, "output_tokens": 129, "seconds": 3.443, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 4219, "cache_read_input_tokens": 46382, "images_in_request": 33, "input_tokens": 2, "output_tokens": 116, "seconds": 3.435, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 4206, "cache_read_input_tokens": 50601, "images_in_request": 36, "input_tokens": 2, "output_tokens": 111, "seconds": 3.355, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 4203, "cache_read_input_tokens": 54807, "images_in_request": 39, "input_tokens": 2, "output_tokens": 174, "seconds": 4.087, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 6982, "cache_read_input_tokens": 59010, "images_in_request": 44, "input_tokens": 2, "output_tokens": 264, "seconds": 7.064, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 11148, "cache_read_input_tokens": 65992, "images_in_request": 52, "input_tokens": 2, "output_tokens": 148, "seconds": 4.442, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 237, "cache_read_input_tokens": 77140, "images_in_request": 53, "input_tokens": 2, "output_tokens": 128, "seconds": 3.183, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 1566, "cache_read_input_tokens": 77377, "images_in_request": 55, "input_tokens": 2, "output_tokens": 114, "seconds": 4.589, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 4206, "cache_read_input_tokens": 78943, "images_in_request": 58, "input_tokens": 2, "output_tokens": 108, "seconds": 4.537, "stop_reason": "tool_use", "turn": 14}, {"cache_creation_input_tokens": 4200, "cache_read_input_tokens": 83149, "images_in_request": 61, "input_tokens": 2, "output_tokens": 98, "seconds": 3.913, "stop_reason": "tool_use", "turn": 15}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 87349, "images_in_request": 64, "input_tokens": 2, "output_tokens": 120, "seconds": 4.46, "stop_reason": "tool_use", "turn": 16}, {"cache_creation_input_tokens": 4212, "cache_read_input_tokens": 91539, "images_in_request": 67, "input_tokens": 2, "output_tokens": 157, "seconds": 4.425, "stop_reason": "tool_use", "turn": 17}], "code_action": null, "correct": true, "cost_usd": 0.437405, "error": null, "field_mismatches": [], "fields_matched": 0, "fields_total": 12, "gates": [], "host_load_1m": [287.9, 386.7], "images_in_requests": 640, "input_tokens": 36, "mismatch_case": true, "model": "claude-sonnet-5-5", "model_calls": 17, "model_seconds": 67.919, "new_charts": 0, "output_tokens": 2549, "program": null, "prompt_tokens": 958113, "referral": "r6", "round": 1, "run_seconds": 101.699, "screenshots_sent": 67, "scripts": null, "started_at": "2026-10-18T20:33:51Z", "tool_errors": 2, "under_load": true} +{"actions": 54, "agent_seconds": 74.223, "agent_summary": "Registered Mr. Arlo Whitcombe (DOB 1966-05-30, pid 4) with all given fields; duplicate check showed no matches, chart saved. Country selected as \"USA\" (matches country_code).", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 78870, "cache_read_input_tokens": 522504, "calls": [{"cache_creation_input_tokens": 6007, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 171, "seconds": 2.261, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 9715, "cache_read_input_tokens": 6007, "images_in_request": 8, "input_tokens": 2, "output_tokens": 95, "seconds": 2.695, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 2828, "cache_read_input_tokens": 15722, "images_in_request": 10, "input_tokens": 2, "output_tokens": 99, "seconds": 2.673, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 2832, "cache_read_input_tokens": 18550, "images_in_request": 12, "input_tokens": 2, "output_tokens": 226, "seconds": 3.107, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 12467, "cache_read_input_tokens": 21382, "images_in_request": 21, "input_tokens": 2, "output_tokens": 103, "seconds": 3.019, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 192, "cache_read_input_tokens": 33849, "images_in_request": 22, "input_tokens": 2, "output_tokens": 128, "seconds": 2.826, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 1566, "cache_read_input_tokens": 34041, "images_in_request": 24, "input_tokens": 2, "output_tokens": 134, "seconds": 2.688, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 2867, "cache_read_input_tokens": 35607, "images_in_request": 26, "input_tokens": 2, "output_tokens": 82, "seconds": 3.502, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 2815, "cache_read_input_tokens": 38474, "images_in_request": 28, "input_tokens": 2, "output_tokens": 401, "seconds": 4.19, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 23516, "cache_read_input_tokens": 41289, "images_in_request": 45, "input_tokens": 2, "output_tokens": 92, "seconds": 5.531, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 2823, "cache_read_input_tokens": 64805, "images_in_request": 47, "input_tokens": 2, "output_tokens": 119, "seconds": 4.2, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 2852, "cache_read_input_tokens": 67628, "images_in_request": 49, "input_tokens": 2, "output_tokens": 98, "seconds": 3.222, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 4190, "cache_read_input_tokens": 70480, "images_in_request": 52, "input_tokens": 2, "output_tokens": 108, "seconds": 5.563, "stop_reason": "tool_use", "turn": 13}, {"cache_creation_input_tokens": 4200, "cache_read_input_tokens": 74670, "images_in_request": 55, "input_tokens": 2, "output_tokens": 118, "seconds": 3.789, "stop_reason": "tool_use", "turn": 14}], "code_action": null, "correct": true, "cost_usd": 0.321476, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [361.0, 154.4], "images_in_requests": 400, "input_tokens": 30, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 14, "model_seconds": 49.265, "new_charts": 1, "output_tokens": 1974, "program": null, "prompt_tokens": 601404, "referral": "r7", "round": 1, "run_seconds": 74.238, "screenshots_sent": 55, "scripts": null, "started_at": "2026-10-18T20:35:37Z", "tool_errors": 2, "under_load": true} +{"actions": 57, "agent_seconds": 76.202, "agent_summary": "Registered Priya Raman (pid 5, DOB 1975-10-14) with all given fields; the duplicate check found no matches, and the chart was saved.", "app": "openemr-referral", "arm": "every_turn", "cache_creation_input_tokens": 84338, "cache_read_input_tokens": 553227, "calls": [{"cache_creation_input_tokens": 5998, "cache_read_input_tokens": 0, "images_in_request": 1, "input_tokens": 4, "output_tokens": 59, "seconds": 1.817, "stop_reason": "tool_use", "turn": 1}, {"cache_creation_input_tokens": 1434, "cache_read_input_tokens": 5998, "images_in_request": 2, "input_tokens": 2, "output_tokens": 188, "seconds": 2.199, "stop_reason": "tool_use", "turn": 2}, {"cache_creation_input_tokens": 9730, "cache_read_input_tokens": 7432, "images_in_request": 9, "input_tokens": 2, "output_tokens": 118, "seconds": 3.081, "stop_reason": "tool_use", "turn": 3}, {"cache_creation_input_tokens": 4211, "cache_read_input_tokens": 17162, "images_in_request": 12, "input_tokens": 2, "output_tokens": 332, "seconds": 3.779, "stop_reason": "tool_use", "turn": 4}, {"cache_creation_input_tokens": 16655, "cache_read_input_tokens": 21373, "images_in_request": 24, "input_tokens": 2, "output_tokens": 182, "seconds": 4.509, "stop_reason": "tool_use", "turn": 5}, {"cache_creation_input_tokens": 8347, "cache_read_input_tokens": 38028, "images_in_request": 30, "input_tokens": 2, "output_tokens": 131, "seconds": 4.242, "stop_reason": "tool_use", "turn": 6}, {"cache_creation_input_tokens": 5580, "cache_read_input_tokens": 46375, "images_in_request": 34, "input_tokens": 2, "output_tokens": 255, "seconds": 5.52, "stop_reason": "tool_use", "turn": 7}, {"cache_creation_input_tokens": 13856, "cache_read_input_tokens": 51955, "images_in_request": 44, "input_tokens": 2, "output_tokens": 169, "seconds": 4.202, "stop_reason": "tool_use", "turn": 8}, {"cache_creation_input_tokens": 5621, "cache_read_input_tokens": 65811, "images_in_request": 48, "input_tokens": 2, "output_tokens": 134, "seconds": 3.503, "stop_reason": "tool_use", "turn": 9}, {"cache_creation_input_tokens": 222, "cache_read_input_tokens": 71432, "images_in_request": 49, "input_tokens": 2, "output_tokens": 116, "seconds": 2.959, "stop_reason": "tool_use", "turn": 10}, {"cache_creation_input_tokens": 4216, "cache_read_input_tokens": 71654, "images_in_request": 52, "input_tokens": 2, "output_tokens": 175, "seconds": 3.977, "stop_reason": "tool_use", "turn": 11}, {"cache_creation_input_tokens": 4267, "cache_read_input_tokens": 75870, "images_in_request": 55, "input_tokens": 2, "output_tokens": 109, "seconds": 3.887, "stop_reason": "tool_use", "turn": 12}, {"cache_creation_input_tokens": 4201, "cache_read_input_tokens": 80137, "images_in_request": 58, "input_tokens": 2, "output_tokens": 97, "seconds": 4.17, "stop_reason": "tool_use", "turn": 13}], "code_action": null, "correct": true, "cost_usd": 0.342196, "error": null, "field_mismatches": [], "fields_matched": 12, "fields_total": 12, "gates": [{"control": "Confirm Create New Patient", "decision": "passed_no_hold", "missing": []}], "host_load_1m": [143.8, 208.8], "images_in_requests": 418, "input_tokens": 28, "mismatch_case": false, "model": "claude-sonnet-5-5", "model_calls": 13, "model_seconds": 47.844, "new_charts": 1, "output_tokens": 2065, "program": null, "prompt_tokens": 637593, "referral": "r8", "round": 1, "run_seconds": 76.217, "screenshots_sent": 58, "scripts": null, "started_at": "2026-10-18T20:36:57Z", "tool_errors": 1, "under_load": true} diff --git a/packages/flow/benchmark/code_as_action/results/openemr/summary.json b/packages/flow/benchmark/code_as_action/results/openemr/summary.json new file mode 100644 index 0000000000..30d5471ad7 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/results/openemr/summary.json @@ -0,0 +1,115 @@ +{ + "arms": { + "code": { + "correct": 8, + "median_actions": 36.0, + "median_cost_usd": 0.055, + "median_model_calls": 12.5, + "median_model_seconds": 18.542, + "median_output_tokens": 1023.5, + "median_prompt_tokens": 90777.0, + "median_run_seconds": 45.395, + "median_screenshots_sent": 2.0, + "runs": 8, + "runs_under_load": 8, + "total_cost_usd": 0.4463 + }, + "every_turn": { + "correct": 6, + "median_actions": 55.5, + "median_cost_usd": 0.332, + "median_model_calls": 13.5, + "median_model_seconds": 48.555, + "median_output_tokens": 2028.5, + "median_prompt_tokens": 619498.5, + "median_run_seconds": 75.227, + "median_screenshots_sent": 56.5, + "runs": 8, + "runs_under_load": 8, + "total_cost_usd": 2.6388 + } + }, + "arms_by_case": { + "duplicate": { + "code": { + "correct": 1, + "median_actions": 35.0, + "median_cost_usd": 0.048, + "median_model_calls": 10.0, + "median_model_seconds": 17.401, + "median_output_tokens": 887.0, + "median_prompt_tokens": 68795.0, + "median_run_seconds": 45.673, + "median_screenshots_sent": 2.0, + "runs": 1, + "runs_under_load": 1, + "total_cost_usd": 0.0476 + }, + "every_turn": { + "correct": 1, + "median_actions": 66.0, + "median_cost_usd": 0.437, + "median_model_calls": 17.0, + "median_model_seconds": 67.919, + "median_output_tokens": 2549.0, + "median_prompt_tokens": 958113.0, + "median_run_seconds": 101.699, + "median_screenshots_sent": 67.0, + "runs": 1, + "runs_under_load": 1, + "total_cost_usd": 0.4374 + } + }, + "normal": { + "code": { + "correct": 7, + "median_actions": 36.0, + "median_cost_usd": 0.062, + "median_model_calls": 14.0, + "median_model_seconds": 18.551, + "median_output_tokens": 1099.0, + "median_prompt_tokens": 103679.0, + "median_run_seconds": 45.116, + "median_screenshots_sent": 2.0, + "runs": 7, + "runs_under_load": 7, + "total_cost_usd": 0.3986 + }, + "every_turn": { + "correct": 5, + "median_actions": 54.0, + "median_cost_usd": 0.321, + "median_model_calls": 13.0, + "median_model_seconds": 47.844, + "median_output_tokens": 1992.0, + "median_prompt_tokens": 601404.0, + "median_run_seconds": 74.238, + "median_screenshots_sent": 55.0, + "runs": 7, + "runs_under_load": 7, + "total_cost_usd": 2.2014 + } + } + }, + "environment": { + "app": "OpenEMR local fixture (pinned image), New Patient form, 1280x800, headless", + "blocks": [ + { + "arm": "every_turn", + "at": "2026-10-18T20:28:52Z", + "baseline_sha256": "fdcab903921f0d2f518317e659ab79528a31c8b9639365dc1e3758376afc0d12", + "round": 1, + "seeded_existing_chart_status": 201 + } + ], + "effort": "medium", + "engine_commit": "24313234", + "load_limit_1m": 12.0, + "measured_at": "2026-10-18T20:38:15Z", + "model": "claude-sonnet-5-5", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "python": "3.12.7", + "spend_total_usd": 3.860689, + "started_at": "2026-10-18T20:28:47Z" + } +} diff --git a/packages/flow/benchmark/code_as_action/results/spend.json b/packages/flow/benchmark/code_as_action/results/spend.json index 407f33f158..586469c7c0 100644 --- a/packages/flow/benchmark/code_as_action/results/spend.json +++ b/packages/flow/benchmark/code_as_action/results/spend.json @@ -114,8 +114,83 @@ "label": "openemr every_turn r1", "usd": 0.426621, "at": "2026-10-18T20:08:39Z" + }, + { + "label": "openemr code r1", + "usd": 0.065987, + "at": "2026-10-18T20:11:53Z" + }, + { + "label": "openemr code r2", + "usd": 0.046911, + "at": "2026-10-18T20:13:59Z" + }, + { + "label": "openemr code r3", + "usd": 0.045941, + "at": "2026-10-18T20:15:47Z" + }, + { + "label": "openemr code r4", + "usd": 0.048208, + "at": "2026-10-18T20:18:14Z" + }, + { + "label": "openemr code r5", + "usd": 0.061664, + "at": "2026-10-18T20:21:30Z" + }, + { + "label": "openemr code r6", + "usd": 0.047633, + "at": "2026-10-18T20:24:20Z" + }, + { + "label": "openemr code r7", + "usd": 0.065492, + "at": "2026-10-18T20:26:39Z" + }, + { + "label": "openemr code r8", + "usd": 0.064428, + "at": "2026-10-18T20:28:46Z" + }, + { + "label": "openemr every_turn r2", + "usd": 0.364369, + "at": "2026-10-18T20:30:26Z" + }, + { + "label": "openemr every_turn r3", + "usd": 0.227369, + "at": "2026-10-18T20:31:27Z" + }, + { + "label": "openemr every_turn r4", + "usd": 0.311481, + "at": "2026-10-18T20:32:47Z" + }, + { + "label": "openemr every_turn r5", + "usd": 0.207888, + "at": "2026-10-18T20:33:48Z" + }, + { + "label": "openemr every_turn r6", + "usd": 0.437405, + "at": "2026-10-18T20:35:34Z" + }, + { + "label": "openemr every_turn r7", + "usd": 0.321476, + "at": "2026-10-18T20:36:53Z" + }, + { + "label": "openemr every_turn r8", + "usd": 0.342196, + "at": "2026-10-18T20:38:15Z" } ], - "total_usd": 1.202241, + "total_usd": 3.860689, "cap_usd": 15.0 } diff --git a/packages/flow/benchmark/code_as_action/summarize.py b/packages/flow/benchmark/code_as_action/summarize.py new file mode 100644 index 0000000000..73176e9e56 --- /dev/null +++ b/packages/flow/benchmark/code_as_action/summarize.py @@ -0,0 +1,98 @@ +"""Rebuild results//summary.json from the committed runs and print the tables. + + python benchmark/code_as_action/summarize.py + +Medians are over all runs of an arm; "correct" is the read-back verdict. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import common # noqa: E402 + +KEYS = [ + "model_calls", + "actions", + "prompt_tokens", + "output_tokens", + "cost_usd", + "model_seconds", + "run_seconds", + "screenshots_sent", +] +ROWS = [ + ("Correct (read back)", None), + ("Model calls", "model_calls"), + ("Actions", "actions"), + ("Prompt tokens", "prompt_tokens"), + ("Output tokens", "output_tokens"), + ("Cost", "cost_usd"), + ("Model time", "model_seconds"), + ("Run time (under load)", "run_seconds"), + ("Screenshots sent", "screenshots_sent"), +] + + +def _fmt(key: str, value: object) -> str: + if value is None: + return "n/a" + v = float(value) # type: ignore[arg-type] + if key == "cost_usd": + return f"${v:.3f}" + if key.endswith("seconds"): + return f"{v:.1f} s" + return f"{v:,.0f}" + + +def table(summary: dict, arms: list[str]) -> str: + head = "| median per run | " + " | ".join(arms) + " |" + lines = [head, "| --- |" + " ---: |" * len(arms)] + for label, key in ROWS: + cells = [] + for arm in arms: + s = summary[arm] + if key is None: + cells.append(f"{s['correct']} / {s['runs']}") + else: + cells.append(_fmt(key, s.get(f"median_{key}"))) + lines.append(f"| {label} | " + " | ".join(cells) + " |") + return "\n".join(lines) + + +def main() -> int: + for app, arms in ( + ("mockmed", ["code", "selective", "every_turn"]), + ("openemr", ["code", "every_turn"]), + ): + path = common.RESULTS / app / "runs.jsonl" + if not path.is_file(): + continue + rows = [json.loads(line) for line in path.read_text().splitlines() if line] + arms = [a for a in arms if any(r["arm"] == a for r in rows)] + summary_path = common.RESULTS / app / "summary.json" + old = json.loads(summary_path.read_text()) if summary_path.is_file() else {} + old["arms"] = common.summarize(rows, arms, KEYS) + if app == "openemr": + old["arms_by_case"] = { + case: common.summarize( + [r for r in rows if r["mismatch_case"] == (case == "duplicate")], + arms, + KEYS, + ) + for case in ("normal", "duplicate") + } + old.setdefault("environment", {})["spend_total_usd"] = common.spent() + summary_path.write_text(json.dumps(old, indent=2, sort_keys=True) + "\n") + print(f"## {app}\n") + print(table(old["arms"], arms)) + print() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/openadapt_flow/compile_agent/code_action.py b/packages/flow/openadapt_flow/compile_agent/code_action.py index ed3b257a82..14dc67859b 100644 --- a/packages/flow/openadapt_flow/compile_agent/code_action.py +++ b/packages/flow/openadapt_flow/compile_agent/code_action.py @@ -1433,14 +1433,42 @@ def _watch(request: Any, harness: Harness) -> dict[str, str]: # -- the compiled program -------------------------------------------------------------- -def check_trace_against_ir(trace: Mapping[str, Any], workflow: Any) -> dict[str, Any]: +def distilled_events(recording_dir: Optional[Path]) -> dict[int, str]: + """``{event index: rule}`` for the steps distill dropped from a recording + (its ``distill.json``), or an empty dict.""" + + import json + + if recording_dir is None: + return {} + path = Path(recording_dir) / "distill.json" + if not path.is_file(): + return {} + try: + report = json.loads(path.read_text()) + except (OSError, ValueError): + return {} + return {int(d["index"]): str(d["rule"]) for d in report.get("dropped") or []} + + +def check_trace_against_ir( + trace: Mapping[str, Any], + workflow: Any, + *, + distilled: Optional[Mapping[int, str]] = None, +) -> dict[str, Any]: """Does the compiled program carry the identity the agent acted on? For each step the agent took by structure, the recorder's locator for its - target must be on one of the program's steps, in order. Returns counts and - the steps that are missing. + target must be on one of the program's steps, in order. A step whose + recorded event distill dropped (``distilled``, from + :func:`distilled_events`: a click with no effect, a detour that ended + where it started) is reported as distilled, not missing. Returns counts + and the steps that are missing. """ + distilled = dict(distilled or {}) + def key(locator: Mapping[str, Any]) -> tuple[Any, ...]: return ( locator.get("selector"), @@ -1456,11 +1484,16 @@ def key(locator: Mapping[str, Any]) -> tuple[Any, ...]: structural = getattr(anchor, "structural", None) if anchor else None if structural is not None: program.append(key(structural.model_dump(exclude_none=True))) - wanted = [ - (int(step["i"]), key(step["recorded"])) - for step in trace.get("steps", []) - if step.get("recorded") - ] + wanted = [] + dropped: list[dict[str, Any]] = [] + for step in trace.get("steps", []): + if not step.get("recorded"): + continue + events = [int(e) for e in step.get("events") or []] + if events and events[0] in distilled: + dropped.append({"step": int(step["i"]), "rule": distilled[events[0]]}) + continue + wanted.append((int(step["i"]), key(step["recorded"]))) missing: list[int] = [] cursor = 0 for index, locator in wanted: @@ -1472,7 +1505,8 @@ def key(locator: Mapping[str, Any]) -> tuple[Any, ...]: else: cursor = found + 1 return { - "trace_steps_with_locator": len(wanted), + "trace_steps_with_locator": len(wanted) + len(dropped), + "distilled_steps": dropped, "in_program": len(wanted) - len(missing), "missing_steps": missing, "program_structural_steps": len(program), diff --git a/packages/flow/openadapt_flow/compile_agent/pipeline.py b/packages/flow/openadapt_flow/compile_agent/pipeline.py index d844c6f48c..532376e9ea 100644 --- a/packages/flow/openadapt_flow/compile_agent/pipeline.py +++ b/packages/flow/openadapt_flow/compile_agent/pipeline.py @@ -600,14 +600,22 @@ def _check_trace(jit: CompiledAgent, trace: Mapping[str, Any]) -> dict[str, Any] """Whether the draft program carries the locator of every element the agent acted on by structure.""" - from openadapt_flow.compile_agent.code_action import check_trace_against_ir + from openadapt_flow.compile_agent.code_action import ( + check_trace_against_ir, + distilled_events, + ) from openadapt_flow.ir import Workflow try: workflow = Workflow.load(jit.bundle_dir) except Exception as exc: # noqa: BLE001 - reported, never admitted on return {"passed": False, "error": f"{type(exc).__name__}: {exc}"} - return check_trace_against_ir(trace, workflow) + recording = (jit.candidate() or {}).get("compiled_recording") + return check_trace_against_ir( + trace, + workflow, + distilled=distilled_events(Path(recording) if recording else None), + ) def _set_aside(self, jit: CompiledAgent, status: str, reasons: list[str]) -> Path: program = jit.program_dir diff --git a/packages/flow/tests/test_code_action.py b/packages/flow/tests/test_code_action.py index d74e2b887d..ea1218f738 100644 --- a/packages/flow/tests/test_code_action.py +++ b/packages/flow/tests/test_code_action.py @@ -472,6 +472,41 @@ def test_the_program_must_carry_every_locator_in_order() -> None: assert not swapped["passed"] and swapped["missing_steps"] == [1] +def test_a_detour_distill_dropped_is_not_missing(tmp_path: Path) -> None: + import json + + from openadapt_flow.compile_agent.code_action import distilled_events + + trace = { + "steps": [ + {"i": 0, "events": [0], "recorded": {"role": "button", "name": "Contact"}}, + {"i": 1, "events": [1], "recorded": {"role": "button", "name": "Contact"}}, + {"i": 2, "events": [2], "recorded": {"selector": "#save"}}, + ] + } + (tmp_path / "distill.json").write_text( + json.dumps( + { + "dropped": [ + {"index": 0, "kind": "click", "rule": "dead_end"}, + {"index": 1, "kind": "click", "rule": "dead_end"}, + ] + } + ) + ) + dropped = distilled_events(tmp_path) + assert dropped == {0: "dead_end", 1: "dead_end"} + program = _workflow({"selector": "#save"}) + assert not check_trace_against_ir(trace, program)["passed"] + ok = check_trace_against_ir(trace, program, distilled=dropped) + assert ok["passed"] and ok["in_program"] == 1 + assert ok["distilled_steps"] == [ + {"step": 0, "rule": "dead_end"}, + {"step": 1, "rule": "dead_end"}, + ] + assert distilled_events(tmp_path / "none") == {} + + # -- the scripted agent and the Claude loop ----------------------------------------- From a4dbc19ec9523b40f83315e691ff1eb3cc7af985 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 16:46:00 -0400 Subject: [PATCH 07/13] bench(code-as-action): OpenEMR referral results, README, docs tables Co-Authored-By: Claude Opus 5.5 --- .../flow/benchmark/code_as_action/README.md | 118 ++++++++++++++++++ .../results/mockmed/summary.json | 2 +- .../results/openemr-distill-check/runs.jsonl | 2 + .../openemr-distill-check/summary.json | 39 ++++++ .../results/openemr/summary.json | 2 +- .../code_as_action/results/spend.json | 12 +- packages/flow/docs/CAPABILITIES.md | 7 ++ packages/flow/docs/CODE_AS_ACTION.md | 54 +++++++- packages/flow/docs/COMPILE_AGENT.md | 6 + 9 files changed, 233 insertions(+), 9 deletions(-) create mode 100644 packages/flow/benchmark/code_as_action/README.md create mode 100644 packages/flow/benchmark/code_as_action/results/openemr-distill-check/runs.jsonl create mode 100644 packages/flow/benchmark/code_as_action/results/openemr-distill-check/summary.json diff --git a/packages/flow/benchmark/code_as_action/README.md b/packages/flow/benchmark/code_as_action/README.md new file mode 100644 index 0000000000..713cd646bb --- /dev/null +++ b/packages/flow/benchmark/code_as_action/README.md @@ -0,0 +1,118 @@ +# Code as action against the screenshot loop + +Measured 2026-10-18 with `claude-sonnet-5-5` at effort `medium`, on the +operator's own key, in the compile-agent harness. Every arm runs through the +same harness, save gate, JIT capture and compiler; only what the agent sees +and how it acts differ: + +- **code**: `ClaudeAgent(vision="code")`, one `run` tool that takes a script + ([docs/CODE_AS_ACTION.md](../../docs/CODE_AS_ACTION.md)). +- **every_turn**: Claude's computer toolset (`computer_toolset_20260801`), a + screenshot after every action. The screenshot loop. +- **selective** (MockMed only): the structural listing after every action, a + screenshot on request (compile-agent's default). + +Correct is decided by reading the system of record back, never by the agent's +own report. Synthetic data only. + +**Load.** The host (12 cores) was shared with other sessions; its 1-minute +load was between 115 and 420 at the start and end of every run, so every row is marked +`under_load: true` and every time below was taken under load. Tokens, model +calls, actions and cost do not depend on load. + +## MockMed (5 trials per arm, `results/mockmed/`) + +Task: "Open the referral for Jane Sample, start a Triage encounter, and save +this note: {note}", with a distinct note per trial, arms in rotating order, +MockMed reset before each run. Correct = the record check passed (exactly one +new encounter, on patient p1, type Triage, with the note). + +| median per run | code | selective | every_turn | +| --- | ---: | ---: | ---: | +| Correct (read back) | 5 / 5 | 5 / 5 | 5 / 5 | +| Model calls | 6 | 5 | 5 | +| Actions | 6 | 6 | 13 | +| Prompt tokens | 12,738 | 10,332 | 73,303 | +| Output tokens | 350 | 359 | 533 | +| Cost | $0.012 | $0.014 | $0.075 | +| Model time | 5.9 s | 5.8 s | 9.7 s | +| Run time (under load) | 21.5 s | 24.0 s | 26.0 s | +| Screenshots sent | 0 | 1 | 14 | + +On a five-step task, code as action and the structural listing are about the +same; both read about a sixth of the screenshot loop's tokens and cost about +a sixth as much. The program compiled from a code-arm run was admitted and +replayed twice: VERIFIED both times, 0 model calls (`replays.jsonl`). + +## OpenEMR referral (8 referrals per arm, `results/openemr/`) + +The referral task from [openemr_referral_timing](../openemr_referral_timing/): +register a referred, made-up patient from 12 structured fields in OpenEMR's +New Patient form (it sits in an iframe; a duplicate-check dialog opens in a +second one), and do not create a duplicate. r6 matches an existing chart's +name with a different birth date: correct means no new chart. Sign-in and the +blank form are set up before the clock starts. One block per arm from a reset +fixture (pinned image, `realrun` state), referrals r1 to r8 in order. Correct += SQL read-back by the test setup: exactly one new chart with 12 of 12 fields +matching (r6: no new chart). + +| median per run | code | every_turn | +| --- | ---: | ---: | +| Correct (read back) | 8 / 8 | 6 / 8 | +| Model calls | 12 | 14 | +| Actions | 36 | 56 | +| Prompt tokens | 90,777 | 619,498 | +| Output tokens | 1,024 | 2,028 | +| Cost | $0.055 | $0.332 | +| Model time | 18.5 s | 48.6 s | +| Run time (under load) | 45.4 s | 75.2 s | +| Screenshots sent | 2 | 56 | + +- The code agent read 6.8 times fewer prompt tokens, cost 6 times less, and + took 36 actions to the screenshot loop's 56. It asked for a screenshot in + every run (1 to 4 each), mostly to find a section below the fold. Both + stopped correctly on r6. +- The two screenshot-loop misses were stopped by the save gate before any + save: Enter on the "Contact" button (a `