diff --git a/packages/flow/benchmark/openemr_api_tier/approval_durable.py b/packages/flow/benchmark/openemr_api_tier/approval_durable.py new file mode 100644 index 000000000..7b3e57f92 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/approval_durable.py @@ -0,0 +1,445 @@ +"""The OpenEMR referral with the API step under each_item approval and a durable run. + +Label every number from this harness exactly as: "API step under each_item +approval + durable, no run gate; harness answers approvals; approval wait +excluded". It is not the governed production profile. The referral bundle +doesn't pass the run gate yet (GUI-path admission; see docs/API_TIER.md and +the admission list in the session notes), so this measures the parts that +do run: the person approving the exact request (docs/APPROVAL_MODES.md) and +the durable checkpoints (docs/API_TIER.md "In a durable run"). + +- Arms: ``browser`` (API steps off) and ``api`` (API steps on), both under + ``ApprovalGate("each_item")`` with ``durable=True``. Interleaved blocks; each + block resets the fixture and seeds the chart r6 must not duplicate. +- Approvals: the harness stands in for the person and answers each request + as soon as it is written (``decided_by="human"``, route ``desktop_run_it``, + approver ``harness-approver``). Time spent waiting for the answer is + measured and subtracted: ``engine_s`` excludes it, ``approval_wait_s`` is it. +- Read-back: the test setup's own direct SQL check (``common.record_check``), + never the arm under test. r6 is right when no chart is created. +- Load: each item waits until the 1-minute load is under GATE_LOAD1 (12). + + PYTHONPATH=packages/flow python -m benchmark.openemr_api_tier.approval_durable \\ + compile --recording --out + PYTHONPATH=packages/flow python -m benchmark.openemr_api_tier.approval_durable \\ + run --bundle --out [--blocks 5 --per-block 4] +""" + +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import statistics +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from benchmark.openemr_api_tier.common import ( + REFERRALS, + load_demo, + record_check, + reset_and_seed, + spec_for, + sql_patients, +) +from benchmark.openemr_api_tier.feature_e2e import NO_MATCH_TEXT, _load1, wait_quiet +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + +LABEL = ( + "API step under each_item approval + durable, no run gate; harness answers " + "approvals; approval wait excluded" +) +APPROVER = "harness-approver" +COLS = [ + "arm", + "block", + "pos", + "ref", + "mismatch", + "correct", + "readback", + "new_charts", + "engine_success", + "transaction_outcome", + "actuation", + "api_fallback", + "approvals", + "approval_kinds", + "approval_wait_s", + "engine_s", + "wall_s", + "checkpoints", + "error", + "load_start", + "load_end", + "gate_waited_s", + "engine_sha", +] + + +def cmd_compile(a: argparse.Namespace) -> None: + """The harness compile (feature_e2e), admitted with a right-record parameter.""" + + from openadapt_flow.compiler.api_proposal import ( + admit_api_step, + describe_api_step, + propose_api_step, + ) + from openadapt_flow.ir import Postcondition, PostconditionKind, Workflow + + demo = load_demo() + rec, bundle = Path(a.recording), Path(a.out) + demo.compile_recording(rec, bundle) + wf = Workflow.load(bundle) + dialog = [ + s + for s in wf.steps + if s.anchor is not None + and s.anchor.structural is not None + and s.anchor.structural.selector == "#create" + ] + assert len(dialog) == 1 + dialog[0].expect = list(dialog[0].expect) + [ + Postcondition( + kind=PostconditionKind.TEXT_PRESENT, text=NO_MATCH_TEXT, timeout_s=10.0 + ) + ] + wf, step, reasons = propose_api_step(wf, rec, replace=True) + if step is None: + raise SystemExit(f"no API step: {reasons}") + # The harness admits under the fixture's name so the matrix runs + # unattended; in use a person admits with `openadapt-flow api admit`. + wf, _ = admit_api_step( + wf, admitted_by="benchmark-fixture", identity={"subject_name": "lname"} + ) + wf.save(bundle) + print(describe_api_step(next(s for s in wf.steps if s.id == step.id))) + + +class HarnessPerson: + """Answers every waiting approval request at once, and times the wait.""" + + def __init__(self, store: Any) -> None: + self.store = store + self.waited_s = 0.0 + self.answered: list[str] = [] + + def sleep(self, seconds: float) -> None: + from openadapt_flow.approval_mode.item_approval import make_answer + + t0 = time.monotonic() + for request in self.store.requests(): + if request.request_id in self.answered: + continue + if self.store.status(request) != "waiting": + continue + self.store.write_answer( + make_answer( + request, + decision="approve", + decided_by="human", + route="desktop_run_it", + approver_id=APPROVER, + auth_method="cloud_aal2", + ) + ) + self.answered.append(request.request_id) + time.sleep(min(seconds, 0.02)) + self.waited_s += time.monotonic() - t0 + + +def run_one( + env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path +) -> dict[str, Any]: + from openadapt_flow.approval_mode.gate import ApprovalGate + from openadapt_flow.approval_mode.item_approval import ItemApprovalStore + from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + from openadapt_flow.runtime.durable.checkpoint import CheckpointStore + + demo, f, browser, ocr = env["demo"], env["f"], env["browser"], env["ocr"] + ref = REFERRALS[ref_idx - 1] + spec = spec_for(ref) + out.mkdir(parents=True) + engine_dir = out / "engine" + before_ids = {r["id"] for r in sql_patients(f)} + oracle = OpenEMRPatientOracle(f.api_base_url, f.token_session("oracle"), spec) + res: dict[str, Any] = { + "arm": arm, + "ref": ref.key, + "mismatch": ref.mismatch, + "error": None, + } + store = ItemApprovalStore(engine_dir) + person = HarnessPerson(store) + gate = ApprovalGate( + "each_item", + approval_store=store, + approval_wait_s=120.0, + approval_poll_s=0.05, + sleep=person.sleep, + monotonic=time.monotonic, + ) + t0 = time.time() + ctx = browser.new_context( + viewport={"width": 1280, "height": 800}, device_scale_factor=1 + ) + try: + page = ctx.new_page() + page.goto(f.ui_base_url) + backend = demo._BasePlaywrightBackend(page) + demo._preauthenticate_browser(backend, f) + t_setup = time.time() + ocr.clear_ocr_memo() + rep = Replayer( + backend, + effect_verifier=oracle.verifier, + api_steps=(arm == "api"), + approval_gate=gate, + durable=True, + ).run( + Workflow.load(bundle), + params=spec.params(), + bundle_dir=bundle, + run_dir=engine_dir, + ) + t_task = time.time() + raw = rep.model_dump(mode="json") + last = raw["results"][-1] if raw["results"] else {} + res.update( + engine_success=raw.get("success"), + transaction_outcome=raw.get("transaction_outcome"), + actuation=last.get("actuation"), + api_fallback=next( + (r["api_fallback"] for r in raw["results"] if r.get("api_fallback")), + None, + ), + step_errors=[ + (r["step_id"], (r.get("error") or "")[:300]) + for r in raw["results"] + if r.get("error") + ], + ) + except Exception as exc: # noqa: BLE001 + t_setup = locals().get("t_setup", time.time()) + t_task = time.time() + res["error"] = f"{type(exc).__name__}: {exc}"[:500] + time.sleep(demo.ORACLE_SETTLE_S) + res.update(record_check(f, ref, before_ids)) + t_end = time.time() + ctx.close() + requests = store.requests() + res.update( + approvals=len(requests), + approval_kinds="+".join(r.form_state_kind for r in requests), + approval_wait_s=round(person.waited_s, 3), + engine_s=round(t_task - t_setup - person.waited_s, 3), + wall_s=round(t_end - t0, 3), + checkpoints=len(CheckpointStore(engine_dir).checkpoints()), + ) + (out / "result.json").write_text(json.dumps(res, indent=1, default=str)) + shutil.rmtree(engine_dir / "steps", ignore_errors=True) + return res + + +def cmd_run(a: argparse.Namespace) -> None: + from playwright.sync_api import sync_playwright + + import openadapt_flow.vision.ocr # noqa: F401 + + ocr = sys.modules["openadapt_flow.vision.ocr"] + out, bundle = Path(a.out), Path(a.bundle) + out.mkdir(parents=True, exist_ok=True) + sha = subprocess.run( + ["git", "rev-parse", "HEAD"], capture_output=True, text=True + ).stdout.strip()[:9] + demo, f = load_demo(), SpikeFixture() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + ocr._get_engine() + env = {"demo": demo, "f": f, "browser": browser, "ocr": ocr} + csv_path = out / "runs.csv" + rows: list[dict[str, Any]] = [] + done = ( + { + d.name + for d in (Path(a.skip_done) / "items").iterdir() + if (d / "result.json").is_file() + } + if a.skip_done + else set() + ) + try: + for b in range(a.from_block - 1, a.blocks): + for arm in ["browser", "api"] if b % 2 == 0 else ["api", "browser"]: + names = [ + f"b{b + 1:02d}-{arm}-{pos + 1}-" + f"{REFERRALS[(b * a.per_block + pos) % len(REFERRALS)].key}" + for pos in range(a.per_block) + ] + if all(n in done for n in names): + continue + reset_and_seed(f) + for pos in range(a.per_block): + idx = ((b * a.per_block + pos) % len(REFERRALS)) + 1 + if names[pos] in done: + continue + gate = wait_quiet() + load_start = _load1() + res = run_one( + env, + arm, + idx, + bundle, + out + / "items" + / f"b{b + 1:02d}-{arm}-{pos + 1}-{REFERRALS[idx - 1].key}", + ) + row = {k: res.get(k) for k in COLS} | { + "block": b + 1, + "pos": pos + 1, + "load_start": load_start, + "load_end": _load1(), + "gate_waited_s": gate["gate_waited_s"], + "engine_sha": sha, + } + rows.append(row) + print( + json.dumps( + { + k: row[k] + for k in ( + "arm", + "block", + "ref", + "correct", + "actuation", + "approvals", + "engine_s", + "approval_wait_s", + "error", + ) + } + ), + flush=True, + ) + finally: + browser.close() + pw.stop() + with open(csv_path, "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text( + json.dumps(summarize(rows, sha), indent=1) + "\n" + ) + + +def summarize(rows: list[dict[str, Any]], sha: str) -> dict[str, Any]: + """Per arm: correctness by read-back, engine time, approvals.""" + + out: dict[str, Any] = {"label": LABEL, "engine_sha": sha, "arms": {}} + for arm in ("browser", "api"): + mine = [r for r in rows if r["arm"] == arm] + if not mine: + continue + times = [r["engine_s"] for r in mine if r["engine_s"] is not None] + out["arms"][arm] = { + "runs": len(mine), + "correct": sum(bool(r["correct"]) for r in mine), + "errors": sum(bool(r["error"]) for r in mine), + "api_saves": sum(r["actuation"] == "api" for r in mine), + "approvals_per_item": sorted({r["approvals"] for r in mine}), + "approval_kinds": sorted({r["approval_kinds"] for r in mine}), + "median_engine_s": round(statistics.median(times), 2) if times else None, + "median_approval_wait_s": round( + statistics.median(r["approval_wait_s"] for r in mine), 3 + ), + "max_load_start": max( + (r["load_start"] for r in mine if r["load_start"] is not None), + default=None, + ), + # Stopped before saving, nothing written (a fail-closed halt or a + # refused duplicate): not a wrong save. + "safe_stops": sum( + not r["correct"] and r.get("new_charts") == 0 for r in mine + ), + "wrong_saves": sum( + not r["correct"] and (r.get("new_charts") or 0) > 0 for r in mine + ), + } + return out + + +def rows_from_items(out: Path, sha: str) -> list[dict[str, Any]]: + """Rebuild the rows from each item's result.json (an interrupted run).""" + + rows = [] + for item in sorted((out / "items").iterdir()): + result = item / "result.json" + if not result.is_file(): + continue # interrupted before the item finished + res = json.loads(result.read_text()) + block, arm, pos = item.name.split("-")[:3] + rows.append( + {k: res.get(k) for k in COLS} + | { + "block": int(block[1:]), + "pos": int(pos), + "load_start": None, + "load_end": None, + "gate_waited_s": None, + "engine_sha": sha, + } + ) + return rows + + +def cmd_summarize(a: argparse.Namespace) -> None: + """Combine the finished items of one or more --out directories into --to.""" + + out = Path(a.to) + out.mkdir(parents=True, exist_ok=True) + rows = [row for d in a.out for row in rows_from_items(Path(d), a.engine_sha)] + with open(out / "runs.csv", "w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS) + w.writeheader() + w.writerows(rows) + (out / "summary.json").write_text( + json.dumps(summarize(rows, a.engine_sha), indent=1) + "\n" + ) + print(json.dumps(summarize(rows, a.engine_sha), indent=1)) + + +def main() -> None: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + sm = sub.add_parser( + "summarize", help="Rebuild runs.csv and summary.json from items/" + ) + sm.add_argument("--out", required=True, action="append") + sm.add_argument("--to", required=True) + sm.add_argument("--engine-sha", required=True) + c = sub.add_parser("compile") + c.add_argument("--recording", required=True) + c.add_argument("--out", required=True) + r = sub.add_parser("run") + r.add_argument("--bundle", required=True) + r.add_argument("--out", required=True) + r.add_argument("--blocks", type=int, default=5) + r.add_argument("--per-block", type=int, default=4) + r.add_argument("--from-block", type=int, default=1) + r.add_argument( + "--skip-done", default=None, help="An earlier --out; skip its finished items" + ) + a = ap.parse_args() + {"compile": cmd_compile, "run": cmd_run, "summarize": cmd_summarize}[a.cmd](a) + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..f716662ab --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.023, + "engine_s": 1.114, + "wall_s": 4.369, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..21282d886 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.023, + "engine_s": 1.092, + "wall_s": 3.806, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..96a6aa09f --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.021, + "engine_s": 1.053, + "wall_s": 3.758, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..b7d10d91c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.026, + "engine_s": 1.1, + "wall_s": 3.783, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..78095f21c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.043, + "engine_s": 18.324, + "wall_s": 21.29, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..1f8d8bbe9 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json @@ -0,0 +1,99 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": null, + "api_fallback": "API steps are off for this run", + "step_errors": [ + [ + "step_040", + "Identity check failed for step 'step_040' (click recorded visual target): a target was found positionally (rung 'structural', confidence 1.00) but its surrounding text does not match the recorded target's \u2014 expected '', observed 'Emergency Contact: Confirm Create New Patient Cance" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": false, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Marcus", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Feld", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Male", + "got": null, + "match": false + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0124", + "got": null, + "match": false + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.048, + "engine_s": 14.588, + "wall_s": 17.402, + "checkpoints": 40 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..df48c52c9 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.05, + "engine_s": 16.038, + "wall_s": 18.852, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..43307b2f2 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.049, + "engine_s": 15.285, + "wall_s": 18.056, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..d6f3067dc --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.025, + "engine_s": 1.188, + "wall_s": 4.307, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json new file mode 100644 index 000000000..fa684296b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "api", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.026, + "engine_s": 22.666, + "wall_s": 25.59, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..8c2bd8ca1 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.03, + "engine_s": 1.125, + "wall_s": 3.803, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json new file mode 100644 index 000000000..5d47989f4 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.025, + "engine_s": 1.152, + "wall_s": 3.842, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..b13c98f80 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.055, + "engine_s": 18.26, + "wall_s": 21.718, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json new file mode 100644 index 000000000..2c2218b94 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "browser", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.031, + "engine_s": 24.871, + "wall_s": 28.107, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..ee05a2c9e --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.047, + "engine_s": 16.847, + "wall_s": 19.804, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json new file mode 100644 index 000000000..5cfcac1c3 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.048, + "engine_s": 15.714, + "wall_s": 18.419, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..da326917a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.027, + "engine_s": 1.822, + "wall_s": 5.114, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..8dc00a019 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.028, + "engine_s": 1.102, + "wall_s": 3.921, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..e06609ff5 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.031, + "engine_s": 1.081, + "wall_s": 3.904, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..86214e809 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.025, + "engine_s": 1.106, + "wall_s": 3.773, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..fe0d8e75c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.047, + "engine_s": 15.583, + "wall_s": 18.693, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..59874bcad --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.052, + "engine_s": 15.871, + "wall_s": 18.792, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..6ee11a79c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.047, + "engine_s": 22.049, + "wall_s": 25.165, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..d485d931d --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.053, + "engine_s": 15.59, + "wall_s": 18.753, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..e0ac70c6c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.028, + "engine_s": 1.082, + "wall_s": 4.212, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json new file mode 100644 index 000000000..d7d80d8e3 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "api", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.03, + "engine_s": 22.625, + "wall_s": 25.706, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..a7173128c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.03, + "engine_s": 1.123, + "wall_s": 3.806, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json new file mode 100644 index 000000000..5d8d09a2f --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.027, + "engine_s": 1.057, + "wall_s": 3.865, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..986266edf --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.048, + "engine_s": 15.514, + "wall_s": 18.949, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json new file mode 100644 index 000000000..901860315 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json @@ -0,0 +1,99 @@ +{ + "arm": "browser", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "transaction_outcome": "RECONCILIATION_REQUIRED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click recorded visual target): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "approvals": 1, + "approval_kinds": "dom", + "approval_wait_s": 0.022, + "engine_s": 22.555, + "wall_s": 25.298, + "checkpoints": 39 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..858ce9b65 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.053, + "engine_s": 15.174, + "wall_s": 17.925, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json new file mode 100644 index 000000000..0ead31498 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.043, + "engine_s": 15.596, + "wall_s": 18.571, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..2974ad6af --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.028, + "engine_s": 1.3, + "wall_s": 4.671, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..d824da622 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.029, + "engine_s": 1.363, + "wall_s": 4.065, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..060a97af0 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.026, + "engine_s": 1.495, + "wall_s": 4.167, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..e6bc09193 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "api", + "api_fallback": null, + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 1, + "approval_kinds": "api_request", + "approval_wait_s": 0.026, + "engine_s": 1.533, + "wall_s": 4.193, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..f00af8a91 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.049, + "engine_s": 20.218, + "wall_s": 23.385, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..672a01468 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.053, + "engine_s": 18.634, + "wall_s": 21.533, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..213defb27 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.06, + "engine_s": 16.615, + "wall_s": 19.53, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..dbfeff64e --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,94 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "transaction_outcome": "COMPLETED_UNVERIFIED", + "actuation": "dom", + "api_fallback": "API steps are off for this run", + "step_errors": [], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "approvals": 2, + "approval_kinds": "dom+dom", + "approval_wait_s": 0.052, + "engine_s": 15.536, + "wall_s": 18.458, + "checkpoints": 41 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv new file mode 100644 index 000000000..0835c9741 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv @@ -0,0 +1,41 @@ +arm,block,pos,ref,mismatch,correct,readback,new_charts,engine_success,transaction_outcome,actuation,api_fallback,approvals,approval_kinds,approval_wait_s,engine_s,wall_s,checkpoints,error,load_start,load_end,gate_waited_s,engine_sha +api,1,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.023,1.114,4.369,41,,,,,352af585b +api,1,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.023,1.092,3.806,41,,,,,352af585b +api,1,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.021,1.053,3.758,41,,,,,352af585b +api,1,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.026,1.1,3.783,41,,,,,352af585b +browser,1,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.043,18.324,21.29,41,,,,,352af585b +browser,1,2,r2-marcus-feld,False,False,0/12,0,False,RECONCILIATION_REQUIRED,,API steps are off for this run,2,dom+dom,0.048,14.588,17.402,40,,,,,352af585b +browser,1,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.05,16.038,18.852,41,,,,,352af585b +browser,1,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.049,15.285,18.056,41,,,,,352af585b +api,2,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.025,1.188,4.307,41,,,,,352af585b +api,2,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent,1,dom,0.026,22.666,25.59,39,,,,,352af585b +api,2,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.03,1.125,3.803,41,,,,,352af585b +api,2,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.025,1.152,3.842,41,,,,,352af585b +browser,2,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.055,18.26,21.718,41,,,,,352af585b +browser,2,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,API steps are off for this run,1,dom,0.031,24.871,28.107,39,,,,,352af585b +browser,2,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.047,16.847,19.804,41,,,,,352af585b +browser,2,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.048,15.714,18.419,41,,,,,352af585b +browser,3,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.047,15.583,18.693,41,,,,,352af585b +browser,3,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.052,15.871,18.792,41,,,,,352af585b +browser,3,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.047,22.049,25.165,41,,,,,352af585b +api,3,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.027,1.822,5.114,41,,,,,352af585b +api,3,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.028,1.102,3.921,41,,,,,352af585b +api,3,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.031,1.081,3.904,41,,,,,352af585b +api,3,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.025,1.106,3.773,41,,,,,352af585b +browser,3,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.053,15.59,18.753,41,,,,,352af585b +api,4,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.028,1.082,4.212,41,,,,,352af585b +api,4,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,the check before saving (step_039) did not pass: missing ['No matches were found.']; the save was not sent,1,dom,0.03,22.625,25.706,39,,,,,352af585b +api,4,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.03,1.123,3.806,41,,,,,352af585b +api,4,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.027,1.057,3.865,41,,,,,352af585b +browser,4,1,r5-grace-adeyemi,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.048,15.514,18.949,41,,,,,352af585b +browser,4,2,r6-rosa-delgado,True,True,0/12,0,False,RECONCILIATION_REQUIRED,dom,API steps are off for this run,1,dom,0.022,22.555,25.298,39,,,,,352af585b +browser,4,3,r7-arlo-whitcombe,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.053,15.174,17.925,41,,,,,352af585b +browser,4,4,r8-priya-raman,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.043,15.596,18.571,41,,,,,352af585b +api,5,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.028,1.3,4.671,41,,,,,352af585b +api,5,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.029,1.363,4.065,41,,,,,352af585b +api,5,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.026,1.495,4.167,41,,,,,352af585b +api,5,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,api,,1,api_request,0.026,1.533,4.193,41,,,,,352af585b +browser,5,1,r1-nadia-brennan,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.049,20.218,23.385,41,,,,,352af585b +browser,5,2,r2-marcus-feld,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.053,18.634,21.533,41,,,,,352af585b +browser,5,3,r3-ines-carvalho,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.06,16.615,19.53,41,,,,,352af585b +browser,5,4,r4-theo-lindqvist,False,True,12/12,1,True,COMPLETED_UNVERIFIED,dom,API steps are off for this run,2,dom+dom,0.052,15.536,18.458,41,,,,,352af585b diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json new file mode 100644 index 000000000..ce3187035 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json @@ -0,0 +1,43 @@ +{ + "label": "API step under each_item approval + durable, no run gate; harness answers approvals; approval wait excluded", + "engine_sha": "352af585b", + "arms": { + "browser": { + "runs": 20, + "correct": 19, + "errors": 0, + "api_saves": 0, + "approvals_per_item": [ + 1, + 2 + ], + "approval_kinds": [ + "dom", + "dom+dom" + ], + "median_engine_s": 15.95, + "median_approval_wait_s": 0.049, + "max_load_start": null, + "safe_stops": 1, + "wrong_saves": 0 + }, + "api": { + "runs": 20, + "correct": 20, + "errors": 0, + "api_saves": 18, + "approvals_per_item": [ + 1 + ], + "approval_kinds": [ + "api_request", + "dom" + ], + "median_engine_s": 1.12, + "median_approval_wait_s": 0.026, + "max_load_start": null, + "safe_stops": 0, + "wrong_saves": 0 + } + } +} diff --git a/packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md new file mode 100644 index 000000000..8061b73cc --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/approval_durable/README.md @@ -0,0 +1,55 @@ +# OpenEMR referral: API step under per-item approval and a durable run + +**Label: API step under each_item approval + durable, no run gate; harness +answers approvals; approval wait excluded.** These are not numbers for the +governed production profile. The referral bundle doesn't pass the run gate +yet: it is refused on its GUI path, for reasons unrelated to the API step +(see the admission list in the session notes, and the local-edit proposal, +PR #1366). + +Harness: [`approval_durable.py`](../../approval_durable.py). Data: +[`2026-10-19-combined/`](2026-10-19-combined/) (`runs.csv`, `summary.json`, +and each item's `result.json` under `items/`). + +## What ran + +- The OpenEMR new-patient referral, compiled from a recording with network + capture and admitted with `--identity subject_name=lname`, on a local + OpenEMR test copy with synthetic patients only. +- Two arms with the same bundle and record check: API steps off (`browser`) + and on (`api`). Both use `ApprovalGate("each_item")` and `durable=True`. +- 5 blocks of 4 referrals per arm, interleaved, with the arm order alternating + by block. Each block resets the fixture and seeds the existing chart that + referral r6 must not duplicate. +- The harness answers each approval request as soon as it is written, + standing in for the person. The time spent waiting for that answer is + measured (`approval_wait_s`) and left out of `engine_s`. +- Correctness is checked by the test setup's own direct SQL read-back. A new + referral is correct when exactly one new chart exists and it matches all 12 + fields. r6 is correct when no chart is created. +- Each item waited until the machine's 1-minute load was under 12. The run + was paused for a shared quiet window after 19 of 40 runs and resumed into a + second directory. The interrupted item was run again from the start. Load + per item was not kept for the rebuilt rows. +- Engine: the API-step stack (PRs #1358, #1361, #1362, #1364) at `352af585b`. + It does not include the review fixes #1367–#1372. With #1370 and #1371 this + referral's API step is refused: OpenEMR sends a background write while the + form is filled, and its form runs script on submit and on the name fields. + +## Results + +| Arm | Runs | Correct by read-back | Saved through | Median engine time | Approvals per item | +| --- | --- | --- | --- | --- | --- | +| API steps off | 20 | 19, plus 1 safe stop (nothing saved) | GUI | 15.95 s | 2 (1 for r6) | +| API step | 20 | 20 | API 18, GUI 2 (r6) | 1.12 s | 1 | + +- **API step.** Each new referral sent one request the person approved, with + the operation digest bound to the request bytes, and read back 12 of 12. + For r6 the API step's duplicate pre-check found the existing chart, so the + API step was not used. The GUI steps ran instead and created nothing, as + they should. +- **API steps off.** The GUI asks twice per new referral: once at "Create New + Patient", and once at the confirming save. One item (b01, r2) stopped itself + at the save's identity check (coverage 0.81) and saved nothing. The + earlier spike saw the same fail-closed stop once in 20 runs. +- No run saved a wrong or duplicate chart. diff --git a/packages/flow/docs/API_TIER.md b/packages/flow/docs/API_TIER.md index 2f56c5a93..e6fac0e78 100644 --- a/packages/flow/docs/API_TIER.md +++ b/packages/flow/docs/API_TIER.md @@ -11,6 +11,12 @@ the benchmark scoreboard, against the same program on the GUI (`program`), with the same record check. The harness and the earlier spike are in [benchmark/openemr_api_tier](../benchmark/openemr_api_tier/README.md). +Under per-item approval and a durable run (no run gate; the harness answers +the approvals and their wait is left out), the API step saved each new +referral correctly with one approval at a median 1.12 s of engine time, +against 15.95 s and two approvals with API steps off (20 runs each, +2026-10-19): [results](../benchmark/openemr_api_tier/results/approval_durable/README.md). + ## How to use it ```bash diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 079a128bb..77c7c4931 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -11794,11 +11794,15 @@ def _web_form_approval_view( if isinstance(prepared, ApiActuationResult): return prepared.reason or "the API step is unavailable" effects = self._resolve_effects(list(binding.effects), params) - record = [ - (key, str(expr)) - for effect in effects - for key, expr in sorted(effect.match.items()) - ] + # Several record checks usually select the same record (one per + # field): the person reads, and the digest binds, each pair once. + record = list( + dict.fromkeys( + (key, str(expr)) + for effect in effects + for key, expr in sorted(effect.match.items()) + ) + ) digest = operation_digest( prepared, record_check=[effect.contract_hash() for effect in effects], diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 9b88e04d4..a4abb3059 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -371,6 +371,174 @@ "path": "benchmark/openemr/results.json", "sha256": "30de6dbb43ad0db4fd8b55737d765d06747b896ffae04522b6957d567dcfd89c" }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-1-r1-nadia-brennan.json", + "sha256": "720f7b21df440c2736ef2e7a8b1fcc95647d94413f9d65ba1bab0f957095a967" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-2-r2-marcus-feld.json", + "sha256": "0ffab27cdb11594c779424d3bdbc9b498ddf740d480042f7370b727e820da030" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-3-r3-ines-carvalho.json", + "sha256": "f7721ce8ae1e57ef52638138e920a1758ac61e62699e954310cd5e21c607b240" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-api-4-r4-theo-lindqvist.json", + "sha256": "bc7fd5d2037d605dbf242ce4005ea784cd63c8e4b62b173362c664efb0a48f9b" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-1-r1-nadia-brennan.json", + "sha256": "f818fd9dd2a964c10a88ff69ce4c10c5e3b27af08c7f7f8d60c6a9aa665ee761" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-2-r2-marcus-feld.json", + "sha256": "13a6ba67f0fde89c5ef8370ff2c2661434d16d5947a5f42a401a3fc218bbc648" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-3-r3-ines-carvalho.json", + "sha256": "d93aa3ff2703c9eae85ce4c7de73ad3aad9ec24965c46711b06d292af9b14991" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b01-browser-4-r4-theo-lindqvist.json", + "sha256": "b79c859009098cfe8e526b5775601697c1612a3280af1011d879273d97ccecc7" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-1-r5-grace-adeyemi.json", + "sha256": "ac05089531d7c3d0bc7d0a5513ac9789e9594983c24e476dd5300047633fb3aa" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-2-r6-rosa-delgado.json", + "sha256": "0e27df1fc55e4952a5e5bc7ee842005d54a7e8ccd2ab66f486a724febf7a9d72" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-3-r7-arlo-whitcombe.json", + "sha256": "4a2dbf32430ce5f4c064854472658486aa56eaffd63eb446191535c4c81b1c41" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-api-4-r8-priya-raman.json", + "sha256": "903fc03903e2ed8ab7a1be8cd004ae09b0fd6f16cfdd82c0fb1902667052e7b9" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-1-r5-grace-adeyemi.json", + "sha256": "2fd6d6a3fabc7987a854f925d84d9c709ce95f2c3d992bf8d04e0c18a520803c" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-2-r6-rosa-delgado.json", + "sha256": "8bd71537bf0737a2cb932753c4dc8e4e12c9fcf3012a85495f20ba4e1d9f132e" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-3-r7-arlo-whitcombe.json", + "sha256": "3e74161adf06a0b692153da436ac1d289aa65d74976989134d6189bf71291dd1" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b02-browser-4-r8-priya-raman.json", + "sha256": "4e690e3e561a421130fdfee2d88e8cf0005d604b741bcc7ebcad123f4b4c92d4" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-1-r1-nadia-brennan.json", + "sha256": "56f90f6c2e1c29e542c3da5c7fa02785f471bdcbf38a0cd285889b675058bdc7" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-2-r2-marcus-feld.json", + "sha256": "a3ea8fdc6b2f25feded5e257e8f64cd644783ea8b784daaef2c9c3a15c09d674" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-3-r3-ines-carvalho.json", + "sha256": "192eb11514aa498c79ae2997243084f214f0a903d7c614b88e9a4d867f072977" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-api-4-r4-theo-lindqvist.json", + "sha256": "0e04941fea9db21558e7c75c37c37fe2244006fe21af5723a22984d86211dd39" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-1-r1-nadia-brennan.json", + "sha256": "f6436902635448d7014d3fb8b03f9fc5bc5026925d3bba205ae23f58772fb0fc" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-2-r2-marcus-feld.json", + "sha256": "149c85d593c8b304343b89742095fc0be38bd5dd3ba5dba65756223a42b6bccd" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-3-r3-ines-carvalho.json", + "sha256": "7713f65b351b3430e5efe337fffa1ec593357fe38301616538b9113fb79b2229" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b03-browser-4-r4-theo-lindqvist.json", + "sha256": "74d05b07758e22d02bf58e8bea3bada953ba500fe5a4af69b54067630b1cbde8" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-1-r5-grace-adeyemi.json", + "sha256": "5f64b48414a8026590043ab2c4d27c1a5bb527e8f2e9cfce34ec3ff60db84b33" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-2-r6-rosa-delgado.json", + "sha256": "171e98fe4617407dc0fba49228a183caff66e0d82170cfb39a1c8692d7b0b2dc" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-3-r7-arlo-whitcombe.json", + "sha256": "484e0ddc68ceb665f1a0871d3c0c1bfae0b65ff69ccdc855d2600f0b4a5beb5f" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-api-4-r8-priya-raman.json", + "sha256": "6562554d027ea6771f9bc3c28c291b08de875389b0608cec71105b1a0def869b" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-1-r5-grace-adeyemi.json", + "sha256": "dacfb7eb2d7432c22cb4f713f0221b016de9bab3e7c58498b6bf775b1cccd4bd" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-2-r6-rosa-delgado.json", + "sha256": "06d6abe039bacf185863912246796197cd5c581d462fd13e80dbfff359e2b152" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-3-r7-arlo-whitcombe.json", + "sha256": "110529a961ce6a86d12c3d0bd46ff7734bcc0e3d50e5962adafe83ceecbc4c49" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b04-browser-4-r8-priya-raman.json", + "sha256": "64b7250dc37ac9d15f82f49d242d1530cc0ee02b9c85d0af762a74f565aa8370" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-1-r1-nadia-brennan.json", + "sha256": "8b568206d6543fef092f12f11ecf0c07b5a1651fa31196ba6cd441f408ff71e6" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-2-r2-marcus-feld.json", + "sha256": "d5e8925e416023b05c4c2168efb63423f36e10ddd62b884f4fce88d590279a1f" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-3-r3-ines-carvalho.json", + "sha256": "41ee9b1f36bfedf2eaa22710b2b59a34fa96da02fae36e6b2d044927624243d4" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-api-4-r4-theo-lindqvist.json", + "sha256": "89fddbcadbb47fa601b728164e9890d7ce4d375c0e199df868843e3daa73c75a" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-1-r1-nadia-brennan.json", + "sha256": "2968a9309a5c111bc5cdbb4fe752c127b147096bc8ab6bd04d66f84e635401ff" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-2-r2-marcus-feld.json", + "sha256": "c58de6296dc893f69f5d9508e6a4348f71a2cecdd7e311b11bfa6f7b77043ae2" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-3-r3-ines-carvalho.json", + "sha256": "f7ec8cc86c8bfeaa8f7bdc8dff0b5f62cc00e735f249a0ca4a3991c14b7e304e" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/items/b05-browser-4-r4-theo-lindqvist.json", + "sha256": "09046626ea521bed7edfc1bbc316293fea2d58c9cf6726f2cc181d0252d9f604" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/runs.csv", + "sha256": "4ce59146852f10c9a076912f7d7acbfcfb6815f8ee84181edf80bd217d3e3645" + }, + { + "path": "benchmark/openemr_api_tier/results/approval_durable/2026-10-19-combined/summary.json", + "sha256": "dcff06ae9d8273191d18afdf042fb21746f4c5f30149d62d61187b603abe6dd0" + }, { "path": "benchmark/openemr_api_tier/spike/api_bundle/compile_report.json", "sha256": "9bea1fdb1e7caa3cd4e25d2dcf6c46bb82871a35aa18c4dbb471d2c4fb6321bc" diff --git a/packages/flow/tests/test_api_step_approval.py b/packages/flow/tests/test_api_step_approval.py index 98f69e5e7..61a4a1893 100644 --- a/packages/flow/tests/test_api_step_approval.py +++ b/packages/flow/tests/test_api_step_approval.py @@ -315,3 +315,30 @@ def test_operation_digest_binds_the_record_check(): ) assert base != operation_digest(changed, record_check=["h1"], record=[("id", "p1")]) assert up.parse_qs(prepared.encoded()) == {"lname": ["Marsh"]} + + +def test_the_record_rows_are_listed_once_per_field(tmp_path): + from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + + wf = _workflow() + save = wf.steps[-1] + match = {"patient_id": ValueExpr(param="lname")} + effects = [ + Effect(kind=EffectKind.RECORD_WRITTEN, match=match, expected_count=1), + Effect( + kind=EffectKind.FIELD_EQUALS, + match=match, + field="note", + value=ValueExpr(literal="Referred"), + ), + ] + binding = save.api_binding.model_copy(update={"effects": effects}) + save = save.model_copy(update={"effects": effects, "api_binding": binding}) + replayer = Replayer( + EnterWritesBackend("http://127.0.0.1:9"), + vision=FakeVision(), + web_form_actuator=WebFormActuator(SessionTransport("http://127.0.0.1:9")), + ) + view = replayer._web_form_approval_view(save, PARAMS) + record_rows = [row for row in view.rows if row[0].startswith("Record check:")] + assert record_rows == [("Record check: patient_id", "Marsh")]