From b8d9183e145a70f75a862c18981a5b351ccb48b0 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 12:26:29 -0400 Subject: [PATCH 1/2] spike(flow): compile the OpenEMR referral save to the app's own HTTP requests Engineering spike, synthetic data only, not a feature. - capture_har.py: replay the replay-dom referral bundle once with HAR on - formtier.py / compile_api.py: explain every save-body field (recorded param, select label, session CSRF token, form default) or refuse; emit a one-step bundle whose ApiBinding (on_unavailable=halt) keeps the GUI save step's effect contract - SessionFormActuator: fresh sign-in, fresh form read (CSRF, defaults, drift and client-rule checks), duplicate check, one save POST; runs through the unchanged Replayer with a backend that refuses GUI calls - bench.py: 20 browser vs 20 API runs, interleaved, same record check - probes.py: CSRF, skipped duplicate check, double submit, no session, form and rule drift - results/: runs.csv, per-run JSON, figures.json, probes.json, redacted HAR summary Browser 19/20 correct, median 23.9 s; API 20/20, median 2.14 s (11.2x). Co-Authored-By: Claude Opus 5.5 --- .../flow/benchmark/openemr_api_tier/README.md | 187 +++ .../api_bundle/compile_report.json | 65 + .../openemr_api_tier/api_bundle/manifest.json | 20 + .../api_bundle/session_form.json | 843 ++++++++++++ .../openemr_api_tier/api_bundle/workflow.json | 1202 +++++++++++++++++ .../flow/benchmark/openemr_api_tier/bench.py | 219 +++ .../benchmark/openemr_api_tier/capture_har.py | 85 ++ .../flow/benchmark/openemr_api_tier/common.py | 127 ++ .../benchmark/openemr_api_tier/compile_api.py | 27 + .../openemr_api_tier/compute_figures.py | 89 ++ .../benchmark/openemr_api_tier/formtier.py | 500 +++++++ .../benchmark/openemr_api_tier/har_summary.py | 55 + .../flow/benchmark/openemr_api_tier/probes.py | 172 +++ .../openemr_api_tier/results/capture_r1.json | 267 ++++ .../openemr_api_tier/results/figures.json | 132 ++ .../openemr_api_tier/results/har_summary.json | 955 +++++++++++++ .../openemr_api_tier/results/probes.json | 91 ++ .../openemr_api_tier/results/runs.csv | 41 + .../runs/b01-api-1-r1-nadia-brennan.json | 126 ++ .../runs/b01-api-2-r2-marcus-feld.json | 126 ++ .../runs/b01-api-3-r3-ines-carvalho.json | 126 ++ .../runs/b01-api-4-r4-theo-lindqvist.json | 126 ++ .../runs/b01-browser-1-r1-nadia-brennan.json | 97 ++ .../runs/b01-browser-2-r2-marcus-feld.json | 97 ++ .../runs/b01-browser-3-r3-ines-carvalho.json | 97 ++ .../runs/b01-browser-4-r4-theo-lindqvist.json | 97 ++ .../runs/b02-api-1-r5-grace-adeyemi.json | 126 ++ .../runs/b02-api-2-r6-rosa-delgado.json | 126 ++ .../runs/b02-api-3-r7-arlo-whitcombe.json | 126 ++ .../runs/b02-api-4-r8-priya-raman.json | 126 ++ .../runs/b02-browser-1-r5-grace-adeyemi.json | 97 ++ .../runs/b02-browser-2-r6-rosa-delgado.json | 102 ++ .../runs/b02-browser-3-r7-arlo-whitcombe.json | 97 ++ .../runs/b02-browser-4-r8-priya-raman.json | 97 ++ .../runs/b03-api-1-r1-nadia-brennan.json | 126 ++ .../runs/b03-api-2-r2-marcus-feld.json | 126 ++ .../runs/b03-api-3-r3-ines-carvalho.json | 126 ++ .../runs/b03-api-4-r4-theo-lindqvist.json | 126 ++ .../runs/b03-browser-1-r1-nadia-brennan.json | 97 ++ .../runs/b03-browser-2-r2-marcus-feld.json | 102 ++ .../runs/b03-browser-3-r3-ines-carvalho.json | 97 ++ .../runs/b03-browser-4-r4-theo-lindqvist.json | 97 ++ .../runs/b04-api-1-r5-grace-adeyemi.json | 126 ++ .../runs/b04-api-2-r6-rosa-delgado.json | 126 ++ .../runs/b04-api-3-r7-arlo-whitcombe.json | 126 ++ .../runs/b04-api-4-r8-priya-raman.json | 126 ++ .../runs/b04-browser-1-r5-grace-adeyemi.json | 97 ++ .../runs/b04-browser-2-r6-rosa-delgado.json | 102 ++ .../runs/b04-browser-3-r7-arlo-whitcombe.json | 97 ++ .../runs/b04-browser-4-r8-priya-raman.json | 97 ++ .../runs/b05-api-1-r1-nadia-brennan.json | 126 ++ .../runs/b05-api-2-r2-marcus-feld.json | 126 ++ .../runs/b05-api-3-r3-ines-carvalho.json | 126 ++ .../runs/b05-api-4-r4-theo-lindqvist.json | 126 ++ .../runs/b05-browser-1-r1-nadia-brennan.json | 97 ++ .../runs/b05-browser-2-r2-marcus-feld.json | 97 ++ .../runs/b05-browser-3-r3-ines-carvalho.json | 97 ++ .../runs/b05-browser-4-r4-theo-lindqvist.json | 97 ++ .../results/runs/block-01-api.json | 8 + .../results/runs/block-01-browser.json | 8 + .../results/runs/block-02-api.json | 8 + .../results/runs/block-02-browser.json | 8 + .../results/runs/block-03-api.json | 8 + .../results/runs/block-03-browser.json | 8 + .../results/runs/block-04-api.json | 8 + .../results/runs/block-04-browser.json | 8 + .../results/runs/block-05-api.json | 8 + .../results/runs/block-05-browser.json | 8 + .../openemr_api_tier/spike_fixture.py | 103 ++ 69 files changed, 9735 insertions(+) create mode 100644 packages/flow/benchmark/openemr_api_tier/README.md create mode 100644 packages/flow/benchmark/openemr_api_tier/api_bundle/compile_report.json create mode 100644 packages/flow/benchmark/openemr_api_tier/api_bundle/manifest.json create mode 100644 packages/flow/benchmark/openemr_api_tier/api_bundle/session_form.json create mode 100644 packages/flow/benchmark/openemr_api_tier/api_bundle/workflow.json create mode 100644 packages/flow/benchmark/openemr_api_tier/bench.py create mode 100644 packages/flow/benchmark/openemr_api_tier/capture_har.py create mode 100644 packages/flow/benchmark/openemr_api_tier/common.py create mode 100644 packages/flow/benchmark/openemr_api_tier/compile_api.py create mode 100644 packages/flow/benchmark/openemr_api_tier/compute_figures.py create mode 100644 packages/flow/benchmark/openemr_api_tier/formtier.py create mode 100644 packages/flow/benchmark/openemr_api_tier/har_summary.py create mode 100644 packages/flow/benchmark/openemr_api_tier/probes.py create mode 100644 packages/flow/benchmark/openemr_api_tier/results/capture_r1.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/figures.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/har_summary.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/probes.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs.csv create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-1-r5-grace-adeyemi.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-2-r6-rosa-delgado.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-3-r7-arlo-whitcombe.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-4-r8-priya-raman.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-1-r1-nadia-brennan.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-2-r2-marcus-feld.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-3-r3-ines-carvalho.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-4-r4-theo-lindqvist.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-01-api.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-01-browser.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-02-api.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-02-browser.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-03-api.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-03-browser.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-04-api.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-04-browser.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-05-api.json create mode 100644 packages/flow/benchmark/openemr_api_tier/results/runs/block-05-browser.json create mode 100644 packages/flow/benchmark/openemr_api_tier/spike_fixture.py diff --git a/packages/flow/benchmark/openemr_api_tier/README.md b/packages/flow/benchmark/openemr_api_tier/README.md new file mode 100644 index 000000000..6dfe7a286 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/README.md @@ -0,0 +1,187 @@ +# Spike: compiling the OpenEMR referral save to the app's own HTTP requests + +Status: engineering spike, 18 October 2026. Local OpenEMR 8.0.0.3 test copy, +synthetic patients only. Not a product feature and not a published claim. + +## Question + +Can the save step of the recorded OpenEMR referral be compiled into the HTTP +requests the browser made, replayed without a browser using a fresh sign-in +and the run's values, and still pass the same independent record check? How +much faster is it? + +## What was built + +1. **Capture** (`capture_har.py`). One replay of the published referral + program (the DOM-identity bundle, `replay-dom` in + `../openemr_referral_timing`) with Playwright HAR recording on. + `results/har_summary.json` is a redacted list of the 29 non-static requests. +2. **Compile** (`formtier.py`, `compile_api.py`). From the HAR, the compiler + finds the save, the duplicate check and the sign-in. It then explains each + of the 105 fields in the save body by where its value came from, or it + refuses to compile (`api_bundle/compile_report.json`): + + | Source | Fields | Example | + | --- | ---: | --- | + | Recorded parameter | 11 | `form_lname` = `{lname}` | + | Select option chosen by a recorded label | 1 | `form_state`: label `{state_label}` ("Massachusetts") gives value `MA` | + | Server-issued session value | 1 | `csrf_token_form` | + | The form's own default | 92 | `form_care_team_status` = `active` | + + The output is a one-step bundle (`api_bundle/`). Its step carries the + engine's existing `ApiBinding` with `on_unavailable="halt"` and the same + system-of-record effect contract as the GUI save step, so it never falls + back to the GUI. The 41 GUI steps are folded into one request. +3. **Run** (`SessionFormActuator` in `formtier.py`). It implements the + engine's API actuator protocol and runs through the unchanged `Replayer` + with a backend that raises on any screen or input call. For each run it: + signs in with a new session, reads the New Patient form (for the CSRF + token, the current defaults and options, and a drift check), checks the + form's client-side required fields, runs the duplicate check, and then + sends the save once. The engine's API tier then confirms the record with + the same REST `EffectVerifier` the GUI run uses. + +### What the capture showed + +| Step | Request | Auth | +| --- | --- | --- | +| Sign in | `POST /interface/main/main_screen.php?auth=login`, form-encoded user and password | Returns the `OpenEMR` session cookie; no CSRF token on this request | +| Duplicate check | `GET /interface/new/new_search_popup.php?mf_fname=…&mf_lname=…&mf_DOB=…` | Session cookie | +| Save | `POST /interface/new/new_comprehensive_save.php`, 105 form fields | Session cookie plus `csrf_token_form`, one token per session | +| Success | 200 with a script that redirects to `demographics.php?set_pid=` | | + +The GUI program's check before Confirm (the text "No matches were found." +must be on screen) carries over directly. The server writes that string into +the duplicate-check response only when the search finds no one. The compiled +step requires it in the response, so the pre-save check still works without +a screen. + +## Results: 20 browser runs and 20 API runs + +Five blocks. Each block resets the database and seeds the existing Rosa +Delgado chart, then runs four referrals in rotation; r6 (possible duplicate) +came up twice per arm. The arms were interleaved, and the order alternated by +block. Both arms used the same engine commit (8d0758b9) in warm mode, with +imports, Chromium launch and the OCR engine set up once beforehand. The clock +runs from a new browser context or HTTP session to the end of the record +check. The record check is the test setup's own SQL read-back after a 1.0 s +settle, the same rule as the October study: exactly one new chart with 12 of +12 fields, or no new chart for r6. No screen recorder ran in either arm. + +| | Browser program | API step | +| --- | --- | --- | +| Correct | 19 of 20 | 20 of 20 | +| New referrals saved, 12 of 12 | 17 of 18 | 18 of 18 | +| r6 possible duplicate: no new chart | 2 of 2 | 2 of 2 | +| Median time per run, correct new referrals | 23.9 s (p10–p90 23.0–27.1) | 2.14 s (2.04–2.37) | +| Sign-in and form, median | 1.48 s | 0.49 s (sign-in 0.32 s, form read 0.17 s) | +| Engine time after setup, median | 21.0 s | 0.34 s (duplicate check 0.035 s, save 0.036 s; the rest is the engine's record snapshot before the save and its check after) | +| Record check (settle + SQL), median | 1.34 s | 1.29 s | + +- **Speedup:** the median run is 11.2 times faster (bootstrap 95% CI 10.8 to + 12.1), 21.8 s less per run. Without the shared 1.0 s settle and SQL check, + it is 22.5 s against 0.85 s, about 27 times. +- **The one browser miss** (block 3, r2) was a fail-closed stop with nothing + saved. At the final Confirm click, the engine's identity check found the + button by position but saw "Emergency Contact" text around it, so it + refused. The one OpenAdapt miss in the October data was also a stop with + nothing saved, at the step before. The API step has no screen to misread. +- The browser arm's 23.9 s is faster than the published 29.7 s. That figure + also counted the screen recorder and the audit-quiescence wait, and it ran + on an older engine. + +Raw data: `results/runs.csv` (one row per run), `results/runs/*.json` (per-run +results, including the API stage timings and every field compared), and +`results/figures.json` (from `compute_figures.py`). + +## Where a recorded request is not enough + +`results/probes.json`, run against the same test copy: + +| Case | What the server does | What the API step does | +| --- | --- | --- | +| CSRF token from another session | Rejects ("Authentication Error"), nothing written | Never sends one: it reads a fresh token from the form in its own session | +| No CSRF token | Rejects, nothing written | As above | +| Token lifetime | Same token for the whole session, also after a save | One sign-in per run; a recorded token is never reused | +| Duplicate check skipped, r6 posted directly | **Saves a second Rosa Delgado chart.** The duplicate check runs only in the browser's script | Runs the check itself and stops before the save when the server finds a possible match | +| Same save sent twice | **Two charts**; the endpoint has no idempotency | Sends once; after the POST starts it never retries (engine HALT rule) | +| No signed-in session | "Site ID is missing from session data!", nothing written | Treated as a rejected write (HALT), confirmed by the record check | +| Form fields changed since the recording | n/a | Refuses before sending | +| Admin makes Middle Name required | **The server still saves a record with Middle Name empty**; only the browser's script enforces the rule | Refuses before sending because the form's rules changed | +| State label the form doesn't offer | n/a | Refuses before sending | + +Every refusal before the save is reported to the engine as "not sent", and +the step is API-only, so the run halts with nothing written. A real feature +could fall back to the GUI program at that point instead. + +## Where this can't apply + +- **No web form underneath.** Citrix, RDP and native desktop apps have no + page requests to compile. Pixels stay the only layer there. +- **Rules that live in the page's script.** OpenEMR's duplicate check and + required fields run in browser JavaScript, and the server enforces neither. + A direct POST that skips them writes records the GUI would have blocked. The + spike handles the two rules it found by name. An app with logic in the page + script that the compiler doesn't model is unsafe at this tier, and finding + that logic is the hard part. +- **Sign-in the session can't repeat by itself.** This copy accepts a plain + user and password. SSO, MFA, device-bound sessions, CAPTCHA, or a sign-in + that needs its own CSRF token or a JavaScript challenge would need the + browser, or the vendor's real API, for at least the sign-in. +- **Multi-step server state.** This save needs no state from earlier pages. A + workflow whose server keeps wizard state, uploads, or locks between pages + needs the whole request sequence and its order, not one POST. +- **Request bodies built by script.** Signed, encrypted or computed fields, + GraphQL or JSON bodies assembled at submit time, or tokens rotated on every + request need more than value matching. +- **Success that isn't in the response.** Here a `set_pid` redirect is the + success signal. Some apps return 200 for errors with nothing to match; then + only the record check can decide. + +## What a real feature would need + +1. **Recording that captures requests by default.** The recorder would store + request/response pairs next to the GUI events, so a later compile doesn't + need a second capture replay. Credentials and tokens must be redacted at + capture time, not later. +2. **Request bindings in the IR.** `ApiBinding` today sends JSON. The spike's + form encoding, session sign-in, CSRF source, read-only pre-checks and + success signature live in a separate `session_form.json`. They should be + first-class, hashed fields so qualification can pin them. +3. **Pre-checks as steps.** The duplicate check should be a read step with + its own pass condition, carried over from the GUI postcondition, rather + than hidden in the actuator. +4. **Page-logic discovery.** Find the client-side checks the server doesn't + repeat (validation rules, duplicate dialogs, confirmation prompts) and + either compile them or refuse the tier. Today this is done by hand for two + rules. +5. **Drift detection and fallback.** Compare the live form (fields, defaults, + options, rules) with the recording on every run, and fall back to the GUI + program before sending anything when they differ. The probes show this + works for field and rule changes; a real feature needs it qualified per + app. +6. **Qualification per app and endpoint.** Run the same effect contract, fault + injection and false-stop measurements as for the GUI tier before the API + tier may run unattended. Write `ApiIdentityBinding` contracts for + form-field names; today's identity pointers expect the field name to match + the effect field (`lname` against `form_lname`). +7. **Secrets from the deployment.** Sign-in values come from the deployment's + secret store at run time. The spike reads them from the test copy's + runtime file. + +## Reproduce + +```bash +export PYTHONPATH=packages/flow DOCKER_HOST= +python -m benchmark.openemr_api_tier.spike_fixture create # once: private fixture on 9811/9810 +python -m benchmark.openemr_api_tier.capture_har --ref 1 --out +python -m benchmark.openemr_api_tier.compile_api --har /referral.har --gui-bundle --out +python -m benchmark.openemr_api_tier.bench matrix --blocks 5 --per-block 4 --out +python -m benchmark.openemr_api_tier.probes --out +python -m benchmark.openemr_api_tier.compute_figures +``` + +The GUI bundle and the raw HAR are not in the repository. The bundle stays in +the referral study's scratch area. The HAR holds the session cookie and the +sign-in password of the disposable test copy. The OpenEMR compose file is +fetched at run time by `../openemr_local/upstream.py` and is not stored here. diff --git a/packages/flow/benchmark/openemr_api_tier/api_bundle/compile_report.json b/packages/flow/benchmark/openemr_api_tier/api_bundle/compile_report.json new file mode 100644 index 000000000..a4aca0837 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/api_bundle/compile_report.json @@ -0,0 +1,65 @@ +{ + "save": { + "method": "POST", + "path": "/interface/new/new_comprehensive_save.php", + "fields": 105, + "by_source": { + "session": 1, + "param": 11, + "form_default": 92, + "select_label": 1 + } + }, + "precheck": { + "method": "GET", + "path": "/interface/new/new_search_popup.php", + "query": [ + [ + "mf_fname", + "{fname}" + ], + [ + "mf_lname", + "{lname}" + ], + [ + "mf_DOB", + "{DOB}" + ], + [ + "close", + "" + ] + ], + "require_text": [ + "Confirm Create New Patient", + "No matches were found." + ], + "from_gui_step": "step_039" + }, + "login": { + "path": "/interface/main/main_screen.php", + "fields": [ + "new_login_session_management", + "languageChoice", + "authUser", + "clearPass", + "languageChoice" + ] + }, + "success_regex": "set_pid=(\\d+)", + "param_fields": { + "form_title": "title", + "form_fname": "fname", + "form_lname": "lname", + "form_DOB": "DOB", + "form_sex": "sex", + "form_street": "street", + "form_city": "city", + "form_state": "state_label", + "form_postal_code": "postal_code", + "form_country_code": "country_code", + "form_phone_home": "phone_home", + "form_email": "email" + } +} diff --git a/packages/flow/benchmark/openemr_api_tier/api_bundle/manifest.json b/packages/flow/benchmark/openemr_api_tier/api_bundle/manifest.json new file mode 100644 index 000000000..2dc5eea0a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/api_bundle/manifest.json @@ -0,0 +1,20 @@ +{ + "schema_version": 2, + "content_digest": "e5c038601681335df2537e2816dcb8d613193d88781aada637bc038961c1a2b1", + "file_hashes": {}, + "provenance": { + "compiler_version": "2.2.0", + "source_recording_sha256": "241803615513807b7f5236413f1efaaa520f1edc831dbd9e2c86ce9d144f10b9", + "compiler_config_sha256": "b12e6c6e0943d452f357b90258b55555593fd95e2fcdb6026330a0222204ed24", + "created_at": "2026-10-15T01:01:43.647330+00:00", + "policy_name": null, + "certified": false, + "certification_status": null, + "certified_at": null, + "expires_at": null, + "certification_invalidated_at": null, + "certification_invalidation_reason": null, + "governed_authorization_template": null + }, + "encrypted": false +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/api_bundle/session_form.json b/packages/flow/benchmark/openemr_api_tier/api_bundle/session_form.json new file mode 100644 index 000000000..9d5c68837 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/api_bundle/session_form.json @@ -0,0 +1,843 @@ +{ + "schema": "openadapt.spike.session_form/1", + "login": { + "method": "POST", + "path": "/interface/main/main_screen.php", + "query": "auth=login&site=default", + "body": [ + [ + "new_login_session_management", + "1" + ], + [ + "languageChoice", + "1" + ], + [ + "authUser", + "{__secret_username}" + ], + [ + "clearPass", + "{__secret_password}" + ], + [ + "languageChoice", + "1" + ] + ], + "expect_status": 302, + "expect_location_path": "/interface/main/tabs/main.php" + }, + "form_page": { + "path": "/interface/new/new_comprehensive.php", + "action": "new_comprehensive_save.php", + "controls": [ + "csrf_token_form", + "form_DOB", + "form_additional_addresses[__A-INDEX__][addresses_id]", + "form_additional_addresses[__A-INDEX__][city]", + "form_additional_addresses[__A-INDEX__][contact_address_id]", + "form_additional_addresses[__A-INDEX__][contact_id]", + "form_additional_addresses[__A-INDEX__][country]", + "form_additional_addresses[__A-INDEX__][data_action]", + "form_additional_addresses[__A-INDEX__][district]", + "form_additional_addresses[__A-INDEX__][line_1]", + "form_additional_addresses[__A-INDEX__][line_2]", + "form_additional_addresses[__A-INDEX__][notes]", + "form_additional_addresses[__A-INDEX__][period_end]", + "form_additional_addresses[__A-INDEX__][period_start]", + "form_additional_addresses[__A-INDEX__][priority]", + "form_additional_addresses[__A-INDEX__][state]", + "form_additional_addresses[__A-INDEX__][status]", + "form_additional_addresses[__A-INDEX__][type]", + "form_additional_addresses[__A-INDEX__][use]", + "form_additional_addresses[__A-INDEX__][zip]", + "form_allow_health_info_ex", + "form_allow_imm_info_share", + "form_allow_imm_reg_use", + "form_allow_patient_portal", + "form_billing_note", + "form_birth_fname", + "form_birth_lname", + "form_birth_mname", + "form_care_team_facility[]", + "form_care_team_provider[]", + "form_care_team_status", + "form_city", + "form_cmsportal_login", + "form_contact_relationship", + "form_country_code", + "form_county", + "form_deceased_date", + "form_deceased_reason", + "form_drivers_license", + "form_em_city", + "form_em_country", + "form_em_end_date", + "form_em_name", + "form_em_postal_code", + "form_em_start_date", + "form_em_state", + "form_em_street", + "form_em_street_line_2", + "form_email", + "form_email_direct", + "form_ethnicity[]", + "form_family_size", + "form_financial_review", + "form_fname", + "form_gender_identity", + "form_genericname1", + "form_genericname2", + "form_genericval1", + "form_genericval2", + "form_guardianaddress", + "form_guardiancity", + "form_guardiancountry", + "form_guardianemail", + "form_guardianphone", + "form_guardianpostalcode", + "form_guardianrelationship", + "form_guardiansex", + "form_guardiansname", + "form_guardianstate", + "form_guardianworkphone", + "form_hipaa_allowemail", + "form_hipaa_allowsms", + "form_hipaa_mail", + "form_hipaa_message", + "form_hipaa_notice", + "form_hipaa_voice", + "form_homeless", + "form_imm_reg_stat_effdate", + "form_imm_reg_status", + "form_industry", + "form_interpreter_needed", + "form_interpretter", + "form_language", + "form_lname", + "form_migrantseasonal", + "form_mname", + "form_monthly_income", + "form_mothersname", + "form_name_history[]", + "form_nationality_country", + "form_occupation", + "form_patient_groups[]", + "form_person_gender", + "form_pharmacy_id", + "form_phone_biz", + "form_phone_cell", + "form_phone_contact", + "form_phone_home", + "form_postal_code", + "form_preferred_name", + "form_prevent_portal_apps", + "form_pronoun", + "form_prot_indi_effdate", + "form_protect_indicator", + "form_providerID", + "form_provider_since_date", + "form_publ_code_eff_date", + "form_publicity_code", + "form_pubpid", + "form_race[]", + "form_ref_providerID", + "form_referral_source", + "form_related_persons[__RP-INDEX__][active]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][addresses_id]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][city]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][contact_address_id]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][contact_id]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][country]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][data_action]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][district]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][line_1]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][line_2]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][notes]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][period_end]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][period_start]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][priority]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][state]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][status]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][type]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][use]", + "form_related_persons[__RP-INDEX__][addresses][__A-INDEX__][zip]", + "form_related_persons[__RP-INDEX__][birth_date]", + "form_related_persons[__RP-INDEX__][can_make_medical_decisions]", + "form_related_persons[__RP-INDEX__][can_receive_medical_info]", + "form_related_persons[__RP-INDEX__][contact_priority]", + "form_related_persons[__RP-INDEX__][data_action]", + "form_related_persons[__RP-INDEX__][end_date]", + "form_related_persons[__RP-INDEX__][first_name]", + "form_related_persons[__RP-INDEX__][gender]", + "form_related_persons[__RP-INDEX__][is_emergency_contact]", + "form_related_persons[__RP-INDEX__][is_primary_contact]", + "form_related_persons[__RP-INDEX__][last_name]", + "form_related_persons[__RP-INDEX__][middle_name]", + "form_related_persons[__RP-INDEX__][notes]", + "form_related_persons[__RP-INDEX__][owner_contact_id]", + "form_related_persons[__RP-INDEX__][owner_contact_relation_id]", + "form_related_persons[__RP-INDEX__][relationship]", + "form_related_persons[__RP-INDEX__][role]", + "form_related_persons[__RP-INDEX__][start_date]", + "form_related_persons[__RP-INDEX__][target_id]", + "form_related_persons[__RP-INDEX__][target_table]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][contact_id]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][data_action]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][id]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][is_primary]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][notes]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][period_start]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][rank]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][status]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][system]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][use]", + "form_related_persons[__RP-INDEX__][telecoms][__T-INDEX__][value]", + "form_religion", + "form_sex", + "form_sex_identified", + "form_sexual_orientation", + "form_ss", + "form_state", + "form_status", + "form_street", + "form_street_line_2", + "form_suffix", + "form_text_gender_identity", + "form_text_sexual_orientation", + "form_title", + "form_tribal_affiliations", + "form_vfc" + ], + "client_rules": { + "form_fname": { + "presence": true + }, + "form_lname": { + "presence": true + }, + "form_DOB": { + "presence": true, + "pastDate": { + "message": "must be past date" + } + }, + "form_sex": { + "presence": true + } + } + }, + "precheck": { + "method": "GET", + "path": "/interface/new/new_search_popup.php", + "query": [ + [ + "mf_fname", + "{fname}" + ], + [ + "mf_lname", + "{lname}" + ], + [ + "mf_DOB", + "{DOB}" + ], + [ + "close", + "" + ] + ], + "require_text": [ + "Confirm Create New Patient", + "No matches were found." + ], + "from_gui_step": "step_039" + }, + "success_regex": "set_pid=(\\d+)", + "fields": [ + { + "name": "csrf_token_form", + "recorded": "", + "source": "session" + }, + { + "name": "form_title", + "recorded": "Ms.", + "source": "param", + "param": "title" + }, + { + "name": "form_fname", + "recorded": "Nadia", + "source": "param", + "param": "fname" + }, + { + "name": "form_mname", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_lname", + "recorded": "Brennan", + "source": "param", + "param": "lname" + }, + { + "name": "form_suffix", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_preferred_name", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_birth_fname", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_birth_mname", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_birth_lname", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_DOB", + "recorded": "1971-04-19", + "source": "param", + "param": "DOB" + }, + { + "name": "form_sex", + "recorded": "Female", + "source": "param", + "param": "sex" + }, + { + "name": "form_gender_identity", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_text_gender_identity", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_sex_identified", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_sexual_orientation", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_text_sexual_orientation", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_pronoun", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_pubpid", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_ss", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_drivers_license", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_status", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_genericname1", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_genericval1", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_genericname2", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_genericval2", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_billing_note", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_street", + "recorded": "27 WILLOW BEND ROAD", + "source": "param", + "param": "street" + }, + { + "name": "form_street_line_2", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_city", + "recorded": "EXAMPLETON", + "source": "param", + "param": "city" + }, + { + "name": "form_state", + "recorded": "MA", + "source": "select_label", + "param": "state_label", + "label": "Massachusetts" + }, + { + "name": "form_postal_code", + "recorded": "02139", + "source": "param", + "param": "postal_code" + }, + { + "name": "form_country_code", + "recorded": "USA", + "source": "param", + "param": "country_code" + }, + { + "name": "form_county", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_mothersname", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_contact_relationship", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_phone_contact", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_phone_home", + "recorded": "555-0121", + "source": "param", + "param": "phone_home" + }, + { + "name": "form_phone_biz", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_phone_cell", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_email", + "recorded": "nadia.brennan@example.invalid", + "source": "param", + "param": "email" + }, + { + "name": "form_email_direct", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_providerID", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_provider_since_date", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_ref_providerID", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_pharmacy_id", + "recorded": "0", + "source": "form_default" + }, + { + "name": "form_hipaa_notice", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_hipaa_voice", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_hipaa_message", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_hipaa_mail", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_hipaa_allowsms", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_hipaa_allowemail", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_allow_imm_reg_use", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_allow_imm_info_share", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_allow_health_info_ex", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_allow_patient_portal", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_cmsportal_login", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_imm_reg_status", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_imm_reg_stat_effdate", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_publicity_code", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_publ_code_eff_date", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_protect_indicator", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_prot_indi_effdate", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_care_team_status", + "recorded": "active", + "source": "form_default" + }, + { + "name": "form_patient_groups[]", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_occupation", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_industry", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_name", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_street", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_street_line_2", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_city", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_state", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_postal_code", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_country", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_start_date", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_em_end_date", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_language", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_ethnicity[]", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_race[]", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_nationality_country", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_financial_review", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_family_size", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_monthly_income", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_homeless", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_interpreter_needed", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_interpretter", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_migrantseasonal", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_referral_source", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_vfc", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_religion", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_tribal_affiliations", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_deceased_date", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_deceased_reason", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardiansname", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianrelationship", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardiansex", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianaddress", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardiancity", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianstate", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianpostalcode", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardiancountry", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianphone", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianworkphone", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_guardianemail", + "recorded": "", + "source": "form_default" + }, + { + "name": "form_person_gender", + "recorded": "", + "source": "form_default" + } + ], + "folded_gui_steps": [ + "step_000", + "step_001", + "step_002", + "step_003", + "step_004", + "step_005", + "step_006", + "step_007", + "step_008", + "step_009", + "step_010", + "step_011", + "step_012", + "step_013", + "step_014", + "step_015", + "step_016", + "step_017", + "step_018", + "step_019", + "step_020", + "step_021", + "step_022", + "step_023", + "step_024", + "step_025", + "step_026", + "step_027", + "step_028", + "step_029", + "step_030", + "step_031", + "step_032", + "step_033", + "step_034", + "step_035", + "step_036", + "step_037", + "step_038", + "step_039", + "step_040" + ] +} diff --git a/packages/flow/benchmark/openemr_api_tier/api_bundle/workflow.json b/packages/flow/benchmark/openemr_api_tier/api_bundle/workflow.json new file mode 100644 index 000000000..53d4f3c09 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/api_bundle/workflow.json @@ -0,0 +1,1202 @@ +{ + "schema_version": 2, + "name": "openemr-create-synthetic-patient (api tier)", + "recording_id": "241803615513807b7f5236413f1efaaa520f1edc831dbd9e2c86ce9d144f10b9", + "contains_phi": false, + "phi_scrubbed": false, + "encrypted": false, + "created_at": "2026-10-15T01:01:43.646413+00:00", + "viewport": [ + 1280, + 800 + ], + "params": { + "title": "Ms.", + "fname": "Elena", + "lname": "Marsh", + "DOB": "1979-06-14", + "sex": "Female", + "street": "42 HARBOR LANE", + "city": "EXAMPLETON", + "state_label": "Massachusetts", + "postal_code": "02139", + "phone_home": "555-0142", + "email": "elena.marsh@example.invalid", + "country_code": "USA" + }, + "param_specs": { + "title": { + "name": "title", + "type": "string", + "example": "Ms.", + "required": true, + "choices": [] + }, + "fname": { + "name": "fname", + "type": "string", + "example": "Elena", + "required": true, + "choices": [] + }, + "lname": { + "name": "lname", + "type": "string", + "example": "Marsh", + "required": true, + "choices": [] + }, + "DOB": { + "name": "DOB", + "type": "string", + "example": "1979-06-14", + "required": true, + "choices": [] + }, + "sex": { + "name": "sex", + "type": "string", + "example": "Female", + "required": true, + "choices": [] + }, + "street": { + "name": "street", + "type": "string", + "example": "42 HARBOR LANE", + "required": true, + "choices": [] + }, + "city": { + "name": "city", + "type": "string", + "example": "EXAMPLETON", + "required": true, + "choices": [] + }, + "state_label": { + "name": "state_label", + "type": "string", + "example": "Massachusetts", + "required": true, + "choices": [] + }, + "postal_code": { + "name": "postal_code", + "type": "string", + "example": "02139", + "required": true, + "choices": [] + }, + "phone_home": { + "name": "phone_home", + "type": "string", + "example": "555-0142", + "required": true, + "choices": [] + }, + "email": { + "name": "email", + "type": "string", + "example": "elena.marsh@example.invalid", + "required": true, + "choices": [] + }, + "country_code": { + "name": "country_code", + "type": "string", + "example": "USA", + "required": true, + "choices": [] + } + }, + "secret_params": [], + "steps": [ + { + "id": "api_save", + "intent": "save the form via POST /interface/new/new_comprehensive_save.php (folds 41 GUI steps)", + "action": "click", + "anchor": { + "template": "templates/step_040.png", + "structural": { + "selector": "#confirmCreate", + "frame_path": [ + "#modalframe" + ], + "role": "button", + "name": "Confirm Create New Patient", + "automation_id": null, + "window_name": null + }, + "region": [ + 520, + 248, + 160, + 64 + ], + "click_point": [ + 600, + 280 + ], + "ocr_text": "Confirm Create New Patient", + "context_text": null, + "structured_identity": null, + "identity_template": { + "schema_version": 1, + "salt": "2bd0bacdef913775b67ed50575a98ebf", + "band_len": 36, + "tokens": [ + { + "c": "680dcba873ad4b9a", + "r": "680dcba873ad4b9a", + "n": 7, + "alpha": true, + "name": true, + "digit": false, + "idsh": false, + "glyph": false, + "gen": false + }, + { + "c": "6d8e313e89033bfa", + "r": "977f18e495c1ad69", + "n": 7, + "alpha": true, + "name": true, + "digit": false, + "idsh": false, + "glyph": false, + "gen": false + }, + { + "c": "42f37d83c28df96a", + "r": "42f37d83c28df96a", + "n": 6, + "alpha": true, + "name": true, + "digit": false, + "idsh": false, + "glyph": false, + "gen": false + }, + { + "c": "5fa49123dab7ff4f", + "r": "5fa49123dab7ff4f", + "n": 3, + "alpha": true, + "name": true, + "digit": false, + "idsh": false, + "glyph": false, + "gen": false + }, + { + "c": "f6c24be2e622c022", + "r": "d4e50c84680d1cb5", + "n": 7, + "alpha": true, + "name": true, + "digit": false, + "idsh": false, + "glyph": false, + "gen": false + }, + { + "c": "62406d441cd5a20d", + "r": "62406d441cd5a20d", + "n": 6, + "alpha": true, + "name": true, + "digit": false, + "idsh": false, + "glyph": false, + "gen": false + } + ], + "concats": [ + { + "i": 0, + "size": 2, + "c": "1ec34e510dd2d380", + "r": "a2e61fe08bab6169", + "digit": false, + "name": true, + "n": 14 + }, + { + "i": 0, + "size": 3, + "c": "cbc7e1a89ae97faf", + "r": "de0fc25dc0cedcbb", + "digit": false, + "name": true, + "n": 20 + }, + { + "i": 0, + "size": 4, + "c": "af49d64c7963cd7a", + "r": "70c25ae790ce8cdd", + "digit": false, + "name": true, + "n": 23 + }, + { + "i": 1, + "size": 2, + "c": "a00ed16fd07342c3", + "r": "b1c2e6251dd03f3d", + "digit": false, + "name": true, + "n": 13 + }, + { + "i": 1, + "size": 3, + "c": "99d7f40ddc501806", + "r": "03a6e52c8e6eb829", + "digit": false, + "name": true, + "n": 16 + }, + { + "i": 1, + "size": 4, + "c": "11205f3a616cc78c", + "r": "d34f221c714e753e", + "digit": false, + "name": true, + "n": 23 + }, + { + "i": 2, + "size": 2, + "c": "4e43ce50af303389", + "r": "4e43ce50af303389", + "digit": false, + "name": true, + "n": 9 + }, + { + "i": 2, + "size": 3, + "c": "b44145ef5c8201e1", + "r": "34485d0e4f32b42f", + "digit": false, + "name": true, + "n": 16 + }, + { + "i": 2, + "size": 4, + "c": "c5f1aeaa9fca2746", + "r": "011d96e0a779dd51", + "digit": false, + "name": true, + "n": 22 + }, + { + "i": 3, + "size": 2, + "c": "59f9a02ecff8defd", + "r": "536bad54abec8e0b", + "digit": false, + "name": true, + "n": 10 + }, + { + "i": 3, + "size": 3, + "c": "b83931ad5de66017", + "r": "526f318b33f2d71e", + "digit": false, + "name": true, + "n": 16 + }, + { + "i": 4, + "size": 2, + "c": "ec560bc2be6a2825", + "r": "8c075b9d9ac827bb", + "digit": false, + "name": true, + "n": 13 + } + ], + "structured": null, + "signal_hashes": { + "captured_context|exact|": "75b854d4741d82d0300c078b36770da7228e7d52045d180570f413e81a83ca84", + "captured_context|normalized|unicode_nfkc": "75b854d4741d82d0300c078b36770da7228e7d52045d180570f413e81a83ca84", + "captured_context|normalized|casefold": "0f2c89186c04c26a37f3080f567cc691e798e8098e239914872b6fa75987eb15", + "captured_context|normalized|collapse_whitespace": "75b854d4741d82d0300c078b36770da7228e7d52045d180570f413e81a83ca84", + "captured_context|normalized|strip_punctuation": "ce6bb7f01356215676099c1ecdf970ad034b73e107876c4d64d01666db31250d", + "captured_context|normalized|unicode_nfkc,casefold": "0f2c89186c04c26a37f3080f567cc691e798e8098e239914872b6fa75987eb15", + "captured_context|normalized|unicode_nfkc,collapse_whitespace": "75b854d4741d82d0300c078b36770da7228e7d52045d180570f413e81a83ca84", + "captured_context|normalized|unicode_nfkc,strip_punctuation": "ce6bb7f01356215676099c1ecdf970ad034b73e107876c4d64d01666db31250d", + "captured_context|normalized|casefold,collapse_whitespace": "0f2c89186c04c26a37f3080f567cc691e798e8098e239914872b6fa75987eb15", + "captured_context|normalized|casefold,strip_punctuation": "4fbff88ddb88c39fb5b129f56db93999e44fc48bab72323a3c8e7b7e5be8e173", + "captured_context|normalized|collapse_whitespace,strip_punctuation": "ce6bb7f01356215676099c1ecdf970ad034b73e107876c4d64d01666db31250d", + "captured_context|normalized|unicode_nfkc,casefold,collapse_whitespace": "0f2c89186c04c26a37f3080f567cc691e798e8098e239914872b6fa75987eb15", + "captured_context|normalized|unicode_nfkc,casefold,strip_punctuation": "4fbff88ddb88c39fb5b129f56db93999e44fc48bab72323a3c8e7b7e5be8e173", + "captured_context|normalized|unicode_nfkc,collapse_whitespace,strip_punctuation": "ce6bb7f01356215676099c1ecdf970ad034b73e107876c4d64d01666db31250d", + "captured_context|normalized|casefold,collapse_whitespace,strip_punctuation": "4fbff88ddb88c39fb5b129f56db93999e44fc48bab72323a3c8e7b7e5be8e173", + "captured_context|normalized|unicode_nfkc,casefold,collapse_whitespace,strip_punctuation": "4fbff88ddb88c39fb5b129f56db93999e44fc48bab72323a3c8e7b7e5be8e173" + }, + "param_token_indices": {}, + "rests_on_confusable_identifier": false + }, + "identifier_crop": "templates/identifiers/step_040.png", + "identifier_region": [ + 107, + 258, + 678, + 34 + ], + "landmarks": [ + { + "relation": "above", + "ocr_text": "Name", + "distance_px": 151, + "dx_px": 50, + "dy_px": 142 + }, + { + "relation": "above", + "ocr_text": "Name", + "distance_px": 152, + "dx_px": -54, + "dy_px": 142 + } + ], + "search_pad": 80 + }, + "text": null, + "param": null, + "secret": false, + "field_label": null, + "selection_commit_key": null, + "selection_region": null, + "key": null, + "modifiers": [], + "drag_end_anchor": null, + "scroll_dx": null, + "scroll_dy": null, + "expect": [], + "effects": [ + { + "kind": "record_written", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": null, + "value": null, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "exactly one synthetic OpenEMR patient exists", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "title", + "value": { + "literal": null, + "param": "title" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'title' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "fname", + "value": { + "literal": null, + "param": "fname" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'fname' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "DOB", + "value": { + "literal": null, + "param": "DOB" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'DOB' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "sex", + "value": { + "literal": null, + "param": "sex" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'sex' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "street", + "value": { + "literal": null, + "param": "street" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'street' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "city", + "value": { + "literal": null, + "param": "city" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'city' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "state", + "value": { + "literal": "MA", + "param": null + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'state' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "postal_code", + "value": { + "literal": null, + "param": "postal_code" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'postal_code' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "phone_home", + "value": { + "literal": null, + "param": "phone_home" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'phone_home' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "country_code", + "value": { + "literal": null, + "param": "country_code" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'country_code' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + } + ], + "api_binding": { + "kind": "rest", + "on_unavailable": "halt", + "method": "POST", + "url_template": "/interface/new/new_comprehensive_save.php", + "body_template": { + "csrf_token_form": "{__session_csrf_token_form}", + "form_title": "{title}", + "form_fname": "{fname}", + "form_mname": "", + "form_lname": "{lname}", + "form_suffix": "", + "form_preferred_name": "", + "form_birth_fname": "", + "form_birth_mname": "", + "form_birth_lname": "", + "form_DOB": "{DOB}", + "form_sex": "{sex}", + "form_gender_identity": "", + "form_text_gender_identity": "", + "form_sex_identified": "", + "form_sexual_orientation": "", + "form_text_sexual_orientation": "", + "form_pronoun": "", + "form_pubpid": "", + "form_ss": "", + "form_drivers_license": "", + "form_status": "", + "form_genericname1": "", + "form_genericval1": "", + "form_genericname2": "", + "form_genericval2": "", + "form_billing_note": "", + "form_street": "{street}", + "form_street_line_2": "", + "form_city": "{city}", + "form_state": "{state_label}", + "form_postal_code": "{postal_code}", + "form_country_code": "{country_code}", + "form_county": "", + "form_mothersname": "", + "form_contact_relationship": "", + "form_phone_contact": "", + "form_phone_home": "{phone_home}", + "form_phone_biz": "", + "form_phone_cell": "", + "form_email": "{email}", + "form_email_direct": "", + "form_providerID": "", + "form_provider_since_date": "", + "form_ref_providerID": "", + "form_pharmacy_id": "0", + "form_hipaa_notice": "", + "form_hipaa_voice": "", + "form_hipaa_message": "", + "form_hipaa_mail": "", + "form_hipaa_allowsms": "", + "form_hipaa_allowemail": "", + "form_allow_imm_reg_use": "", + "form_allow_imm_info_share": "", + "form_allow_health_info_ex": "", + "form_allow_patient_portal": "", + "form_cmsportal_login": "", + "form_imm_reg_status": "", + "form_imm_reg_stat_effdate": "", + "form_publicity_code": "", + "form_publ_code_eff_date": "", + "form_protect_indicator": "", + "form_prot_indi_effdate": "", + "form_care_team_status": "active", + "form_patient_groups[]": "", + "form_occupation": "", + "form_industry": "", + "form_em_name": "", + "form_em_street": "", + "form_em_street_line_2": "", + "form_em_city": "", + "form_em_state": "", + "form_em_postal_code": "", + "form_em_country": "", + "form_em_start_date": "", + "form_em_end_date": "", + "form_language": "", + "form_ethnicity[]": "", + "form_race[]": "", + "form_nationality_country": "", + "form_financial_review": "", + "form_family_size": "", + "form_monthly_income": "", + "form_homeless": "", + "form_interpreter_needed": "", + "form_interpretter": "", + "form_migrantseasonal": "", + "form_referral_source": "", + "form_vfc": "", + "form_religion": "", + "form_tribal_affiliations": "", + "form_deceased_date": "", + "form_deceased_reason": "", + "form_guardiansname": "", + "form_guardianrelationship": "", + "form_guardiansex": "", + "form_guardianaddress": "", + "form_guardiancity": "", + "form_guardianstate": "", + "form_guardianpostalcode": "", + "form_guardiancountry": "", + "form_guardianphone": "", + "form_guardianworkphone": "", + "form_guardianemail": "", + "form_person_gender": "" + }, + "query": {}, + "headers": { + "Content-Type": "application/x-www-form-urlencoded" + }, + "expected_status": [ + 200 + ], + "timeout_s": 30.0, + "effects": [ + { + "kind": "record_written", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": null, + "value": null, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "exactly one synthetic OpenEMR patient exists", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "title", + "value": { + "literal": null, + "param": "title" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'title' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "fname", + "value": { + "literal": null, + "param": "fname" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'fname' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "DOB", + "value": { + "literal": null, + "param": "DOB" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'DOB' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "sex", + "value": { + "literal": null, + "param": "sex" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'sex' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "street", + "value": { + "literal": null, + "param": "street" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'street' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "city", + "value": { + "literal": null, + "param": "city" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'city' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "state", + "value": { + "literal": "MA", + "param": null + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'state' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "postal_code", + "value": { + "literal": null, + "param": "postal_code" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'postal_code' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "phone_home", + "value": { + "literal": null, + "param": "phone_home" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'phone_home' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + }, + { + "kind": "field_equals", + "match": { + "email": { + "literal": null, + "param": "email" + }, + "lname": { + "literal": null, + "param": "lname" + } + }, + "field": "country_code", + "value": { + "literal": null, + "param": "country_code" + }, + "expected_count": 1, + "new_records": [], + "identity_field": "id", + "idempotency_key": null, + "key_field": "key", + "count_new_only": false, + "forbid_collateral_loss": true, + "risk": "reversible", + "probe": "OpenEMR patient field 'country_code' read-back", + "timeout_s": 5.0, + "needs_operator_confirmation": false, + "readback": null + } + ], + "identity": [] + }, + "risk": "irreversible", + "risk_explanation": "the marked #confirmCreate click creates the patient chart", + "risk_review_required": false, + "wait_until": null, + "guard": null, + "timeout_s": 10.0, + "identity_armed": true, + "identity_unarmed_reason": null, + "identifier_crop_missing_reason": null, + "submit_binding": null, + "submit_binding_unbound_reason": null + } + ], + "interstitials": [], + "program": null, + "subflows": {}, + "data_sources": {}, + "manifest": { + "schema_version": 2, + "content_digest": "e5c038601681335df2537e2816dcb8d613193d88781aada637bc038961c1a2b1", + "file_hashes": {}, + "provenance": { + "compiler_version": "2.2.0", + "source_recording_sha256": "241803615513807b7f5236413f1efaaa520f1edc831dbd9e2c86ce9d144f10b9", + "compiler_config_sha256": "b12e6c6e0943d452f357b90258b55555593fd95e2fcdb6026330a0222204ed24", + "created_at": "2026-10-15T01:01:43.647330+00:00", + "policy_name": null, + "certified": false, + "certification_status": null, + "certified_at": null, + "expires_at": null, + "certification_invalidated_at": null, + "certification_invalidation_reason": null, + "governed_authorization_template": null + }, + "encrypted": false + } +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/bench.py b/packages/flow/benchmark/openemr_api_tier/bench.py new file mode 100644 index 000000000..3e281a742 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/bench.py @@ -0,0 +1,219 @@ +"""Steps 3-4: run the referral through the browser program or the API step. + + python -m benchmark.openemr_api_tier.bench one --arm api --ref 1 --out + python -m benchmark.openemr_api_tier.bench matrix --blocks 5 --per-block 4 --out + +Arms (both through the same ``Replayer`` and the same ``EffectVerifier``): + +* ``browser``: the DOM-identity referral bundle (the published ``replay-dom`` + arm). Clock: new browser context -> sign-in -> open the New Patient form -> + 41 replayed steps -> record check. +* ``api``: the compiled one-step bundle with :class:`SessionFormActuator`. + No browser is started; the backend refuses every GUI call. Clock: new HTTP + session -> sign-in -> read the form (CSRF token, defaults, drift check) -> + duplicate check -> save POST -> record check. + +Warm mode for both: Python imports, Chromium launch and the OCR engine are +set up once before the matrix (untimed), as in the October timing study. +The record check is the test setup's own SQL read-back, the same rule as that +study. Fixture reset and seeding are outside the clock. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import subprocess +import time +from pathlib import Path +from typing import Any + +from benchmark.openemr_api_tier.capture_har import BUNDLE as GUI_BUNDLE +from benchmark.openemr_api_tier.common import ( + REFERRALS, + load_demo, + record_check, + reset_and_seed, + spec_for, + sql_patients, +) +from benchmark.openemr_api_tier.formtier import SessionFormActuator +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + +API_BUNDLE = Path(os.environ.get("API_BUNDLE", str(Path(__file__).resolve().parent / "api_bundle"))) +SETTLE_S = 1.0 # same oracle settle as the October study (demo.ORACLE_SETTLE_S) + + +class NoGuiBackend: + """A backend that fails loudly if the API run looks at or acts on the GUI. + + Optional backend capabilities are simply absent (AttributeError); every + observation or input method raises. + """ + + def _refuse(self, *_a: Any, **_k: Any) -> Any: + raise RuntimeError("API-tier run touched the GUI backend") + + screenshot = click = double_click = right_click = drag = type_text = press = hotkey = scroll = _refuse + move = mouse_move = key = select_option = _refuse + + +class Env: + def __init__(self, browser: bool) -> None: + from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + self.demo = load_demo() + self.f = SpikeFixture() + self.Oracle, self.Workflow, self.Replayer = OpenEMRPatientOracle, Workflow, Replayer + self.tier = json.loads((API_BUNDLE / "session_form.json").read_text()) + self.pw = self.browser = None + if browser: + import sys + + from playwright.sync_api import sync_playwright + + import openadapt_flow.vision.ocr # noqa: F401 + + ocr = sys.modules["openadapt_flow.vision.ocr"] # the package re-exports a function named ocr + self.ocr = ocr + self.pw = sync_playwright().start() + self.browser = self.pw.chromium.launch(headless=True) + ocr._get_engine() # warm the OCR engine once, untimed + ctx = self.browser.new_context(viewport={"width": 1280, "height": 800}) + page = ctx.new_page() + page.goto(self.f.ui_base_url) + ocr.ocr(page.screenshot(type="png")) + ctx.close() + ocr.clear_ocr_memo() + + def close(self) -> None: + if self.browser: + self.browser.close() + self.pw.stop() + + +def run_one(env: Env, arm: str, ref_idx: int, out: Path) -> dict[str, Any]: + ref = REFERRALS[ref_idx - 1] + spec = spec_for(ref) + out.mkdir(parents=True, exist_ok=False) + before_ids = {r["id"] for r in sql_patients(env.f)} + oracle = env.Oracle(env.f.api_base_url, env.f.token_session("oracle"), spec) + res: dict[str, Any] = {"arm": arm, "ref": ref.key, "mismatch": ref.mismatch, "error": None} + m: dict[str, float] = {} + ctx = None + try: + if arm == "browser": + m["t0"] = time.time() + ctx = env.browser.new_context(viewport={"width": 1280, "height": 800}, device_scale_factor=1) + page = ctx.new_page() + page.goto(env.f.ui_base_url) + backend = env.demo._BasePlaywrightBackend(page) + env.demo._preauthenticate_browser(backend, env.f) + m["setup_end"] = time.time() + env.ocr.clear_ocr_memo() + rep = env.Replayer(backend, effect_verifier=oracle.verifier).run( + env.Workflow.load(GUI_BUNDLE), params=spec.params(), bundle_dir=GUI_BUNDLE, run_dir=out / "engine") + else: + vals = env.f._runtime_values() + m["t0"] = time.time() + act = SessionFormActuator(env.f.ui_base_url, env.tier, username=vals["OPENEMR_ACTOR_USER"], + password=vals["OPENEMR_ACTOR_PASSWORD"]) + rep = env.Replayer(NoGuiBackend(), effect_verifier=oracle.verifier, api_actuator=act).run( + env.Workflow.load(API_BUNDLE), params=spec.params(), bundle_dir=API_BUNDLE, run_dir=out / "engine") + m["setup_end"] = act.marks.get("form_read", m["t0"]) + res["api_marks"] = {k: round(v - m["t0"], 3) for k, v in act.marks.items()} + res["api_log"] = act.log + m["task_end"] = time.time() + raw = rep.model_dump(mode="json") + res.update(engine_success=raw.get("success"), outcome=str(raw.get("execution_outcome")), + steps=len(raw.get("results") or []), + step_errors=[(r["step_id"], (r.get("error") or "")[:300]) for r in raw["results"] if r.get("error")], + actuation=[r.get("actuation") for r in raw["results"]][-1:], + effect_verified=[r.get("effect_verified") for r in raw["results"]][-1:]) + except Exception as exc: # noqa: BLE001 + m.setdefault("task_end", time.time()) + res["error"] = f"{type(exc).__name__}: {exc}"[:500] + time.sleep(SETTLE_S) + res.update(record_check(env.f, ref, before_ids)) + m["verified"] = time.time() + if ctx is not None: + ctx.close() + res["wall_s"] = round(m["verified"] - m["t0"], 3) + res["setup_s"] = round(m.get("setup_end", m["t0"]) - m["t0"], 3) + res["task_s"] = round(m["task_end"] - m["t0"], 3) # from the clock start, so it includes setup_s + res["readback_s"] = round(m["verified"] - m["task_end"], 3) + (out / "result.json").write_text(json.dumps(res, indent=1, default=str)) + return res + + +COLS = ["arm", "block", "pos", "ref", "mismatch", "correct", "new_charts", "readback", "engine_success", "outcome", + "error", "wall_s", "setup_s", "task_s", "readback_s", "load1", "start_local", "engine_sha"] + + +def load1() -> float: + return float(subprocess.run(["sysctl", "-n", "vm.loadavg"], capture_output=True, text=True).stdout.split()[1]) + + +def cmd_matrix(a: argparse.Namespace) -> None: + out = Path(a.out) + out.mkdir(parents=True, exist_ok=True) + sha = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()[:8] + env = Env(browser=True) + csv_path = out / "runs.csv" + new = not csv_path.exists() + try: + for b in range(a.blocks): + # Interleave arms in time and alternate which goes first. + arms = ["browser", "api"] if b % 2 == 0 else ["api", "browser"] + for arm in arms: + meta = reset_and_seed(env.f) + meta.update(arm=arm, block=b + 1, started=time.strftime("%Y-%m-%d %H:%M:%S")) + (out / f"block-{b + 1:02d}-{arm}.json").write_text(json.dumps(meta, indent=1)) + for pos in range(a.per_block): + idx = ((b * a.per_block + pos) % len(REFERRALS)) + 1 + start = time.strftime("%H:%M:%S") + res = run_one(env, arm, idx, out / f"b{b + 1:02d}-{arm}-{pos + 1}-{REFERRALS[idx - 1].key}") + row = {k: res.get(k) for k in COLS} + row.update(block=b + 1, pos=pos + 1, load1=load1(), start_local=start, engine_sha=sha) + with open(csv_path, "a", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS) + if new: + w.writeheader() + new = False + w.writerow(row) + print(json.dumps({k: row[k] for k in ("arm", "block", "ref", "correct", "readback", "wall_s", "task_s", "error")}), flush=True) + finally: + env.close() + + +def main() -> None: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + o = sub.add_parser("one") + o.add_argument("--arm", required=True, choices=["browser", "api"]) + o.add_argument("--ref", type=int, default=1) + o.add_argument("--out", required=True) + o.add_argument("--reset", action="store_true") + mtx = sub.add_parser("matrix") + mtx.add_argument("--blocks", type=int, default=5) + mtx.add_argument("--per-block", type=int, default=4) + mtx.add_argument("--out", required=True) + a = ap.parse_args() + if a.cmd == "matrix": + return cmd_matrix(a) + env = Env(browser=a.arm == "browser") + try: + if a.reset: + reset_and_seed(env.f) + res = run_one(env, a.arm, a.ref, Path(a.out)) + print(json.dumps({k: v for k, v in res.items() if k != "field_check"}, indent=1, default=str)) + finally: + env.close() + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/openemr_api_tier/capture_har.py b/packages/flow/benchmark/openemr_api_tier/capture_har.py new file mode 100644 index 000000000..80408472a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/capture_har.py @@ -0,0 +1,85 @@ +"""Step 1: replay the compiled referral bundle once with network capture on. + +Signs in, opens the New Patient form, replays the DOM-identity referral bundle +(the published ``replay-dom`` arm) for one referral, and writes a Playwright HAR +of every request the browser made, with bodies. The HAR is the input to +``compile_api.py``. + + python -m benchmark.openemr_api_tier.capture_har --ref 1 --out +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +from pathlib import Path + +from benchmark.openemr_api_tier.common import ( + REFERRALS, + load_demo, + record_check, + reset_and_seed, + spec_for, + sql_patients, +) +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + +BUNDLE = Path(os.environ.get("REFERRAL_BUNDLE", str(Path.home() / "oa/scratch/replay-speed/dom-identity/out/bundle"))) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--ref", type=int, default=1) + ap.add_argument("--out", required=True) + a = ap.parse_args() + out = Path(a.out) + out.mkdir(parents=True, exist_ok=False) + + from playwright.sync_api import sync_playwright + + from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + demo = load_demo() + f = SpikeFixture() + meta = reset_and_seed(f) + ref = REFERRALS[a.ref - 1] + spec = spec_for(ref) + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + ctx = browser.new_context(viewport={"width": 1280, "height": 800}, device_scale_factor=1, + record_har_path=str(out / "referral.har"), record_har_content="embed") + page = ctx.new_page() + page.goto(f.ui_base_url) + backend = demo._BasePlaywrightBackend(page) + try: + demo._preauthenticate_browser(backend, f) + demo._stable_pre_trial_counts(f) + before_ids = {r["id"] for r in sql_patients(f)} + oracle = OpenEMRPatientOracle(f.api_base_url, f.token_session("oracle"), spec) + marks = {"task_start": time.time()} + rep = Replayer(backend, effect_verifier=oracle.verifier).run( + Workflow.load(BUNDLE), params=spec.params(), bundle_dir=BUNDLE, run_dir=out / "engine") + marks["task_end"] = time.time() + time.sleep(demo.ORACLE_SETTLE_S) + check = record_check(f, ref, before_ids) + raw = rep.model_dump(mode="json") + (out / "capture.json").write_text(json.dumps({ + "ref": ref.key, "params": spec.params(), "fixture": meta, "bundle": str(BUNDLE), "success": raw.get("success"), + "outcome": str(raw.get("execution_outcome")), "task_s": round(marks["task_end"] - marks["task_start"], 3), + "steps": len(raw.get("results") or []), "check": check, + "step_marks": [(r["step_id"], r["intent"][:40]) for r in raw["results"]], + }, indent=1, default=str)) + print(json.dumps({"ref": ref.key, "success": raw.get("success"), "check": check["readback"], + "correct": check["correct"]})) + finally: + ctx.close() # flushes the HAR + browser.close() + pw.stop() + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/openemr_api_tier/common.py b/packages/flow/benchmark/openemr_api_tier/common.py new file mode 100644 index 000000000..19bb06454 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/common.py @@ -0,0 +1,127 @@ +"""Shared pieces of the API-tier spike: referrals, fixture, browser, record check. + +The eight referrals are the made-up patients of the October referral timing +benchmark (``benchmark/openemr_referral_timing``): seven new patients and r6, +a possible duplicate of the seeded chart Rosa Delgado 1958-03-27 whose correct +outcome is no new chart. Phones use 555-01xx and emails ``example.invalid``. +""" + +from __future__ import annotations + +import importlib.util +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + +FLOW = Path(__file__).resolve().parents[2] +HERE = Path(__file__).resolve().parent + +_MA = dict(city="EXAMPLETON", state="MA", state_label="Massachusetts", postal_code="02139") + + +@dataclass(frozen=True) +class Referral: + key: str + kwargs: dict[str, str] + mismatch: bool = False + + +REFERRALS: list[Referral] = [ + Referral("r1-nadia-brennan", dict(title="Ms.", fname="Nadia", lname="Brennan", dob="1971-04-19", sex="Female", + street="27 WILLOW BEND ROAD", phone_home="555-0121", + email="nadia.brennan@example.invalid", **_MA)), + Referral("r2-marcus-feld", dict(title="Mr.", fname="Marcus", lname="Feld", dob="1958-09-02", sex="Male", + street="114 QUARRY HILL DRIVE", phone_home="555-0124", + email="marcus.feld@example.invalid", **_MA)), + Referral("r3-ines-carvalho", dict(title="Mrs.", fname="Ines", lname="Carvalho", dob="1949-12-11", sex="Female", + street="9 LANTERN COURT", phone_home="555-0131", + email="ines.carvalho@example.invalid", **_MA)), + Referral("r4-theo-lindqvist", dict(title="Mr.", fname="Theo", lname="Lindqvist", dob="1983-07-26", sex="Male", + street="63 FERRY STREET", phone_home="555-0136", + email="theo.lindqvist@example.invalid", **_MA)), + Referral("r5-grace-adeyemi", dict(title="Ms.", fname="Grace", lname="Adeyemi", dob="1990-02-08", sex="Female", + street="402 CEDAR ROW", phone_home="555-0145", + email="grace.adeyemi@example.invalid", **_MA)), + Referral("r6-rosa-delgado", dict(title="Mrs.", fname="Rosa", lname="Delgado", dob="1958-03-22", sex="Female", + street="7 MILL ROAD", phone_home="555-0188", + email="rosa.delgado@example.invalid", **_MA), mismatch=True), + Referral("r7-arlo-whitcombe", dict(title="Mr.", fname="Arlo", lname="Whitcombe", dob="1966-05-30", sex="Male", + street="88 BEACON TERRACE", phone_home="555-0152", + email="arlo.whitcombe@example.invalid", **_MA)), + Referral("r8-priya-raman", dict(title="Ms.", fname="Priya", lname="Raman", dob="1975-10-14", sex="Female", + street="15 STATION AVENUE", phone_home="555-0167", + email="priya.raman@example.invalid", **_MA)), +] +EXISTING_ROSA = dict(title="Mrs.", fname="Rosa", lname="Delgado", dob="1958-03-27", sex="Female", + street="311 BIRCH COURT", phone_home="555-0119", + email="r.delgado.prior@example.invalid", **_MA) + + +def load_demo() -> Any: + """The OpenEMR reference harness (sign-in, form frame, Playwright backend).""" + name = "openemr_local_demo" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, FLOW / "scripts/openemr_local_demo.py") + mod = importlib.util.module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + +def spec_for(ref: Referral) -> Any: + from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + + return SyntheticPatientSpec(**ref.kwargs) + + +def q(v: str) -> str: + assert "'" not in v and "\\" not in v + return "'" + v + "'" + + +SQL_FIELDS = ("id", "pid", "title", "fname", "lname", "DOB", "sex", "street", "city", "state", + "postal_code", "phone_home", "email", "country_code") + + +def sql_patients(f: SpikeFixture, where: str = "1=1") -> list[dict[str, str]]: + out = f._db("SELECT " + ",".join(SQL_FIELDS) + " FROM openemr.patient_data WHERE " + where + " ORDER BY id").decode() + return [{k: ("" if v == "NULL" else v) for k, v in zip(SQL_FIELDS, line.split("\t"))} for line in out.splitlines()] + + +def reset_and_seed(f: SpikeFixture) -> dict[str, Any]: + """Restore the baseline and seed the existing chart r6 must not duplicate.""" + t = time.time() + digest = f.reset() + seed = f.token_session("actor").post( + f.api_base_url + "/apis/default/api/patient", + json=dict(spec_for(Referral("rosa", EXISTING_ROSA)).fields), + ).status_code + return {"baseline": digest, "seed_status": seed, "reset_seed_s": round(time.time() - t, 2)} + + +def record_check(f: SpikeFixture, ref: Referral, before_ids: set[str]) -> dict[str, Any]: + """The test setup's own read-back (direct SQL), never the arm under test. + + Same rule as the referral timing benchmark: a new-patient referral is + correct iff exactly one new chart exists and it matches the referral on + all 12 fields; r6 is correct iff no new chart exists. + """ + spec = spec_for(ref) + after = sql_patients(f) + new_rows = [r for r in after if r["id"] not in before_ids] + target = sql_patients(f, f"lname={q(spec.lname)} AND email={q(spec.email)}") + row = new_rows[0] if len(new_rows) == 1 else None + fields = [{"field": k, "want": v, "got": None if row is None else row.get(k), + "match": row is not None and row.get(k) == v} for k, v in spec.fields.items()] + nmatch = sum(x["match"] for x in fields) + if ref.mismatch: + correct = len(new_rows) == 0 + else: + correct = len(new_rows) == 1 and len(target) == 1 and nmatch == len(fields) + return {"new_charts": len(new_rows), "readback": f"{nmatch}/{len(fields)}", "correct": correct, + "field_check": fields} diff --git a/packages/flow/benchmark/openemr_api_tier/compile_api.py b/packages/flow/benchmark/openemr_api_tier/compile_api.py new file mode 100644 index 000000000..e4001828d --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/compile_api.py @@ -0,0 +1,27 @@ +"""Step 2: compile the captured referral save into an API-tier bundle. + + python -m benchmark.openemr_api_tier.compile_api --har --gui-bundle --out +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from benchmark.openemr_api_tier.formtier import compile_from_har + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--har", required=True) + ap.add_argument("--gui-bundle", required=True) + ap.add_argument("--out", required=True) + a = ap.parse_args() + capture = Path(a.har).with_name("capture.json") + run_params = json.loads(capture.read_text()).get("params") if capture.exists() else None + print(json.dumps(compile_from_har(Path(a.har), Path(a.gui_bundle), Path(a.out), run_params), indent=1)) + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/openemr_api_tier/compute_figures.py b/packages/flow/benchmark/openemr_api_tier/compute_figures.py new file mode 100644 index 000000000..1a8c6e887 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/compute_figures.py @@ -0,0 +1,89 @@ +"""Derive figures.json from runs.csv (the 20 + 20 matrix). + + python -m benchmark.openemr_api_tier.compute_figures [--check] +""" + +from __future__ import annotations + +import csv +import json +import random +import statistics +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +RUNS = HERE / "results" / "runs.csv" +OUT = HERE / "results" / "figures.json" + + +def pct(xs: list[float], p: float) -> float: + xs = sorted(xs) + k = (len(xs) - 1) * p + lo, hi = int(k), min(int(k) + 1, len(xs) - 1) + return xs[lo] + (xs[hi] - xs[lo]) * (k - lo) + + +def summary(xs: list[float]) -> dict[str, float | int]: + return {"n": len(xs), "median": round(statistics.median(xs), 3), "p10": round(pct(xs, 0.1), 3), + "p90": round(pct(xs, 0.9), 3), "min": round(min(xs), 3), "max": round(max(xs), 3), + "mean": round(statistics.fmean(xs), 3)} + + +def boot_ratio(a: list[float], b: list[float], n: int = 10_000, seed: int = 0) -> list[float]: + rnd = random.Random(seed) + rs = sorted(statistics.median(rnd.choices(a, k=len(a))) / statistics.median(rnd.choices(b, k=len(b))) + for _ in range(n)) + return [round(rs[int(0.025 * n)], 2), round(rs[int(0.975 * n)], 2)] + + +def compute() -> dict: + rows = list(csv.DictReader(RUNS.open())) + fig: dict = {"source": "results/runs.csv", "rows": len(rows), "arms": {}} + for arm in ("browser", "api"): + rs = [r for r in rows if r["arm"] == arm] + normal = [r for r in rs if r["mismatch"] == "False"] + dup = [r for r in rs if r["mismatch"] == "True"] + ok = [r for r in normal if r["correct"] == "True"] + fig["arms"][arm] = { + "runs": len(rs), + "correct": sum(r["correct"] == "True" for r in rs), + "normal_runs": len(normal), "normal_saved_12_of_12": len(ok), + "duplicate_runs": len(dup), "duplicate_no_new_chart": sum(r["correct"] == "True" for r in dup), + "wall_s_correct_normal": summary([float(r["wall_s"]) for r in ok]), + "task_s_correct_normal": summary([float(r["task_s"]) for r in ok]), + "setup_s_correct_normal": summary([float(r["setup_s"]) for r in ok]), + # runs.csv task_s runs from the clock start; engine time is after setup. + "engine_s_correct_normal": summary([float(r["task_s"]) - float(r["setup_s"]) for r in ok]), + "wall_s_all_runs": summary([float(r["wall_s"]) for r in rs]), + "failures": [{k: r[k] for k in ("block", "ref", "readback", "outcome", "error")} + for r in rs if r["correct"] != "True"], + } + b, a = fig["arms"]["browser"], fig["arms"]["api"] + bw = [float(r["wall_s"]) for r in rows if r["arm"] == "browser" and r["mismatch"] == "False" and r["correct"] == "True"] + aw = [float(r["wall_s"]) for r in rows if r["arm"] == "api" and r["mismatch"] == "False" and r["correct"] == "True"] + fig["speedup"] = { + "wall_median_ratio": round(b["wall_s_correct_normal"]["median"] / a["wall_s_correct_normal"]["median"], 2), + "wall_median_ratio_ci95": boot_ratio(bw, aw), + "wall_median_diff_s": round(b["wall_s_correct_normal"]["median"] - a["wall_s_correct_normal"]["median"], 3), + "task_median_ratio": round(b["task_s_correct_normal"]["median"] / a["task_s_correct_normal"]["median"], 2), + "engine_median_ratio": round(b["engine_s_correct_normal"]["median"] / a["engine_s_correct_normal"]["median"], 2), + } + return fig + + +def main() -> int: + fig = compute() + text = json.dumps(fig, indent=1) + "\n" + if "--check" in sys.argv: + if OUT.read_text() != text: + print("figures.json is stale; rerun compute_figures.py") + return 1 + return 0 + OUT.write_text(text) + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/openemr_api_tier/formtier.py b/packages/flow/benchmark/openemr_api_tier/formtier.py new file mode 100644 index 000000000..04b446632 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/formtier.py @@ -0,0 +1,500 @@ +"""Spike: compile a recorded browser form save into an API step, and run it. + +Two halves, both model-free: + +* :func:`compile_from_har` reads the HAR captured while the compiled GUI + program replayed one referral, finds the requests that did the work (sign-in, + the duplicate check, the save), and explains every field of the save body by + where its value came from: a recorded parameter, a select option chosen by a + recorded label, a server-issued session value (the CSRF token), or the + form's own default. A field it cannot explain stops the compile; that + workflow stays on the GUI. +* :class:`SessionFormActuator` implements the engine's ``ApiActuator`` + protocol (``actuate(binding, params) -> ApiActuationResult``) for a web + app's own form endpoints: it signs in with a fresh session, fetches the form + page for the session's CSRF token and current defaults, refuses before + sending anything if the form changed since the recording, runs the + compiled duplicate check, then posts the save once. The engine's API tier + then confirms the write against the system of record with the same + ``EffectVerifier`` the GUI path uses. + +This is spike code for one application (OpenEMR 8.0.0.3) on synthetic data. +""" + +from __future__ import annotations + +import json +import re +import urllib.parse as up +from html.parser import HTMLParser +from pathlib import Path +from typing import Any + +# -- HTML form parsing --------------------------------------------------------- + + +class _FormParser(HTMLParser): + """Named controls of each
: defaults and select options.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.forms: list[dict[str, Any]] = [] + self._form: dict[str, Any] | None = None + self._select: dict[str, Any] | None = None + self._option: dict[str, Any] | None = None + self._textarea: dict[str, Any] | None = None + self._nested = 0 # start tags ignored inside an open form + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + a = {k: (v if v is not None else "") for k, v in attrs} + if tag == "form": + if self._form is not None: + # HTML parsing ignores a inside an open form; its + # controls belong to (and are posted with) the outer form. + self._nested += 1 + return + self._form = {"action": a.get("action", ""), "name": a.get("name", ""), "controls": {}} + self.forms.append(self._form) + return + if self._form is None: + return + name = a.get("name") + if tag == "input" and name: + kind = a.get("type", "text").lower() + if kind in ("checkbox", "radio") and "checked" not in a: + value = None + elif kind in ("submit", "button", "image", "reset", "file"): + return + else: + value = a.get("value", "on" if kind in ("checkbox", "radio") else "") + self._form["controls"].setdefault(name, {"kind": kind, "default": value, "options": None}) + elif tag == "select" and name: + self._select = {"kind": "select", "default": None, "options": [], "multiple": "multiple" in a} + self._form["controls"].setdefault(name, self._select) + elif tag == "option" and self._select is not None: + self._option = {"value": a.get("value"), "label": "", "selected": "selected" in a} + self._select["options"].append(self._option) + elif tag == "textarea" and name: + self._textarea = {"kind": "textarea", "default": "", "options": None} + self._form["controls"].setdefault(name, self._textarea) + + def handle_data(self, data: str) -> None: + if self._option is not None: + self._option["label"] += data + elif self._textarea is not None: + self._textarea["default"] += data + + def handle_endtag(self, tag: str) -> None: + if tag == "option" and self._option is not None: + o = self._option + o["label"] = o["label"].strip() + if o["value"] is None: + o["value"] = o["label"] + self._option = None + elif tag == "select" and self._select is not None: + opts = self._select["options"] + chosen = [o for o in opts if o["selected"]] or ([] if self._select["multiple"] else opts[:1]) + self._select["default"] = chosen[0]["value"] if chosen else "" + self._select = None + elif tag == "textarea": + self._textarea = None + elif tag == "form": + if self._nested: + self._nested -= 1 + else: + self._form = None + + +def parse_form(html: str, action_suffix: str) -> dict[str, Any]: + p = _FormParser() + p.feed(html) + for form in p.forms: + if form["action"].split("?")[0].endswith(action_suffix): + return form["controls"] + raise ValueError(f"no posting to {action_suffix}") + + +_CONSTRAINTS = re.compile(r"var constraints = (\{.*?\});") + + +def client_rules(html: str) -> dict[str, Any] | None: + """The page's client-side validation rules (OpenEMR's ``constraints``). + + The browser enforces these before it submits; a direct POST skips them, + so the API step records them, refuses if they change, and enforces them. + """ + m = _CONSTRAINTS.search(html) + return json.loads(m.group(1)) if m else None + + +def option_value_for_label(control: dict[str, Any], label: str) -> str | None: + hits = [o["value"] for o in (control.get("options") or []) if o["label"] == label] + return hits[0] if len(hits) == 1 else None + + +# -- compiler ------------------------------------------------------------------ + +_STATIC = re.compile(r"\.(js|css|png|svg|gif|ico|woff2?|ttf|map)(\?|$)") + + +def _entries(har: dict[str, Any]) -> list[dict[str, Any]]: + return [e for e in har["log"]["entries"] if not _STATIC.search(e["request"]["url"])] + + +def _path(url: str) -> str: + return up.urlsplit(url).path + + +def _body_pairs(entry: dict[str, Any]) -> list[tuple[str, str]]: + return up.parse_qsl(entry["request"].get("postData", {}).get("text") or "", keep_blank_values=True) + + +def compile_from_har(har_path: Path, gui_bundle: Path, out_dir: Path, + run_params: dict[str, str] | None = None) -> dict[str, Any]: + """Compile the save of a recorded GUI workflow into one API step. + + ``run_params`` are the values the captured run used (default: the + bundle's recorded demonstration values). + + Returns the compile report (also written to ``out_dir``). Raises + ``CompileRefused`` when any field cannot be explained. + """ + from openadapt_flow.ir import ApiBinding, Workflow + + har = json.loads(Path(har_path).read_text()) + wf = Workflow.load(gui_bundle) + params = dict(run_params or wf.params) + if sorted(params) != sorted(wf.params): + raise CompileRefused("run params do not match the workflow's parameters") + entries = _entries(har) + + def find(method: str, suffix: str) -> list[dict[str, Any]]: + return [e for e in entries if e["request"]["method"] == method and _path(e["request"]["url"]).endswith(suffix)] + + # The save is the one form POST whose body carries the recorded values. + posts = [e for e in entries if e["request"]["method"] == "POST" + and "x-www-form-urlencoded" in (e["request"].get("postData", {}).get("mimeType") or "")] + scored = sorted(posts, key=lambda e: -sum(v in params.values() for _, v in _body_pairs(e) if v)) + save = scored[0] + save_url = save["request"]["url"] + hits = sum(v in params.values() for _, v in _body_pairs(save) if v) + if hits < len(params) // 2: + raise CompileRefused("no form POST carries the recorded parameter values") + + # The form page the save posts from: the last GET whose HTML has a form + # with that action, before the save. + save_i = har["log"]["entries"].index(save) + form_page = None + for e in har["log"]["entries"][:save_i]: + t = e["response"].get("content", {}).get("text") or "" + if e["request"]["method"] == "GET" and "= 2: + qt = [] + for k, v in q: + qt.append([k, "{" + by_value[v][0] + "}" if v in by_value and len(by_value[v]) == 1 else v]) + precheck = { + "method": "GET", "path": _path(e["request"]["url"]), "query": qt, + "require_text": [p.text for p in gui_dialog.expect] if gui_dialog else [], + "from_gui_step": gui_dialog.id if gui_dialog else None, + } + break + + # Sign-in: the form POST that set the session cookie. Secrets are never + # copied from the capture; the deployment supplies them at run time. + login = None + for e in entries: + if e["request"]["method"] != "POST": + continue + pairs = _body_pairs(e) + if any("pass" in k.lower() for k, _ in pairs) and any( + h["name"].lower() == "set-cookie" for h in e["response"]["headers"]): + lb = [] + for k, v in pairs: + if "pass" in k.lower(): + lb.append([k, "{__secret_password}"]) + elif k.lower() in ("authuser", "username", "user", "login"): + lb.append([k, "{__secret_username}"]) + else: + lb.append([k, v]) + loc = next((h["value"] for h in e["response"]["headers"] if h["name"].lower() == "location"), "") + login = {"method": "POST", "path": _path(e["request"]["url"]), + "query": up.urlsplit(e["request"]["url"]).query, "body": lb, + "expect_status": e["response"]["status"], "expect_location_path": _path(loc)} + break + if login is None: + raise CompileRefused("no sign-in request in the capture") + + resp_text = save["response"].get("content", {}).get("text") or "" + m = re.search(r"set_pid=(\d+)", resp_text) + success = re.escape(resp_text[m.start():m.end()]).replace(re.escape(m.group(1)), r"(\d+)") if m else None + + save_step = next(s for s in wf.steps if s.effects) + binding = ApiBinding( + kind="rest", + on_unavailable="halt", + method="POST", + url_template=_path(save_url), + body_template=body, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + expected_status=[save["response"]["status"]], + timeout_s=30.0, + effects=list(save_step.effects), + ) + api_step = save_step.model_copy(update={ + "id": "api_save", + "intent": f"save the form via {binding.method} {binding.url_template} (folds {len(wf.steps)} GUI steps)", + "api_binding": binding, + "expect": [], + }) + api_wf = wf.model_copy(update={"steps": [api_step], "name": wf.name + " (api tier)", + "interstitials": [], "program": None, "subflows": {}}) + tier = { + "schema": "openadapt.spike.session_form/1", + "login": login, + "form_page": {"path": _path(form_entry["request"]["url"]), + "action": _path(save_url).rsplit("/", 1)[-1], + "controls": sorted(controls), + "client_rules": rules}, + "precheck": precheck, + "success_regex": success, + "fields": fields, + "folded_gui_steps": [s.id for s in wf.steps], + } + out_dir.mkdir(parents=True, exist_ok=True) + api_wf.save(out_dir) + (out_dir / "session_form.json").write_text(json.dumps(tier, indent=1) + "\n") + counts: dict[str, int] = {} + for f in fields: + counts[f["source"]] = counts.get(f["source"], 0) + 1 + report = {"save": {"method": "POST", "path": _path(save_url), "fields": len(fields), "by_source": counts}, + "precheck": precheck, "login": {"path": login["path"], "fields": [k for k, _ in login["body"]]}, + "success_regex": success, "param_fields": {f["name"]: f.get("param") for f in fields if f.get("param")}} + (out_dir / "compile_report.json").write_text(json.dumps(report, indent=1) + "\n") + return report + + +class CompileRefused(Exception): + """The capture cannot be compiled to the API tier; keep the GUI program.""" + + +# -- runtime ------------------------------------------------------------------- + + +class SessionFormActuator: + """Run a compiled session-form save: sign in, re-read the form, check, post. + + Implements the engine's API actuator protocol. Every refusal before the + save POST is ``UNAVAILABLE`` (nothing was sent that writes); anything after + the POST began follows ``ApiActuator``'s no-double-write rule (``HALT``). + """ + + substrate = "rest" + + def __init__(self, base_url: str, tier: dict[str, Any], *, username: str, password: str, + timeout_s: float = 30.0) -> None: + import requests + + self.base_url = base_url.rstrip("/") + self.tier = tier + self.secrets = {"__secret_username": username, "__secret_password": password} + self.session = requests.Session() + self.timeout_s = timeout_s + self.marks: dict[str, float] = {} + self.log: list[dict[str, Any]] = [] + + # Each pre-save stage returns an error string or None. + + def sign_in(self) -> str | None: + import time + + lg = self.tier["login"] + data = [(k, v.format_map(self.secrets) if "{" in v else v) for k, v in lg["body"]] + url = f"{self.base_url}{lg['path']}" + (f"?{lg['query']}" if lg["query"] else "") + self.session.get(self.base_url + "/", timeout=self.timeout_s) # App cookie, as the browser does + r = self.session.post(url, data=data, allow_redirects=False, timeout=self.timeout_s) + loc = up.urlsplit(r.headers.get("location", "")).path + self.log.append({"stage": "sign_in", "status": r.status_code, "location_path": loc}) + self.marks["signed_in"] = time.time() + if r.status_code != lg["expect_status"] or loc != lg["expect_location_path"]: + return f"sign-in did not reach {lg['expect_location_path']} (got {r.status_code} {loc})" + return None + + def read_form(self) -> tuple[dict[str, Any] | None, str | None]: + import time + + fp = self.tier["form_page"] + r = self.session.get(self.base_url + fp["path"], timeout=self.timeout_s) + self.marks["form_read"] = time.time() + self.log.append({"stage": "read_form", "status": r.status_code, "bytes": len(r.content)}) + if r.status_code != 200: + return None, f"form page returned {r.status_code}" + try: + controls = parse_form(r.text, fp["action"]) + except ValueError: + return None, "form page has no save form (session or page changed)" + if client_rules(r.text) != fp.get("client_rules"): + return None, "the form's client-side validation rules changed since the recording" + if sorted(controls) != fp["controls"]: + added = sorted(set(controls) - set(fp["controls"])) + removed = sorted(set(fp["controls"]) - set(controls)) + return None, f"form changed since the recording (added {added[:5]}, removed {removed[:5]})" + return controls, None + + def run_precheck(self, params: dict[str, str]) -> str | None: + import time + + pc = self.tier.get("precheck") + if not pc: + return None + q = [(k, v.format_map(params) if "{" in v else v) for k, v in pc["query"]] + qs = "&".join(up.quote(k) + ("=" + up.quote(v) if v else "") for k, v in q) + r = self.session.get(f"{self.base_url}{pc['path']}?{qs}", timeout=self.timeout_s) + self.marks["prechecked"] = time.time() + missing = [t for t in pc["require_text"] if t not in r.text] + self.log.append({"stage": "precheck", "status": r.status_code, "missing": missing}) + if r.status_code != 200 or missing: + return f"duplicate check did not pass: missing {missing!r} (HTTP {r.status_code})" + return None + + def build_body(self, binding: Any, params: dict[str, str], controls: dict[str, Any], *, + enforce_rules: bool = True) -> tuple[list[tuple[str, str]] | None, str | None]: + body: list[tuple[str, str]] = [] + by_name = {f["name"]: f for f in self.tier["fields"]} + for name, template in binding.body_template.items(): + f = by_name[name] + ctl = controls.get(name) or {} + src = f["source"] + if src == "param": + body.append((name, params[f["param"]])) + elif src == "select_label": + value = option_value_for_label(ctl, params[f["param"]]) + if value is None: + return None, f"{name}: no single option labelled {params[f['param']]!r}" + body.append((name, value)) + elif src == "session": + body.append((name, ctl.get("default") or "")) + if not body[-1][1]: + return None, f"{name}: the fresh form carries no session value" + elif src == "form_default": + if (ctl.get("default") or "") != template: + return None, f"{name}: form default changed since the recording" + body.append((name, template)) + # Enforce the recorded client-side rules the browser would have applied. + sent = dict(body) + rules = (self.tier["form_page"].get("client_rules") or {}) if enforce_rules else {} + for name, rule in rules.items(): + if rule.get("presence") and not sent.get(name): + return None, f"{name} is required by the form and would be empty" + return body, None + + def actuate(self, binding: Any, params: dict[str, str]) -> Any: + import time + + import requests + + from openadapt_flow.runtime.actuators import ( + ActuationStatus, + ApiActuationResult, + ApiHaltKind, + ) + + summary = f"{binding.method} {binding.url_template}" + + def unavailable(reason: str) -> Any: + return ApiActuationResult(status=ActuationStatus.UNAVAILABLE, reason=reason + " -- save not sent", + request_summary=summary) + + try: + err = self.sign_in() + if err: + return unavailable(err) + controls, err = self.read_form() + if err: + return unavailable(err) + body, err = self.build_body(binding, params, controls) + if err: + return unavailable(err) + err = self.run_precheck(params) + if err: + return unavailable(err) + except (KeyError, requests.RequestException) as exc: + # Only sign-in, a page read and the read-only duplicate search were + # sent; none of them writes the record. + return unavailable(f"pre-save stage failed ({type(exc).__name__})") + + self.marks["save_sent"] = time.time() + try: + r = self.session.post(self.base_url + binding.url_template, data=body, + headers={"Referer": self.base_url + self.tier["form_page"]["path"]}, + allow_redirects=False, timeout=binding.timeout_s or self.timeout_s) + except Exception as exc: # noqa: BLE001 - after dispatch began + return ApiActuationResult(status=ActuationStatus.HALT, halt_kind=ApiHaltKind.DELIVERY_UNCERTAIN, + reason=f"save transport failed after dispatch ({type(exc).__name__})", + request_summary=summary) + self.marks["save_done"] = time.time() + sig = self.tier.get("success_regex") + ok = r.status_code in (binding.expected_status or [200]) and (sig is None or re.search(sig, r.text)) + self.log.append({"stage": "save", "status": r.status_code, "success_signature": bool(ok)}) + if ok: + return ApiActuationResult(status=ActuationStatus.ACTUATED, reason=f"{summary} -> {r.status_code}", + http_status=r.status_code, request_summary=summary) + return ApiActuationResult(status=ActuationStatus.HALT, halt_kind=ApiHaltKind.RESPONSE_REJECTED, + reason=f"{summary} -> {r.status_code} without the recorded success response", + http_status=r.status_code, request_summary=summary) diff --git a/packages/flow/benchmark/openemr_api_tier/har_summary.py b/packages/flow/benchmark/openemr_api_tier/har_summary.py new file mode 100644 index 000000000..a957823ef --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/har_summary.py @@ -0,0 +1,55 @@ +"""Write a redacted summary of the capture HAR (safe to commit). + +Keeps method, path, query, status, content type, sizes and timing for every +non-static request, and the full field list of the duplicate check and the +save. Drops cookies and headers, and replaces the sign-in username and +password and every CSRF/session token with placeholders. All patient values +are synthetic. + + python -m benchmark.openemr_api_tier.har_summary +""" + +from __future__ import annotations + +import json +import re +import sys +import urllib.parse as up +from pathlib import Path + +from benchmark.openemr_api_tier.formtier import _STATIC + +_TOKEN_KEYS = re.compile(r"csrf|token|pass|authuser|session", re.I) + + +def _clean(pairs: list[tuple[str, str]]) -> list[list[str]]: + return [[k, "" if _TOKEN_KEYS.search(k) else v] for k, v in pairs] + + +def main(har_path: str, out_path: str) -> None: + har = json.loads(Path(har_path).read_text()) + t0 = None + rows = [] + for e in har["log"]["entries"]: + r, s = e["request"], e["response"] + if _STATIC.search(r["url"]): + continue + u = up.urlsplit(r["url"]) + started = e["startedDateTime"] + t0 = t0 or started + pd = r.get("postData") or {} + body = up.parse_qsl(pd.get("text") or "", keep_blank_values=True) if "urlencoded" in (pd.get("mimeType") or "") else [] + rows.append({ + "started": started[11:23], "ms": round(e.get("time") or 0, 1), "method": r["method"], "path": u.path, + "query": _clean(up.parse_qsl(u.query, keep_blank_values=True)), "status": s["status"], + "request_type": (pd.get("mimeType") or "").split(";")[0], "request_bytes": len(pd.get("text") or ""), + "response_type": (s.get("content", {}).get("mimeType") or "").split(";")[0], + "response_bytes": s.get("content", {}).get("size"), + "body": _clean(body) if body else None, + }) + Path(out_path).write_text(json.dumps({"requests": rows, "static_requests_dropped": + len(har["log"]["entries"]) - len(rows)}, indent=1) + "\n") + + +if __name__ == "__main__": + main(*sys.argv[1:3]) diff --git a/packages/flow/benchmark/openemr_api_tier/probes.py b/packages/flow/benchmark/openemr_api_tier/probes.py new file mode 100644 index 000000000..c7ad53957 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/probes.py @@ -0,0 +1,172 @@ +"""Step 4b: the cases where a recorded request is not enough on its own. + +Each probe resets nothing by itself; ``main`` resets and seeds the fixture +once before and once after. Every probe patient is made up +(``probe-*@example.invalid``). For each probe we record what the server did +(HTTP status, whether a chart was written) and what the API step did. + + python -m benchmark.openemr_api_tier.probes --out +""" + +from __future__ import annotations + +import argparse +import copy +import json +import time +from pathlib import Path +from typing import Any + +from benchmark.openemr_api_tier.bench import API_BUNDLE +from benchmark.openemr_api_tier.common import ( + REFERRALS, + reset_and_seed, + spec_for, + sql_patients, +) +from benchmark.openemr_api_tier.formtier import SessionFormActuator, client_rules +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + + +def probe_params(tag: str, **over: str) -> dict[str, str]: + p = spec_for(REFERRALS[0]).params() + # A distinct made-up name per probe, so OpenEMR's fuzzy duplicate search + # does not match one probe's chart against another's. + p.update(fname=tag.capitalize(), lname=tag.capitalize() + "ova", email=f"probe-{tag}@example.invalid", + DOB="1980-01-0" + str(len(tag) % 9 + 1)) + p.update(over) + return p + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--out", required=True) + a = ap.parse_args() + from openadapt_flow.ir import Workflow + + f = SpikeFixture() + vals = f._runtime_values() + tier = json.loads((API_BUNDLE / "session_form.json").read_text()) + binding = Workflow.load(API_BUNDLE).steps[0].api_binding + creds = dict(username=vals["OPENEMR_ACTOR_USER"], password=vals["OPENEMR_ACTOR_PASSWORD"]) + results: dict[str, Any] = {"fixture_before": reset_and_seed(f), "probes": []} + + def act(t: dict[str, Any] = tier) -> SessionFormActuator: + return SessionFormActuator(f.ui_base_url, t, **creds) + + def count() -> int: + return len(sql_patients(f)) + + def post(a_: SessionFormActuator, body: list[tuple[str, str]]) -> Any: + return a_.session.post(f.ui_base_url + binding.url_template, data=body, allow_redirects=False, timeout=30) + + def record(name: str, question: str, n0: int, **obs: Any) -> None: + time.sleep(0.5) + row = {"probe": name, "question": question, "new_charts": count() - n0, **obs} + results["probes"].append(row) + print(json.dumps(row), flush=True) + + def fresh(t: dict[str, Any] = tier) -> tuple[SessionFormActuator, dict[str, Any]]: + """A newly signed-in session and its freshly read form.""" + a_ = act(t) + a_.sign_in() + controls, err = a_.read_form() + assert err is None, err + return a_, controls + + # 1. CSRF token from another session. + a1, c1 = fresh() + a2, c2 = fresh() + tok1 = c1["csrf_token_form"]["default"] + tok2 = c2["csrf_token_form"]["default"] + body, _ = a2.build_body(binding, probe_params("stalecsrf"), c2) + body = [(k, tok1 if k == "csrf_token_form" else v) for k, v in body] + n0 = count() + r = post(a2, body) + record("stale_csrf", "Does a save with another session's CSRF token write?", n0, http=r.status_code, + tokens_differ_between_sessions=tok1 != tok2, response_has_success=("set_pid=" in r.text), + response_head=r.text[:120].replace("\n", " ")) + + # 2. No CSRF token at all. + body, _ = a2.build_body(binding, probe_params("nocsrf"), c2) + body = [(k, v) for k, v in body if k != "csrf_token_form"] + n0 = count() + r = post(a2, body) + record("missing_csrf", "Does a save without a CSRF token write?", n0, http=r.status_code, + response_has_success=("set_pid=" in r.text)) + + # 3. Token lifetime inside one session: same token after a save? + a3, c3 = fresh() + body, _ = a3.build_body(binding, probe_params("rotation"), c3) + n0 = count() + r = post(a3, body) + c3b, _ = a3.read_form() + record("token_rotation", "Does the CSRF token change after a save in the same session?", n0, http=r.status_code, + token_same_after_save=c3["csrf_token_form"]["default"] == c3b["csrf_token_form"]["default"], + response_has_success=("set_pid=" in r.text)) + + # 4. Skip the duplicate check and post the r6 possible duplicate directly. + a4, c4 = fresh() + body, _ = a4.build_body(binding, spec_for(REFERRALS[5]).params(), c4) + n0 = count() + r = post(a4, body) + record("skip_duplicate_check", "Does the server itself stop a possible duplicate (r6) when the " + "duplicate-check request is skipped?", n0, http=r.status_code, response_has_success=("set_pid=" in r.text)) + + # 5. The same save sent twice (what a naive retry would do). + a5, c5 = fresh() + body, _ = a5.build_body(binding, probe_params("doublesubmit"), c5) + n0 = count() + r1 = post(a5, body) + r2 = post(a5, body) + record("double_submit", "Does sending the same save twice write twice?", n0, http=[r1.status_code, r2.status_code]) + + # 6. A session that never signed in (or whose session expired). + body, _ = a5.build_body(binding, probe_params("nosession"), c5) + n0 = count() + r = post(act(), body) + record("no_session", "Does a save without a signed-in session write, and what comes back?", n0, http=r.status_code, + location=r.headers.get("location"), response_head=r.text[:160].replace("\n", " ")) + + # 7. Through the actuator: the form gained a control since the recording. + t7 = copy.deepcopy(tier) + t7["form_page"]["controls"] = [c for c in t7["form_page"]["controls"] if c != "form_mname"] + n0 = count() + out = act(t7).actuate(binding, probe_params("formdrift")) + record("form_controls_changed", "Does the API step refuse before sending when the form's fields differ " + "from the recording?", n0, status=out.status.value, reason=out.reason) + + # 8. An admin makes Middle Name required in the layout. + f._db("UPDATE openemr.layout_options SET uor=2 WHERE form_id='DEM' AND field_id='mname';") + n0 = count() + out = act().actuate(binding, probe_params("rulechange")) + record("validation_rule_changed", "Middle Name made required in the layout: does the API step refuse " + "instead of saving a record the browser form would have blocked?", n0, status=out.status.value, + reason=out.reason) + # Same change: what would the recorded rules say, and does the server enforce the new one? + t8 = copy.deepcopy(tier) + a8 = act(t8) + a8.sign_in() + t8["form_page"]["client_rules"] = client_rules(a8.session.get(f.ui_base_url + tier["form_page"]["path"]).text) + c8, _ = a8.read_form() + _, rule_err = a8.build_body(binding, probe_params("rulechange"), c8) + body, _ = a8.build_body(binding, probe_params("rulechange"), c8, enforce_rules=False) + n0 = count() + r = post(a8, body) + record("server_enforces_rule", "With Middle Name required in the layout, does the server reject a POST " + "that leaves it empty?", n0, http=r.status_code, response_has_success=("set_pid=" in r.text), + api_step_refusal=rule_err) + f._db("UPDATE openemr.layout_options SET uor=1 WHERE form_id='DEM' AND field_id='mname';") + + # 9. A select label the form does not offer. + n0 = count() + out = act().actuate(binding, probe_params("badlabel", state_label="Atlantis")) + record("unknown_select_label", "state_label not among the form's options: refuse before sending?", n0, + status=out.status.value, reason=out.reason) + + results["fixture_after"] = reset_and_seed(f) + Path(a.out).write_text(json.dumps(results, indent=1) + "\n") + + +if __name__ == "__main__": + main() diff --git a/packages/flow/benchmark/openemr_api_tier/results/capture_r1.json b/packages/flow/benchmark/openemr_api_tier/results/capture_r1.json new file mode 100644 index 000000000..1b86a0836 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/capture_r1.json @@ -0,0 +1,267 @@ +{ + "ref": "r1-nadia-brennan", + "bundle": "/Users/abrichr/oa/scratch/replay-speed/dom-identity/out/bundle", + "success": true, + "outcome": "COMPLETED_UNVERIFIED", + "task_s": 21.894, + "steps": 41, + "check": { + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ] + }, + "step_marks": [ + [ + "step_000", + "click 'Unassigned'" + ], + [ + "step_001", + "type " + ], + [ + "step_002", + "press Enter" + ], + [ + "step_003", + "click 'FirstName'" + ], + [ + "step_004", + "press Control+a" + ], + [ + "step_005", + "type <fname>" + ], + [ + "step_006", + "click 'Last Name'" + ], + [ + "step_007", + "press Control+a" + ], + [ + "step_008", + "type <lname>" + ], + [ + "step_009", + "click recorded visual target" + ], + [ + "step_010", + "press Control+a" + ], + [ + "step_011", + "type <DOB>" + ], + [ + "step_012", + "click 'Unassigned'" + ], + [ + "step_013", + "type <sex>" + ], + [ + "step_014", + "press Enter" + ], + [ + "step_015", + "click recorded visual target" + ], + [ + "step_016", + "scroll by (0, 600)" + ], + [ + "step_017", + "click recorded visual target" + ], + [ + "step_018", + "press Control+a" + ], + [ + "step_019", + "type <street>" + ], + [ + "step_020", + "click recorded visual target" + ], + [ + "step_021", + "press Control+a" + ], + [ + "step_022", + "type <city>" + ], + [ + "step_023", + "click 'Jnassigned'" + ], + [ + "step_024", + "type <state_label>" + ], + [ + "step_025", + "press Enter" + ], + [ + "step_026", + "click recorded visual target" + ], + [ + "step_027", + "press Control+a" + ], + [ + "step_028", + "type <postal_code>" + ], + [ + "step_029", + "click recorded visual target" + ], + [ + "step_030", + "press Control+a" + ], + [ + "step_031", + "type <phone_home>" + ], + [ + "step_032", + "click recorded visual target" + ], + [ + "step_033", + "press Control+a" + ], + [ + "step_034", + "type <email>" + ], + [ + "step_035", + "click 'Jnassigned'" + ], + [ + "step_036", + "type <country_code>" + ], + [ + "step_037", + "press Enter" + ], + [ + "step_038", + "scroll by (0, 600)" + ], + [ + "step_039", + "click 'Create New Patient'" + ], + [ + "step_040", + "click 'Confirm Create New Patient'" + ] + ], + "params": { + "title": "Ms.", + "fname": "Nadia", + "lname": "Brennan", + "DOB": "1971-04-19", + "sex": "Female", + "street": "27 WILLOW BEND ROAD", + "city": "EXAMPLETON", + "state_label": "Massachusetts", + "postal_code": "02139", + "phone_home": "555-0121", + "email": "nadia.brennan@example.invalid", + "country_code": "USA" + } +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/figures.json b/packages/flow/benchmark/openemr_api_tier/results/figures.json new file mode 100644 index 000000000..f07a0fc80 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/figures.json @@ -0,0 +1,132 @@ +{ + "source": "results/runs.csv", + "rows": 40, + "arms": { + "browser": { + "runs": 20, + "correct": 19, + "normal_runs": 18, + "normal_saved_12_of_12": 17, + "duplicate_runs": 2, + "duplicate_no_new_chart": 2, + "wall_s_correct_normal": { + "n": 17, + "median": 23.899, + "p10": 22.997, + "p90": 27.111, + "min": 22.736, + "max": 28.391, + "mean": 24.671 + }, + "task_s_correct_normal": { + "n": 17, + "median": 22.539, + "p10": 21.666, + "p90": 25.6, + "min": 21.395, + "max": 27.013, + "mean": 23.284 + }, + "setup_s_correct_normal": { + "n": 17, + "median": 1.479, + "p10": 1.412, + "p90": 1.889, + "min": 1.406, + "max": 1.999, + "mean": 1.597 + }, + "engine_s_correct_normal": { + "n": 17, + "median": 21.019, + "p10": 20.043, + "p90": 23.933, + "min": 19.957, + "max": 25.388, + "mean": 21.687 + }, + "wall_s_all_runs": { + "n": 20, + "median": 24.091, + "p10": 22.823, + "p90": 28.559, + "min": 21.115, + "max": 34.35, + "mean": 25.247 + }, + "failures": [ + { + "block": "3", + "ref": "r2-marcus-feld", + "readback": "0/12", + "outcome": "HALTED", + "error": "" + } + ] + }, + "api": { + "runs": 20, + "correct": 20, + "normal_runs": 18, + "normal_saved_12_of_12": 18, + "duplicate_runs": 2, + "duplicate_no_new_chart": 2, + "wall_s_correct_normal": { + "n": 18, + "median": 2.136, + "p10": 2.035, + "p90": 2.368, + "min": 1.996, + "max": 2.54, + "mean": 2.17 + }, + "task_s_correct_normal": { + "n": 18, + "median": 0.846, + "p10": 0.731, + "p90": 1.052, + "min": 0.708, + "max": 1.171, + "mean": 0.866 + }, + "setup_s_correct_normal": { + "n": 18, + "median": 0.485, + "p10": 0.416, + "p90": 0.58, + "min": 0.412, + "max": 0.698, + "mean": 0.499 + }, + "engine_s_correct_normal": { + "n": 18, + "median": 0.338, + "p10": 0.296, + "p90": 0.47, + "min": 0.292, + "max": 0.549, + "mean": 0.368 + }, + "wall_s_all_runs": { + "n": 20, + "median": 2.123, + "p10": 1.976, + "p90": 2.343, + "min": 1.763, + "max": 2.54, + "mean": 2.131 + }, + "failures": [] + } + }, + "speedup": { + "wall_median_ratio": 11.19, + "wall_median_ratio_ci95": [ + 10.78, + 12.14 + ], + "wall_median_diff_s": 21.763, + "task_median_ratio": 26.64, + "engine_median_ratio": 62.19 + } +} diff --git a/packages/flow/benchmark/openemr_api_tier/results/har_summary.json b/packages/flow/benchmark/openemr_api_tier/results/har_summary.json new file mode 100644 index 000000000..4b7e1fded --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/har_summary.json @@ -0,0 +1,955 @@ +{ + "requests": [ + { + "started": "16:03:47.203", + "ms": 13.8, + "method": "GET", + "path": "/", + "query": [], + "status": 302, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": -1, + "body": null + }, + { + "started": "16:03:47.215", + "ms": 150.6, + "method": "GET", + "path": "/interface/login/login.php", + "query": [ + [ + "site", + "default" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 7909, + "body": null + }, + { + "started": "16:03:47.510", + "ms": 1.6, + "method": "GET", + "path": "/", + "query": [], + "status": 302, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": -1, + "body": null + }, + { + "started": "16:03:47.511", + "ms": 40.1, + "method": "GET", + "path": "/interface/login/login.php", + "query": [ + [ + "site", + "default" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 7909, + "body": null + }, + { + "started": "16:03:47.724", + "ms": 276.7, + "method": "POST", + "path": "/interface/main/main_screen.php", + "query": [ + [ + "auth", + "login" + ], + [ + "site", + "default" + ] + ], + "status": 302, + "request_type": "application/x-www-form-urlencoded", + "request_bytes": 143, + "response_type": "text/html", + "response_bytes": -1, + "body": [ + [ + "new_login_session_management", + "<redacted>" + ], + [ + "languageChoice", + "1" + ], + [ + "authUser", + "<redacted>" + ], + [ + "clearPass", + "<redacted>" + ], + [ + "languageChoice", + "1" + ] + ] + }, + { + "started": "16:03:48.001", + "ms": 429.6, + "method": "GET", + "path": "/interface/main/tabs/main.php", + "query": [ + [ + "token_main", + "<redacted>" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 66566, + "body": null + }, + { + "started": "16:03:48.136", + "ms": 320.8, + "method": "GET", + "path": "/library/ajax/i18n_generator.php", + "query": [ + [ + "lang_id", + "1" + ], + [ + "csrf_token_form", + "<redacted>" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 2, + "body": null + }, + { + "started": "16:03:48.454", + "ms": 26.9, + "method": "GET", + "path": "/interface/main/main_info.php", + "query": [], + "status": 302, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": -1, + "body": null + }, + { + "started": "16:03:48.455", + "ms": 216.4, + "method": "GET", + "path": "/interface/main/messages/messages.php", + "query": [ + [ + "form_active", + "1" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 50368, + "body": null + }, + { + "started": "16:03:48.473", + "ms": 25.4, + "method": "GET", + "path": "/interface/product_registration/product_registration_controller.php", + "query": [], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "application/json", + "response_bytes": 25, + "body": null + }, + { + "started": "16:03:48.476", + "ms": 21.9, + "method": "POST", + "path": "/library/ajax/dated_reminders_counter.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 674, + "response_type": "text/html", + "response_bytes": 19, + "body": null + }, + { + "started": "16:03:48.480", + "ms": 277.0, + "method": "GET", + "path": "/interface/main/calendar/index.php", + "query": [ + [ + "module", + "PostCalendar" + ], + [ + "viewtype", + "day" + ], + [ + "func", + "view" + ], + [ + "pc_username", + "openadapt_actor" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 24057, + "body": null + }, + { + "started": "16:03:48.668", + "ms": 65.9, + "method": "GET", + "path": "/Documentation/help_files/message_center_help.php", + "query": [], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 21830, + "body": null + }, + { + "started": "16:03:48.679", + "ms": 27.6, + "method": "POST", + "path": "/interface/main/dated_reminders/dated_reminders.php", + "query": [], + "status": 200, + "request_type": "application/x-www-form-urlencoded", + "request_bytes": 83, + "response_type": "text/html", + "response_bytes": 78, + "body": [ + [ + "drR", + "0" + ], + [ + "skip_timeout_reset", + "1" + ], + [ + "csrf_token_form", + "<redacted>" + ] + ] + }, + { + "started": "16:03:48.859", + "ms": 186.2, + "method": "GET", + "path": "/interface/new/new_comprehensive.php", + "query": [], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 341515, + "body": null + }, + { + "started": "16:03:58.477", + "ms": 66.8, + "method": "POST", + "path": "/library/ajax/execute_background_services.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 384, + "response_type": "text/html", + "response_bytes": -1, + "body": null + }, + { + "started": "16:04:08.637", + "ms": 44.5, + "method": "GET", + "path": "/interface/new/new_search_popup.php", + "query": [ + [ + "mf_fname", + "Nadia" + ], + [ + "mf_lname", + "Brennan" + ], + [ + "mf_DOB", + "1971-04-19" + ], + [ + "close", + "" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 4703, + "body": null + }, + { + "started": "16:04:10.908", + "ms": 42.8, + "method": "POST", + "path": "/interface/new/new_comprehensive_save.php", + "query": [], + "status": 200, + "request_type": "application/x-www-form-urlencoded", + "request_bytes": 2173, + "response_type": "text/html", + "response_bytes": 137, + "body": [ + [ + "csrf_token_form", + "<redacted>" + ], + [ + "form_title", + "Ms." + ], + [ + "form_fname", + "Nadia" + ], + [ + "form_mname", + "" + ], + [ + "form_lname", + "Brennan" + ], + [ + "form_suffix", + "" + ], + [ + "form_preferred_name", + "" + ], + [ + "form_birth_fname", + "" + ], + [ + "form_birth_mname", + "" + ], + [ + "form_birth_lname", + "" + ], + [ + "form_DOB", + "1971-04-19" + ], + [ + "form_sex", + "Female" + ], + [ + "form_gender_identity", + "" + ], + [ + "form_text_gender_identity", + "" + ], + [ + "form_sex_identified", + "" + ], + [ + "form_sexual_orientation", + "" + ], + [ + "form_text_sexual_orientation", + "" + ], + [ + "form_pronoun", + "" + ], + [ + "form_pubpid", + "" + ], + [ + "form_ss", + "" + ], + [ + "form_drivers_license", + "" + ], + [ + "form_status", + "" + ], + [ + "form_genericname1", + "" + ], + [ + "form_genericval1", + "" + ], + [ + "form_genericname2", + "" + ], + [ + "form_genericval2", + "" + ], + [ + "form_billing_note", + "" + ], + [ + "form_street", + "27 WILLOW BEND ROAD" + ], + [ + "form_street_line_2", + "" + ], + [ + "form_city", + "EXAMPLETON" + ], + [ + "form_state", + "MA" + ], + [ + "form_postal_code", + "02139" + ], + [ + "form_country_code", + "USA" + ], + [ + "form_county", + "" + ], + [ + "form_mothersname", + "" + ], + [ + "form_contact_relationship", + "" + ], + [ + "form_phone_contact", + "" + ], + [ + "form_phone_home", + "555-0121" + ], + [ + "form_phone_biz", + "" + ], + [ + "form_phone_cell", + "" + ], + [ + "form_email", + "nadia.brennan@example.invalid" + ], + [ + "form_email_direct", + "" + ], + [ + "form_providerID", + "" + ], + [ + "form_provider_since_date", + "" + ], + [ + "form_ref_providerID", + "" + ], + [ + "form_pharmacy_id", + "0" + ], + [ + "form_hipaa_notice", + "" + ], + [ + "form_hipaa_voice", + "" + ], + [ + "form_hipaa_message", + "" + ], + [ + "form_hipaa_mail", + "" + ], + [ + "form_hipaa_allowsms", + "" + ], + [ + "form_hipaa_allowemail", + "" + ], + [ + "form_allow_imm_reg_use", + "" + ], + [ + "form_allow_imm_info_share", + "" + ], + [ + "form_allow_health_info_ex", + "" + ], + [ + "form_allow_patient_portal", + "" + ], + [ + "form_cmsportal_login", + "" + ], + [ + "form_imm_reg_status", + "" + ], + [ + "form_imm_reg_stat_effdate", + "" + ], + [ + "form_publicity_code", + "" + ], + [ + "form_publ_code_eff_date", + "" + ], + [ + "form_protect_indicator", + "" + ], + [ + "form_prot_indi_effdate", + "" + ], + [ + "form_care_team_status", + "active" + ], + [ + "form_patient_groups[]", + "" + ], + [ + "form_occupation", + "" + ], + [ + "form_industry", + "" + ], + [ + "form_em_name", + "" + ], + [ + "form_em_street", + "" + ], + [ + "form_em_street_line_2", + "" + ], + [ + "form_em_city", + "" + ], + [ + "form_em_state", + "" + ], + [ + "form_em_postal_code", + "" + ], + [ + "form_em_country", + "" + ], + [ + "form_em_start_date", + "" + ], + [ + "form_em_end_date", + "" + ], + [ + "form_language", + "" + ], + [ + "form_ethnicity[]", + "" + ], + [ + "form_race[]", + "" + ], + [ + "form_nationality_country", + "" + ], + [ + "form_financial_review", + "" + ], + [ + "form_family_size", + "" + ], + [ + "form_monthly_income", + "" + ], + [ + "form_homeless", + "" + ], + [ + "form_interpreter_needed", + "" + ], + [ + "form_interpretter", + "" + ], + [ + "form_migrantseasonal", + "" + ], + [ + "form_referral_source", + "" + ], + [ + "form_vfc", + "" + ], + [ + "form_religion", + "" + ], + [ + "form_tribal_affiliations", + "" + ], + [ + "form_deceased_date", + "" + ], + [ + "form_deceased_reason", + "" + ], + [ + "form_guardiansname", + "" + ], + [ + "form_guardianrelationship", + "" + ], + [ + "form_guardiansex", + "" + ], + [ + "form_guardianaddress", + "" + ], + [ + "form_guardiancity", + "" + ], + [ + "form_guardianstate", + "" + ], + [ + "form_guardianpostalcode", + "" + ], + [ + "form_guardiancountry", + "" + ], + [ + "form_guardianphone", + "" + ], + [ + "form_guardianworkphone", + "" + ], + [ + "form_guardianemail", + "" + ], + [ + "form_person_gender", + "" + ] + ] + }, + { + "started": "16:04:10.954", + "ms": 466.7, + "method": "GET", + "path": "/interface/patient_file/summary/demographics.php", + "query": [ + [ + "set_pid", + "2" + ], + [ + "is_new", + "1" + ] + ], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 148505, + "body": null + }, + { + "started": "16:04:11.422", + "ms": 31.8, + "method": "GET", + "path": "/Documentation/help_files/medical_dashboard_help.php", + "query": [], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 17495, + "body": null + }, + { + "started": "16:04:11.423", + "ms": 52.3, + "method": "POST", + "path": "/interface/patient_file/summary/stats.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 291, + "response_type": "text/html", + "response_bytes": 2244, + "body": null + }, + { + "started": "16:04:11.424", + "ms": 45.9, + "method": "POST", + "path": "/interface/patient_file/summary/pnotes_fragment.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 186, + "response_type": "text/html", + "response_bytes": 397, + "body": null + }, + { + "started": "16:04:11.424", + "ms": 36.6, + "method": "POST", + "path": "/interface/patient_file/summary/disc_fragment.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 186, + "response_type": "text/html", + "response_bytes": 548, + "body": null + }, + { + "started": "16:04:11.424", + "ms": 23.1, + "method": "POST", + "path": "/interface/patient_file/summary/labdata_fragment.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 186, + "response_type": "text/html", + "response_bytes": 185, + "body": null + }, + { + "started": "16:04:11.424", + "ms": 28.1, + "method": "POST", + "path": "/interface/patient_file/summary/vitals_fragment.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 186, + "response_type": "text/html", + "response_bytes": 109, + "body": null + }, + { + "started": "16:04:11.425", + "ms": 383.6, + "method": "POST", + "path": "/interface/patient_file/summary/clinical_reminders_fragment.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 291, + "response_type": "text/html", + "response_bytes": 4312, + "body": null + }, + { + "started": "16:04:11.425", + "ms": 30.7, + "method": "POST", + "path": "/interface/patient_file/summary/patient_reminders_fragment.php", + "query": [], + "status": 200, + "request_type": "multipart/form-data", + "request_bytes": 186, + "response_type": "text/html", + "response_bytes": 28, + "body": null + }, + { + "started": "16:04:11.505", + "ms": 47.2, + "method": "GET", + "path": "/interface/patient_file/history/encounters.php", + "query": [], + "status": 200, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 8800, + "body": null + }, + { + "started": "16:04:11.490", + "ms": 56.2, + "method": "GET", + "path": "/controller.php", + "query": [ + [ + "document", + "" + ], + [ + "retrieve", + "" + ], + [ + "patient_id", + "2" + ], + [ + "document_id", + "-1" + ], + [ + "as_file", + "false" + ], + [ + "original_file", + "true" + ], + [ + "disable_exit", + "false" + ], + [ + "show_original", + "true" + ], + [ + "context", + "patient_picture" + ] + ], + "status": 403, + "request_type": "", + "request_bytes": 0, + "response_type": "text/html", + "response_bytes": 37, + "body": null + } + ], + "static_requests_dropped": 135 +} diff --git a/packages/flow/benchmark/openemr_api_tier/results/probes.json b/packages/flow/benchmark/openemr_api_tier/results/probes.json new file mode 100644 index 000000000..b5961e43d --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/probes.json @@ -0,0 +1,91 @@ +{ + "fixture_before": { + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.43 + }, + "probes": [ + { + "probe": "stale_csrf", + "question": "Does a save with another session's CSRF token write?", + "new_charts": 0, + "http": 200, + "tokens_differ_between_sessions": true, + "response_has_success": false, + "response_head": "Authentication Error" + }, + { + "probe": "missing_csrf", + "question": "Does a save without a CSRF token write?", + "new_charts": 0, + "http": 200, + "response_has_success": false + }, + { + "probe": "token_rotation", + "question": "Does the CSRF token change after a save in the same session?", + "new_charts": 1, + "http": 200, + "token_same_after_save": true, + "response_has_success": true + }, + { + "probe": "skip_duplicate_check", + "question": "Does the server itself stop a possible duplicate (r6) when the duplicate-check request is skipped?", + "new_charts": 1, + "http": 200, + "response_has_success": true + }, + { + "probe": "double_submit", + "question": "Does sending the same save twice write twice?", + "new_charts": 2, + "http": [ + 200, + 200 + ] + }, + { + "probe": "no_session", + "question": "Does a save without a signed-in session write, and what comes back?", + "new_charts": 0, + "http": 200, + "location": null, + "response_head": "Site ID is missing from session data!" + }, + { + "probe": "form_controls_changed", + "question": "Does the API step refuse before sending when the form's fields differ from the recording?", + "new_charts": 0, + "status": "unavailable", + "reason": "form changed since the recording (added ['form_mname'], removed []) -- save not sent" + }, + { + "probe": "validation_rule_changed", + "question": "Middle Name made required in the layout: does the API step refuse instead of saving a record the browser form would have blocked?", + "new_charts": 0, + "status": "unavailable", + "reason": "the form's client-side validation rules changed since the recording -- save not sent" + }, + { + "probe": "server_enforces_rule", + "question": "With Middle Name required in the layout, does the server reject a POST that leaves it empty?", + "new_charts": 1, + "http": 200, + "response_has_success": true, + "api_step_refusal": "form_mname is required by the form and would be empty" + }, + { + "probe": "unknown_select_label", + "question": "state_label not among the form's options: refuse before sending?", + "new_charts": 0, + "status": "unavailable", + "reason": "form_state: no single option labelled 'Atlantis' -- save not sent" + } + ], + "fixture_after": { + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.25 + } +} diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs.csv b/packages/flow/benchmark/openemr_api_tier/results/runs.csv new file mode 100644 index 000000000..39a612099 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs.csv @@ -0,0 +1,41 @@ +arm,block,pos,ref,mismatch,correct,new_charts,readback,engine_success,outcome,error,wall_s,setup_s,task_s,readback_s,load1,start_local,engine_sha +browser,1,1,r1-nadia-brennan,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,27.119,1.833,25.487,1.633,8.34,12:12:43,8d0758b9 +browser,1,2,r2-marcus-feld,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,24.761,1.559,23.281,1.481,12.9,12:13:11,8d0758b9 +browser,1,3,r3-ines-carvalho,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,22.736,1.438,21.395,1.341,10.74,12:13:36,8d0758b9 +browser,1,4,r4-theo-lindqvist,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.899,1.413,22.432,1.467,9.49,12:13:59,8d0758b9 +api,1,1,r1-nadia-brennan,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.54,0.698,1.171,1.369,9.27,12:14:30,8d0758b9 +api,1,2,r2-marcus-feld,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.453,0.505,1.054,1.399,10.29,12:14:33,8d0758b9 +api,1,3,r3-ines-carvalho,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.085,0.496,0.806,1.278,10.29,12:14:36,8d0758b9 +api,1,4,r4-theo-lindqvist,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.231,0.549,0.946,1.286,10.18,12:14:38,8d0758b9 +api,2,1,r5-grace-adeyemi,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.232,0.579,0.913,1.319,10.73,12:14:45,8d0758b9 +api,2,2,r6-rosa-delgado,True,True,0,0/12,False,HALTED,,1.763,0.44,0.477,1.286,10.73,12:14:48,8d0758b9 +api,2,3,r7-arlo-whitcombe,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.159,0.448,0.864,1.295,10.75,12:14:50,8d0758b9 +api,2,4,r8-priya-raman,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.127,0.445,0.825,1.303,10.75,12:14:53,8d0758b9 +browser,2,1,r5-grace-adeyemi,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,26.879,1.932,25.384,1.495,12.31,12:15:01,8d0758b9 +browser,2,2,r6-rosa-delgado,True,True,0,0/12,False,HALTED,,34.35,1.581,32.767,1.584,21.56,12:15:29,8d0758b9 +browser,2,3,r7-arlo-whitcombe,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,28.391,1.625,27.013,1.378,19.92,12:16:04,8d0758b9 +browser,2,4,r8-priya-raman,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,26.131,1.466,24.753,1.377,16.53,12:16:32,8d0758b9 +browser,3,1,r1-nadia-brennan,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,24.456,1.86,23.106,1.349,12.82,12:17:04,8d0758b9 +browser,3,2,r2-marcus-feld,False,False,0,0/12,False,HALTED,,21.115,1.445,19.788,1.326,11.47,12:17:29,8d0758b9 +browser,3,3,r3-ines-carvalho,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.617,1.406,22.287,1.33,13.43,12:17:50,8d0758b9 +browser,3,4,r4-theo-lindqvist,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,22.833,1.413,21.499,1.334,13.25,12:18:15,8d0758b9 +api,3,1,r1-nadia-brennan,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.16,0.557,0.868,1.293,12.67,12:18:42,8d0758b9 +api,3,2,r2-marcus-feld,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.12,0.483,0.837,1.283,11.98,12:18:45,8d0758b9 +api,3,3,r3-ines-carvalho,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,1.996,0.412,0.709,1.287,11.98,12:18:47,8d0758b9 +api,3,4,r4-theo-lindqvist,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.054,0.426,0.767,1.286,11.5,12:18:50,8d0758b9 +api,4,1,r5-grace-adeyemi,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.145,0.56,0.855,1.29,11.22,12:18:56,8d0758b9 +api,4,2,r6-rosa-delgado,True,True,0,0/12,False,HALTED,,1.797,0.445,0.485,1.312,10.4,12:18:59,8d0758b9 +api,4,3,r7-arlo-whitcombe,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.036,0.415,0.74,1.297,10.4,12:19:01,8d0758b9 +api,4,4,r8-priya-raman,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.103,0.487,0.815,1.288,9.81,12:19:04,8d0758b9 +browser,4,1,r5-grace-adeyemi,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.164,1.854,21.832,1.333,9.83,12:19:10,8d0758b9 +browser,4,2,r6-rosa-delgado,True,True,0,0/12,False,HALTED,,30.068,1.423,28.345,1.722,26.43,12:19:34,8d0758b9 +browser,4,3,r7-arlo-whitcombe,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.447,1.479,22.105,1.342,41.66,12:20:05,8d0758b9 +browser,4,4,r8-priya-raman,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.613,1.587,22.276,1.337,41.76,12:20:28,8d0758b9 +browser,5,1,r1-nadia-brennan,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.871,1.999,22.539,1.332,34.89,12:20:57,8d0758b9 +browser,5,2,r2-marcus-feld,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,23.106,1.456,21.777,1.329,28.3,12:21:21,8d0758b9 +browser,5,3,r3-ines-carvalho,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,24.284,1.41,22.887,1.397,23.55,12:21:45,8d0758b9 +browser,5,4,r4-theo-lindqvist,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,27.106,1.419,25.77,1.336,20.79,12:22:09,8d0758b9 +api,5,1,r1-nadia-brennan,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.331,0.583,1.051,1.28,20.0,12:22:42,8d0758b9 +api,5,2,r2-marcus-feld,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.054,0.449,0.774,1.28,18.8,12:22:44,8d0758b9 +api,5,3,r3-ines-carvalho,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.033,0.416,0.708,1.325,18.8,12:22:47,8d0758b9 +api,5,4,r4-theo-lindqvist,False,True,1,12/12,True,COMPLETED_UNVERIFIED,,2.201,0.471,0.894,1.307,17.86,12:22:49,8d0758b9 diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..13c758683 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-1-r1-nadia-brennan.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.484, + "form_read": 0.698, + "prechecked": 0.745, + "save_sent": 0.745, + "save_done": 0.808 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.54, + "setup_s": 0.698, + "task_s": 1.171, + "readback_s": 1.369 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..d316b3615 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-2-r2-marcus-feld.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.325, + "form_read": 0.505, + "prechecked": 0.552, + "save_sent": 0.552, + "save_done": 0.599 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.453, + "setup_s": 0.505, + "task_s": 1.054, + "readback_s": 1.399 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..70cc1715b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-3-r3-ines-carvalho.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.357, + "form_read": 0.496, + "prechecked": 0.533, + "save_sent": 0.533, + "save_done": 0.566 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.085, + "setup_s": 0.496, + "task_s": 0.806, + "readback_s": 1.278 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..b7dcabbcc --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-api-4-r4-theo-lindqvist.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.39, + "form_read": 0.549, + "prechecked": 0.583, + "save_sent": 0.583, + "save_done": 0.616 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.231, + "setup_s": 0.549, + "task_s": 0.946, + "readback_s": 1.286 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..9b38eec66 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-1-r1-nadia-brennan.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 27.119, + "setup_s": 1.833, + "task_s": 25.487, + "readback_s": 1.633 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..cd25bb3f6 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-2-r2-marcus-feld.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 24.761, + "setup_s": 1.559, + "task_s": 23.281, + "readback_s": 1.481 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..f776b058f --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-3-r3-ines-carvalho.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 22.736, + "setup_s": 1.438, + "task_s": 21.395, + "readback_s": 1.341 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..050441cd2 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b01-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.899, + "setup_s": 1.413, + "task_s": 22.432, + "readback_s": 1.467 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..cc76fdcb9 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-1-r5-grace-adeyemi.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.387, + "form_read": 0.579, + "prechecked": 0.616, + "save_sent": 0.616, + "save_done": 0.649 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.232, + "setup_s": 0.579, + "task_s": 0.913, + "readback_s": 1.319 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-2-r6-rosa-delgado.json new file mode 100644 index 000000000..2c17b1554 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-2-r6-rosa-delgado.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "api_marks": { + "signed_in": 0.308, + "form_read": 0.44, + "prechecked": 0.474 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [ + "No matches were found." + ] + } + ], + "engine_success": false, + "outcome": "HALTED", + "steps": 1, + "step_errors": [ + [ + "api_save", + "API-only step 'api_save' (save the form via POST /interface/new/new_comprehensive_save.php (folds 41 GUI steps)) is unavailable before delivery; refusing GUI fallback \u2014 run aborted" + ] + ], + "actuation": [ + null + ], + "effect_verified": [ + false + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "wall_s": 1.763, + "setup_s": 0.44, + "task_s": 0.477, + "readback_s": 1.286 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..63530ff69 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-3-r7-arlo-whitcombe.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.313, + "form_read": 0.448, + "prechecked": 0.482, + "save_sent": 0.482, + "save_done": 0.52 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.159, + "setup_s": 0.448, + "task_s": 0.864, + "readback_s": 1.295 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-4-r8-priya-raman.json new file mode 100644 index 000000000..17335e500 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-api-4-r8-priya-raman.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.301, + "form_read": 0.445, + "prechecked": 0.482, + "save_sent": 0.482, + "save_done": 0.52 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.127, + "setup_s": 0.445, + "task_s": 0.825, + "readback_s": 1.303 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..7ae011326 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-1-r5-grace-adeyemi.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 26.879, + "setup_s": 1.932, + "task_s": 25.384, + "readback_s": 1.495 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-2-r6-rosa-delgado.json new file mode 100644 index 000000000..c327af058 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-2-r6-rosa-delgado.json @@ -0,0 +1,102 @@ +{ + "arm": "browser", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "outcome": "HALTED", + "steps": 40, + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click 'Create New Patient'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "actuation": [ + "dom" + ], + "effect_verified": [ + null + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "wall_s": 34.35, + "setup_s": 1.581, + "task_s": 32.767, + "readback_s": 1.584 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..260cac7e4 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-3-r7-arlo-whitcombe.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 28.391, + "setup_s": 1.625, + "task_s": 27.013, + "readback_s": 1.378 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-4-r8-priya-raman.json new file mode 100644 index 000000000..af02546a4 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b02-browser-4-r8-priya-raman.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 26.131, + "setup_s": 1.466, + "task_s": 24.753, + "readback_s": 1.377 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..67866fa86 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-1-r1-nadia-brennan.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.389, + "form_read": 0.557, + "prechecked": 0.599, + "save_sent": 0.599, + "save_done": 0.63 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.16, + "setup_s": 0.557, + "task_s": 0.868, + "readback_s": 1.293 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..8e2753a7b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-2-r2-marcus-feld.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.287, + "form_read": 0.483, + "prechecked": 0.517, + "save_sent": 0.517, + "save_done": 0.557 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.12, + "setup_s": 0.483, + "task_s": 0.837, + "readback_s": 1.283 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..3e2e0f361 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-3-r3-ines-carvalho.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.294, + "form_read": 0.412, + "prechecked": 0.455, + "save_sent": 0.455, + "save_done": 0.483 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 1.996, + "setup_s": 0.412, + "task_s": 0.709, + "readback_s": 1.287 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..affaf6ca5 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-api-4-r4-theo-lindqvist.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.297, + "form_read": 0.426, + "prechecked": 0.461, + "save_sent": 0.461, + "save_done": 0.492 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.054, + "setup_s": 0.426, + "task_s": 0.767, + "readback_s": 1.286 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..9b1079318 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-1-r1-nadia-brennan.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 24.456, + "setup_s": 1.86, + "task_s": 23.106, + "readback_s": 1.349 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..6f2f6b49a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-2-r2-marcus-feld.json @@ -0,0 +1,102 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": false, + "outcome": "HALTED", + "steps": 41, + "step_errors": [ + [ + "step_040", + "Identity check failed for step 'step_040' (click 'Confirm Create New Patient'): a target was found positionally (rung 'structural', confidence 1.00) but its surrounding text does not match the recorded target's \u2014 expected '<identity template>', observed 'Emergency Contact: Confirm Create New Patient" + ] + ], + "actuation": [ + null + ], + "effect_verified": [ + null + ], + "new_charts": 0, + "readback": "0/12", + "correct": false, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Marcus", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Feld", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Male", + "got": null, + "match": false + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0124", + "got": null, + "match": false + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "wall_s": 21.115, + "setup_s": 1.445, + "task_s": 19.788, + "readback_s": 1.326 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..ec884db4b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-3-r3-ines-carvalho.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.617, + "setup_s": 1.406, + "task_s": 22.287, + "readback_s": 1.33 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..597a0803c --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b03-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 22.833, + "setup_s": 1.413, + "task_s": 21.499, + "readback_s": 1.334 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..42f846c13 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-1-r5-grace-adeyemi.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.398, + "form_read": 0.56, + "prechecked": 0.599, + "save_sent": 0.599, + "save_done": 0.629 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.145, + "setup_s": 0.56, + "task_s": 0.855, + "readback_s": 1.29 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-2-r6-rosa-delgado.json new file mode 100644 index 000000000..382dad9b8 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-2-r6-rosa-delgado.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "api_marks": { + "signed_in": 0.305, + "form_read": 0.445, + "prechecked": 0.483 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [ + "No matches were found." + ] + } + ], + "engine_success": false, + "outcome": "HALTED", + "steps": 1, + "step_errors": [ + [ + "api_save", + "API-only step 'api_save' (save the form via POST /interface/new/new_comprehensive_save.php (folds 41 GUI steps)) is unavailable before delivery; refusing GUI fallback \u2014 run aborted" + ] + ], + "actuation": [ + null + ], + "effect_verified": [ + false + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "wall_s": 1.797, + "setup_s": 0.445, + "task_s": 0.485, + "readback_s": 1.312 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..179af90fd --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-3-r7-arlo-whitcombe.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.286, + "form_read": 0.415, + "prechecked": 0.448, + "save_sent": 0.448, + "save_done": 0.478 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.036, + "setup_s": 0.415, + "task_s": 0.74, + "readback_s": 1.297 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-4-r8-priya-raman.json new file mode 100644 index 000000000..aefc51975 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-api-4-r8-priya-raman.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.297, + "form_read": 0.487, + "prechecked": 0.523, + "save_sent": 0.523, + "save_done": 0.556 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.103, + "setup_s": 0.487, + "task_s": 0.815, + "readback_s": 1.288 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-1-r5-grace-adeyemi.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-1-r5-grace-adeyemi.json new file mode 100644 index 000000000..821288d81 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-1-r5-grace-adeyemi.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r5-grace-adeyemi", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Grace", + "got": "Grace", + "match": true + }, + { + "field": "lname", + "want": "Adeyemi", + "got": "Adeyemi", + "match": true + }, + { + "field": "DOB", + "want": "1990-02-08", + "got": "1990-02-08", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "402 CEDAR ROW", + "got": "402 CEDAR ROW", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0145", + "got": "555-0145", + "match": true + }, + { + "field": "email", + "want": "grace.adeyemi@example.invalid", + "got": "grace.adeyemi@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.164, + "setup_s": 1.854, + "task_s": 21.832, + "readback_s": 1.333 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-2-r6-rosa-delgado.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-2-r6-rosa-delgado.json new file mode 100644 index 000000000..d7b59a027 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-2-r6-rosa-delgado.json @@ -0,0 +1,102 @@ +{ + "arm": "browser", + "ref": "r6-rosa-delgado", + "mismatch": true, + "error": null, + "engine_success": false, + "outcome": "HALTED", + "steps": 40, + "step_errors": [ + [ + "step_039", + "Postconditions failed for step 'step_039' (click 'Create New Patient'): expected screen state not reached (semantic drift) \u2014 failed: text_present 'No matches were found.' \u2014 run aborted" + ] + ], + "actuation": [ + "dom" + ], + "effect_verified": [ + null + ], + "new_charts": 0, + "readback": "0/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": null, + "match": false + }, + { + "field": "fname", + "want": "Rosa", + "got": null, + "match": false + }, + { + "field": "lname", + "want": "Delgado", + "got": null, + "match": false + }, + { + "field": "DOB", + "want": "1958-03-22", + "got": null, + "match": false + }, + { + "field": "sex", + "want": "Female", + "got": null, + "match": false + }, + { + "field": "street", + "want": "7 MILL ROAD", + "got": null, + "match": false + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": null, + "match": false + }, + { + "field": "state", + "want": "MA", + "got": null, + "match": false + }, + { + "field": "postal_code", + "want": "02139", + "got": null, + "match": false + }, + { + "field": "phone_home", + "want": "555-0188", + "got": null, + "match": false + }, + { + "field": "email", + "want": "rosa.delgado@example.invalid", + "got": null, + "match": false + }, + { + "field": "country_code", + "want": "USA", + "got": null, + "match": false + } + ], + "wall_s": 30.068, + "setup_s": 1.423, + "task_s": 28.345, + "readback_s": 1.722 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-3-r7-arlo-whitcombe.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-3-r7-arlo-whitcombe.json new file mode 100644 index 000000000..8ece06c99 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-3-r7-arlo-whitcombe.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r7-arlo-whitcombe", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Arlo", + "got": "Arlo", + "match": true + }, + { + "field": "lname", + "want": "Whitcombe", + "got": "Whitcombe", + "match": true + }, + { + "field": "DOB", + "want": "1966-05-30", + "got": "1966-05-30", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "88 BEACON TERRACE", + "got": "88 BEACON TERRACE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0152", + "got": "555-0152", + "match": true + }, + { + "field": "email", + "want": "arlo.whitcombe@example.invalid", + "got": "arlo.whitcombe@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.447, + "setup_s": 1.479, + "task_s": 22.105, + "readback_s": 1.342 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-4-r8-priya-raman.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-4-r8-priya-raman.json new file mode 100644 index 000000000..3357142c5 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b04-browser-4-r8-priya-raman.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r8-priya-raman", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Priya", + "got": "Priya", + "match": true + }, + { + "field": "lname", + "want": "Raman", + "got": "Raman", + "match": true + }, + { + "field": "DOB", + "want": "1975-10-14", + "got": "1975-10-14", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "15 STATION AVENUE", + "got": "15 STATION AVENUE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0167", + "got": "555-0167", + "match": true + }, + { + "field": "email", + "want": "priya.raman@example.invalid", + "got": "priya.raman@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.613, + "setup_s": 1.587, + "task_s": 22.276, + "readback_s": 1.337 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-1-r1-nadia-brennan.json new file mode 100644 index 000000000..40238392b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-1-r1-nadia-brennan.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.403, + "form_read": 0.583, + "prechecked": 0.623, + "save_sent": 0.623, + "save_done": 0.687 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.331, + "setup_s": 0.583, + "task_s": 1.051, + "readback_s": 1.28 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-2-r2-marcus-feld.json new file mode 100644 index 000000000..9e622744f --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-2-r2-marcus-feld.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.322, + "form_read": 0.449, + "prechecked": 0.481, + "save_sent": 0.481, + "save_done": 0.511 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.054, + "setup_s": 0.449, + "task_s": 0.774, + "readback_s": 1.28 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-3-r3-ines-carvalho.json new file mode 100644 index 000000000..cb26c58bc --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-3-r3-ines-carvalho.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.291, + "form_read": 0.416, + "prechecked": 0.45, + "save_sent": 0.45, + "save_done": 0.48 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.033, + "setup_s": 0.416, + "task_s": 0.708, + "readback_s": 1.325 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..87febabfe --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-api-4-r4-theo-lindqvist.json @@ -0,0 +1,126 @@ +{ + "arm": "api", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "api_marks": { + "signed_in": 0.315, + "form_read": 0.471, + "prechecked": 0.516, + "save_sent": 0.516, + "save_done": 0.551 + }, + "api_log": [ + { + "stage": "sign_in", + "status": 302, + "location_path": "/interface/main/tabs/main.php" + }, + { + "stage": "read_form", + "status": 200, + "bytes": 341515 + }, + { + "stage": "precheck", + "status": 200, + "missing": [] + }, + { + "stage": "save", + "status": 200, + "success_signature": true + } + ], + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 1, + "step_errors": [], + "actuation": [ + "api" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 2.201, + "setup_s": 0.471, + "task_s": 0.894, + "readback_s": 1.307 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-1-r1-nadia-brennan.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-1-r1-nadia-brennan.json new file mode 100644 index 000000000..034598f4a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-1-r1-nadia-brennan.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r1-nadia-brennan", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Ms.", + "got": "Ms.", + "match": true + }, + { + "field": "fname", + "want": "Nadia", + "got": "Nadia", + "match": true + }, + { + "field": "lname", + "want": "Brennan", + "got": "Brennan", + "match": true + }, + { + "field": "DOB", + "want": "1971-04-19", + "got": "1971-04-19", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "27 WILLOW BEND ROAD", + "got": "27 WILLOW BEND ROAD", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0121", + "got": "555-0121", + "match": true + }, + { + "field": "email", + "want": "nadia.brennan@example.invalid", + "got": "nadia.brennan@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.871, + "setup_s": 1.999, + "task_s": 22.539, + "readback_s": 1.332 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-2-r2-marcus-feld.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-2-r2-marcus-feld.json new file mode 100644 index 000000000..da9f08fd7 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-2-r2-marcus-feld.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r2-marcus-feld", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Marcus", + "got": "Marcus", + "match": true + }, + { + "field": "lname", + "want": "Feld", + "got": "Feld", + "match": true + }, + { + "field": "DOB", + "want": "1958-09-02", + "got": "1958-09-02", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "114 QUARRY HILL DRIVE", + "got": "114 QUARRY HILL DRIVE", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0124", + "got": "555-0124", + "match": true + }, + { + "field": "email", + "want": "marcus.feld@example.invalid", + "got": "marcus.feld@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 23.106, + "setup_s": 1.456, + "task_s": 21.777, + "readback_s": 1.329 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-3-r3-ines-carvalho.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-3-r3-ines-carvalho.json new file mode 100644 index 000000000..9550981c0 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-3-r3-ines-carvalho.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r3-ines-carvalho", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mrs.", + "got": "Mrs.", + "match": true + }, + { + "field": "fname", + "want": "Ines", + "got": "Ines", + "match": true + }, + { + "field": "lname", + "want": "Carvalho", + "got": "Carvalho", + "match": true + }, + { + "field": "DOB", + "want": "1949-12-11", + "got": "1949-12-11", + "match": true + }, + { + "field": "sex", + "want": "Female", + "got": "Female", + "match": true + }, + { + "field": "street", + "want": "9 LANTERN COURT", + "got": "9 LANTERN COURT", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0131", + "got": "555-0131", + "match": true + }, + { + "field": "email", + "want": "ines.carvalho@example.invalid", + "got": "ines.carvalho@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 24.284, + "setup_s": 1.41, + "task_s": 22.887, + "readback_s": 1.397 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-4-r4-theo-lindqvist.json b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-4-r4-theo-lindqvist.json new file mode 100644 index 000000000..7f75273e6 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/b05-browser-4-r4-theo-lindqvist.json @@ -0,0 +1,97 @@ +{ + "arm": "browser", + "ref": "r4-theo-lindqvist", + "mismatch": false, + "error": null, + "engine_success": true, + "outcome": "COMPLETED_UNVERIFIED", + "steps": 41, + "step_errors": [], + "actuation": [ + "dom" + ], + "effect_verified": [ + true + ], + "new_charts": 1, + "readback": "12/12", + "correct": true, + "field_check": [ + { + "field": "title", + "want": "Mr.", + "got": "Mr.", + "match": true + }, + { + "field": "fname", + "want": "Theo", + "got": "Theo", + "match": true + }, + { + "field": "lname", + "want": "Lindqvist", + "got": "Lindqvist", + "match": true + }, + { + "field": "DOB", + "want": "1983-07-26", + "got": "1983-07-26", + "match": true + }, + { + "field": "sex", + "want": "Male", + "got": "Male", + "match": true + }, + { + "field": "street", + "want": "63 FERRY STREET", + "got": "63 FERRY STREET", + "match": true + }, + { + "field": "city", + "want": "EXAMPLETON", + "got": "EXAMPLETON", + "match": true + }, + { + "field": "state", + "want": "MA", + "got": "MA", + "match": true + }, + { + "field": "postal_code", + "want": "02139", + "got": "02139", + "match": true + }, + { + "field": "phone_home", + "want": "555-0136", + "got": "555-0136", + "match": true + }, + { + "field": "email", + "want": "theo.lindqvist@example.invalid", + "got": "theo.lindqvist@example.invalid", + "match": true + }, + { + "field": "country_code", + "want": "USA", + "got": "USA", + "match": true + } + ], + "wall_s": 27.106, + "setup_s": 1.419, + "task_s": 25.77, + "readback_s": 1.336 +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-01-api.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-01-api.json new file mode 100644 index 000000000..143bae412 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-01-api.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 6.04, + "arm": "api", + "block": 1, + "started": "2026-10-18 12:14:30" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-01-browser.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-01-browser.json new file mode 100644 index 000000000..a8c14ad85 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-01-browser.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.67, + "arm": "browser", + "block": 1, + "started": "2026-10-18 12:12:43" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-02-api.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-02-api.json new file mode 100644 index 000000000..45f7b57a8 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-02-api.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.46, + "arm": "api", + "block": 2, + "started": "2026-10-18 12:14:45" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-02-browser.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-02-browser.json new file mode 100644 index 000000000..18ea8ea05 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-02-browser.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 5.95, + "arm": "browser", + "block": 2, + "started": "2026-10-18 12:15:01" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-03-api.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-03-api.json new file mode 100644 index 000000000..453f0b0c5 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-03-api.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.39, + "arm": "api", + "block": 3, + "started": "2026-10-18 12:18:42" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-03-browser.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-03-browser.json new file mode 100644 index 000000000..eb11df180 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-03-browser.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 5.02, + "arm": "browser", + "block": 3, + "started": "2026-10-18 12:17:04" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-04-api.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-04-api.json new file mode 100644 index 000000000..e5d5a3633 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-04-api.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.15, + "arm": "api", + "block": 4, + "started": "2026-10-18 12:18:56" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-04-browser.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-04-browser.json new file mode 100644 index 000000000..22979f44a --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-04-browser.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.4, + "arm": "browser", + "block": 4, + "started": "2026-10-18 12:19:10" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-05-api.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-05-api.json new file mode 100644 index 000000000..ba65dba32 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-05-api.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.74, + "arm": "api", + "block": 5, + "started": "2026-10-18 12:22:42" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/results/runs/block-05-browser.json b/packages/flow/benchmark/openemr_api_tier/results/runs/block-05-browser.json new file mode 100644 index 000000000..8a0bf0b12 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/results/runs/block-05-browser.json @@ -0,0 +1,8 @@ +{ + "baseline": "359322296c24ca08859a70a9ba29efb673fbf6b43d96879c981b811e1276e1d0", + "seed_status": 201, + "reset_seed_s": 4.28, + "arm": "browser", + "block": 5, + "started": "2026-10-18 12:20:57" +} \ No newline at end of file diff --git a/packages/flow/benchmark/openemr_api_tier/spike_fixture.py b/packages/flow/benchmark/openemr_api_tier/spike_fixture.py new file mode 100644 index 000000000..6f91ece9b --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/spike_fixture.py @@ -0,0 +1,103 @@ +"""A private copy of the pinned local OpenEMR fixture for the API-tier spike. + +Same pinned images, upstream compose file, bootstrap and reset as +``benchmark/openemr_local``; its own compose project, ports and state +directory, so it does not disturb fixtures other studies are using. + +State (runtime secrets, OAuth client IDs, SQL baseline) lives outside the +repository in ``$API_SPIKE_STATE`` (default +``~/oa/scratch/api-spike-fixture/state``). Every record in it is synthetic. + + python -m benchmark.openemr_api_tier.spike_fixture create # once + python -m benchmark.openemr_api_tier.spike_fixture up # start + reset +""" + +from __future__ import annotations + +import os +import secrets +import sys +from pathlib import Path + +import benchmark.openemr_local.fixture as fx + +PROJECT = "openadapt-openemr-apispike" +UI_PORT = os.environ.get("API_SPIKE_UI_PORT", "9811") +API_PORT = os.environ.get("API_SPIKE_API_PORT", "9810") +STATE = Path( + os.environ.get( + "API_SPIKE_STATE", + str(Path.home() / "oa/scratch/api-spike-fixture/state"), + ) +) + + +class SpikeFixture(fx.OpenEMRFixture): + def __init__(self, state_dir: Path = STATE) -> None: + super().__init__(state_dir) + self.lock = dict(self.lock, project=PROJECT) + + @property + def ui_base_url(self) -> str: + return f"http://127.0.0.1:{UI_PORT}" + + @property + def api_base_url(self) -> str: + return f"https://127.0.0.1:{API_PORT}" + + def _validate_runtime_env(self) -> None: + v = self._runtime_values() + assert v.get("OPENEMR_HTTP_PORT") == UI_PORT + assert v.get("OPENEMR_HTTPS_PORT") == API_PORT + assert v.get("OPENEMR_IMAGE") == self.lock["services"]["openemr"] + assert v.get("MARIADB_IMAGE") == self.lock["services"]["mariadb"] + self.runtime_env.chmod(0o600) + + def write_runtime_env(self) -> None: + values = { + "OPENEMR_IMAGE": self.lock["services"]["openemr"], + "MARIADB_IMAGE": self.lock["services"]["mariadb"], + "MARIADB_ROOT_PASSWORD": secrets.token_urlsafe(32), + "OPENEMR_DB_PASSWORD": secrets.token_urlsafe(32), + "OPENEMR_ACTOR_USER": "openadapt_actor", + "OPENEMR_ACTOR_PASSWORD": secrets.token_urlsafe(32), + "OPENEMR_HTTP_PORT": UI_PORT, + "OPENEMR_HTTPS_PORT": API_PORT, + } + self._protect_state_dir() + self._write_private_exclusive( + self.runtime_env, + ( + "# generated local synthetic fixture; never commit\n" + + "".join(f"{k}={v}\n" for k, v in values.items()) + ).encode(), + ) + + def start(self) -> None: + """Start without pulling (the pinned images are already local).""" + self.image_identity() + self._compose("up", "-d", timeout_s=600.0) + self._wait_ready() + self._verify_running_source() + + +def main(argv: list[str]) -> int: + f = SpikeFixture() + cmd = argv[0] if argv else "up" + if cmd == "create": + f.write_runtime_env() + f.start() + f.bootstrap() + print("baseline", f.snapshot()) + elif cmd == "up": + f.start() + print("reset", f.reset()[:12]) + elif cmd == "down": + f._compose("stop", timeout_s=120.0) + else: + raise SystemExit(f"unknown command {cmd}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) From 839a0936f3736f63bd4ba4a0145e5b4fb7041967 Mon Sep 17 00:00:00 2001 From: abrichr <richard.abrich@mldsai.com> Date: Sun, 18 Oct 2026 13:26:27 -0400 Subject: [PATCH 2/2] feat(flow): API steps compiled from a recording's network capture - record --capture-network (web): NetworkCapture keeps the page's requests in network.jsonl; headers, secrets, session tokens and echoed cookie values are withheld - compile proposes a web_form ApiBinding on the save when every field of its request is explained (param, select label, session value, form default); reads whose responses carry the GUI's pre-save text become pre-checks; api_proposal.json says why when nothing is proposed - api show / propose / admit: a person admits the API step at a terminal; the admission is bound to the exact binding and copies the save's record check - WebFormActuator sends through the replaying browser's own session, re-reads the form (CSRF, defaults, drift, client rules), runs the pre-checks, posts once; refusals before the post fall back to GUI - Replayer runs an admitted API step in place of its GUI steps in plain linear runs (not under each_item/problems_only approval, governed, durable or resumed runs) and reports api_folded / api_fallback - docs/API_TIER.md; tests/test_api_tier_web_form.py Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .../benchmark/openemr_api_tier/feature_e2e.py | 269 +++++++ packages/flow/docs/API_TIER.md | 154 ++++ packages/flow/openadapt_flow/__main__.py | 18 + .../openadapt_flow/approval_mode/coverage.py | 7 + packages/flow/openadapt_flow/cli_api.py | 188 +++++ .../openadapt_flow/compiler/api_proposal.py | 507 +++++++++++++ .../flow/openadapt_flow/compiler/compile.py | 31 + .../openadapt_flow/interactive_recorder.py | 14 + packages/flow/openadapt_flow/ir.py | 137 +++- .../flow/openadapt_flow/network_capture.py | 302 ++++++++ .../openadapt_flow/record_check_binding.py | 5 +- packages/flow/openadapt_flow/recorder.py | 14 + .../runtime/actuators/web_form.py | 435 +++++++++++ .../flow/openadapt_flow/runtime/replayer.py | 366 ++++++++-- packages/flow/tests/test_api_tier_web_form.py | 686 ++++++++++++++++++ 15 files changed, 3066 insertions(+), 67 deletions(-) create mode 100644 packages/flow/benchmark/openemr_api_tier/feature_e2e.py create mode 100644 packages/flow/docs/API_TIER.md create mode 100644 packages/flow/openadapt_flow/cli_api.py create mode 100644 packages/flow/openadapt_flow/compiler/api_proposal.py create mode 100644 packages/flow/openadapt_flow/network_capture.py create mode 100644 packages/flow/openadapt_flow/runtime/actuators/web_form.py create mode 100644 packages/flow/tests/test_api_tier_web_form.py diff --git a/packages/flow/benchmark/openemr_api_tier/feature_e2e.py b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py new file mode 100644 index 000000000..5addb7390 --- /dev/null +++ b/packages/flow/benchmark/openemr_api_tier/feature_e2e.py @@ -0,0 +1,269 @@ +"""The API-step feature end to end on the local OpenEMR referral. + +Unlike the spike (``bench.py``), this uses only the shipped feature: + + record a scripted person registers referral fax A with network capture + on (NetworkCapture attached before sign-in, so the form page is + captured; Recorder writes network.jsonl) + compile openadapt-flow's compiler (via the OpenEMR harness, which adds + the record check and the duplicate-check rule), then + ``propose_api_step(replace=True)`` because the rule was added + after compile, then the admission + matrix the same bundle, replayed with API steps off (``browser``) and + on (``api``), interleaved, same record check + +The admission in this harness is made by the harness under the name +``benchmark-fixture`` so the matrix can run unattended. In use, a person +admits with ``openadapt-flow api admit`` at a terminal. + + python -m benchmark.openemr_api_tier.feature_e2e record --out <rec> + python -m benchmark.openemr_api_tier.feature_e2e compile --recording <rec> --out <bundle> + python -m benchmark.openemr_api_tier.feature_e2e matrix --bundle <bundle> --out <dir> +""" + +from __future__ import annotations + +import argparse +import csv +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +from benchmark.openemr_api_tier.common import ( + REFERRALS, + load_demo, + record_check, + reset_and_seed, + spec_for, + sql_patients, +) +from benchmark.openemr_api_tier.spike_fixture import SpikeFixture + +FAX_A = dict(title="Ms.", fname="Elena", lname="Marsh", dob="1979-06-14", sex="Female", + street="42 HARBOR LANE", city="EXAMPLETON", state="MA", state_label="Massachusetts", + postal_code="02139", phone_home="555-0142", email="elena.marsh@example.invalid") +NO_MATCH_TEXT = "No matches were found." + + +def cmd_record(a: argparse.Namespace) -> None: + from openadapt_flow.benchmark.openemr_local import SyntheticPatientSpec + from openadapt_flow.network_capture import NetworkCapture + from openadapt_flow.recorder import Recorder + + demo = load_demo() + out = Path(a.out) + if out.exists(): + raise SystemExit(f"exists: {out}") + f = SpikeFixture() + reset_and_seed(f) + backend, close = demo._BasePlaywrightBackend.launch(f.ui_base_url, headless=True) + try: + capture = None if a.no_capture else NetworkCapture(backend.page.context).start() + demo._preauthenticate_browser(backend, f) + page = backend.page + form = demo._form_frame(page) + rec = Recorder(backend, out, app_url=page.url, network_capture=capture, **demo.SETTLE) + spec = SyntheticPatientSpec(**FAX_A) + + def text_field(name: str, value: str, *, param: str) -> None: + rec.click(*demo._center(form.locator(f"#form_{name}"))) + rec.press("Control+a") + rec.type_text(value, param=param) + + def scroll_form() -> None: + # The wheel scrolls whatever is under the pointer: put it over the + # form (not recorded; a pointer move is not an event). + page.mouse.move(640, 500) + rec.scroll(0, 600) + + def select_by_label(name: str, label: str, *, param: str) -> None: + rec.click(*demo._center(form.locator(f"#form_{name}"))) + rec.type_text(label, param=param) + rec.press("Enter") + + select_by_label("title", spec.title, param="title") + text_field("fname", spec.fname, param="fname") + text_field("lname", spec.lname, param="lname") + text_field("DOB", spec.dob, param="DOB") + select_by_label("sex", spec.sex, param="sex") + rec.click(*demo._center(form.get_by_text("Contact", exact=True))) + scroll_form() + text_field("street", spec.street, param="street") + text_field("city", spec.city, param="city") + select_by_label("state", spec.state_label, param="state_label") + text_field("postal_code", spec.postal_code, param="postal_code") + text_field("phone_home", spec.phone_home, param="phone_home") + text_field("email", spec.email, param="email") + select_by_label("country_code", "USA", param="country_code") + scroll_form() + rec.click(*demo._center(form.locator("#create"))) + try: + confirm = demo._confirm_create_locator(page) + except Exception: + page.screenshot(path=str(out.parent / "confirm-not-ready.png")) + raise + rec.click(*demo._center(confirm)) + recording = rec.finish() + events = [json.loads(x) for x in (recording / "events.jsonl").read_text().splitlines() if x.strip()] + (recording / "openemr-save-event.json").write_text(json.dumps({ + "schema_version": 1, "event_index": events[-1]["i"], + "dom_target": "#confirmCreate inside duplicate-check iframe", + "effect_contract": "create exactly one synthetic patient"}, indent=2, sort_keys=True) + "\n") + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and not sql_patients(f, "email='elena.marsh@example.invalid'"): + time.sleep(0.2) + print(json.dumps({"recording": str(recording), "events": len(events), + "exchanges": None if capture is None else capture.count, + "saved": len(sql_patients(f, "email='elena.marsh@example.invalid'"))})) + finally: + close() + + +def cmd_compile(a: argparse.Namespace) -> None: + from openadapt_flow.compiler.api_proposal import ( + admit_api_step, + describe_api_step, + propose_api_step, + ) + from openadapt_flow.ir import Postcondition, PostconditionKind, Workflow + + demo = load_demo() + rec, bundle = Path(a.recording), Path(a.out) + demo.compile_recording(rec, bundle) + first = json.loads((bundle / "api_proposal.json").read_text()) + wf = Workflow.load(bundle) + # The referral's business rule (as in the published bundle): confirm a new + # chart only when OpenEMR's duplicate check finds no existing patient. + dialog = [s for s in wf.steps if s.anchor is not None and s.anchor.structural is not None + and s.anchor.structural.selector == "#create"] + assert len(dialog) == 1 + dialog[0].expect = list(dialog[0].expect) + [ + Postcondition(kind=PostconditionKind.TEXT_PRESENT, text=NO_MATCH_TEXT, timeout_s=10.0)] + wf, step, reasons = propose_api_step(wf, rec, replace=True) + if step is None: + raise SystemExit(f"no API step: {reasons}") + wf, _ = admit_api_step(wf, admitted_by="benchmark-fixture") + wf.save(bundle) + print(json.dumps({"first_compile": first}, indent=1)) + print(describe_api_step(next(s for s in wf.steps if s.id == step.id))) + + +COLS = ["arm", "block", "pos", "ref", "mismatch", "correct", "new_charts", "readback", "engine_success", + "outcome", "actuation", "api_fallback", "error", "wall_s", "setup_s", "engine_s", "readback_s", + "load1", "start_local", "engine_sha"] + + +def run_one(env: dict[str, Any], arm: str, ref_idx: int, bundle: Path, out: Path) -> dict[str, Any]: + from openadapt_flow.benchmark.openemr_local import OpenEMRPatientOracle + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime import Replayer + + demo, f, browser, ocr = env["demo"], env["f"], env["browser"], env["ocr"] + ref = REFERRALS[ref_idx - 1] + spec = spec_for(ref) + out.mkdir(parents=True) + before_ids = {r["id"] for r in sql_patients(f)} + oracle = OpenEMRPatientOracle(f.api_base_url, f.token_session("oracle"), spec) + res: dict[str, Any] = {"arm": arm, "ref": ref.key, "mismatch": ref.mismatch, "error": None} + t0 = time.time() + ctx = browser.new_context(viewport={"width": 1280, "height": 800}, device_scale_factor=1) + try: + page = ctx.new_page() + page.goto(f.ui_base_url) + backend = demo._BasePlaywrightBackend(page) + demo._preauthenticate_browser(backend, f) + t_setup = time.time() + ocr.clear_ocr_memo() + rep = Replayer(backend, effect_verifier=oracle.verifier, api_steps=(arm == "api")).run( + Workflow.load(bundle), params=spec.params(), bundle_dir=bundle, run_dir=out / "engine") + t_task = time.time() + raw = rep.model_dump(mode="json") + last = raw["results"][-1] if raw["results"] else {} + res.update(engine_success=raw.get("success"), outcome=str(raw.get("execution_outcome")), + actuation=last.get("actuation"), + api_fallback=next((r["api_fallback"] for r in raw["results"] if r.get("api_fallback")), None), + steps=len(raw["results"]), + step_errors=[(r["step_id"], (r.get("error") or "")[:300]) for r in raw["results"] if r.get("error")]) + except Exception as exc: # noqa: BLE001 + t_setup = locals().get("t_setup", time.time()) + t_task = time.time() + res["error"] = f"{type(exc).__name__}: {exc}"[:500] + time.sleep(demo.ORACLE_SETTLE_S) + res.update(record_check(f, ref, before_ids)) + t_end = time.time() + ctx.close() + res.update(wall_s=round(t_end - t0, 3), setup_s=round(t_setup - t0, 3), engine_s=round(t_task - t_setup, 3), + readback_s=round(t_end - t_task, 3)) + (out / "result.json").write_text(json.dumps(res, indent=1, default=str)) + return res + + +def cmd_matrix(a: argparse.Namespace) -> None: + from playwright.sync_api import sync_playwright + + import openadapt_flow.vision.ocr # noqa: F401 + + ocr = sys.modules["openadapt_flow.vision.ocr"] + out, bundle = Path(a.out), Path(a.bundle) + out.mkdir(parents=True, exist_ok=True) + sha = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()[:8] + demo, f = load_demo(), SpikeFixture() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + ocr._get_engine() + ctx = browser.new_context(viewport={"width": 1280, "height": 800}) + page = ctx.new_page() + page.goto(f.ui_base_url) + ocr.ocr(page.screenshot(type="png")) + ctx.close() + env = {"demo": demo, "f": f, "browser": browser, "ocr": ocr} + csv_path = out / "runs.csv" + new = not csv_path.exists() + try: + for b in range(a.blocks): + for arm in (["browser", "api"] if b % 2 == 0 else ["api", "browser"]): + meta = reset_and_seed(f) | {"arm": arm, "block": b + 1, "started": time.strftime("%Y-%m-%d %H:%M:%S")} + (out / f"block-{b + 1:02d}-{arm}.json").write_text(json.dumps(meta, indent=1)) + for pos in range(a.per_block): + idx = ((b * a.per_block + pos) % len(REFERRALS)) + 1 + start = time.strftime("%H:%M:%S") + res = run_one(env, arm, idx, bundle, out / f"b{b + 1:02d}-{arm}-{pos + 1}-{REFERRALS[idx - 1].key}") + load1 = float(subprocess.run(["sysctl", "-n", "vm.loadavg"], capture_output=True, text=True).stdout.split()[1]) + row = {k: res.get(k) for k in COLS} | {"block": b + 1, "pos": pos + 1, "load1": load1, + "start_local": start, "engine_sha": sha} + with open(csv_path, "a", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=COLS) + if new: + w.writeheader() + new = False + w.writerow(row) + print(json.dumps({k: row[k] for k in ("arm", "block", "ref", "correct", "readback", "actuation", + "api_fallback", "wall_s", "error")}), flush=True) + finally: + browser.close() + pw.stop() + + +def main() -> None: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("record") + r.add_argument("--out", required=True) + r.add_argument("--no-capture", action="store_true", help="Diagnostics: record without capture") + c = sub.add_parser("compile") + c.add_argument("--recording", required=True) + c.add_argument("--out", required=True) + m = sub.add_parser("matrix") + m.add_argument("--bundle", required=True) + m.add_argument("--out", required=True) + m.add_argument("--blocks", type=int, default=5) + m.add_argument("--per-block", type=int, default=4) + a = ap.parse_args() + {"record": cmd_record, "compile": cmd_compile, "matrix": cmd_matrix}[a.cmd](a) + + +if __name__ == "__main__": + main() diff --git a/packages/flow/docs/API_TIER.md b/packages/flow/docs/API_TIER.md new file mode 100644 index 000000000..3c2f9f943 --- /dev/null +++ b/packages/flow/docs/API_TIER.md @@ -0,0 +1,154 @@ +# API steps from a recording's network traffic + +A web form save that OpenAdapt can repeat as one HTTP request runs as one +request instead of as the clicks and keystrokes that filled the form. The +right-record check before the save and the record check after it stay the +same. If anything the API step depends on has changed, the recorded GUI +steps run instead. + +On the local OpenEMR referral, the API step took the median run from +[MEASURED_BROWSER] to [MEASURED_API], with the same record check +([benchmark data](../benchmark/openemr_api_tier/README.md)). + +## How to use it + +```bash +openadapt-flow record --backend web --url https://app.example/ --capture-network --out rec/ +openadapt-flow compile rec/ --out bundle/ --name referral +openadapt-flow api show bundle/ +openadapt-flow api admit bundle/ # a person, at a terminal +openadapt-flow run bundle/ --approval-mode not_gated ... +``` + +1. **Record with `--capture-network`.** Flow keeps the page's requests next + to the recorded events in `network.jsonl`. Start the recording before you + open the form, so the page that serves the form is captured. +2. **Compile.** If the save can be repeated as one request, `compile` + proposes an API step on the save step. `api_proposal.json` in the bundle + says either which step got the proposal, or why there is none. +3. **Admit.** `api show` lists what the API step sends and checks. + `api admit` asks a person to type `admit`. Until then, the API step never + runs. An agent, model, or service name can't admit it, and the admission + is bound to the exact proposal. If the binding changes, it needs a new + admission. +4. **Run.** An admitted API step runs in place of the GUI steps it covers. + The run report marks those steps `actuation: "api_folded"`, and the save + step is marked `actuation: "api"`. When the API step doesn't run, the + first of those steps carries `api_fallback` with the reason, and the GUI + steps run as recorded. + +If you change a check on one of the covered steps after compiling (for +example, you add a required on-screen text), the API step stops running +because it would skip that check. Run `openadapt-flow api propose bundle/ +--recording rec/` to propose it again, then admit it. + +## What the compiler checks + +The compiler finds the form request that the save step sent and explains +every field it sent: + +| Source | Meaning | At run time | +| --- | --- | --- | +| `param` | The value of a recorded parameter | The run's value | +| `select_label` | A select whose option the person picked by its label | The option with that label in the live form; refuse if it isn't exactly one | +| `session` | A CSRF or session value the server issued | Read from the live form in this session | +| `form_default` | The form's own default | Must still equal the recorded default | + +If a field can't be explained, nothing is proposed. The compiler also +doesn't propose an API step in these cases: + +- the save sends a secret field; +- the form page wasn't captured; +- the save isn't the program's last step; +- the program has branches; +- a covered step has a guard, a wait rule, or a record-identifying region the + person marked with `record --identifier`; +- a covered step made a request whose response must show text the GUI checks + on screen, and no response carries that text. + +Read-only requests that a covered step made, and whose response carries the +text the GUI program checks before saving, become **pre-checks**. For +example, OpenEMR's duplicate-patient search must answer "No matches were +found." The API step sends these requests and requires the same text before +it sends the save. On-screen checks of steps that made no request (layout +and region checks) are screen-only. They are listed for the person who +admits the step and aren't repeated. + +## What happens at run time + +The API step uses the replaying browser's own session: its requests go +through the browser context's request API, which shares the browser's +cookies. If the browser is signed in, the API step is signed in too. No +credential is stored in the bundle. + +Before it sends the save, the API step: + +1. reads the form page again in this session; +2. refuses if the form's fields or its client-side validation rules (for + example, OpenEMR's `constraints` object) differ from the recording; +3. builds the body, and refuses if a field the form requires would be + empty; +4. runs the pre-checks. + +Each of these refusals means the save wasn't sent, so the GUI steps run. +After the save request starts, the API tier's no-double-write rule +applies: an error or a response without the recorded success signal stops +the run. That save is never retried, and it's never redone through the GUI. +The record check (`effects`) then confirms the save with the same verifier +a GUI save uses. + +## When it doesn't run + +The API step runs only in a plain linear run. It doesn't run, and the GUI +steps run instead, when: + +- approval mode is `each_item` (the default) or `problems_only`. The person + approves what the screen shows, and an API step leaves the form empty, so + API steps run only with `--approval-mode not_gated`; +- the run is governed by qualification, is durable, or is resumed; +- the step isn't admitted, or the program's steps, on-screen checks or the + save's record check changed since it was proposed or admitted; +- there's no live browser session (`--backend web` only). + +`Replayer(api_steps=False)` keeps a run on the GUI. + +## What the capture keeps and withholds + +`network.jsonl` holds, for each page request except static assets: the +method, the URL, the form or text body, the status, the content type, the +redirect target, and the text of HTML, JSON and text responses. It +withholds: + +- every header, so no cookie or authorization value; +- the values of password-like fields and of `--secret` fields, everywhere + they appear (`<secret>`); +- CSRF and session tokens, found by field name (`csrf`, `xsrf`, `token`, + `nonce`, `authenticity`), everywhere they appear (`<session:NAME>`); +- cookie values the browser sent, where a page echoes them + (`<session:cookie>`). + +Business values the person typed stay in the file, as they do in +`events.jsonl`. A recording with capture is as sensitive as any other +recording. + +## Limits + +- Web only. A desktop app, an RDP session or a Citrix session has no page + requests to capture (`record --capture-network` refuses those backends). +- One form save, posted as `application/x-www-form-urlencoded`. JSON bodies + built by script, multipart uploads, signed or computed fields, and saves + over a websocket aren't proposed. +- The server may enforce less than the page. On OpenEMR, the duplicate + check and the required fields run only in the browser. The API step + carries over the checks it can see (pre-checks and `constraints` + presence rules). Logic in the page's script that it can't see is the main + risk of this tier, and it's why a person admits each API step. +- Multi-step server state (wizards, uploads, locks held between pages) needs + the whole request sequence, and isn't proposed. + +The code: `openadapt_flow/network_capture.py` (capture), +`openadapt_flow/compiler/api_proposal.py` (proposal and admission), +`openadapt_flow/runtime/actuators/web_form.py` (the live-session actuator), +`openadapt_flow/cli_api.py` (`api show`, `api propose`, `api admit`), and the +span logic in `runtime/replayer.py` (`_web_form_span_at`, +`_run_web_form_span`). diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 77040e094..b58ea2b09 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -1140,6 +1140,11 @@ def _cmd_record_capture(args: argparse.Namespace) -> int: or getattr(args, "browser_page_url", None) ) if backend in ("windows", "macos", "linux", "rdp", "citrix"): + if getattr(args, "capture_network", False): + raise SystemExit( + "record: --capture-network applies only to --backend web; a " + "desktop, RDP or Citrix session has no page requests to capture" + ) if browser_attach_requested: raise SystemExit( "record: --browser-cdp-endpoint and --browser-page-url apply " @@ -1197,6 +1202,7 @@ def _cmd_record_capture(args: argparse.Namespace) -> int: browser_page_url=getattr(args, "browser_page_url", None), stop_when=_record_until_stop(args), surface="web", + capture_network=bool(getattr(args, "capture_network", False)), ) except BrowserAttachError as exc: raise SystemExit(f"record: browser attachment refused: {exc}") from exc @@ -6989,6 +6995,16 @@ def build_parser() -> argparse.ArgumentParser: "pre-profile default." ), ) + p.add_argument( + "--capture-network", + action="store_true", + help=( + "Web only: also keep the page's requests (network.jsonl; cookies, " + "passwords and session tokens withheld). compile then proposes an " + "API step for a save it can repeat as one request; a person admits " + "it with 'api admit'" + ), + ) p.add_argument( "--with-check", action="store_true", @@ -9933,10 +9949,12 @@ def _repair_store_flag(rp: argparse.ArgumentParser) -> None: pr.set_defaults(func=_cmd_connector) from openadapt_flow.cli_admit import register_admit_parser + from openadapt_flow.cli_api import register_api_parser from openadapt_flow.cli_check import register_check_parser register_admit_parser(sub) register_check_parser(sub) + register_api_parser(sub) from openadapt_flow.jit.cli import add_parser as _add_jit_parser diff --git a/packages/flow/openadapt_flow/approval_mode/coverage.py b/packages/flow/openadapt_flow/approval_mode/coverage.py index 9e75c39c4..93c47bec8 100644 --- a/packages/flow/openadapt_flow/approval_mode/coverage.py +++ b/packages/flow/openadapt_flow/approval_mode/coverage.py @@ -121,6 +121,13 @@ def walk(node: object) -> None: for value in binding.headers.values(): scan(value) walk(binding.body_template) + if binding.web_form is not None: + # A recorded form save names its inputs per field, and its + # pre-checks template their queries. + names.update(f.param for f in binding.web_form.fields if f.param) + for check in binding.web_form.prechecks: + for _name, value in check.query: + scan(value) except ValueError: return None return names diff --git a/packages/flow/openadapt_flow/cli_api.py b/packages/flow/openadapt_flow/cli_api.py new file mode 100644 index 000000000..fed1177b4 --- /dev/null +++ b/packages/flow/openadapt_flow/cli_api.py @@ -0,0 +1,188 @@ +"""CLI: ``openadapt-flow api``, API steps compiled from network capture. + +- ``api show``: list a bundle's proposed API steps, what each sends and + whether a person has admitted it. +- ``api admit``: a person admits them. It needs an interactive terminal and + a typed confirmation, so neither a script nor the agent that drove the + recording can admit one. + +``record --capture-network`` keeps the page's requests; ``compile`` proposes +the API step (see ``docs/API_TIER.md``). +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Callable, Optional + +from openadapt_flow.compiler.api_proposal import ( + ApiProposalError, + admit_api_step, + describe_api_step, + proposed_api_steps, +) + +#: What the person types to admit an API step. +ADMIT_WORD = "admit" + + +def register_api_parser(sub: argparse._SubParsersAction) -> None: + parser = sub.add_parser( + "api", + help=( + "Show or admit API steps compiled from a recording's network " + "capture. None runs until a person admits it." + ), + ) + verbs = parser.add_subparsers(dest="api_cmd", required=True) + show = verbs.add_parser("show", help="List a bundle's proposed API steps") + show.add_argument("bundle", help="Bundle directory") + show.add_argument("--json", action="store_true", dest="as_json") + show.set_defaults(func=cmd_api_show) + propose = verbs.add_parser( + "propose", + help=( + "Propose the API step again from the recording (after editing the " + "program's checks); drops any earlier admission" + ), + ) + propose.add_argument("bundle", help="Bundle directory") + propose.add_argument( + "--recording", required=True, help="The recording made with --capture-network" + ) + propose.set_defaults(func=cmd_api_propose) + admit = verbs.add_parser( + "admit", help="Admit a bundle's API steps (a person, at a terminal)" + ) + admit.add_argument("bundle", help="Bundle directory") + admit.add_argument("--step", default=None, help="Admit only this save step") + admit.add_argument("--by", default=None, help="Your name (default: the login name)") + admit.set_defaults(func=cmd_api_admit) + + +def cmd_api_show(args: argparse.Namespace) -> int: + from openadapt_flow.ir import Workflow + + bundle = Path(args.bundle) + found = proposed_api_steps(Workflow.load(bundle)) + reasons: list[str] = [] + sidecar = bundle / "api_proposal.json" + if sidecar.is_file(): + reasons = json.loads(sidecar.read_text()).get("reasons") or [] + if args.as_json: + rows = [ + { + "step": s.id, + "method": s.api_binding.method, + "path": s.api_binding.url_template, + "replaces_steps": s.api_binding.web_form.replaces_steps, + "admitted": s.api_binding.admitted(), + "admitted_by": s.api_binding.admission.admitted_by + if s.api_binding.admission + else None, + } + for s in found + ] + print( + json.dumps( + {"api_steps": rows, "not_proposed_because": reasons}, + indent=2, + sort_keys=True, + ) + ) + return 0 + if not found: + print( + "No API step in this bundle." + + (f" Not proposed: {'; '.join(reasons)}." if reasons else "") + ) + return 0 + for step in found: + print(describe_api_step(step)) + binding = step.api_binding + state = ( + f"admitted by {binding.admission.admitted_by}" + if binding.admitted() + else "NOT admitted" + ) + print(f" -> {state}\n") + return 0 + + +def cmd_api_propose(args: argparse.Namespace) -> int: + from openadapt_flow.compiler.api_proposal import propose_api_step + from openadapt_flow.ir import Workflow + + bundle = Path(args.bundle) + workflow, step, reasons = propose_api_step( + Workflow.load(bundle), Path(args.recording), replace=True + ) + (bundle / "api_proposal.json").write_text( + json.dumps( + { + "proposed": step is not None, + "step": None if step is None else step.id, + "reasons": reasons, + }, + indent=2, + ) + + "\n" + ) + if step is None: + print(f"No API step proposed: {'; '.join(reasons)}.") + return 1 + workflow.save(bundle) + print(describe_api_step(step)) + print("\nNot admitted yet: a person runs 'openadapt-flow api admit'.") + return 0 + + +def cmd_api_admit( + args: argparse.Namespace, + *, + ask: Callable[[str], str] = input, + interactive: Optional[Callable[[], bool]] = None, +) -> int: + is_tty = interactive or (lambda: sys.stdin.isatty() and sys.stdout.isatty()) + if not is_tty(): + print( + "api admit: a person admits an API step at an interactive terminal; " + "it can't be admitted by a script or by the recording", + file=sys.stderr, + ) + return 2 + from openadapt_flow.ir import Workflow + + bundle = Path(args.bundle) + workflow = Workflow.load(bundle) + found = [ + s + for s in proposed_api_steps(workflow) + if args.step is None or s.id == args.step + ] + if not found: + print("api admit: no proposed API step to admit", file=sys.stderr) + return 2 + for step in found: + print(describe_api_step(step)) + print() + answer = ask( + f"Type '{ADMIT_WORD}' to let these requests save in place of the GUI steps " + "(the record check still confirms every save): " + ) + if answer.strip().lower() != ADMIT_WORD: + print("Not admitted. Nothing changed.") + return 1 + try: + updated, steps = admit_api_step( + workflow, admitted_by=args.by, step_id=args.step + ) + except ApiProposalError as exc: + print(f"api admit: {exc}", file=sys.stderr) + return 2 + updated.save(bundle) + print(f"Admitted the API step on {', '.join(steps)}.") + return 0 diff --git a/packages/flow/openadapt_flow/compiler/api_proposal.py b/packages/flow/openadapt_flow/compiler/api_proposal.py new file mode 100644 index 000000000..8be653a45 --- /dev/null +++ b/packages/flow/openadapt_flow/compiler/api_proposal.py @@ -0,0 +1,507 @@ +"""Propose an API step for a recorded web save (``record --capture-network``). + +When a web recording carries ``network.jsonl``, ``compile`` looks for the one +form request that did the save and tries to explain every field it sent: + +- ``param``: the value a recorded parameter typed into the form; +- ``select_label``: a select whose option the person chose by its label (the + value is looked up in the live form each run); +- ``session``: a CSRF or session value the server issued (withheld at + capture as ``<session:NAME>``; read from the live form each run); +- ``form_default``: the form's own default value. + +If any field can't be explained, nothing is proposed and the program stays on +the GUI. Read-only requests that a folded step made, and whose response +carries text the GUI program checks on screen before saving (for example a +duplicate search that must say "No matches were found."), become pre-checks +the API step runs before it sends the save. So the right-record check before +the save and the record check after it both stay. + +The proposal is an ``ApiBinding(kind="web_form", on_unavailable="gui")`` on +the save step with no admission. It does nothing until a person admits it +(:func:`admit_api_step`, ``openadapt-flow api admit``), the same rule as a +demonstrated record check. The binding copies the save step's record check +(``effects``); admitting refuses a save that has none. + +What is not proposed (the reasons are returned): + +- no captured form POST carries the recorded values (a JSON or script-built + request, a save done over a websocket, a desktop app); +- the form page was not captured (start the recording before opening the + form); +- the save sends a secret field, or a field the compiler can't explain; +- the save is not the program's last step (steps after it would expect the + page the save left); +- a folded step checks record identity on a region the person marked + (``record --identifier``), or has a guard or wait rule; +- a folded step made a read whose response must show text the GUI checks, + and that text is in no response. + +Model-free and deterministic. +""" + +from __future__ import annotations + +import getpass +import json +import re +import urllib.parse as up +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +from openadapt_flow import network_capture as nc +from openadapt_flow.ir import ( + ActionKind, + ApiAdmission, + ApiBinding, + Step, + WebFormField, + WebFormPrecheck, + WebFormSpec, + Workflow, +) +from openadapt_flow.runtime.actuators.web_form import ( + client_rules, + folded_checks_digest, + form_controls, +) + + +class ApiProposalError(ValueError): + """The API step can't be admitted (or the proposal input is malformed).""" + + +def _event_index(step: Step) -> Optional[int]: + prefix, _, number = step.id.rpartition("_") + return int(number) if prefix == "step" and number.isdigit() else None + + +def _events(recording: Path) -> list[dict[str, Any]]: + path = recording / "events.jsonl" + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def _path(url: str) -> str: + return up.urlsplit(url).path + + +def _templated(value: str, by_value: dict[str, list[str]]) -> Optional[str]: + names = by_value.get(value) or [] + return "{" + names[0] + "}" if len(names) == 1 else None + + +def _success_pattern(line: dict[str, Any]) -> Optional[str]: + """Where the accepted save sent the browser, with record numbers generalized.""" + + target = line.get("location") + if not target: + m = re.search( + r"""(?:window\.)?location(?:\.href)?\s*=\s*['"]([^'"]+)['"]""", + line.get("response_text") or "", + ) + target = m.group(1) if m else None + if not target: + return None + path = up.urlsplit(target).path.rsplit("/", 1)[-1] + query = up.urlsplit(target).query + pieces = [re.escape(path)] + for name, value in up.parse_qsl(query, keep_blank_values=True): + pieces.append( + re.escape(name) + "=" + (r"\d+" if value.isdigit() else re.escape(value)) + ) + return r".*?".join(pieces) + + +def propose_api_step( + workflow: Workflow, recording_dir: Path | str, *, replace: bool = False +) -> tuple[Workflow, Optional[Step], list[str]]: + """Attach a proposed ``web_form`` binding to the save step, or say why not. + + Returns the workflow (unchanged when nothing is proposed), the save step + that got the proposal, and the reasons when there is none. ``replace`` + proposes again over an earlier ``web_form`` proposal (its admission is + dropped), for example after a check was added to a folded step. + """ + + recording = Path(recording_dir) + lines = nc.load(recording) + if not lines: + return workflow, None, ["the recording has no network capture"] + if workflow.program is not None: + return ( + workflow, + None, + ["the program has branches; API steps cover linear programs only"], + ) + meta = json.loads((recording / "meta.json").read_text(encoding="utf-8")) + params = {k: str(v) for k, v in (meta.get("params") or {}).items()} + if not params: + return workflow, None, ["the recording has no parameters to map into a request"] + events = _events(recording) + by_value: dict[str, list[str]] = {} + for name, value in params.items(): + if value: + by_value.setdefault(value, []).append(name) + + steps = list(workflow.steps) + save = steps[-1] + save_i = _event_index(save) + if ( + save_i is None + or save_i >= len(events) + or save.action + not in (ActionKind.CLICK, ActionKind.DOUBLE_CLICK, ActionKind.KEY) + ): + return ( + workflow, + None, + ["the last step is not a recorded click or key that could be the save"], + ) + if save.api_binding is not None and not ( + replace and save.api_binding.kind == "web_form" + ): + return workflow, None, ["the save step already has an API binding"] + # The save is the last step, so its request may finish after the + # recorder stamped the event: take anything after the previous event. + t_end = float("inf") + t_prev = float(events[save_i - 1]["t"]) if save_i > 0 else float("-inf") + + def carried(line: dict[str, Any]) -> int: + return sum(1 for _, v in line.get("form") or [] if v in by_value) + + posts = [ + ln + for ln in lines + if ln["method"] == "POST" + and ln.get("form") is not None + and t_prev < float(ln["t"]) <= t_end + and not ln.get("failed") + ] + if not posts: + return workflow, None, ["no form request was sent by the save step"] + post = max(posts, key=carried) + if carried(post) < max(1, len(by_value) // 2): + return workflow, None, ["the save's request does not carry the recorded values"] + action = _path(post["url"]) + if any(v == nc.SECRET for _, v in post["form"]): + return workflow, None, ["the save sends a secret field; it stays on the GUI"] + + # The page that served the form: the last captured document with that form. + form_line, controls = None, None + for ln in lines: + if float(ln["t"]) >= float(post["t"]): + break + if ( + ln["method"] == "GET" + and ln.get("status") == 200 + and "<form" in (ln.get("response_text") or "") + ): + found = form_controls(ln["response_text"], action) + if found is not None: + form_line, controls = ln, found + if form_line is None or controls is None: + return ( + workflow, + None, + [ + "the page that serves the form is not in the capture; start recording before opening it" + ], + ) + form_t = float(form_line["t"]) + + fields: list[WebFormField] = [] + for name, value in post["form"]: + ctl = controls.get(name) + template = _templated(value, by_value) if value else None + if template is not None: + fields.append(WebFormField(name=name, source="param", param=template[1:-1])) + continue + if ( + value.startswith("<session:") + and ctl is not None + and ctl.get("default") == value + ): + fields.append(WebFormField(name=name, source="session")) + continue + if value and ctl is not None and ctl.get("options"): + label = next( + (o["label"] for o in ctl["options"] if o["value"] == value), None + ) + names = by_value.get(label or "") or [] + if len(names) == 1: + fields.append( + WebFormField(name=name, source="select_label", param=names[0]) + ) + continue + if ctl is not None and (ctl.get("default") or "") == value: + fields.append(WebFormField(name=name, source="form_default", value=value)) + continue + return ( + workflow, + None, + [ + f"field {name!r} of the save is not a recorded value, a form default or a session value" + ], + ) + + # Steps the API step stands in for: those recorded on the form page. + folded: list[Step] = [] + for step in steps: + i = _event_index(step) + if i is not None and i < len(events) and float(events[i]["t"]) > form_t: + folded.append(step) + if not folded or folded[-1] is not save: + return ( + workflow, + None, + ["the save is not part of the steps recorded on the form page"], + ) + for step in folded: + if step.guard is not None or step.wait_until is not None: + return ( + workflow, + None, + [f"{step.id} has a guard or wait rule; the API step can't carry it"], + ) + i = _event_index(step) + if i is not None and events[i].get("identifier_region"): + # The person marked a record-identifying region (record + # --identifier): the GUI checks which record it is on screen. + return ( + workflow, + None, + [f"{step.id} checks record identity on screen; keep the GUI program"], + ) + + # Checks the GUI program makes on screen before saving, carried as reads. + prechecks: list[WebFormPrecheck] = [] + screen_only: list[str] = [] + for step in folded[:-1]: + kinds = {getattr(p.kind, "value", p.kind) for p in step.expect} + texts = [ + p.text + for p in step.expect + if getattr(p.kind, "value", p.kind) == "text_present" and p.text + ] + i = _event_index(step) + if i is None or not step.expect: + continue + lo = float(events[i - 1]["t"]) if i > 0 else form_t + # A read the step caused can finish just after its event was stamped. + hi = float(events[i + 1]["t"]) if i + 1 < len(events) else float("inf") + reads = [ + ln + for ln in lines + if ln["method"] == "GET" + and lo < float(ln["t"]) <= hi + and ln is not post + and float(ln["t"]) < float(post["t"]) + ] + if not reads or not texts: + if reads and "text_absent" in kinds: + return ( + workflow, + None, + [ + f"{step.id} checks that text is absent after a request; the API step can't carry it" + ], + ) + screen_only.append( + f"{step.id}: " + ", ".join(sorted(str(k) for k in kinds)) + ) + continue # a screen-only change; nothing the server said + hit = next( + ( + ln + for ln in reads + if all(t in (ln.get("response_text") or "") for t in texts) + ), + None, + ) + if hit is None: + return ( + workflow, + None, + [ + f"{step.id} checks {texts!r} on screen and no response it caused carries it" + ], + ) + query = [] + for k, v in up.parse_qsl(up.urlsplit(hit["url"]).query, keep_blank_values=True): + if v.startswith("<session:"): + return ( + workflow, + None, + [f"the check at {step.id} sends a session value in its URL"], + ) + query.append((k, _templated(v, by_value) or v)) + prechecks.append( + WebFormPrecheck( + path=_path(hit["url"]), + query=query, + require_text=texts, + from_step=step.id, + ) + ) + + others = sorted( + { + f"{ln['method']} {_path(ln['url'])}" + for ln in lines + if ln["method"] not in ("GET", "HEAD", "OPTIONS") + and ln is not post + and form_t < float(ln["t"]) < float(post["t"]) + } + ) + spec = WebFormSpec( + form_path=_path(form_line["url"]) + + ( + f"?{up.urlsplit(form_line['url']).query}" + if up.urlsplit(form_line["url"]).query + else "" + ), + fields=fields, + controls=sorted(controls), + client_rules=client_rules(form_line["response_text"]), + prechecks=prechecks, + success_pattern=_success_pattern(post), + replaces_steps=[s.id for s in folded], + other_writes_seen=others, + screen_only_checks=screen_only, + folded_checks_digest=folded_checks_digest(folded[:-1]), + ) + binding = ApiBinding( + kind="web_form", + on_unavailable="gui", + method="POST", + url_template=action, + expected_status=[int(post["status"])] if post.get("status") else [], + timeout_s=30.0, + effects=list(save.effects), + web_form=spec, + ) + new_steps = [ + s.model_copy(update={"api_binding": binding}) if s is save else s for s in steps + ] + workflow = workflow.model_copy(update={"steps": new_steps}) + return workflow, new_steps[-1], [] + + +def proposed_api_steps(workflow: Workflow) -> list[Step]: + return [ + s + for s in workflow.steps + if s.api_binding is not None and s.api_binding.kind == "web_form" + ] + + +def describe_api_step(step: Step) -> str: + b = step.api_binding + assert b is not None and b.web_form is not None + spec = b.web_form + counts: dict[str, int] = {} + for f in spec.fields: + counts[f.source] = counts.get(f.source, 0) + 1 + lines = [ + f"{step.id}: save with one request, {b.method} {b.url_template}, in place of " + f"{len(spec.replaces_steps)} GUI steps ({spec.replaces_steps[0]} to {spec.replaces_steps[-1]}).", + f" sign-in: the replaying browser's own session; form read again from {spec.form_path}", + " fields: " + ", ".join(f"{n} {k}" for k, n in sorted(counts.items())), + " run values: " + + ", ".join(f"{f.name}={{{f.param}}}" for f in spec.fields if f.param), + ] + for check in spec.prechecks: + lines.append( + f" before saving: GET {check.path} must show {check.require_text!r} (from {check.from_step})" + ) + if spec.client_rules: + lines.append( + f" required by the form: {sorted(k for k, r in spec.client_rules.items() if isinstance(r, dict) and r.get('presence'))}" + ) + lines.append( + f" accepted when: HTTP {b.expected_status or '2xx/3xx'}" + + ( + f" and the response matches {spec.success_pattern!r}" + if spec.success_pattern + else "" + ) + ) + lines.append( + f" record check after: {len(b.effects)} effect(s) copied from the save step" + ) + if spec.screen_only_checks: + lines.append(f" screen-only checks, not repeated: {spec.screen_only_checks}") + if spec.other_writes_seen: + lines.append( + f" NOT repeated (other writes seen during these steps): {spec.other_writes_seen}" + ) + lines.append( + " if the form or a check differs at run time, the GUI steps run instead" + ) + return "\n".join(lines) + + +def admit_api_step( + workflow: Workflow, + *, + admitted_by: Optional[str] = None, + step_id: Optional[str] = None, + now: Optional[datetime] = None, +) -> tuple[Workflow, list[str]]: + """Record a person's admission of the proposed API steps. + + Copies the save step's current record check into the binding first (so a + check added after compile is the one admitted), then binds the admission + to the exact binding. The caller saves the workflow and asks the person. + """ + + from openadapt_flow.runtime.effects.effect import is_agent_identity + + who = (admitted_by or getpass.getuser() or "").strip() + if not who: + raise ApiProposalError("an admission needs the name of the person admitting it") + if is_agent_identity(who): + raise ApiProposalError( + f"{who!r} is not a person; an agent or model can't admit an API step" + ) + stamp = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + when = stamp.replace(microsecond=0).isoformat().replace("+00:00", "Z") + admitted: list[str] = [] + steps = [] + for step in workflow.steps: + b = step.api_binding + if ( + b is not None + and b.kind == "web_form" + and (step_id is None or step.id == step_id) + ): + if not step.effects: + raise ApiProposalError( + f"{step.id} has no record check; an API step must be confirmed by one " + "(add it, for example with record --verify or check mark, then admit)" + ) + clean = b.model_copy( + update={"effects": list(step.effects), "admission": None} + ) + b = clean.model_copy( + update={ + "admission": ApiAdmission( + admitted_by=who, + admitted_at=when, + spec_digest=clean.admission_digest(), + ) + } + ) + step = step.model_copy(update={"api_binding": b}) + admitted.append(step.id) + steps.append(step) + if not admitted: + raise ApiProposalError( + "no proposed API step to admit" + (f" on {step_id}" if step_id else "") + ) + return workflow.model_copy(update={"steps": steps}), admitted diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index 916085ffd..b3635dd0f 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -2929,6 +2929,37 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: _check.tier, ) + # A web recording made with --capture-network: propose an API step for + # the save when its request can be explained field by field. It runs only + # after a person admits it (openadapt-flow api admit). + from openadapt_flow.network_capture import NETWORK_FILE + + if (recording / NETWORK_FILE).is_file(): + from openadapt_flow.compiler.api_proposal import propose_api_step + + workflow, _api_step, _api_reasons = propose_api_step(workflow, recording) + if _api_step is not None: + logger.info( + "proposed an API step for %s (%d GUI steps); a person admits it " + "with 'api admit'", + _api_step.id, + len(_api_step.api_binding.web_form.replaces_steps), + ) + else: + logger.info("no API step proposed: %s", "; ".join(_api_reasons)) + # Say what happened either way, so "no API step" is never silent. + (bundle / "api_proposal.json").write_text( + json.dumps( + { + "proposed": _api_step is not None, + "step": None if _api_step is None else _api_step.id, + "reasons": _api_reasons, + }, + indent=2, + ) + + "\n" + ) + # Bind each consequential click on an unlabeled control to the form whose # fields the demonstration filled, or record why it cannot be bound (the # replayer refuses that click). Runs after every risk decision, model diff --git a/packages/flow/openadapt_flow/interactive_recorder.py b/packages/flow/openadapt_flow/interactive_recorder.py index a8074c63e..60fce050f 100644 --- a/packages/flow/openadapt_flow/interactive_recorder.py +++ b/packages/flow/openadapt_flow/interactive_recorder.py @@ -2203,9 +2203,14 @@ def __init__( ] = None, stop_when: Optional[Callable[[], bool]] = None, surface: str = "web", + capture_network: bool = False, ) -> None: self._url = url self._surface = surface + # ``record --capture-network``: keep the page's requests next to the + # events (openadapt_flow.network_capture). Off by default. + self._capture_network = capture_network + self._network = None self._out_dir = Path(out_dir) self._secret_fields = set(secret_fields) self._param_fields = set(param_fields) @@ -2333,6 +2338,14 @@ def start(self) -> None: ) self._context = self.page.context + if self._capture_network: + from openadapt_flow.network_capture import NetworkCapture + + # Attach before navigating, so the page that serves the form + # is in the capture. + self._network = NetworkCapture( + self._context, secret_names=self._secret_fields + ).start() if self._owns_browser: self._context.on("page", self._context_page_listener) self._context_page_latches = [ @@ -2417,6 +2430,7 @@ def start(self) -> None: self._recording_dir, app_url=self._url, system_of_record_reader=self._system_of_record_reader, + network_capture=self._network, **self._settle, ) if self._owns_browser: diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index e4117021a..f5e0f288d 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -522,6 +522,100 @@ def _validate_request_pointers(cls, pointers: list[str]) -> list[str]: return pointers +class WebFormField(BaseModel): + """One field of a recorded form save, and where its value comes from. + + ``param``: a run parameter typed into the form. ``select_label``: a select + whose option is chosen by the label in ``param`` (the value is looked up in + the live form each run). ``session``: a value the server issued to this + session, such as a CSRF token, read from the live form each run. + ``form_default``: the form's own default, which must still equal ``value`` + in the live form or the API step does not run. + """ + + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + source: Literal["param", "select_label", "session", "form_default"] + param: Optional[str] = None + value: Optional[str] = None + + @model_validator(mode="after") + def _source_fields(self) -> "WebFormField": + if self.source in ("param", "select_label") and not self.param: + raise ValueError(f"{self.source} field {self.name!r} needs a param") + if self.source == "form_default" and self.value is None: + raise ValueError(f"form_default field {self.name!r} needs its value") + return self + + +class WebFormPrecheck(BaseModel): + """A read-only request the page made before the save, and its pass rule. + + Carries over a check the GUI program made on screen before saving (for + example a duplicate search that must say "No matches were found."). The + response must contain every ``require_text`` or the API step does not + send the save. + """ + + model_config = ConfigDict(extra="forbid") + + path: str = Field(min_length=1) + query: list[tuple[str, str]] = Field(default_factory=list) + require_text: list[str] = Field(min_length=1) + from_step: Optional[str] = None + + +class WebFormSpec(BaseModel): + """How to repeat a recorded browser form save as one HTTP request. + + Compiled from the network traffic captured during ``record + --capture-network``. The request goes through the replaying browser's + own session (its cookies), so no credential is stored or needed. Before + sending, the runtime reads ``form_path`` again and refuses (falls back to + the GUI steps) if the form's fields, defaults, options or client-side + rules differ from the recording. + """ + + model_config = ConfigDict(extra="forbid") + + form_path: str = Field(min_length=1) + fields: list[WebFormField] = Field(min_length=1) + controls: list[str] = Field(default_factory=list) + client_rules: Optional[dict[str, Any]] = None + prechecks: list[WebFormPrecheck] = Field(default_factory=list) + #: Regular expression the save response must match to count as accepted. + success_pattern: Optional[str] = None + #: GUI steps the API step stands in for, in order. The last one is the save. + replaces_steps: list[str] = Field(min_length=1) + #: Other non-GET requests seen while the recorded steps ran. They are not + #: repeated; they are listed so the person admitting the step can judge. + other_writes_seen: list[str] = Field(default_factory=list) + #: On-screen checks of folded steps that caused no request (screen-only, + #: not repeated), listed for the person admitting the step. + screen_only_checks: list[str] = Field(default_factory=list) + #: Digest of the folded steps' on-screen checks when this was proposed. + #: If a check is added or changed later, the API step does not run (it + #: would skip that check) until it is proposed again. + folded_checks_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + + +class ApiAdmission(BaseModel): + """A person's acceptance of a compiler-proposed API step. + + Bound to the exact binding by :attr:`spec_digest`; a changed binding needs + a new admission. Only a person can admit (an agent, model or service name + admits nothing). + """ + + model_config = ConfigDict(extra="forbid") + + admitted_by: str = Field(min_length=1) + admitted_at: str = Field(min_length=1) + spec_digest: str = Field(pattern=r"^sha256:[0-9a-f]{64}$") + role: Literal["person"] = "person" + + class ApiBinding(BaseModel): """A declarative API/tool call that performs a step's write WITHOUT the GUI. @@ -553,9 +647,12 @@ class ApiBinding(BaseModel): caller's values) at actuation time. """ - kind: Literal["rest", "fhir", "mcp", "tool"] = Field( + kind: Literal["rest", "fhir", "mcp", "tool", "web_form"] = Field( default="rest", - description="Substrate: 'rest'/'fhir' HTTP, or an 'mcp'/'tool' call", + description=( + "Substrate: 'rest'/'fhir' HTTP, an 'mcp'/'tool' call, or " + "'web_form': a browser form save repeated through the live session" + ), ) on_unavailable: Literal["gui", "halt"] = Field( default="gui", @@ -619,13 +716,46 @@ class ApiBinding(BaseModel): ), ) + web_form: Optional[WebFormSpec] = Field( + default=None, + description="The recorded form save, for kind='web_form' only", + ) + admission: Optional[ApiAdmission] = Field( + default=None, + description=( + "A person's acceptance of a compiler-proposed binding. A " + "web_form binding never runs without one." + ), + ) + @model_validator(mode="after") def _unique_identity_bindings(self) -> "ApiBinding": keys = [binding.key for binding in self.identity] if len(keys) != len(set(keys)): raise ValueError("API identity semantic keys must be unique") + if (self.kind == "web_form") != (self.web_form is not None): + raise ValueError("web_form is required for, and only for, kind='web_form'") return self + def admission_digest(self) -> str: + """SHA-256 of what a person admits: the binding without its admission.""" + + payload = self.model_copy(update={"admission": None}).model_dump(mode="json") + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def admitted(self) -> bool: + """True when a person admitted this exact binding.""" + + from openadapt_flow.runtime.effects.effect import is_agent_identity + + return ( + self.admission is not None + and self.admission.role == "person" + and not is_agent_identity(self.admission.admitted_by) + and self.admission.spec_digest == self.admission_digest() + ) + # -- Workflow-program IR, Phase 1 (RFC docs/design/WORKFLOW_PROGRAM_IR.md §6) -- # @@ -3151,6 +3281,9 @@ class StepResult(BaseModel): # ``guard`` was unmet with ``on_unmet="skip"`` (a no-op success -- the # step did not act). False for every executed step; additive. skipped: bool = False + # API steps (docs/API_TIER.md): on the first step an API step stands in + # for, why it did not run, so the GUI steps ran instead. None otherwise. + api_fallback: Optional[str] = None # Workflow-program IR, Phase 2: True when this (failed) action state was # routed to its ``State.on_exception`` handler instead of aborting the run # (the graph analog of a caught try/except). The result stays ``ok=False`` diff --git a/packages/flow/openadapt_flow/network_capture.py b/packages/flow/openadapt_flow/network_capture.py new file mode 100644 index 000000000..5f98bca52 --- /dev/null +++ b/packages/flow/openadapt_flow/network_capture.py @@ -0,0 +1,302 @@ +"""Capture a web recording's network traffic (``record --capture-network``). + +Opt-in. While a browser recording runs, :class:`NetworkCapture` keeps one +line per finished page request: method, URL, form or text body, status, +content type, redirect target and, for HTML, JSON and text, the response +body. ``compile`` reads the file (``network.jsonl``) and may propose an API +step for a save whose requests it can explain +(:mod:`openadapt_flow.compiler.api_proposal`). + +What is never written: + +- headers, so no cookie or authorization value. Cookie values the browser + sent are also withheld where a page echoes them in a body or script + (``<session:cookie>``); +- values of password fields and of the recording's declared secret fields, + in bodies, URLs and response text (replaced by ``<secret>``); +- session tokens. A body or query field whose name looks like a CSRF or + session token (``csrf``, ``xsrf``, ``token``, ``nonce``, ``authenticity``) + has its value replaced by ``<session:NAME>`` everywhere it appears, + including the HTML that carried it. The API step reads a fresh token from + the live page each run. + +Patient or other business values the person typed stay in the file, as they +do in ``events.jsonl``. A recording made with capture is as sensitive as any +other recording. + +Static assets (images, fonts, style sheets, scripts, media) are skipped. +Request and response bodies are cut at :data:`MAX_REQUEST_BYTES` and +:data:`MAX_RESPONSE_BYTES`; a cut body is marked ``truncated``. +""" + +from __future__ import annotations + +import json +import re +import threading +import time +import urllib.parse as up +from pathlib import Path +from typing import Any, Iterable, Optional + +NETWORK_FILE = "network.jsonl" +SCHEMA = "openadapt.network-capture/1" + +MAX_REQUEST_BYTES = 256 * 1024 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 + +_SKIP_TYPES = frozenset( + {"image", "font", "stylesheet", "script", "media", "manifest", "texttrack"} +) +_TEXT_MIME = re.compile( + r"text/html|application/json|text/plain|application/xhtml|\+json", re.I +) +TOKEN_NAME = re.compile(r"csrf|xsrf|token|nonce|authenticity", re.I) +SECRET_NAME = re.compile(r"pass(word|wd)?\b|clearpass|pwd|secret", re.I) +SECRET = "<secret>" + + +def session_placeholder(name: str) -> str: + return f"<session:{name}>" + + +class NetworkCapture: + """Record a Playwright browser context's requests for one recording. + + Start it before the browser opens the form, so the form page itself is + captured. Each line is stamped on the monotonic clock; :meth:`write` + converts that to the recorder's ``t`` (seconds since the recording + started), so lines and events share one timeline. Lines from before the + recording started have a negative ``t``. + + ``secret_names`` are field names whose values are withheld in addition + to password-like names (the recorder's declared secret fields). + """ + + def __init__(self, context: Any, *, secret_names: Iterable[str] = ()) -> None: + self._context = context + self._secret_names = frozenset(secret_names) + # Cookie values the browser sent, kept in memory only so they can be + # withheld wherever a page echoes one. Never written. + self._cookie_values: set[str] = set() + self._lines: list[dict[str, Any]] = [] + self._lock = threading.Lock() + self._attached = False + + def start(self) -> "NetworkCapture": + if not self._attached: + self._context.on("requestfinished", self._on_finished) + self._context.on("requestfailed", self._on_failed) + self._attached = True + return self + + def stop(self) -> None: + if self._attached: + for event, handler in ( + ("requestfinished", self._on_finished), + ("requestfailed", self._on_failed), + ): + try: + self._context.remove_listener(event, handler) + except Exception: # noqa: BLE001 - a closed context has no listeners + pass + self._attached = False + + @property + def count(self) -> int: + with self._lock: + return len(self._lines) + + # -- Playwright callbacks ------------------------------------------------ + + def _on_failed(self, request: Any) -> None: + if request.resource_type in _SKIP_TYPES: + return + self._append(self._request_line(request) | {"status": None, "failed": True}) + + def _on_finished(self, request: Any) -> None: + if request.resource_type in _SKIP_TYPES: + return + line = self._request_line(request) + try: + response = request.response() + except Exception: # noqa: BLE001 - the page may already be gone + response = None + if response is not None: + headers = {k.lower(): v for k, v in (response.headers or {}).items()} + line["status"] = response.status + line["response_type"] = ( + (headers.get("content-type") or "").split(";")[0].strip() + ) + if headers.get("location"): + line["location"] = headers["location"] + if _TEXT_MIME.search(line["response_type"]): + try: + body = response.body() + except Exception: # noqa: BLE001 - redirect or evicted body + body = b"" + if len(body) > MAX_RESPONSE_BYTES: + line["response_truncated"] = True + body = body[:MAX_RESPONSE_BYTES] + line["response_text"] = body.decode("utf-8", errors="replace") + self._append(line) + + def _request_line(self, request: Any) -> dict[str, Any]: + try: + headers = {k.lower(): v for k, v in request.all_headers().items()} + except Exception: # noqa: BLE001 - the request is gone + headers = {k.lower(): v for k, v in (request.headers or {}).items()} + for part in (headers.get("cookie") or "").split(";"): + value = part.partition("=")[2].strip() + if len(value) >= 8: + self._cookie_values.add(value) + line: dict[str, Any] = { + "_mono": time.monotonic(), + "method": request.method, + "url": request.url, + "resource_type": request.resource_type, + "frame_url": _frame_url(request), + } + content_type = (headers.get("content-type") or "").split(";")[0].strip() + if content_type: + line["request_type"] = content_type + try: + raw = request.post_data_buffer + except Exception: # noqa: BLE001 - binary or unavailable body + raw = None + if raw: + if len(raw) > MAX_REQUEST_BYTES: + line["request_truncated"] = True + raw = raw[:MAX_REQUEST_BYTES] + text = raw.decode("utf-8", errors="replace") + if content_type == "application/x-www-form-urlencoded": + line["form"] = [ + list(p) for p in up.parse_qsl(text, keep_blank_values=True) + ] + elif content_type.startswith("multipart/"): + line["body_omitted"] = "multipart" + else: + line["body"] = text + return line + + def _append(self, line: dict[str, Any]) -> None: + with self._lock: + line["seq"] = len(self._lines) + self._lines.append(line) + + # -- writing --------------------------------------------------------------- + + def lines(self, t0_monotonic: float) -> list[dict[str, Any]]: + """The captured lines, timed from ``t0_monotonic``, with secrets and + session tokens withheld.""" + + with self._lock: + lines = [dict(line) for line in self._lines] + for line in lines: + line["t"] = round(line.pop("_mono") - t0_monotonic, 3) + return redact( + lines, + secret_names=self._secret_names, + session_values=set(self._cookie_values), + ) + + def write(self, recording_dir: Path | str, t0_monotonic: float) -> Path: + path = Path(recording_dir) / NETWORK_FILE + lines = self.lines(t0_monotonic) + with path.open("w", encoding="utf-8") as fh: + fh.write(json.dumps({"schema": SCHEMA, "exchanges": len(lines)}) + "\n") + for line in lines: + fh.write(json.dumps(line, sort_keys=True) + "\n") + return path + + +def _frame_url(request: Any) -> Optional[str]: + try: + return request.frame.url + except Exception: # noqa: BLE001 - service worker or detached frame + return None + + +def redact( + lines: list[dict[str, Any]], + *, + secret_names: Iterable[str] = (), + session_values: Iterable[str] = (), +) -> list[dict[str, Any]]: + """Withhold secret values and session tokens from captured lines. + + Values are found by field name in request bodies and queries, then + replaced wherever the same value appears (other requests, URLs, response + text): password-like and declared secret names become ``<secret>``, + token-like names become ``<session:NAME>``. + """ + + declared = {n.lower() for n in secret_names} + replace: dict[str, str] = { + v: session_placeholder("cookie") for v in session_values if v + } + for line in lines: + pairs = list(line.get("form") or []) + up.parse_qsl( + up.urlsplit(line["url"]).query, keep_blank_values=True + ) + for name, value in pairs: + if not value or len(value) < 4: + continue + if SECRET_NAME.search(name) or name.lower() in declared: + replace[value] = SECRET + elif TOKEN_NAME.search(name) and len(value) >= 8: + replace.setdefault(value, session_placeholder(name)) + if not replace: + return lines + # Longest first, so a value inside another is not replaced half-way. + order = sorted(replace, key=len, reverse=True) + pattern = re.compile("|".join(re.escape(v) for v in order)) + encoded = {up.quote_plus(v): replace[v] for v in order if up.quote_plus(v) != v} + enc_pattern = ( + re.compile( + "|".join(re.escape(v) for v in sorted(encoded, key=len, reverse=True)) + ) + if encoded + else None + ) + + def scrub(text: str) -> str: + text = pattern.sub(lambda m: replace[m.group(0)], text) + if enc_pattern is not None: + text = enc_pattern.sub(lambda m: encoded[m.group(0)], text) + return text + + out = [] + for line in lines: + line = dict(line) + for key in ("url", "frame_url", "location", "body", "response_text"): + if isinstance(line.get(key), str): + line[key] = scrub(line[key]) + if line.get("form"): + line["form"] = [ + [ + name, + SECRET + if (SECRET_NAME.search(name) or name.lower() in declared) and value + else replace.get(value, scrub(value)), + ] + for name, value in line["form"] + ] + out.append(line) + return out + + +def load(recording_dir: Path | str) -> list[dict[str, Any]]: + """Read ``network.jsonl``; an absent file is an empty capture.""" + + path = Path(recording_dir) / NETWORK_FILE + if not path.is_file(): + return [] + rows = [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + if not rows or rows[0].get("schema") != SCHEMA: + raise ValueError(f"{path} is not a {SCHEMA} capture") + return rows[1:] diff --git a/packages/flow/openadapt_flow/record_check_binding.py b/packages/flow/openadapt_flow/record_check_binding.py index b35173048..30dfe2d5f 100644 --- a/packages/flow/openadapt_flow/record_check_binding.py +++ b/packages/flow/openadapt_flow/record_check_binding.py @@ -407,7 +407,10 @@ def _validate( f"step {sid!r} intent is {target.intent!r}, binding expects " f"{step_binding.intent!r}" ) - if target.api_binding is not None: + if target.api_binding is not None and target.api_binding.kind != "web_form": + # A web_form API step takes the save step's record check when a + # person admits it (openadapt-flow api admit), so binding here is + # the way to give it one. problems.append( f"step {sid!r} has an API binding; its API-path contracts are " "bound with that binding, not here" diff --git a/packages/flow/openadapt_flow/recorder.py b/packages/flow/openadapt_flow/recorder.py index 43f5f3f77..5d3307d7d 100644 --- a/packages/flow/openadapt_flow/recorder.py +++ b/packages/flow/openadapt_flow/recorder.py @@ -92,8 +92,13 @@ def __init__( settle_interval_s: float = 0.1, settle_stable_frames: int = 2, settle_timeout_s: float = 3.0, + network_capture: Optional[Any] = None, ) -> None: self._backend = backend + # Opt-in ``record --capture-network``: a started + # :class:`openadapt_flow.network_capture.NetworkCapture`, written beside + # the events by :meth:`finish`. + self._network_capture = network_capture self._dir = Path(out_dir) self._frames_dir = self._dir / "frames" self._frames_dir.mkdir(parents=True, exist_ok=True) @@ -222,6 +227,15 @@ def finish(self) -> Path: "params": dict(self._params), "secret_params": sorted(self._secret_params), } + if self._network_capture is not None: + from openadapt_flow.network_capture import NETWORK_FILE + + self._network_capture.stop() + self._network_capture.write(self._dir, self._t0) + meta["network_capture"] = { + "file": NETWORK_FILE, + "exchanges": self._network_capture.count, + } (self._dir / "meta.json").write_text(json.dumps(meta, indent=2)) return self._dir diff --git a/packages/flow/openadapt_flow/runtime/actuators/web_form.py b/packages/flow/openadapt_flow/runtime/actuators/web_form.py new file mode 100644 index 000000000..2c0285b6f --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/actuators/web_form.py @@ -0,0 +1,435 @@ +"""Repeat a recorded browser form save as one request: :class:`WebFormActuator`. + +The ``web_form`` leaf of the API tier. ``compile`` proposes it from a +recording made with ``record --capture-network`` +(:mod:`openadapt_flow.compiler.api_proposal`); a person admits it +(``openadapt-flow api admit``); the replayer then runs it in place of the GUI +steps it stands for. + +Auth comes from the live session. Requests go through the replaying +browser's own request context, which shares its cookies, so the API step is +signed in exactly when the browser is. No credential is stored in the bundle +or read by this module. + +Every run, before anything that writes is sent: + +1. read the form page again, in this session, for the current CSRF or + session value, defaults and options; +2. refuse if the form's fields or its client-side validation rules differ + from the recording (the page changed, so the recorded request may no + longer be what the page would send); +3. build the body: run values for recorded parameters, select values looked + up by label in the live form, the live session value, and the form's + defaults (each must still equal the recorded default); refuse if a field + the page requires would be empty; +4. run the read-only pre-checks the GUI program made on screen (for example + a duplicate search that must report no match), and refuse if a required + text is missing. + +A refusal at any of these is ``UNAVAILABLE``: the save was not sent, so the +replayer falls back to the GUI steps. After the save request starts, the +no-double-write rule of :mod:`openadapt_flow.runtime.actuators.api` applies: +any error or unaccepted response is ``HALT`` and is never retried or redone +through the GUI. The replayer then confirms the record with the same +``EffectVerifier`` as a GUI save. +""" + +from __future__ import annotations + +import json +import re +import urllib.parse as up +from html.parser import HTMLParser +from typing import Any, Optional, Protocol + +from openadapt_flow.runtime.actuators.api import ( + ActuationStatus, + ApiActuationResult, + ApiHaltKind, +) + +# -- HTML forms ------------------------------------------------------------------ + + +class _FormParser(HTMLParser): + """Named controls of each <form>, with defaults and select options.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.forms: list[dict[str, Any]] = [] + self._form: Optional[dict[str, Any]] = None + self._nested = 0 + self._select: Optional[dict[str, Any]] = None + self._option: Optional[dict[str, Any]] = None + self._textarea: Optional[dict[str, Any]] = None + + def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: + a = {k: (v if v is not None else "") for k, v in attrs} + if tag == "form": + if self._form is not None: + # HTML parsing ignores a <form> inside an open form; its + # controls belong to, and are posted with, the outer form. + self._nested += 1 + return + self._form = {"action": a.get("action", ""), "controls": {}} + self.forms.append(self._form) + return + if self._form is None: + return + name = a.get("name") + if tag == "input" and name: + kind = a.get("type", "text").lower() + if kind in ("submit", "button", "image", "reset", "file"): + return + if kind in ("checkbox", "radio") and "checked" not in a: + value: Optional[str] = None + else: + value = a.get("value", "on" if kind in ("checkbox", "radio") else "") + self._form["controls"].setdefault( + name, {"kind": kind, "default": value, "options": None} + ) + elif tag == "select" and name: + self._select = { + "kind": "select", + "default": None, + "options": [], + "multiple": "multiple" in a, + } + self._form["controls"].setdefault(name, self._select) + elif tag == "option" and self._select is not None: + self._option = { + "value": a.get("value"), + "label": "", + "selected": "selected" in a, + } + self._select["options"].append(self._option) + elif tag == "textarea" and name: + self._textarea = {"kind": "textarea", "default": "", "options": None} + self._form["controls"].setdefault(name, self._textarea) + + def handle_data(self, data: str) -> None: + if self._option is not None: + self._option["label"] += data + elif self._textarea is not None: + self._textarea["default"] += data + + def handle_endtag(self, tag: str) -> None: + if tag == "option" and self._option is not None: + self._option["label"] = self._option["label"].strip() + if self._option["value"] is None: + self._option["value"] = self._option["label"] + self._option = None + elif tag == "select" and self._select is not None: + opts = self._select["options"] + chosen = [o for o in opts if o["selected"]] or ( + [] if self._select["multiple"] else opts[:1] + ) + self._select["default"] = chosen[0]["value"] if chosen else "" + self._select = None + elif tag == "textarea": + self._textarea = None + elif tag == "form": + if self._nested: + self._nested -= 1 + else: + self._form = None + + +def form_controls(html: str, action_path: str) -> Optional[dict[str, Any]]: + """The named controls of the form in ``html`` that posts to ``action_path``.""" + + parser = _FormParser() + parser.feed(html) + leaf = up.urlsplit(action_path).path.rsplit("/", 1)[-1] + for form in parser.forms: + action = up.urlsplit(form["action"]).path + if action == action_path or (leaf and action.rsplit("/", 1)[-1] == leaf): + return form["controls"] + return None + + +_RULES = re.compile(r"var constraints = (\{.*?\});") + + +def client_rules(html: str) -> Optional[dict[str, Any]]: + """The page's client-side validation rules, when it declares them. + + Recognizes the ``var constraints = {...}`` object that validate.js-style + pages (OpenEMR among them) emit. The browser enforces these before it + submits; a direct request skips them, so the API step records them, + refuses when they change, and enforces their ``presence`` rules. + """ + + m = _RULES.search(html) + if not m: + return None + try: + return json.loads(m.group(1)) + except ValueError: + return None + + +def option_for_label(control: dict[str, Any], label: str) -> Optional[str]: + hits = [o["value"] for o in (control.get("options") or []) if o["label"] == label] + return hits[0] if len(hits) == 1 else None + + +def folded_checks_digest(steps: Any) -> str: + """SHA-256 over the on-screen checks (``expect``) of the folded steps.""" + + import hashlib + + payload = [[s.id, [p.model_dump(mode="json") for p in s.expect]] for s in steps] + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +# -- transport ------------------------------------------------------------------- + + +class Response(Protocol): + status: int + text: str + headers: dict[str, str] + + +class _Resp: + def __init__(self, status: int, text: str, headers: dict[str, str]) -> None: + self.status, self.text, self.headers = status, text, headers + + +class Transport(Protocol): + """Sends requests in the live session. ``base_url`` is the app origin.""" + + base_url: str + + def get(self, path_and_query: str) -> Response: ... + + def post_form(self, path: str, body: str, *, referer: str) -> Response: ... + + +class PlaywrightSessionTransport: + """The browser context's own request API: same cookies, same session.""" + + def __init__(self, page: Any, *, timeout_s: float = 30.0) -> None: + parts = up.urlsplit(page.url) + self.base_url = f"{parts.scheme}://{parts.netloc}" + self._request = page.context.request + self._timeout_ms = timeout_s * 1000.0 + + def _wrap(self, resp: Any) -> Response: + return _Resp( + resp.status, resp.text(), {k.lower(): v for k, v in resp.headers.items()} + ) + + def get(self, path_and_query: str) -> Response: + return self._wrap( + self._request.get( + self.base_url + path_and_query, + timeout=self._timeout_ms, + max_redirects=0, + ) + ) + + def post_form(self, path: str, body: str, *, referer: str) -> Response: + return self._wrap( + self._request.post( + self.base_url + path, + data=body, + headers={ + "Content-Type": "application/x-www-form-urlencoded", + "Referer": referer, + "Origin": self.base_url, + }, + timeout=self._timeout_ms, + max_redirects=0, + ) + ) + + +# -- actuator -------------------------------------------------------------------- + + +class WebFormActuator: + """Run an admitted ``web_form`` binding through a live-session transport.""" + + substrate = "web_form" + + def __init__(self, transport: Transport) -> None: + self.transport = transport + #: One line per stage, for the run report and diagnostics. Never holds + #: body values. + self.log: list[dict[str, Any]] = [] + + def actuate(self, binding: Any, params: dict[str, str]) -> ApiActuationResult: + spec = binding.web_form + summary = f"{binding.method} {binding.url_template}" + + def unavailable(reason: str) -> ApiActuationResult: + self.log.append({"stage": "refused", "reason": reason}) + return ApiActuationResult( + status=ActuationStatus.UNAVAILABLE, + substrate=self.substrate, + reason=f"{reason}; the save was not sent", + request_summary=summary, + ) + + if spec is None: + return unavailable("the binding has no recorded form") + try: + page = self.transport.get(spec.form_path) + self.log.append({"stage": "read_form", "status": page.status}) + if page.status != 200: + return unavailable( + f"the form page returned HTTP {page.status} (signed out?)" + ) + controls = form_controls(page.text, binding.url_template) + if controls is None: + return unavailable("the form page no longer has the recorded form") + if spec.controls and sorted(controls) != sorted(spec.controls): + added = sorted(set(controls) - set(spec.controls))[:5] + removed = sorted(set(spec.controls) - set(controls))[:5] + return unavailable( + f"the form changed since the recording (added {added}, removed {removed})" + ) + if client_rules(page.text) != spec.client_rules: + return unavailable( + "the form's validation rules changed since the recording" + ) + body, problem = self.build_body(spec, params, controls) + if problem: + return unavailable(problem) + for check in spec.prechecks: + query = [(k, _fill(v, params)) for k, v in check.query] + qs = "&".join( + up.quote(k) + ("=" + up.quote(v) if v else "") for k, v in query + ) + resp = self.transport.get(check.path + (f"?{qs}" if qs else "")) + missing = [t for t in check.require_text if t not in resp.text] + self.log.append( + { + "stage": "precheck", + "path": check.path, + "status": resp.status, + "missing": missing, + } + ) + if resp.status != 200 or missing: + return unavailable( + f"the check before saving ({check.from_step or check.path}) did not pass: " + f"missing {missing!r}" + ) + except KeyError as exc: + return unavailable(f"the run has no value for {exc}") + except Exception as exc: # noqa: BLE001 - only reads were sent + return unavailable(f"a read before the save failed ({type(exc).__name__})") + + encoded = up.urlencode(body) + try: + resp = self.transport.post_form( + binding.url_template, + encoded, + referer=self.transport.base_url + spec.form_path, + ) + except Exception as exc: # noqa: BLE001 - after dispatch began + self.log.append({"stage": "save", "error": type(exc).__name__}) + return ApiActuationResult( + status=ActuationStatus.HALT, + substrate=self.substrate, + halt_kind=ApiHaltKind.DELIVERY_UNCERTAIN, + reason=( + f"the save request failed after it was sent ({type(exc).__name__}); " + "it may have saved, so it is not retried" + ), + request_summary=summary, + ) + accepted = resp.status in (binding.expected_status or list(range(200, 400))) + signature = True + if spec.success_pattern: + target = (resp.headers.get("location") or "") + "\n" + resp.text + signature = re.search(spec.success_pattern, target) is not None + self.log.append( + { + "stage": "save", + "status": resp.status, + "accepted": accepted, + "signature": signature, + } + ) + if accepted and signature: + return ApiActuationResult( + status=ActuationStatus.ACTUATED, + substrate=self.substrate, + reason=f"{summary} -> {resp.status}", + http_status=resp.status, + request_summary=summary, + ) + return ApiActuationResult( + status=ActuationStatus.HALT, + substrate=self.substrate, + halt_kind=ApiHaltKind.RESPONSE_REJECTED, + reason=( + f"{summary} -> {resp.status} without the recorded success response; " + "it is not retried" + ), + http_status=resp.status, + request_summary=summary, + ) + + @staticmethod + def build_body( + spec: Any, + params: dict[str, str], + controls: dict[str, Any], + *, + enforce_rules: bool = True, + ) -> tuple[list[tuple[str, str]], Optional[str]]: + body: list[tuple[str, str]] = [] + for field in spec.fields: + control = controls.get(field.name) or {} + if field.source == "param": + body.append((field.name, str(params[field.param]))) + elif field.source == "select_label": + value = option_for_label(control, str(params[field.param])) + if value is None: + return ( + [], + f"{field.name}: the form has no single option {params[field.param]!r}", + ) + body.append((field.name, value)) + elif field.source == "session": + value = control.get("default") or "" + if not value: + return [], f"{field.name}: the live form carries no session value" + body.append((field.name, value)) + else: + if (control.get("default") or "") != field.value: + return ( + [], + f"{field.name}: the form's default changed since the recording", + ) + body.append((field.name, field.value or "")) + if enforce_rules: + sent = dict(body) + for name, rule in (spec.client_rules or {}).items(): + if ( + isinstance(rule, dict) + and rule.get("presence") + and not sent.get(name) + ): + return [], f"{name} is required by the form and would be empty" + return body, None + + +class _Missing(KeyError): + pass + + +def _fill(template: str, params: dict[str, str]) -> str: + def sub(m: re.Match[str]) -> str: + key = m.group(1) + if key not in params: + raise _Missing(key) + return str(params[key]) + + return re.sub(r"\{([A-Za-z_][A-Za-z0-9_]*)\}", sub, template) diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 1412238e0..256b697bd 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -707,6 +707,8 @@ def __init__( effect_verifier: Optional[EffectVerifier] = None, effect_compensator: Optional[Any] = None, api_actuator: Optional[Any] = None, + web_form_actuator: Optional[Any] = None, + api_steps: bool = True, durable: bool = False, checkpoint_key: Optional[str] = None, poll_interval_s: float = 0.05, @@ -930,6 +932,16 @@ def __init__( # step falls through to the GUI ladder unchanged. Deterministic; makes # no model call (the $0 runtime guarantee holds on this path too). self.api_actuator = api_actuator + # API steps compiled from a recording's network capture (web_form + # bindings, docs/API_TIER.md). An admitted one runs in place of the + # GUI steps it stands for, through the live browser session; when a + # precondition fails before the save is sent, those GUI steps run. + # ``api_steps=False`` keeps every run on the GUI. ``web_form_actuator`` + # replaces the live-session actuator (tests, other transports). + self.api_steps = api_steps + self.web_form_actuator = web_form_actuator + self._web_form_span_active = False + self._web_form_fallback_reason: Optional[str] = None # Durable tiered runtime (RFC §5, Tier 3), OFF by default. When True, # run() writes a RunCheckpoint after each VERIFIED step and, on a halt, # a PendingEscalation instead of just dying -- so the run can be @@ -1993,67 +2005,67 @@ def acknowledge_delivery(self_nonlocal) -> None: resume_checkpoint=resume_program, ) else: + span_skip_to = -1 + stop_run = False for step_index, step in enumerate(workflow.steps): # Resume: never re-execute an already-verified step (no # re-performed confirmed writes); its result was pre-loaded. if resume_from is not None and step_index < resume_from: continue - result = self._run_step( - step, - workflow=workflow, - step_index=step_index, - params=params, - bundle_dir=bundle_dir, - run_dir=run_dir, - new_crops=new_crops, + if step_index <= span_skip_to: + continue + # An admitted API step (docs/API_TIER.md) runs in place of the + # GUI steps it stands for. When it can't, the reason lands on + # the first of those steps and the GUI steps run as recorded. + batch: Optional[list[StepResult]] = None + why_not: Optional[str] = None + span = self._web_form_span_at( + workflow, + step_index, + durable=durable_run is not None or resume_from is not None, ) - post_action_refusal = self._workflow_snapshot_refusal(workflow) - if post_action_refusal is None: - post_action_refusal = ( - self._governed_authorization_snapshot_refusal() - ) - if post_action_refusal is not None: - prior_error = ( - f"; earlier result: {result.error}" if result.error else "" - ) - result.ok = False - result.safety_halt = True - result.failure_category = ( - "governed_refusal" - if self.governed_authorization is not None - else "safety_halt" - ) - result.error = ( - f"{post_action_refusal}; the action result was checked, " - f"but the workflow changed during delivery{prior_error}" - ) - report.results.append(result) - self._account_result(report, result) - if durable_run is not None: - self._sync_durable_audit(durable_run, report) - # Tier-3: verified step -> checkpoint; halt -> pending - # escalation (resumable from the last checkpoint). - if post_action_refusal is None: - durable_run.record( + if span is not None: + end_index, _save_step, why_not = span + if why_not is None: + batch = self._run_web_form_span( + workflow, step_index, - step, - result, - params, - workflow=workflow, - transition_observation=( - self._attended_transition_observation() - if not result.ok - else None - ), + end_index, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + new_crops=new_crops, ) - if ( - self._durable_continuation_guard is not None - and result.failure_category != "continuation_preempted" - ): - self._durable_continuation_guard.acknowledge_progress( - terminal=not result.ok - ) - if not result.ok: + if batch is None: + why_not = self._web_form_fallback_reason + else: + span_skip_to = end_index + if batch is None: + result = self._run_step( + step, + workflow=workflow, + step_index=step_index, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + new_crops=new_crops, + ) + if why_not is not None: + result.api_fallback = why_not + batch = [result] + for result in batch: + stop_run = self._record_linear_result( + report, + workflow, + result, + step_index=step_index, + step=step, + params=params, + durable_run=durable_run, + ) + if stop_run: + break + if stop_run: break report.success = len(report.results) == len(workflow.steps) and all( @@ -2161,6 +2173,181 @@ def acknowledge_delivery(self_nonlocal) -> None: ) return finalized + def _record_linear_result( + self, + report: RunReport, + workflow: Workflow, + result: StepResult, + *, + step_index: int, + step: Step, + params: Mapping[str, RuntimeParamScalar], + durable_run: Optional[Any], + ) -> bool: + """Account one linear-loop result; True when the run must stop.""" + + post_action_refusal = self._workflow_snapshot_refusal(workflow) + if post_action_refusal is None: + post_action_refusal = self._governed_authorization_snapshot_refusal() + if post_action_refusal is not None: + prior_error = f"; earlier result: {result.error}" if result.error else "" + result.ok = False + result.safety_halt = True + result.failure_category = ( + "governed_refusal" + if self.governed_authorization is not None + else "safety_halt" + ) + result.error = ( + f"{post_action_refusal}; the action result was checked, " + f"but the workflow changed during delivery{prior_error}" + ) + report.results.append(result) + self._account_result(report, result) + if durable_run is not None: + self._sync_durable_audit(durable_run, report) + # Tier-3: verified step -> checkpoint; halt -> pending + # escalation (resumable from the last checkpoint). + if post_action_refusal is None: + durable_run.record( + step_index, + step, + result, + params, + workflow=workflow, + transition_observation=( + self._attended_transition_observation() + if not result.ok + else None + ), + ) + if ( + self._durable_continuation_guard is not None + and result.failure_category != "continuation_preempted" + ): + self._durable_continuation_guard.acknowledge_progress( + terminal=not result.ok + ) + return not result.ok + + def _web_form_span_at( + self, workflow: Workflow, step_index: int, *, durable: bool + ) -> Optional[tuple[int, Step, Optional[str]]]: + """The API step whose span starts at ``step_index``, and why it can't run. + + Returns ``(save_index, save_step, reason)``; ``reason`` is None when + the API step may run. None when no API step starts here. + """ + + from openadapt_flow.runtime.actuators.web_form import folded_checks_digest + + first = workflow.steps[step_index].id + for end_index in range(step_index, len(workflow.steps)): + save = workflow.steps[end_index] + binding = save.api_binding + if ( + binding is None + or binding.kind != "web_form" + or binding.web_form is None + or binding.web_form.replaces_steps[0] != first + ): + continue + ids = [s.id for s in workflow.steps[step_index : end_index + 1]] + if not self.api_steps: + reason = "API steps are off for this run" + elif durable: + reason = "a durable or resumed run uses the GUI steps" + elif ( + self.approval_gate is not None + and str(getattr(getattr(self.approval_gate, "mode", None), "value", "")) + != "not_gated" + ): + # A person approves what the screen shows; an API step would + # leave the form empty. Only an ungated run uses API steps. + reason = ( + "approval mode shows the person the screen before the save, " + "so the GUI steps run" + ) + elif self.governed_authorization is not None: + reason = "a governed run uses its qualified GUI path" + elif not binding.admitted(): + reason = "the API step is not admitted (openadapt-flow api admit)" + elif list(binding.effects) != list(save.effects): + reason = ( + "the save step's record check differs from the one admitted " + "with the API step (openadapt-flow api admit)" + ) + elif ids != list(binding.web_form.replaces_steps): + reason = "the program's steps no longer match the API step" + elif ( + folded_checks_digest(workflow.steps[step_index:end_index]) + != binding.web_form.folded_checks_digest + ): + reason = ( + "an on-screen check of the GUI steps changed after the API " + "step was proposed, and the API step would skip it " + "(openadapt-flow api propose)" + ) + elif self._actuator_for(binding) is None: + reason = "there is no live browser session for the API step" + else: + reason = None + return end_index, save, reason + return None + + def _run_web_form_span( + self, + workflow: Workflow, + start_index: int, + end_index: int, + *, + params: Mapping[str, RuntimeParamScalar], + bundle_dir: Path, + run_dir: Path, + new_crops: dict[str, bytes], + ) -> Optional[list[StepResult]]: + """Run the API step for steps ``start_index..end_index``. + + Returns the results (the folded GUI steps, then the save) when the API + step took responsibility: it saved and the record check ran, or it + stopped after sending. Returns None when the save was not sent, with + the reason in ``self._web_form_fallback_reason``; the caller then runs + the GUI steps. + """ + + save = workflow.steps[end_index] + self._web_form_span_active = True + try: + result = self._run_step( + save, + workflow=workflow, + step_index=end_index, + params=params, + bundle_dir=bundle_dir, + run_dir=run_dir, + new_crops=new_crops, + ) + finally: + self._web_form_span_active = False + if result.api_fallback is not None: + self._web_form_fallback_reason = result.api_fallback + return None + folded = [ + StepResult( + step_id=s.id, + intent=s.intent, + ok=True, + skipped=True, + actuation="api_folded", + risk=s.risk, + risk_explanation=s.risk_explanation, + risk_review_required=s.risk_review_required, + delivery_attempted=False, + ) + for s in workflow.steps[start_index:end_index] + ] + return [*folded, result] + def _lookup_capability_refusal( self, workflow: Workflow, @@ -6038,12 +6225,27 @@ def _run_step( result.error = approval_refusal result.elapsed_ms = (time.monotonic() - t0) * 1000.0 return result + # A web_form binding (compiled from a recording's network capture) + # acts only when the linear loop runs it in place of the GUI steps it + # stands for. On the GUI path its save step is an ordinary click. + step_api_binding = step.api_binding + if ( + step_api_binding is not None + and step_api_binding.kind == "web_form" + and not self._web_form_span_active + ): + step_api_binding = None + step_api_actuator = ( + self._actuator_for(step_api_binding) + if step_api_binding is not None + else None + ) if ( qualification_path != "gui" - and step.api_binding is not None - and self.api_actuator is None + and step_api_binding is not None + and step_api_actuator is None and ( - qualification_path == "api" or step.api_binding.on_unavailable == "halt" + qualification_path == "api" or step_api_binding.on_unavailable == "halt" ) ): self._set_api_unavailable_refusal( @@ -6067,11 +6269,11 @@ def _run_step( api_fault_mutation_start = len(self._qualification_fault_mutations) if ( qualification_path != "gui" - and self.api_actuator is not None - and step.api_binding is not None - and step.api_binding.effects + and step_api_actuator is not None + and step_api_binding is not None + and step_api_binding.effects ): - api_effects = tuple(self._resolve_effects(step.api_binding.effects, params)) + api_effects = tuple(self._resolve_effects(step_api_binding.effects, params)) api_effect_verifier = self._apply_effect_fault_mutation( gate="effect_verifier", step=step, @@ -6088,8 +6290,8 @@ def _run_step( ) if ( qualification_path != "gui" - and self.api_actuator is not None - and step.api_binding is not None + and step_api_actuator is not None + and step_api_binding is not None ): if self._try_api_tier( step, @@ -6132,6 +6334,19 @@ def _run_step( result.elapsed_ms = (time.monotonic() - t0) * 1000.0 return result + if self._web_form_span_active: + # The API step did not send the save (a precondition failed or it + # is unavailable). Report why and act on nothing: the linear loop + # runs the GUI steps from the start of the span instead. + reasons = [r for r in result.effect_results if r.startswith("[api]")] + result.api_fallback = ( + reasons[-1][len("[api] ") :] + if reasons + else "the API step is unavailable" + ) + result.elapsed_ms = (time.monotonic() - t0) * 1000.0 + return result + # Default behavior settles exactly as before. The opt-in readiness gate # owns its own single bounded settle call inside ``_apply_step_gates``; # capture only a raw diagnostic frame here so it is not preceded by an @@ -7293,6 +7508,28 @@ def _api_identity_refusal( ) return None + def _actuator_for(self, binding: Any) -> Optional[Any]: + """The actuator that runs ``binding``, or None when none can.""" + + if binding is None: + return None + if binding.kind != "web_form": + return self.api_actuator + if self.web_form_actuator is not None: + return self.web_form_actuator + page = getattr(self.backend, "page", None) + if page is None: + return None + from openadapt_flow.runtime.actuators.web_form import ( + PlaywrightSessionTransport, + WebFormActuator, + ) + + try: + return WebFormActuator(PlaywrightSessionTransport(page)) + except Exception: # noqa: BLE001 - no live session: the GUI runs + return None + def _try_api_tier( self, step: Step, @@ -7333,7 +7570,8 @@ def _try_api_tier( """ binding = step.api_binding assert binding is not None # guaranteed by the caller - assert self.api_actuator is not None # guaranteed by the caller (line ~1140) + api_actuator = self._actuator_for(binding) + assert api_actuator is not None # guaranteed by the caller # An API write MUST be confirmable against its own exact effect # contract. ``step.effects`` belongs exclusively to the GUI fallback; @@ -7502,7 +7740,7 @@ def _try_api_tier( result.delivery_attempted = True try: self._require_qualification_environment_current() - outcome = self.api_actuator.actuate( + outcome = api_actuator.actuate( binding, runtime_params_for_gui(params), ) diff --git a/packages/flow/tests/test_api_tier_web_form.py b/packages/flow/tests/test_api_tier_web_form.py new file mode 100644 index 000000000..4d7d9950d --- /dev/null +++ b/packages/flow/tests/test_api_tier_web_form.py @@ -0,0 +1,686 @@ +"""API steps compiled from a recording's network capture (docs/API_TIER.md). + +Covers the four parts of the feature without a browser or a model: + +- capture redaction (secrets, session tokens, echoed cookies); +- the compiler's proposal from a synthetic capture, and its refusals; +- admission (person only, bound to the exact binding, needs a record check); +- the live-session actuator against a fake transport, and the replayer's + span: an admitted API step replaces its GUI steps, and when a check fails + before the save is sent the GUI steps run instead. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import requests + +from openadapt_flow import network_capture as nc +from openadapt_flow.compiler.api_proposal import ( + ApiProposalError, + admit_api_step, + describe_api_step, + propose_api_step, +) +from openadapt_flow.ir import ( + ActionKind, + ApiBinding, + Postcondition, + PostconditionKind, + Step, + WebFormField, + WebFormPrecheck, + WebFormSpec, + Workflow, +) +from openadapt_flow.mockmed.fault_server import serve as fault_serve +from openadapt_flow.runtime.actuators import ( + ActuationStatus, + ApiActuationResult, + ApiHaltKind, +) +from openadapt_flow.runtime.actuators.web_form import ( + WebFormActuator, + client_rules, + folded_checks_digest, + form_controls, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, RestRecordVerifier +from openadapt_flow.runtime.replayer import Replayer +from tests.test_replayer import FakeBackend, FakeVision + +FORM_HTML = """<html><body> +<script> var constraints = {"form_lname":{"presence":true}}; </script> +<form action='save.php' id='DEM'> + <input type="hidden" name="csrf_token_form" value="<session:csrf_token_form>"> + <input name="form_lname" value=""> + <select name="form_state"><option value="">Unassigned</option> + <option value="MA">Massachusetts</option><option value="CA">California</option></select> + <input name="form_note" value=""> + <form id="nested"><select name="form_extra"><option value="" selected>-</option></select></form> + <select name="form_status"><option value="active" selected>Active</option></select> +</form></body></html>""" + + +def test_form_parser_keeps_nested_controls_and_reads_rules(): + controls = form_controls(FORM_HTML, "/app/save.php") + assert controls is not None + assert set(controls) == { + "csrf_token_form", + "form_lname", + "form_state", + "form_note", + "form_extra", + "form_status", + } + assert controls["form_status"]["default"] == "active" + assert controls["form_state"]["default"] == "" + assert client_rules(FORM_HTML) == {"form_lname": {"presence": True}} + + +# -- capture redaction ----------------------------------------------------------- + + +def test_redact_withholds_secrets_tokens_and_echoed_cookies(): + lines = [ + { + "url": "http://app/login", + "method": "POST", + "form": [["authUser", "nurse"], ["clearPass", "hunter2-pw"]], + }, + { + "url": "http://app/page?token_main=abcdef123456", + "method": "GET", + "response_text": 'document.cookie="SID=cookievalue99"; csrf="tok-0123456789"; pw=hunter2-pw', + }, + { + "url": "http://app/save", + "method": "POST", + "form": [ + ["csrf_token_form", "tok-0123456789"], + ["ssn", "123-45-6789"], + ["lname", "Marsh"], + ], + }, + ] + out = nc.redact(lines, secret_names=["ssn"], session_values=["cookievalue99"]) + text = json.dumps(out) + for leaked in ( + "hunter2-pw", + "tok-0123456789", + "abcdef123456", + "cookievalue99", + "123-45-6789", + ): + assert leaked not in text + assert ["clearPass", nc.SECRET] in out[0]["form"] + assert ["csrf_token_form", "<session:csrf_token_form>"] in out[2]["form"] + assert ["lname", "Marsh"] in out[2][ + "form" + ] # business values stay, as in events.jsonl + assert "<session:cookie>" in out[1]["response_text"] + + +class _FakeResponse: + status = 200 + headers = {"content-type": "text/html; charset=utf-8"} + + def body(self): + return b"<html>ok</html>" + + +class _FakeRequest: + def __init__(self, url, method="GET", body=None, resource_type="document"): + self.url, self.method, self.resource_type = url, method, resource_type + self.post_data_buffer = body + self.headers = ( + {"content-type": "application/x-www-form-urlencoded"} if body else {} + ) + self.frame = type("F", (), {"url": url})() + + def all_headers(self): + return {**self.headers, "cookie": "SID=cookievalue99"} + + def response(self): + return _FakeResponse() + + +class _FakeContext: + def __init__(self): + self.handlers = {} + + def on(self, event, handler): + self.handlers[event] = handler + + def remove_listener(self, event, handler): + self.handlers.pop(event, None) + + +def test_capture_writes_and_loads_without_static_assets(tmp_path): + ctx = _FakeContext() + cap = nc.NetworkCapture(ctx).start() + ctx.handlers["requestfinished"](_FakeRequest("http://app/form")) + ctx.handlers["requestfinished"]( + _FakeRequest("http://app/a.png", resource_type="image") + ) + ctx.handlers["requestfinished"]( + _FakeRequest("http://app/save", "POST", b"lname=Marsh&csrf=tok-0123456789") + ) + cap.stop() + cap.write(tmp_path, t0_monotonic=0.0) + rows = nc.load(tmp_path) + assert [r["url"] for r in rows] == ["http://app/form", "http://app/save"] + assert rows[1]["form"] == [["lname", "Marsh"], ["csrf", "<session:csrf>"]] + assert "cookievalue99" not in (tmp_path / nc.NETWORK_FILE).read_text() + + +# -- proposal -------------------------------------------------------------------- + +PARAMS = {"lname": "Marsh", "state_label": "Massachusetts", "note": "Referred"} + + +def _effect(): + return Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"lname": "Marsh"}, + expected_count=1, + timeout_s=1.0, + ) + + +def _recording( + tmp_path: Path, + *, + save_form=None, + popup_text="No matches were found.", + form_page=True, +) -> Path: + rec = tmp_path / "rec" + rec.mkdir() + (rec / "meta.json").write_text(json.dumps({"params": PARAMS})) + events = [ + {"i": i, "kind": k, "t": float(i + 1)} + for i, k in enumerate( + ["click", "type", "click", "type", "click", "type", "click", "click"] + ) + ] + (rec / "events.jsonl").write_text("".join(json.dumps(e) + "\n" for e in events)) + lines = [] + if form_page: + lines.append( + { + "t": -1.0, + "method": "GET", + "url": "http://app/app/new.php", + "status": 200, + "response_text": FORM_HTML, + "resource_type": "document", + } + ) + lines.append( + { + "t": 7.5, + "method": "GET", + "url": "http://app/app/dup.php?lname=Marsh&close", + "status": 200, + "response_text": f"<td>{popup_text}</td>", + "resource_type": "document", + } + ) + lines.append( + { + "t": 8.2, + "method": "POST", + "url": "http://app/app/save.php", + "status": 200, + "resource_type": "document", + "form": save_form + or [ + ["csrf_token_form", "<session:csrf_token_form>"], + ["form_lname", "Marsh"], + ["form_state", "MA"], + ["form_note", "Referred"], + ["form_extra", ""], + ["form_status", "active"], + ], + "response_text": "<script>window.location='/app/view.php?set_pid=7&is_new=1';</script>", + } + ) + (rec / nc.NETWORK_FILE).write_text( + json.dumps({"schema": nc.SCHEMA, "exchanges": len(lines)}) + + "\n" + + "".join(json.dumps(ln) + "\n" for ln in lines) + ) + return rec + + +def _workflow(*, dialog_expect=True, effects=True) -> Workflow: + steps = [] + for i in range(8): + action = ActionKind.TYPE if i in (1, 3, 5) else ActionKind.CLICK + steps.append( + Step( + id=f"step_{i:03d}", + intent=f"s{i}", + action=action, + param={1: "lname", 3: "state_label", 5: "note"}.get(i), + ) + ) + if dialog_expect: + steps[6] = steps[6].model_copy( + update={ + "expect": [ + Postcondition( + kind=PostconditionKind.TEXT_PRESENT, + text="No matches were found.", + ) + ] + } + ) + if effects: + steps[7] = steps[7].model_copy( + update={"effects": [_effect()], "risk": "irreversible"} + ) + return Workflow(name="referral", steps=steps, params=dict(PARAMS)) + + +def test_proposal_explains_every_field_and_carries_the_duplicate_check(tmp_path): + wf, step, reasons = propose_api_step(_workflow(), _recording(tmp_path)) + assert reasons == [] and step is not None and step.id == "step_007" + b = step.api_binding + assert b.kind == "web_form" and b.on_unavailable == "gui" and b.admission is None + by = {f.name: f for f in b.web_form.fields} + assert by["form_lname"].source == "param" and by["form_lname"].param == "lname" + assert ( + by["form_state"].source == "select_label" + and by["form_state"].param == "state_label" + ) + assert by["csrf_token_form"].source == "session" + assert ( + by["form_status"].source == "form_default" + and by["form_status"].value == "active" + ) + (check,) = b.web_form.prechecks + assert ( + check.require_text == ["No matches were found."] + and ("lname", "{lname}") in check.query + ) + assert b.web_form.replaces_steps == [f"step_{i:03d}" for i in range(8)] + assert b.web_form.success_pattern and "set_pid" in b.web_form.success_pattern + assert b.effects == [_effect()] + assert "in place of 8 GUI steps" in describe_api_step(step) + + +@pytest.mark.parametrize( + "kwargs, reason", + [ + ( + { + "save_form": [ + ["form_lname", "Marsh"], + ["form_note", "Referred"], + ["mystery", "x9"], + ] + }, + "mystery", + ), + ( + { + "save_form": [ + ["form_lname", "Marsh"], + ["form_note", "Referred"], + ["pin", nc.SECRET], + ] + }, + "secret", + ), + ({"form_page": False}, "start recording before opening it"), + ({"popup_text": "1 match: Marsh"}, "no response it caused carries it"), + ], +) +def test_proposal_refuses_what_it_cannot_explain(tmp_path, kwargs, reason): + wf_in = _workflow() + wf, step, reasons = propose_api_step(wf_in, _recording(tmp_path, **kwargs)) + assert step is None and wf == wf_in + assert reason in " ".join(reasons) + + +# -- admission ------------------------------------------------------------------- + + +def test_admission_is_a_person_bound_to_the_exact_binding(tmp_path): + wf, _, _ = propose_api_step(_workflow(), _recording(tmp_path)) + with pytest.raises(ApiProposalError, match="not a person"): + admit_api_step(wf, admitted_by="agent:claude") + admitted, ids = admit_api_step(wf, admitted_by="reviewer") + b = admitted.steps[-1].api_binding + assert ids == ["step_007"] and b.admitted() + tampered = b.model_copy(update={"url_template": "/app/delete.php"}) + assert not tampered.admitted() + + +def test_admission_needs_a_record_check(tmp_path): + wf, _, _ = propose_api_step(_workflow(effects=False), _recording(tmp_path)) + with pytest.raises(ApiProposalError, match="no record check"): + admit_api_step(wf, admitted_by="reviewer") + + +# -- actuator -------------------------------------------------------------------- + + +class _Resp: + def __init__(self, status=200, text="", headers=None): + self.status, self.text, self.headers = status, text, headers or {} + + +class FakeTransport: + base_url = "http://app" + + def __init__( + self, + form=FORM_HTML.replace("<session:csrf_token_form>", "live-token-1"), + dup="No matches were found.", + save=None, + raise_on_post=False, + ): + self.form, self.dup, self.raise_on_post = form, dup, raise_on_post + self.save = save or _Resp( + 200, "<script>window.location='/app/view.php?set_pid=9&is_new=1'</script>" + ) + self.gets: list[str] = [] + self.posts: list[tuple[str, str]] = [] + + def get(self, path): + self.gets.append(path) + return _Resp(200, self.form if path.startswith("/app/new.php") else self.dup) + + def post_form(self, path, body, *, referer): + if self.raise_on_post: + raise requests.exceptions.ConnectionError("reset after send") + self.posts.append((path, body)) + return self.save + + +def _admitted_binding(tmp_path) -> ApiBinding: + wf, _, _ = propose_api_step(_workflow(), _recording(tmp_path)) + wf, _ = admit_api_step(wf, admitted_by="reviewer") + return wf.steps[-1].api_binding + + +RUN = {"lname": "Feld", "state_label": "California", "note": "Referred"} + + +def test_actuator_sends_run_values_and_the_live_token(tmp_path): + t = FakeTransport() + out = WebFormActuator(t).actuate(_admitted_binding(tmp_path), RUN) + assert out.status is ActuationStatus.ACTUATED + ((path, body),) = t.posts + assert path == "/app/save.php" + assert ( + "csrf_token_form=live-token-1" in body + and "form_lname=Feld" in body + and "form_state=CA" in body + ) + assert any(g.startswith("/app/dup.php?lname=Feld") for g in t.gets) + + +@pytest.mark.parametrize( + "transport, run, reason", + [ + (FakeTransport(dup="1 match found"), RUN, "did not pass"), + ( + FakeTransport( + form=FORM_HTML.replace('name="form_note"', 'name="form_note2"') + ), + RUN, + "form changed", + ), + ( + FakeTransport( + form=FORM_HTML.replace( + '"form_lname":{"presence":true}', '"form_note":{"presence":true}' + ) + ), + RUN, + "validation rules changed", + ), + ( + FakeTransport( + form=FORM_HTML.replace( + 'value="active" selected', 'value="inactive" selected' + ) + ), + RUN, + "default changed", + ), + (FakeTransport(), {**RUN, "state_label": "Atlantis"}, "no single option"), + (FakeTransport(), {**RUN, "lname": ""}, "required by the form"), + (FakeTransport(), {"lname": "Feld"}, "no value for"), + ], +) +def test_actuator_refuses_before_sending(tmp_path, transport, run, reason): + out = WebFormActuator(transport).actuate(_admitted_binding(tmp_path), run) + assert out.status is ActuationStatus.UNAVAILABLE and reason in out.reason + assert transport.posts == [] + + +def test_actuator_halts_after_sending(tmp_path): + binding = _admitted_binding(tmp_path) + rejected = FakeTransport(save=_Resp(200, "Authentication Error")) + out = WebFormActuator(rejected).actuate(binding, RUN) + assert ( + out.status is ActuationStatus.HALT + and out.halt_kind is ApiHaltKind.RESPONSE_REJECTED + ) + lost = FakeTransport(raise_on_post=True) + out = WebFormActuator(lost).actuate(binding, RUN) + assert ( + out.status is ActuationStatus.HALT + and out.halt_kind is ApiHaltKind.DELIVERY_UNCERTAIN + ) + + +# -- replayer span --------------------------------------------------------------- + +TARGET = {"patient_id": "p1", "type": "Triage"} + + +class EnterWritesBackend(FakeBackend): + """The GUI path: pressing Enter on the save step writes the record.""" + + def __init__(self, sor_url): + super().__init__(viewport=(300, 200)) + self.sor_url = sor_url + + def press(self, key): + super().press(key) + if key == "Enter": + requests.post( + f"{self.sor_url}/api/encounter", + json={**TARGET, "note": "gui"}, + timeout=5, + ) + + +class ScriptedActuator: + def __init__(self, sor_url, status): + self.sor_url, self.status, self.calls = sor_url, status, 0 + + def actuate(self, binding, params): + self.calls += 1 + if self.status is ActuationStatus.ACTUATED: + requests.post( + f"{self.sor_url}/api/encounter", + json={**TARGET, "note": "api"}, + timeout=5, + ) + return ApiActuationResult( + status=ActuationStatus.ACTUATED, reason="POST /save -> 200" + ) + if self.status is ActuationStatus.HALT: + return ApiActuationResult( + status=ActuationStatus.HALT, + halt_kind=ApiHaltKind.RESPONSE_REJECTED, + reason="POST /save -> 200 without the recorded success response", + ) + return ApiActuationResult( + status=ActuationStatus.UNAVAILABLE, + reason="the check before saving did not pass; the save was not sent", + ) + + +def _span_workflow(*, admitted=True) -> Workflow: + effect = Effect( + kind=EffectKind.RECORD_WRITTEN, match=TARGET, expected_count=1, timeout_s=2.0 + ) + steps = [ + Step(id="step_000", intent="next field", action=ActionKind.KEY, key="Tab"), + Step( + id="step_001", + intent="save", + action=ActionKind.KEY, + key="Enter", + effects=[effect], + ), + ] + spec = WebFormSpec( + form_path="/new.php", + fields=[WebFormField(name="lname", source="param", param="lname")], + prechecks=[ + WebFormPrecheck(path="/dup.php", require_text=["No matches were found."]) + ], + replaces_steps=["step_000", "step_001"], + folded_checks_digest=folded_checks_digest(steps[:1]), + ) + binding = ApiBinding( + kind="web_form", url_template="/save.php", effects=[effect], web_form=spec + ) + steps[1] = steps[1].model_copy(update={"api_binding": binding}) + wf = Workflow(name="span", steps=steps, params={"lname": "Marsh"}) + if admitted: + wf, _ = admit_api_step(wf, admitted_by="reviewer") + return wf + + +def _run(tmp_path, wf, status, **kw): + url, db, stop = fault_serve() + url = url.rstrip("/") + try: + backend = EnterWritesBackend(url) + actuator = ScriptedActuator(url, status) + bundle = tmp_path / "bundle" + (bundle / "templates").mkdir(parents=True, exist_ok=True) + report = Replayer( + backend, + vision=FakeVision(), + effect_verifier=RestRecordVerifier(url), + web_form_actuator=actuator, + poll_interval_s=0.01, + **kw, + ).run(wf, bundle_dir=bundle, run_dir=tmp_path / "run") + return report, backend, actuator, db.snapshot()["records"] + finally: + stop() + + +def test_admitted_api_step_replaces_its_gui_steps(tmp_path): + report, backend, actuator, records = _run( + tmp_path, _span_workflow(), ActuationStatus.ACTUATED + ) + assert report.success is True and actuator.calls == 1 + folded, save = report.results + assert folded.skipped and folded.actuation == "api_folded" + assert save.actuation == "api" and save.effect_verified is True + assert backend.actions == [] + assert [r["note"] for r in records] == ["api"] + + +def test_a_failed_check_before_the_save_runs_the_gui_steps(tmp_path): + report, backend, actuator, records = _run( + tmp_path, _span_workflow(), ActuationStatus.UNAVAILABLE + ) + assert actuator.calls == 1 + assert [a[1] for a in backend.actions if a[0] == "press"] == ["Tab", "Enter"] + assert "did not pass" in report.results[0].api_fallback + assert [r["note"] for r in records] == ["gui"] + + +def test_a_rejected_save_halts_without_gui_retry(tmp_path): + report, backend, actuator, records = _run( + tmp_path, _span_workflow(), ActuationStatus.HALT + ) + assert report.success is False and backend.actions == [] and records == [] + + +@pytest.mark.parametrize( + "mutate, kw, reason", + [ + (lambda wf: _span_workflow(admitted=False), {}, "not admitted"), + (lambda wf: wf, {"api_steps": False}, "off for this run"), + ( + lambda wf: wf.model_copy( + update={ + "steps": [ + wf.steps[0], + wf.steps[1].model_copy( + update={ + "effects": [ + *wf.steps[1].effects, + wf.steps[1].effects[0], + ] + } + ), + ] + } + ), + {}, + "record check differs", + ), + ( + lambda wf: wf.model_copy( + update={ + "steps": [ + wf.steps[0].model_copy( + update={ + "expect": [ + Postcondition( + kind=PostconditionKind.TEXT_PRESENT, + text="Added later", + ) + ] + } + ), + wf.steps[1], + ] + } + ), + {}, + "on-screen check", + ), + ], +) +def test_gui_steps_run_when_the_api_step_may_not(tmp_path, mutate, kw, reason): + wf = mutate(_span_workflow()) + report, backend, actuator, records = _run( + tmp_path, wf, ActuationStatus.ACTUATED, **kw + ) + assert actuator.calls == 0 + assert reason in report.results[0].api_fallback + assert ("press", "Tab") in [(a[0], a[1]) for a in backend.actions] + + +@pytest.mark.parametrize( + "mode, runs", [("each_item", False), ("problems_only", False), ("not_gated", True)] +) +def test_only_an_ungated_run_uses_api_steps(mode, runs): + from openadapt_flow.approval_mode.model import ApprovalMode + + replayer = Replayer(FakeBackend(), vision=FakeVision(), web_form_actuator=object()) + replayer.approval_gate = type("Gate", (), {"mode": ApprovalMode(mode)})() + _end, _save, reason = replayer._web_form_span_at(_span_workflow(), 0, durable=False) + assert (reason is None) is runs + if not runs: + assert "approval mode" in reason