diff --git a/packages/flow/benchmark/effect_e2e/prove_check.py b/packages/flow/benchmark/effect_e2e/prove_check.py new file mode 100644 index 000000000..4963a1673 --- /dev/null +++ b/packages/flow/benchmark/effect_e2e/prove_check.py @@ -0,0 +1,74 @@ +"""The effect_e2e record service as a fixture for proving a proposed check. + +``openadapt-flow scaffold-verifier --prove effect-e2e`` runs a proposed record +check through the real replayer against this service, one run per fault in +:data:`benchmark.effect_e2e.run.SCENARIOS`, judged by this harness's +independent ground truth (a read-only connection to the SQLite file that never +sees the check). See :mod:`openadapt_flow.record_check_proof`. +""" + +from __future__ import annotations + +import tempfile +from pathlib import Path +from typing import Any, Callable, Mapping, Optional + +import requests + +from benchmark.effect_e2e import ground_truth +from benchmark.effect_e2e.record_service import TARGET_PATIENT, TARGET_TYPE, serve +from benchmark.effect_e2e.run import SCENARIOS +from openadapt_flow.record_check_proof import Fixture + + +class _Handle: + def __init__( + self, base_url: str, db_path: Path, tmp: tempfile.TemporaryDirectory + ) -> None: + self.base_url = base_url + self.db_path = db_path + self.tmp = tmp + + +def _serve() -> tuple[str, Optional[Path], Callable[[], None], Any]: + tmp = tempfile.TemporaryDirectory(prefix="record-check-proof-") + db_path = Path(tmp.name) / "record.db" + service = serve(db_path) + handle = _Handle(service.base_url, db_path, tmp) + + def stop() -> None: + service.stop() + tmp.cleanup() + + return service.base_url, db_path, stop, handle + + +def _reset(handle: _Handle, seed: bool) -> None: + requests.post( + f"{handle.base_url}/api/reset", json={"seed_concurrent": seed}, timeout=5.0 + ) + + +def _judge( + before: ground_truth.Snapshot, + after: ground_truth.Snapshot, + params: Mapping[str, str], +) -> tuple[bool, str]: + verdict = ground_truth.judge(before, after, intended_note=params.get("note", "")) + return verdict.correct, verdict.fault_class + + +FIXTURE = Fixture( + name="effect-e2e", + serve=_serve, + reset=_reset, + snapshot=lambda handle: ground_truth.capture(handle.db_path), + judge=_judge, + scenarios=tuple((sc.name, sc.seed_concurrent) for sc in SCENARIOS), + write_path="/api/encounter", + default_body={ + "patient_id": TARGET_PATIENT, + "type": TARGET_TYPE, + "note": "{note}", + }, +) diff --git a/packages/flow/claims.yaml b/packages/flow/claims.yaml index a74343b5b..913bf3a13 100644 --- a/packages/flow/claims.yaml +++ b/packages/flow/claims.yaml @@ -334,6 +334,72 @@ claims: additional opt-in live-OpenEMR test. Per-verifier claims are exactly as strong as these tests. + # ------------------- record checks built from the app's own reads + - id: record-check-from-app-reads + claim: >- + scaffold-verifier builds a record check from the reads the app made + after the save (the recording's network capture, a SQLite database read + path, or a reviewed verifier section), binds the recorded values to run + parameters, guards every other table when a database path is given (no + row added, changed or removed), and tests the check against injected + bad writes before anyone reviews it. A check that misses one is written as a draft + with the misses listed. Every effect stays unconfirmed until a person + removes the flag. + surfaces: [docs] + tier: supported + evidence: + - path: tests/test_record_check_synthesis.py + node: test_database_read_path_covers_every_table_and_passes_the_suite + proves: >- + On the benchmark/effect_e2e record service, a check built from a + database read path catches every injected bad write end to end + through the real replayer (including the stray billing write) and + passes the clean save. + - path: tests/test_record_check_synthesis.py + node: test_one_rest_read_misses_the_billing_write_and_stays_a_draft + proves: >- + A check built from one REST read misses the stray billing write, + both in the recorded-state fault test and end to end, and is + written as a draft with that miss listed. + - path: tests/test_record_check_synthesis.py + node: test_mockmed_needs_a_declared_complete_read + proves: >- + On the MockMed --break-it fault server, the check built from the + app's /api/db read catches every fault end to end once a person + declares the read complete, and stays a draft until then. + - path: tests/test_record_check_synthesis.py + node: test_openemr_fhir_check_refutes_store_level_faults + proves: >- + On the OpenEMR-shaped FHIR R4 fake, the check found from the + Observation search refutes partial, duplicate, phantom, + wrong-patient and lost-update faults and confirms the clean save. + - path: tests/test_record_check_synthesis.py + node: test_guards_catch_a_changed_or_deleted_row_through_the_real_read + proves: >- + The generated SQL reads, run against a real SQLite file, refute a + save that also changed or deleted a row in another table. + - path: tests/test_record_check_synthesis.py + node: test_verify_run_refuses_until_a_person_removes_the_flags + proves: >- + verify-run refuses the proposed check until every + needs_operator_confirmation line is removed. + - path: docs/EFFECT_KIT.md + proves: >- + The operator guide: inputs, outputs, the fault classes, and what + the proof does not cover. + caveats: + - >- + Proven on synthetic fixtures only (MockMed, the effect_e2e SQLite + record service, an OpenEMR-shaped FHIR fake). A check built from API + reads alone cannot see writes to tables it does not read, so it stays + a draft unless a database path covers them or a person declares the + read complete. The fault classes are the effect_e2e taxonomy, not a + measured field incident rate. + - >- + Saves that update an existing record are not handled yet: the check + expects one new record. The read's credential is the operator's job; + the recording's reads used the browser's sign-in. + - id: ui-readback-separate-session claim: >- With no API or database access, a save can be checked through the @@ -410,6 +476,10 @@ claims: MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap. + `scaffold-verifier --bundle` reports coverage before and after + evidence from the recording's DOM / accessibility text at each + unarmed click; it proposes only evidence the runtime checks on that + surface, and a person applies it. # -------------------------- compiler-emitted identifier crops (pixel tier) - id: identifier-crop-compile-emission diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 110312f47..091e5ced2 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -262,6 +262,151 @@ Both reads are kept in `sor_snapshots.json` (mode 0600, raw records). The proposed file is also mode 0600 and copies `verifier:` as written, with `${NAME}` references unexpanded. Without `--verify`, `record` is unchanged. +## A record check built from the app's own reads (`scaffold-verifier`) + +After a save, the app usually reads the saved record back to show it. That +read is the record check, already written by the app. `openadapt-flow +scaffold-verifier ` finds it, builds the check, and tries to break +it before a person reviews it. + +```bash +openadapt-flow scaffold-verifier rec/ # network capture only +openadapt-flow scaffold-verifier rec/ --db app.sqlite # plus every table +openadapt-flow scaffold-verifier rec/ --verifier verify.yaml --bundle bundle/ +openadapt-flow scaffold-verifier rec/ --db app.sqlite --prove effect-e2e +``` + +### Finding the read + +The recording's parameter values (`meta.json`) identify the saved record. +The command looks for a read that returns a record holding them: + +- **Network capture.** `network.jsonl` in the `openadapt.network-capture/1` + format that `record --capture-network` writes. Each `GET` after the save + (by the recording's timeline) whose JSON response holds a record list is a + candidate: a REST list (`{"records": [...]}`, `{"data": [...]}`, a bare + list) or a FHIR `Bundle`. The read whose record holds the most recorded + values wins; a persisted record (one with an `id`) beats an echo of the + form. A read whose URL carries a session token, or whose values sit only in + nested fields, is rejected and reported. +- **A database read path** (`--db`, SQLite). Every table is scanned. A table + with a row holding the recorded values is read, scoped by the recorded + identifiers (`WHERE patient_id = :patient_id`). Every other table gets a + no-new-rows guard. +- **A verifier section** (`--verifier verify.yaml`). The `verifier:` section + is read once. With `sor_snapshots.json` from `record --verify`, its + before read becomes the baseline. + +### What the check holds + +- The `verifier:` section is `kind: surfaces`: one read per table or + resource. Each effect names its read in `probe` (`surface=`), and a + bad write is judged only against the read that can see it. +- For each read that holds the saved record: a `record_written` effect for + exactly one NEW record (`count_new_only`), matched on every field equal to + a recorded value (bound as `{param: NAME}`, so each run is checked against + its own values) and on fields the save sent with a fixed value under the + same name (`type: Triage`). `forbid_collateral_loss` stops the run if a + record that was there before is gone. +- Recorded values in the read's URL become placeholders (`path_params`, + FHIR `search_param_exprs`), so the read is scoped to the run's record. A + path segment that looks like one record's id (an MRN, a number, a UUID) + and is not a recorded value keeps the check a draft: it would read the + demonstration's record every run. Such segments show as `*` in names and + messages. +- For each other table (database path only): a guard that no row is added. +- A read whose records carry an `id` gets a second read, `.content`, + with `records_drop_fields: [id]`. The first read counts new records by id, + so a stray row identical to an existing one still counts. The content read + compares records by their fields, so a save that changes or deletes another + record (same id, new content) stops the run. + +### Proving it before review + +Two tests, both written to `record_check.proof.json`: + +1. **Recorded-state faults** (always). The fault classes of + `benchmark/effect_e2e` are applied to the records the recording saw: + nothing saved, a field dropped, saved twice, saved on another record, + another record of the same subject destroyed, and a stray row in another + table, plus another record changed or deleted. The check must stop each + one and pass the clean save. A fault the check can't be tested on counts + as a miss: if every checked field identifies the subject, another record + of the same subject looks like the saved one, so "destroyed" is a miss. + Every generated read is built, and each `--db` read is run and compared + with the scan, so a read that can't work is a miss too. +2. **End to end** (`--prove mockmed` or `--prove effect-e2e`). On a fixture + that injects bad writes at the persistence boundary, each fault runs + through the real replayer and API actuator with the proposed check as the + effect verifier, judged by the fixture's own ground truth. Free-text values + change on every run, so a check that froze the demonstration's values fails + the clean save. + +A check that misses any fault, or stops the clean save, is written as a +draft: the file starts with `# DRAFT`, lists each `MISS:`, and the command +exits 1. So does a check with a write request no read covers, or a table +`--db` couldn't read (an irregular name, more than 20,000 rows). A working +check exits 0. + +The save is timed from its own write request. The recorder stamps a step +after the screen settles, so the save's request and the app's read-back come +before the Save step's time. Reads before the first save request are the +baseline; reads after the last one are candidates. + +On the effect_e2e service, a check built from one REST read (`/api/records`) +misses the stray billing write, the same blind spot as the REST arm in +[`EFFECT_E2E.md`](../benchmark/effect_e2e/EFFECT_E2E.md), so it stays a +draft. With `--db`, the billing guard catches it and every fault is stopped. + +A check built from API reads alone can't see tables it doesn't read. If one +store really holds everything the save can change (MockMed's `/api/db`), a +person can say so with `--complete-read`. The proof records that as +`closed_world: declared`, not as proven. + +### What a person still does + +Every effect carries `needs_operator_confirmation: true`. `verify-run` +refuses the file and `qualify` reports a problem until a person: + +- confirms each read is the system of record and gives it its own read-only + credential (`auth`, `access_token_env`, `sql_password_env`); the + recording's reads used the browser's sign-in; +- checks each `{param: ...}` and removes match fields that change on every + save; +- decides whether a guarded table is shared by other users (a guard stops a + run when anyone else writes there) and removes that guard if so; +- deletes every `needs_operator_confirmation: true` line. + +### Identity evidence for unarmed clicks + +With `--bundle` (or `--recording` when the source is a bundle) the command +also reports identity coverage: how many clicks have a right-record check +armed now, and how many would with evidence from the recording's DOM or +accessibility text at each unarmed click (the clicked row's structured text, +or the accessible name of a clicked `row`, `listitem`, `option` or +`treeitem`). It proposes only evidence the runtime checks on the bundle's +surface; on RDP or Citrix nothing is proposed. A record named elsewhere on the +page (title, URL) is listed but not counted, because the runtime checks +identity at the click point. The evidence text goes to `identity.proposed.json` +(mode 0600); the proof holds counts and parameter names only. Nothing is +applied to the bundle. + +### Limits + +- Saves that update an existing record are not handled yet; the check + expects one new record. +- A REST read must return its records at the top level of the response, or + under one key; nested record lists are rejected. +- `--db` reads SQLite files. Other databases go through `--verifier` with a + `kind: sql` section. +- The fault classes are a hand-written taxonomy. Passing them is coverage of + that taxonomy, not a field rate. +- No model calls. The only network read is the `--verifier` read you name. +- The end-to-end proof changes free-text values on every run, but keeps + identifiers (the fixture's patient must exist). Recorded identifiers are + always bound as `{param: ...}`, never written as literals. +- A demonstration with no write step is refused, as before. + ## A read-back taught by demonstration ("Now show how you'd check it saved") For an application with no API, database or export (a legacy desktop app, @@ -582,14 +727,20 @@ Per-kind required fields: |---|---|---| | `onscreen` | (none — auto-derived from the demo) | `readback_region`, `readback_min_ratio` (hand-config fallback) | | `ui_readback` | `ui_readback` section: `record_url`, `sign_in` (env-var names for the read-back account), `ready_selector`, `row_selector`, `fields`, `identity` | `url_params`, `empty_selector`, `not_found_selector`, `read_post_paths`, per-field `attribute` / `date_format` / `casefold`, `truncation_markers`, `navigation_timeout_s` | -| `rest` | `base_url` | `records_path` (may contain `{placeholder}`s), `records_key`, `path_params`, `auth` | +| `rest` | `base_url` | `records_path` (may contain `{placeholder}`s), `records_key`, `path_params`, `auth`, `records_drop_fields` | | `graphql` | `base_url` + `graphql_query` | `graphql_variables` (`{param: ...}` entity binding), `graphql_records_path`, `graphql_freshness_field` + `graphql_freshness_window_s`, `auth` | -| `fhir` | `base_url` | `resource_type`, `search_params`, `search_param_exprs`, `field_paths`, `access_token_env` or `smart_backend`, `verify_tls` | -| `sql` | `sql_query` + (`sqlite_database` or `sql_driver`) | `sql_query_params`, `sql_connect_args`, `sql_password_env` | +| `fhir` | `base_url` | `resource_type`, `search_params`, `search_param_exprs`, `field_paths`, `access_token_env` or `smart_backend`, `verify_tls`, `records_drop_fields` | +| `sql` | `sql_query` + (`sqlite_database` or `sql_driver`) | `sql_query_params`, `sql_connect_args`, `sql_password_env`, `records_drop_fields` | | `file` | `root` | `file_pattern`, `file_min_size`, `file_mtime_window_s`, `file_content_probe` | | `email` | `root` | `mail_pattern`, `file_mtime_window_s` (freshness), `file_content_probe` (body regex) | | `document` | `root` | `file_pattern`, `document_format` (`json` \| `text`), `document_field_paths`, `document_text_pattern`, `file_mtime_window_s` | | `document-hash` | `root` | `glob` | +| `surfaces` | `surfaces`: name -> one plain verifier config (any kind above) | each effect's `probe` names its surface (`surface=`); an effect naming none, or an unknown one, halts | + +`records_drop_fields` (`rest`, `sql`, `fhir`) removes fields from every record +before judging. `[id]` compares records by content: a record whose id stayed +the same but whose fields changed reads as one record removed and one added, +so `forbid_collateral_loss` catches it. Any kind additionally accepts the evidence-minimization fields `evidence_redact_fields` / `evidence_keep_fields` (see "Evidence minimization" diff --git a/packages/flow/docs/VERIFICATION.md b/packages/flow/docs/VERIFICATION.md index 2b2073f9e..9bf7cccac 100644 --- a/packages/flow/docs/VERIFICATION.md +++ b/packages/flow/docs/VERIFICATION.md @@ -12,7 +12,7 @@ ## What is bound to required CI vs. being validated -- **Bound to required CI pass evidence (13):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `ui-readback-separate-session`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported`, `compose-recorded-bundles` +- **Bound to required CI pass evidence (14):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `record-check-from-app-reads`, `ui-readback-separate-session`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported`, `compose-recorded-bundles` - **Being validated — opt-in / infra-gated or field (8):** `effect-verifier-kit`, `windows-desktop-validating`, `macos-native-validating`, `linux-native-validating`, `rdp-validating`, `desktop-recording-validating`, `citrix-pixel-validating`, `openemr-field-benchmark` - **Roadmap / research (1):** `win32-window-replay-roadmap` @@ -110,6 +110,28 @@ - CONTRACT-PROVEN, not live-proven: the SQL verifier is exercised against sqlite fixtures (no production MariaDB/Postgres), the SFTP arrival path against a fake transport (no real SFTP server), and the worked-example configs are templates — only the FHIR substrate has an additional opt-in live-OpenEMR test. Per-verifier claims are exactly as strong as these tests. +### `record-check-from-app-reads` — supported — bound to required CI pass evidence + +> scaffold-verifier builds a record check from the reads the app made after the save (the recording's network capture, a SQLite database read path, or a reviewed verifier section), binds the recorded values to run parameters, guards every other table when a database path is given (no row added, changed or removed), and tests the check against injected bad writes before anyone reviews it. A check that misses one is written as a draft with the misses listed. Every effect stays unconfirmed until a person removes the flag. + +- Surfaces: docs +- Strongest evidence strength: **supported** (tier is `supported`) + +| Backing evidence | Kind | Gating / CI stage | Strength | Proves | +|---|---|---|---|---| +| `tests/test_record_check_synthesis.py` | test | ci (required PR gate (test)) | supported | On the benchmark/effect_e2e record service, a check built from a database read path catches every injected bad write end to end through the real replayer (including the stray billing write) and passes the clean save. | +| `tests/test_record_check_synthesis.py` | test | ci (required PR gate (test)) | supported | A check built from one REST read misses the stray billing write, both in the recorded-state fault test and end to end, and is written as a draft with that miss listed. | +| `tests/test_record_check_synthesis.py` | test | ci (required PR gate (test)) | supported | On the MockMed --break-it fault server, the check built from the app's /api/db read catches every fault end to end once a person declares the read complete, and stays a draft until then. | +| `tests/test_record_check_synthesis.py` | test | ci (required PR gate (test)) | supported | On the OpenEMR-shaped FHIR R4 fake, the check found from the Observation search refutes partial, duplicate, phantom, wrong-patient and lost-update faults and confirms the clean save. | +| `tests/test_record_check_synthesis.py` | test | ci (required PR gate (test)) | supported | The generated SQL reads, run against a real SQLite file, refute a save that also changed or deleted a row in another table. | +| `tests/test_record_check_synthesis.py` | test | ci (required PR gate (test)) | supported | verify-run refuses the proposed check until every needs_operator_confirmation line is removed. | +| `docs/EFFECT_KIT.md` | doc | artifact (doc/benchmark) | roadmap | The operator guide: inputs, outputs, the fault classes, and what the proof does not cover. | + +**Caveats (honest limits):** + +- Proven on synthetic fixtures only (MockMed, the effect_e2e SQLite record service, an OpenEMR-shaped FHIR fake). A check built from API reads alone cannot see writes to tables it does not read, so it stays a draft unless a database path covers them or a person declares the read complete. The fault classes are the effect_e2e taxonomy, not a measured field incident rate. +- Saves that update an existing record are not handled yet: the check expects one new record. The read's credential is the operator's job; the recording's reads used the browser's sign-in. + ### `ui-readback-separate-session` — supported — bound to required CI pass evidence > With no API or database access, a save can be checked through the application's own record page: a separate browser with a new, empty context and its own sign-in opens the record by identifier, confirms the page shows that identifier, reads the saved fields from the page structure, and judges them with the shared effect judge. After sign-in the session refuses every request that is not a read. The check declares tier 2 (a separate session) and is never labeled a system-of-record read. @@ -144,7 +166,7 @@ **Caveats (honest limits):** -- Identity verification covers ONLY armed steps. Real bundles arm a MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap. +- Identity verification covers ONLY armed steps. Real bundles arm a MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap. `scaffold-verifier --bundle` reports coverage before and after evidence from the recording's DOM / accessibility text at each unarmed click; it proposes only evidence the runtime checks on that surface, and a person applies it. ### `identifier-crop-compile-emission` — supported — bound to required CI pass evidence diff --git a/packages/flow/docs/verification.json b/packages/flow/docs/verification.json index 210da6474..851b09a30 100644 --- a/packages/flow/docs/verification.json +++ b/packages/flow/docs/verification.json @@ -485,6 +485,107 @@ ], "errors": [] }, + { + "id": "record-check-from-app-reads", + "claim": "scaffold-verifier builds a record check from the reads the app made after the save (the recording's network capture, a SQLite database read path, or a reviewed verifier section), binds the recorded values to run parameters, guards every other table when a database path is given (no row added, changed or removed), and tests the check against injected bad writes before anyone reviews it. A check that misses one is written as a draft with the misses listed. Every effect stays unconfirmed until a person removes the flag.", + "tier": "supported", + "reproducibility": null, + "surfaces": [ + "docs" + ], + "strongest_evidence": "supported", + "caveats": [ + "Proven on synthetic fixtures only (MockMed, the effect_e2e SQLite record service, an OpenEMR-shaped FHIR fake). A check built from API reads alone cannot see writes to tables it does not read, so it stays a draft unless a database path covers them or a person declares the read complete. The fault classes are the effect_e2e taxonomy, not a measured field incident rate.", + "Saves that update an existing record are not handled yet: the check expects one new record. The read's credential is the operator's job; the recording's reads used the browser's sign-in." + ], + "evidence": [ + { + "path": "tests/test_record_check_synthesis.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_database_read_path_covers_every_table_and_passes_the_suite", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "On the benchmark/effect_e2e record service, a check built from a database read path catches every injected bad write end to end through the real replayer (including the stray billing write) and passes the clean save." + }, + { + "path": "tests/test_record_check_synthesis.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_one_rest_read_misses_the_billing_write_and_stays_a_draft", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "A check built from one REST read misses the stray billing write, both in the recorded-state fault test and end to end, and is written as a draft with that miss listed." + }, + { + "path": "tests/test_record_check_synthesis.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_mockmed_needs_a_declared_complete_read", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "On the MockMed --break-it fault server, the check built from the app's /api/db read catches every fault end to end once a person declares the read complete, and stays a draft until then." + }, + { + "path": "tests/test_record_check_synthesis.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_openemr_fhir_check_refutes_store_level_faults", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "On the OpenEMR-shaped FHIR R4 fake, the check found from the Observation search refutes partial, duplicate, phantom, wrong-patient and lost-update faults and confirms the clean save." + }, + { + "path": "tests/test_record_check_synthesis.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_guards_catch_a_changed_or_deleted_row_through_the_real_read", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "The generated SQL reads, run against a real SQLite file, refute a save that also changed or deleted a row in another table." + }, + { + "path": "tests/test_record_check_synthesis.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": "test_verify_run_refuses_until_a_person_removes_the_flags", + "node_found": true, + "ci_job": "test", + "junit_status": null, + "proves": "verify-run refuses the proposed check until every needs_operator_confirmation line is removed." + }, + { + "path": "docs/EFFECT_KIT.md", + "kind": "doc", + "exists": true, + "strength": "roadmap", + "gating": "artifact (doc/benchmark)", + "node": null, + "node_found": null, + "ci_job": null, + "junit_status": null, + "proves": "The operator guide: inputs, outputs, the fault classes, and what the proof does not cover." + } + ], + "errors": [] + }, { "id": "ui-readback-separate-session", "claim": "With no API or database access, a save can be checked through the application's own record page: a separate browser with a new, empty context and its own sign-in opens the record by identifier, confirms the page shows that identifier, reads the saved fields from the page structure, and judges them with the shared effect judge. After sign-in the session refuses every request that is not a read. The check declares tier 2 (a separate session) and is never labeled a system-of-record read.", @@ -561,7 +662,7 @@ ], "strongest_evidence": "supported", "caveats": [ - "Identity verification covers ONLY armed steps. Real bundles arm a MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap." + "Identity verification covers ONLY armed steps. Real bundles arm a MINORITY of clicks (4 of 12 on a recent live OpenEMR bundle); an UNARMED click proceeds with no identity check whatsoever. Coverage is an auditable per-step metric, but disclosure does not close the gap. `scaffold-verifier --bundle` reports coverage before and after evidence from the recording's DOM / accessibility text at each unarmed click; it proposes only evidence the runtime checks on that surface, and a person applies it." ], "evidence": [ { diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index 3effbf62d..ca2069d4d 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -51,9 +51,10 @@ environment, identity, and effect pins from the demo; ``qualify accept`` runs the proposed oracle against a ``--break-it`` fault before it can succeed; refusing a pin HALTs. -- ``scaffold-verifier`` — draft (never approve) an effect-oracle contract - (``effect_contract.yaml``) from a recording or bundle's write-shaped steps; - refuses demonstrations with no consequential step. +- ``scaffold-verifier`` — build a record check from the reads the app made + after the save and fault-test it (``record_check.proposed.yaml``), or, with + no read, draft an effect-oracle contract (``effect_contract.yaml``); never + approves; refuses demonstrations with no consequential step. - ``explain`` — plain-language, read-only read of a completed run directory: what happened, why the outcome is the safe one, and the next suggested command. @@ -3987,10 +3988,34 @@ def _print_lint_epilogue( def _cmd_scaffold_verifier(args: argparse.Namespace) -> int: """Draft (never approve) an effect-oracle contract from retained evidence.""" - from openadapt_flow.scaffold_verifier import NEXT_COMMANDS_TEMPLATE, write_draft - - out, count = write_draft( - Path(args.source), Path(args.out) if getattr(args, "out", None) else None + from openadapt_flow.scaffold_verifier import ( + NEXT_COMMANDS_TEMPLATE, + build_record_check, + write_draft, + ) + + def optional_path(name: str) -> Optional[Path]: + value = getattr(args, name, None) + return Path(value) if value else None + + out_dir = optional_path("out") + built = build_record_check( + Path(args.source), + out_dir, + db=optional_path("db"), + verifier=optional_path("verifier"), + prove=getattr(args, "prove", None), + recording=optional_path("recording"), + bundle=optional_path("bundle"), + complete_read=bool(getattr(args, "complete_read", False)), + ) + if built is not None: + print(built.summary) + return 0 if built.working else 1 + out, count = write_draft(Path(args.source), out_dir) + print( + "No read in the recording returns the saved record (no network capture, " + "--db or --verifier), so this is a TODO draft." ) print(NEXT_COMMANDS_TEMPLATE.format(out=out, count=count)) return 0 @@ -8230,10 +8255,11 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser( "scaffold-verifier", help=( - "Draft an effect-oracle contract (effect_contract.yaml) from a " - "recording or bundle's write-shaped steps. The output is a DRAFT " - "requiring human edit; refuses demonstrations with no " - "consequential (write) step" + "Build a record check from the reads the app made after the save " + "and test it against injected bad writes (record_check.proposed.yaml " + "+ record_check.proof.json; exit 1 when it misses one). Without a " + "read, drafts effect_contract.yaml for a person to fill in. Never " + "approves; refuses demonstrations with no consequential (write) step" ), ) p.add_argument( @@ -8246,7 +8272,58 @@ def build_parser() -> argparse.ArgumentParser: default=None, metavar="DIR", help=( - "Directory for the drafted effect_contract.yaml (default: beside the input)" + "Directory for the proposed check or drafted effect_contract.yaml " + "(default: beside the input)" + ), + ) + p.add_argument( + "--db", + default=None, + metavar="SQLITE", + help=( + "Read-only database read path (SQLite file): find the saved row in " + "every table and guard the rest with a no-new-rows check" + ), + ) + p.add_argument( + "--verifier", + default=None, + metavar="YAML", + help=( + "A verify.yaml whose `verifier:` section reads the system of " + "record (REST, FHIR, SQL); read once to find the saved record" + ), + ) + p.add_argument( + "--prove", + default=None, + choices=("mockmed", "effect-e2e"), + help=( + "Also run the proposed check end to end against this " + "fault-injection fixture (the check must match its record service)" + ), + ) + p.add_argument( + "--complete-read", + action="store_true", + help=( + "Declare that the reads return everything the save can change " + "(one store, no other tables). Recorded as declared, not proven" + ), + ) + p.add_argument( + "--recording", + default=None, + metavar="DIR", + help="The recording a bundle was compiled from (when SOURCE is a bundle)", + ) + p.add_argument( + "--bundle", + default=None, + metavar="DIR", + help=( + "The bundle compiled from this recording, for identity coverage " + "(when SOURCE is a recording)" ), ) p.set_defaults(func=_cmd_scaffold_verifier) diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 1655e64ff..3f7df3b71 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -216,6 +216,7 @@ def _surface_matches_kind(self) -> "BackendConfig": "email", "document", "document-hash", + "surfaces", } ) @@ -320,6 +321,19 @@ class EffectsConfig(BaseModel): #: unambiguous. candidates: list["EffectsConfig"] = Field(default_factory=list) + #: ``kind: surfaces`` only: one reviewed read per table or resource the + #: save touches (surface name -> that read's own ``effects`` settings). + #: Each effect names its surface in ``probe`` (``surface=``) and is + #: judged by that read alone; an effect naming no configured surface + #: halts. See ``runtime/effects/surfaces.py`` and docs/EFFECT_KIT.md. + surfaces: dict[str, "EffectsConfig"] = Field(default_factory=dict) + + #: ``rest`` / ``sql`` / ``fhir``: record fields removed before judging. + #: ``[id]`` judges records by their whole content, so a save that changes + #: another record shows as that record lost (``forbid_collateral_loss``) + #: instead of passing because its id is unchanged. Empty -> as read. + records_drop_fields: list[str] = Field(default_factory=list) + # -- onscreen (no-API screen read-back; auto-derived per-effect region) --- #: Explicit read-back region ``(x, y, w, h)`` for a hand-configured #: deployment. Normally left None — the compiler auto-derives a per-effect @@ -560,6 +574,38 @@ def _validate_candidates(self) -> "EffectsConfig": ) return self + @model_validator(mode="after") + def _validate_records_drop_fields(self) -> "EffectsConfig": + kind = (self.kind or "none").strip().lower() + if self.records_drop_fields and kind not in ("rest", "sql", "fhir"): + raise ValueError( + "effects.records_drop_fields applies only to kinds rest, sql and fhir" + ) + return self + + @model_validator(mode="after") + def _validate_surfaces(self) -> "EffectsConfig": + kind = (self.kind or "none").strip().lower() + if kind != "surfaces": + if self.surfaces: + raise ValueError("effects.surfaces applies only to kind surfaces") + return self + if not self.surfaces: + raise ValueError("effects.kind 'surfaces' needs at least one surface") + for name, surface in self.surfaces.items(): + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", str(name)): + raise ValueError( + f"effects.surfaces name {name!r} must be letters, digits, " + "'_', '.' or '-'" + ) + surface_kind = (surface.kind or "none").strip().lower() + if surface_kind in ("", "none", "surfaces") or surface.candidates: + raise ValueError( + f"effects.surfaces[{name!r}] must configure one plain " + "verifier kind (not none, surfaces or candidates)" + ) + return self + EffectsConfig.model_rebuild() @@ -1426,6 +1472,14 @@ def prune(dumped: dict[str, Any]) -> dict[str, Any]: if not dumped.get(name): dumped.pop(name, None) dumped["candidates"] = [prune(item) for item in dumped.get("candidates", [])] + # Added after identities were first issued: present only when set. + if not dumped.get("records_drop_fields"): + dumped.pop("records_drop_fields", None) + surfaces = dumped.pop("surfaces", None) or {} + if surfaces: + dumped["surfaces"] = { + name: prune(item) for name, item in sorted(surfaces.items()) + } return dumped return prune(cfg.model_dump(mode="json")) @@ -1513,6 +1567,17 @@ def _build_effect_verifier_unredacted( poll_interval_s=cfg.poll_interval_s, ) + if kind == "surfaces": + from openadapt_flow.runtime.effects.surfaces import SurfaceRoutedVerifier + + built: dict[str, Any] = {} + for name, surface in cfg.surfaces.items(): + verifier = build_effect_verifier(surface, params) + if verifier is None: # guarded by EffectsConfig, retained fail-closed + raise ValueError(f"effects.surfaces[{name!r}] built no verifier") + built[name] = verifier + return SurfaceRoutedVerifier(built) + if kind == "rest": if not cfg.base_url: raise ValueError("effects.kind 'rest' requires effects.base_url") @@ -1543,6 +1608,7 @@ def _build_effect_verifier_unredacted( headers=headers, timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, + drop_fields=tuple(cfg.records_drop_fields), ) if kind == "fhir": @@ -1581,6 +1647,7 @@ def _build_effect_verifier_unredacted( verify_tls=cfg.verify_tls, timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, + drop_fields=tuple(cfg.records_drop_fields), ) if kind == "sql": @@ -1618,6 +1685,7 @@ def _build_effect_verifier_unredacted( ), timeout_s=cfg.timeout_s, poll_interval_s=cfg.poll_interval_s, + drop_fields=tuple(cfg.records_drop_fields), ) if kind == "file": diff --git a/packages/flow/openadapt_flow/record_check_identity.py b/packages/flow/openadapt_flow/record_check_identity.py new file mode 100644 index 000000000..ec0ee62c7 --- /dev/null +++ b/packages/flow/openadapt_flow/record_check_identity.py @@ -0,0 +1,176 @@ +"""Propose identity evidence for clicks a bundle runs without a right-record check. + +docs/LIMITS.md: identity protection applies only where it is armed, and a live +OpenEMR bundle armed 4 to 7 of its 12 clicks. ``scaffold-verifier --bundle`` +reports identity coverage before and after evidence taken from the recording's +DOM / accessibility context near each unarmed click. + +Evidence is proposed only where the shipped runtime checks it on the bundle's +surface: the structured-text tier, which compares the recorded text with the +live DOM / accessibility text at the click point. A record named elsewhere on +the page (the title, the URL) is listed but not counted, because the runtime +does not check identity there. Nothing is applied to the bundle; a person +reviews ``identity.proposed.json`` and re-compiles. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Mapping, Optional + +from openadapt_flow.record_check_synthesis import param_lookup + +#: Accessibility roles whose accessible name is the record row itself; the +#: browser and desktop backends read it as the row's structured identity. +ROW_ROLES = frozenset({"row", "listitem", "option", "treeitem"}) + +IDENTITY_PROPOSED_NAME = "identity.proposed.json" + + +def _events(recording_dir: Path) -> list[dict[str, Any]]: + path = Path(recording_dir) / "events.jsonl" + if not path.is_file(): + return [] + out: list[dict[str, Any]] = [] + for line in path.read_text("utf-8").splitlines(): + try: + out.append(json.loads(line) if line.strip() else {}) + except ValueError: + out.append({}) + return out + + +def identity_coverage( + bundle_dir: Path, + recording_dir: Path, + params: Mapping[str, str], +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """Identity-armed clicks now, and with evidence from the recording's + DOM / accessibility context. + + Returns ``(report, proposals)``. The report holds counts and parameter + names only; ``proposals`` hold the evidence text for the private + ``identity.proposed.json``. Evidence is proposed only where the shipped + runtime checks it on this bundle's surface (the structured-text tier, at + the click point); context it can't check is listed, not counted. + """ + + from openadapt_flow.ir import Workflow + from openadapt_flow.policy import is_identity_applicable, is_identity_armed + from openadapt_flow.runtime.identity import verifiable_identity_evidence + + workflow = Workflow.load(Path(bundle_dir)) + surface = getattr(workflow.surface, "value", workflow.surface) + checkable = "structured_text" in verifiable_identity_evidence(surface) + events = _events(recording_dir) + lookup, _ = param_lookup(params) + + applicable = [s for s in workflow.steps if is_identity_applicable(s)] + armed = [s for s in applicable if is_identity_armed(s)] + proposals: list[dict[str, Any]] = [] + unchecked: list[dict[str, Any]] = [] + still_unarmed: list[dict[str, Any]] = [] + for step in applicable: + if is_identity_armed(step): + continue + match = re.fullmatch(r"step_(\d+)", step.id) + event = ( + events[int(match.group(1))] + if match and int(match.group(1)) < len(events) + else {} + ) + text: Optional[str] = None + basis = "" + recorded = event.get("structured_identity") + if isinstance(recorded, str) and recorded.strip(): + text, basis = ( + recorded.strip(), + "DOM / accessibility text of the clicked row", + ) + raw_structural = event.get("structural") + structural: dict[str, Any] = ( + raw_structural if isinstance(raw_structural, dict) else {} + ) + name = str(structural.get("name") or "").strip() + role = str(structural.get("role") or "").strip().lower() + if text is None and name and role in ROW_ROLES: + text, basis = name, f"accessible name of the clicked {role}" + names_params = sorted( + {p for value, p in lookup.items() if text and value in text} + ) + if text is not None and checkable: + proposals.append( + { + "step_id": step.id, + "anchor_field": "structured_identity", + "value": text, + "basis": basis, + "contains_params": names_params, + } + ) + continue + context = [ + key + for key in ("title_before", "url_before") + if isinstance(event.get(key), str) + and any(value in event[key] for value in lookup) + ] + if text is not None or context: + unchecked.append( + { + "step_id": step.id, + "seen": basis or ", ".join(context), + "why_not_proposed": ( + "this surface has no structured text at the click" + if text is not None + else "the record is named on the page, not at the click; " + "the runtime checks identity at the click point" + ), + } + ) + still_unarmed.append( + { + "step_id": step.id, + "reason": step.identity_unarmed_reason or "no evidence", + } + ) + report = { + "surface": surface or "web", + "applicable": len(applicable), + "armed_before": len(armed), + "armed_after": len(armed) + len(proposals), + "proposed": [{k: v for k, v in p.items() if k != "value"} for p in proposals], + "seen_not_checkable": unchecked, + "still_unarmed": still_unarmed, + "approved": False, + } + return report, proposals + + +def write_identity_proposals( + proposals: list[dict[str, Any]], out_dir: Path +) -> Optional[Path]: + """Write the proposed evidence (it holds record text) with mode 0600.""" + + import os + + if not proposals: + return None + path = out_dir / IDENTITY_PROPOSED_NAME + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as fh: + json.dump( + { + "schema": "openadapt.identity-proposal/v1", + "approved": False, + "proposals": proposals, + }, + fh, + indent=2, + sort_keys=True, + ) + fh.write("\n") + os.chmod(path, 0o600) + return path diff --git a/packages/flow/openadapt_flow/record_check_proof.py b/packages/flow/openadapt_flow/record_check_proof.py new file mode 100644 index 000000000..9139588d8 --- /dev/null +++ b/packages/flow/openadapt_flow/record_check_proof.py @@ -0,0 +1,377 @@ +"""Run a proposed record check end to end against a fault-injection fixture. + +:mod:`openadapt_flow.record_check_synthesis` judges a proposed check on bad +writes built from the recorded records. This module goes further on the two +fixtures that inject bad writes at the persistence boundary: + +- ``mockmed``: the MockMed fault server (:mod:`openadapt_flow.mockmed.fault_server`, + the ``--break-it`` fixture), one in-memory store read at ``/api/db``; +- ``effect-e2e``: the SQLite record service of ``benchmark/effect_e2e`` (repo + only), with an encounters table, a billing table the REST read does not see, + and the app's own save banner. + +For each fault, the save goes through the real +:class:`~openadapt_flow.runtime.replayer.Replayer` and +:class:`~openadapt_flow.runtime.actuators.ApiActuator`, the proposed check is +the replayer's effect verifier, and the fixture's own ground truth (which never +sees the check) says whether the save was right. The check passes only when +every wrong save stops the run and the clean save is reported as done. + +Each run uses new values for free-text parameters, so a check that froze the +demonstration's values fails the clean save. No model calls, localhost only. +""" + +from __future__ import annotations + +import struct +import zlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Optional +from urllib.parse import urlsplit + +from openadapt_flow.runtime.effects.effect import Effect + +#: The write is synchronous through the API actuator, so a short poll is not +#: a source of flakiness; a refuted effect polls to this deadline. +PROOF_EFFECT_TIMEOUT_S = 0.3 +ACTUATOR_TIMEOUT_S = 0.6 + + +class ProofUnavailable(RuntimeError): + """The named fixture is not available in this install.""" + + +@dataclass +class Fixture: + name: str + #: ``() -> (origin, sqlite_path_or_None, stop, handle)`` + serve: Callable[[], tuple[str, Optional[Path], Callable[[], None], Any]] + reset: Callable[[Any, bool], None] + snapshot: Callable[[Any], Any] + #: ``(before, after, run_params) -> (correct, fault_class)`` + judge: Callable[[Any, Any, Mapping[str, str]], tuple[bool, str]] + #: ``(name, seed_concurrent)``; the first is the clean save. + scenarios: tuple[tuple[str, bool], ...] + write_path: str + default_body: dict[str, str] + + +# -- MockMed ----------------------------------------------------------------- + + +def _mockmed_serve() -> tuple[str, Optional[Path], Callable[[], None], Any]: + from openadapt_flow.mockmed.fault_server import serve + + url, db, stop = serve() + return url.rstrip("/"), None, stop, db + + +def _mockmed_judge( + before: list[dict[str, Any]], after: list[dict[str, Any]], params: Mapping[str, str] +) -> tuple[bool, str]: + patient = params.get("patient_id", "p1") + note = params.get("note", "") + before_ids = {r["id"] for r in before} + after_ids = {r["id"] for r in after} + if any(r["id"] not in after_ids for r in before): + return False, "collateral_loss" + new = [r for r in after if r["id"] not in before_ids] + mine = [r for r in new if str(r.get("patient_id")) == patient] + if not mine: + return False, "wrong_record" if new else "absent" + if len(mine) > 1: + return False, "duplicate" + if str(mine[0].get("note")) != note: + return False, "partial" + if len(new) > 1: + return False, "collateral_write" + return True, "correct" + + +MOCKMED = Fixture( + name="mockmed", + serve=_mockmed_serve, + reset=lambda db, seed: db.reset(seed_concurrent=seed), + snapshot=lambda db: [dict(r) for r in db.snapshot()["records"]], + judge=_mockmed_judge, + scenarios=( + ("ok", False), + ("no_persist", False), + ("partial", False), + ("wrong_record", False), + ("stale", True), + ("optimistic", False), + ("session", False), + ), + write_path="/api/encounter", + default_body={"patient_id": "{patient_id}", "type": "Triage", "note": "{note}"}, +) + + +def fixture(name: str) -> Fixture: + if name == "mockmed": + return MOCKMED + if name in ("effect-e2e", "effect_e2e"): + try: + from benchmark.effect_e2e.prove_check import FIXTURE + except ImportError as exc: # pragma: no cover - wheel installs + raise ProofUnavailable( + "the effect-e2e fixture lives in the repository " + "(packages/flow/benchmark); run from a checkout" + ) from exc + return FIXTURE + raise ProofUnavailable(f"unknown fault fixture {name!r} (mockmed, effect-e2e)") + + +# -- replayer fakes (the API tier never touches the screen) -------------------- + + +def _tiny_png() -> bytes: + raw = b"\x00\xff\xff\xff" + + def chunk(tag: bytes, data: bytes) -> bytes: + return ( + struct.pack(">I", len(data)) + + tag + + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + ) + + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", zlib.compress(raw)) + + chunk(b"IEND", b"") + ) + + +_PNG = _tiny_png() + + +class _NoScreen: + """Backend and vision stand-in; the API step never reads or acts on it.""" + + viewport = (2, 2) + + def screenshot(self) -> bytes: + return _PNG + + def wait_settled(self, backend: Any, **_kw: Any) -> bytes: + return _PNG + + def text_present(self, *_a: Any, **_kw: Any) -> bool: + return True + + def pixels_changed(self, *_a: Any, **_kw: Any) -> bool: + return True + + def phash_png(self, *_a: Any, **_kw: Any) -> str: + return "aa" + + def phash_distance(self, *_a: Any, **_kw: Any) -> int: + return 0 + + def ocr(self, *_a: Any, **_kw: Any) -> list: + return [] + + def __getattr__(self, name: str) -> Any: + if name.startswith("find_"): + return lambda *_a, **_kw: None + if name in ("click", "type_text", "press", "scroll"): + return lambda *_a, **_kw: None + raise AttributeError(name) + + +# -- rebinding the check to the fixture --------------------------------------- + + +def rebind( + section: Mapping[str, Any], *, origin: str, sqlite: Optional[Path] +) -> dict[str, Any]: + """The verifier section pointed at the live fixture instead of the + recording-time addresses (same paths, queries and fields).""" + + def walk(node: Any) -> Any: + if isinstance(node, dict): + out = {} + for key, value in node.items(): + if key == "base_url" and isinstance(value, str): + parts = urlsplit(value) + out[key] = origin + parts.path + elif key == "sqlite_database": + if sqlite is None: + raise ProofUnavailable( + "the check reads a database this fixture does not have" + ) + out[key] = str(sqlite) + else: + out[key] = walk(value) + return out + if isinstance(node, list): + return [walk(item) for item in node] + return node + + return walk(dict(section)) + + +def body_template( + write_requests: list[dict[str, Any]], + params: Mapping[str, str], + default: Mapping[str, str], +) -> dict[str, str]: + """The save's JSON body with recorded values replaced by ``{param}``.""" + + import json + + lookup = {value: name for name, value in params.items()} + for line in write_requests: + body = line.get("body") + try: + parsed = json.loads(body) if isinstance(body, str) else None + except ValueError: + parsed = None + if isinstance(parsed, dict) and parsed: + return { + str(key): ( + "{" + lookup[str(value)] + "}" + if str(value) in lookup + else str(value) + ) + for key, value in parsed.items() + } + return dict(default) + + +def run_values(params: Mapping[str, str], scenario: str) -> dict[str, str]: + """New values for free-text parameters; identifiers stay (they must exist).""" + + out = {} + for name, value in params.items(): + if any(ch.isspace() for ch in value) or len(value) > 64: + out[name] = f"{value} (proof {scenario})" + else: + out[name] = value + return out + + +# -- the run ------------------------------------------------------------------- + + +def prove_live( + fixture_name: str, + section: Mapping[str, Any], + effects: list[Effect], + params: Mapping[str, str], + *, + write_requests: Optional[list[dict[str, Any]]] = None, + work_dir: Path, +) -> dict[str, Any]: + """Run every fixture fault through the replayer with the proposed check.""" + + from openadapt_flow.deployment import EffectsConfig, build_effect_verifier + from openadapt_flow.ir import ( + ActionKind, + ApiBinding, + Step, + Workflow, + ) + from openadapt_flow.runtime.actuators import ApiActuator + from openadapt_flow.runtime.replayer import Replayer + + fx = fixture(fixture_name) + origin, sqlite_path, stop, handle = fx.serve() + body = body_template(write_requests or [], params, fx.default_body) + checked = [ + effect.model_copy( + update={ + "needs_operator_confirmation": False, + "timeout_s": PROOF_EFFECT_TIMEOUT_S, + } + ) + for effect in effects + ] + rows: list[dict[str, Any]] = [] + try: + live_section = rebind(section, origin=origin, sqlite=sqlite_path) + for index, (scenario, seed) in enumerate(fx.scenarios): + values = run_values(params, scenario) + fx.reset(handle, seed) + before = fx.snapshot(handle) + verifier = build_effect_verifier( + EffectsConfig.model_validate(live_section), values + ) + workflow = Workflow( + name=f"record-check-proof-{scenario}", + steps=[ + Step( + id="save", + intent="save the record", + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + effects=checked, + api_binding=ApiBinding( + method="POST", + url_template=f"{fx.write_path}?fault={scenario}", + body_template=body, + effects=[e.model_copy(deep=True) for e in checked], + timeout_s=ACTUATOR_TIMEOUT_S, + ), + ) + ], + params=values, + ) + run_dir = work_dir / f"{fx.name}-{index:02d}-{scenario}" + (run_dir / "bundle" / "templates").mkdir(parents=True, exist_ok=True) + screen = _NoScreen() + report = Replayer( + screen, + vision=screen, + effect_verifier=verifier, + api_actuator=ApiActuator(origin, timeout_s=ACTUATOR_TIMEOUT_S), + poll_interval_s=0.01, + ).run(workflow, bundle_dir=run_dir / "bundle", run_dir=run_dir / "out") + after = fx.snapshot(handle) + correct, fault_class = fx.judge(before, after, values) + step = report.results[0] if report.results else None + effect_verified = getattr(step, "effect_verified", None) + done = bool(report.success) + if correct and done: + result = "clean pass" + elif correct and effect_verified is False: + result = "false stop" + elif correct: + result = "stopped by the actuator (outcome unknown)" + elif done: + result = "MISSED" + elif effect_verified is False: + result = "caught by the check" + else: + result = "caught by the actuator" + rows.append( + { + "scenario": scenario, + "ground_truth": fault_class, + "reported_done": done, + "effect_verified": effect_verified, + "result": result, + } + ) + finally: + stop() + misses = [ + f"{row['scenario']}: {row['result']} (ground truth {row['ground_truth']})" + for row in rows + if row["result"] in ("MISSED", "false stop") + ] + clean = rows[0]["result"] == "clean pass" if rows else False + if rows and not clean and not misses: + misses.append(f"{rows[0]['scenario']}: the clean save did not finish") + return { + "fixture": fx.name, + "runs": rows, + "misses": misses, + "passed": bool(rows) and clean and not misses, + } diff --git a/packages/flow/openadapt_flow/record_check_synthesis.py b/packages/flow/openadapt_flow/record_check_synthesis.py new file mode 100644 index 000000000..ea9e3673b --- /dev/null +++ b/packages/flow/openadapt_flow/record_check_synthesis.py @@ -0,0 +1,1655 @@ +"""Build a record check from the reads the app made, and fault-test it. + +``openadapt-flow scaffold-verifier`` used to write a draft that checked nothing +until a person filled it in. This module fills it in from evidence the +recording already holds, then tries to break it before anyone reviews it. + +1. **Find the read.** After the save, the app reads the saved record back to + show it: a REST, FHIR or XHR ``GET``. Those reads are in the recording's + network capture (``network.jsonl``, the ``openadapt.network-capture/1`` + format ``record --capture-network`` writes). A database read path + (``--db``) or an existing verifier section (``--verifier``) works too. The + read chosen is the one that returns a record holding the recorded + parameter values. +2. **Fill in the check.** The check reads that path, expects exactly one new + record with the recorded values (bound to run parameters, so each run is + checked against its own values), and must not lose or change a record that + was there before. With a database read path, every other table gets a + guard: no row added, changed or removed. Write targets nothing covers, and + tables that could not be read, are listed and keep the check a draft. +3. **Prove it.** The check is judged against the fault classes of + ``benchmark/effect_e2e`` applied to the recorded records (nothing saved, a + field dropped, saved twice, saved on the wrong record, another record of + the same subject destroyed, a stray write elsewhere), plus another record + changed or deleted. Every generated read is built, and a database read is + run, so a read that can't work is caught too. On a fixture (MockMed, the + effect_e2e record service) the check also runs end to end through the + replayer (:mod:`openadapt_flow.record_check_proof`). A check that misses + any fault, or stops the clean save, is written as a draft with the misses + listed. +4. **A person approves.** Every effect carries + ``needs_operator_confirmation: true``; ``verify-run`` and ``qualify`` + refuse the file until a person removes those lines. + +Reads whose records carry an ``id`` get a second, content read +(``.content``, ``records_drop_fields: [id]``): the id read counts new +records, so a stray row identical to an existing one still counts; the content +read sees a record whose id stayed the same but whose fields changed. + +No model calls, nothing sent anywhere. The only network read is the optional +``--verifier`` read of the system of record the operator named. +""" + +from __future__ import annotations + +import json +import re +import sqlite3 +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional +from urllib.parse import parse_qsl, quote, urlsplit + +from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects.effect import ( + Effect, + EffectKind, + EffectState, + ValueExpr, + Verdict, + stable_id, +) +from openadapt_flow.runtime.effects.surfaces import surface_probe + +#: The network capture ``record --capture-network`` writes. Read here without +#: importing the recorder side, so the format has one owner +#: (``docs/NETWORK_CAPTURE.md``). +NETWORK_FILE = "network.jsonl" +NETWORK_SCHEMA = "openadapt.network-capture/1" + +PROPOSED_NAME = "record_check.proposed.yaml" +PROOF_NAME = "record_check.proof.json" +PROOF_SCHEMA = "openadapt.record-check-proof/v1" + +#: Largest table ``--db`` scans for the saved row. +MAX_SCAN_ROWS = 20000 + +#: The content twin of a read whose records carry an ``id``. +CONTENT_SUFFIX = ".content" +#: A selector no record matches: with ``forbid_collateral_loss`` it makes +#: every record that was there before a record that must still be there. +_NO_RECORD = {"_openadapt_no_such_field": ValueExpr(literal="-")} + +_WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) +_LIST_KEYS = ("records", "data", "items", "results", "entries", "rows") +_SESSION_VALUE = re.compile(r"]*>|") +_PLAIN_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") +#: A URL path segment that looks like it names one record (an MRN, a numeric +#: id, a UUID), as opposed to a resource or version name (``api``, ``R4``). +_RECORD_SEGMENT = re.compile(r"\d{3,}|^[0-9a-fA-F]{8,}$|^[0-9a-fA-F-]{16,}$") + + +class SynthesisError(ValueError): + """The inputs hold nothing a record check can be built from.""" + + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- + + +def load_network(recording_dir: Path) -> list[dict[str, Any]]: + """The capture's request lines; an absent file is an empty capture.""" + + path = Path(recording_dir) / NETWORK_FILE + if not path.is_file(): + return [] + try: + rows = [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + except (OSError, ValueError) as exc: + raise SynthesisError(f"{path} could not be read: {exc}") from exc + if ( + not rows + or not isinstance(rows[0], dict) + or rows[0].get("schema") != NETWORK_SCHEMA + ): + raise SynthesisError(f"{path} is not a {NETWORK_SCHEMA} capture") + return [row for row in rows[1:] if isinstance(row, dict)] + + +def recorded_params(recording_dir: Path) -> dict[str, str]: + """The demonstration's parameter values from ``meta.json``.""" + + try: + meta = json.loads((Path(recording_dir) / "meta.json").read_text("utf-8")) + except (OSError, ValueError): + return {} + params = meta.get("params") if isinstance(meta, dict) else None + if not isinstance(params, dict): + return {} + return { + str(name): str(value) + for name, value in params.items() + if value is not None and str(value).strip() + } + + +def param_lookup(params: Mapping[str, str]) -> tuple[dict[str, str], list[str]]: + """Recorded value -> parameter name; values two parameters share are skipped.""" + + owners: dict[str, list[str]] = {} + for name, value in params.items(): + owners.setdefault(str(value), []).append(name) + lookup = {value: names[0] for value, names in owners.items() if len(names) == 1} + notes = [ + f"parameters {', '.join(sorted(names))} share one value, so neither is checked" + for names in owners.values() + if len(names) > 1 + ] + return lookup, notes + + +def _identity_like(value: str) -> bool: + """A short single-token value (an id, a name), not free text.""" + + return bool(value) and len(value) <= 64 and not any(ch.isspace() for ch in value) + + +def _t(line: Mapping[str, Any]) -> float: + value = line.get("t") + return float(value) if isinstance(value, (int, float)) else 0.0 + + +# --------------------------------------------------------------------------- +# The check being built +# --------------------------------------------------------------------------- + + +@dataclass +class Surface: + """One read the check makes, and what it saw while recording.""" + + name: str + #: The read's ``effects`` settings (a ``deployment.yaml`` ``effects:``). + config: dict[str, Any] + #: Records the read returns after the save, in the shape the check sees. + after: list[dict[str, Any]] + #: Records before the save; derived (after minus the saved record) when + #: no read before the save was recorded. + before: list[dict[str, Any]] + before_recorded: bool + #: Where the read came from, for the person reviewing it (no values). + source: str + #: The record the save wrote, when this surface holds it. + target: Optional[dict[str, Any]] = None + #: Record field -> parameter name, for fields equal to a recorded value. + bound: dict[str, str] = field(default_factory=dict) + #: Record field -> value the save sent that is not a parameter. + constants: dict[str, str] = field(default_factory=dict) + #: Parameters the read itself is scoped by (a URL path or query value, a + #: SQL ``WHERE``): a record saved under another value is not returned. + scope: list[str] = field(default_factory=list) + + @property + def is_target(self) -> bool: + return self.target is not None + + @property + def has_ids(self) -> bool: + return any("id" in record for record in self.after + self.before) + + @property + def content_name(self) -> str: + return self.name + CONTENT_SUFFIX + + +@dataclass +class ReadSearch: + """What the search for the read found, including what it rejected.""" + + surfaces: list[Surface] = field(default_factory=list) + write_requests: list[dict[str, Any]] = field(default_factory=list) + save_t: Optional[float] = None + notes: list[str] = field(default_factory=list) + #: Write targets and tables the check does not read (plain sentences). + #: Any entry keeps the check a draft. + uncovered: list[str] = field(default_factory=list) + #: Reads that would not work as generated. Any entry keeps it a draft. + problems: list[str] = field(default_factory=list) + #: True when every table the save could touch is read (a database path), + #: or a person declared the read complete (``--complete-read``). + closed_world: bool = False + #: ``"database"`` (every table read) or ``"declared"`` (a person said so). + closed_world_basis: Optional[str] = None + #: Surface names in use, across every source. + taken: set[str] = field(default_factory=set) + + +# -- network capture --------------------------------------------------------- + + +def _json_body(line: Mapping[str, Any]) -> Any: + text = line.get("response_text") + if not isinstance(text, str) or not text.strip(): + return None + if line.get("response_truncated"): + return None + try: + return json.loads(text) + except ValueError: + return None + + +def _flat(node: Any, prefix: str = "") -> dict[str, Any]: + """Dotted scalar paths of a JSON node; a list contributes its first item.""" + + out: dict[str, Any] = {} + if isinstance(node, dict): + for key, value in node.items(): + path = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, (dict, list)): + out.update(_flat(value, path)) + elif value is not None: + out[path] = value + elif isinstance(node, list) and node: + out.update(_flat(node[0], prefix)) + return out + + +@dataclass +class _Listing: + kind: str # "rest" | "fhir" + records_key: Optional[str] + resources: list[dict[str, Any]] + resource_type: Optional[str] = None + + +def _listings(body: Any) -> list[_Listing]: + """The record lists a JSON response holds, in a shape a check can read.""" + + if isinstance(body, dict) and body.get("resourceType") == "Bundle": + resources: list[dict[str, Any]] = [ + entry["resource"] + for entry in body.get("entry") or [] + if isinstance(entry, dict) and isinstance(entry.get("resource"), dict) + ] + types = {res.get("resourceType") for res in resources} + resource_type = next(iter(types)) if len(types) == 1 else None + return [_Listing("fhir", None, resources, resource_type)] + if isinstance(body, list): + records = [item for item in body if isinstance(item, dict)] + return [_Listing("rest", None, records)] if records else [] + if isinstance(body, dict): + found = [] + keys = sorted( + body, + key=lambda k: ( + k not in _LIST_KEYS, + _LIST_KEYS.index(k) if k in _LIST_KEYS else 0, + ), + ) + for key in keys: + value = body[key] + if isinstance(value, list) and any(isinstance(v, dict) for v in value): + found.append( + _Listing( + "rest", str(key), [v for v in value if isinstance(v, dict)] + ) + ) + return found + return [] + + +def _bind_fields( + flat: Mapping[str, Any], lookup: Mapping[str, str] +) -> tuple[dict[str, str], dict[str, str]]: + """``(exact, reference)``: fields equal to a recorded value, and FHIR + references (``Patient/``) that end in one.""" + + exact: dict[str, str] = {} + reference: dict[str, str] = {} + for path, value in flat.items(): + if isinstance(value, bool): + continue + text = str(value).strip() + if text in lookup: + exact[path] = lookup[text] + elif "/" in text and text.rsplit("/", 1)[1] in lookup: + reference[path] = lookup[text.rsplit("/", 1)[1]] + return exact, reference + + +def _unique_name(raw: str, taken: set[str]) -> str: + name = re.sub(r"[^A-Za-z0-9_]", "_", raw.replace(".", "_")).strip("_") or "records" + base, n = name, 2 + while name in taken or name + CONTENT_SUFFIX in taken: + name, n = f"{base}_{n}", n + 1 + taken.add(name) + taken.add(name + CONTENT_SUFFIX) + return name + + +def _field_name(path: str, taken: set[str]) -> str: + name = re.sub(r"[^A-Za-z0-9_]", "_", path.replace(".", "_")) or "field" + base, n = name, 2 + while name in taken: + name, n = f"{base}_{n}", n + 1 + taken.add(name) + return name + + +def _origin(url: str) -> str: + parts = urlsplit(url) + return f"{parts.scheme}://{parts.netloc}" + + +@dataclass +class _Templated: + #: The path with recorded values as ``{param}`` placeholders, braces in + #: literal segments doubled only when a placeholder makes it a template. + path: str + path_params: list[str] + literal_query: list[tuple[str, str]] + bound_query: dict[str, str] + #: The path for names and messages: placeholders kept, any segment that + #: looks like a record id replaced by ``*``. + safe: str + #: Literal segments that look like one record's id (a frozen value). + pinned: int + + +def _templated(url: str, lookup: Mapping[str, str]) -> _Templated: + parts = urlsplit(url) + raw = parts.path.split("/") + bound = [lookup.get(segment) if segment else None for segment in raw] + templated = any(bound) + segments: list[str] = [] + safe: list[str] = [] + pinned = 0 + for segment, name in zip(raw, bound): + if name: + segments.append("{" + name + "}") + safe.append("{" + name + "}") + continue + segments.append( + segment.replace("{", "{{").replace("}", "}}") if templated else segment + ) + if _RECORD_SEGMENT.search(segment): + pinned += 1 + safe.append("*") + else: + safe.append(re.sub(r"[^A-Za-z0-9_.-]", "_", segment)) + literal_query: list[tuple[str, str]] = [] + bound_query: dict[str, str] = {} + for key, value in parse_qsl(parts.query, keep_blank_values=True): + name = lookup.get(value) + if name: + bound_query[key] = name + else: + literal_query.append((key, value)) + return _Templated( + path="/".join(segments), + path_params=sorted({n for n in bound if n}), + literal_query=literal_query, + bound_query=bound_query, + safe="/".join(safe), + pinned=pinned, + ) + + +def _write_values(line: Mapping[str, Any]) -> dict[str, str]: + """Field -> value the write sent (form or JSON body).""" + + values: dict[str, str] = {} + for pair in line.get("form") or []: + if isinstance(pair, (list, tuple)) and len(pair) == 2: + values[str(pair[0])] = str(pair[1]) + body = line.get("body") + if isinstance(body, str) and body.strip(): + try: + parsed = json.loads(body) + except ValueError: + parsed = None + if isinstance(parsed, dict): + for key, value in _flat(parsed).items(): + values[key] = str(value) + for key, value in parse_qsl(urlsplit(str(line.get("url") or "")).query): + values.setdefault(key, value) + return values + + +def _save_event_times(recording_dir: Path) -> tuple[Optional[float], Optional[float]]: + """``(save, previous)``: the last write-shaped step's ``t`` and the ``t`` + of the event before it.""" + + from openadapt_flow.scaffold_verifier import candidates_from_recording + + times: dict[int, float] = {} + events_path = Path(recording_dir) / "events.jsonl" + if not events_path.is_file(): + return None, None + for index, line in enumerate(events_path.read_text("utf-8").splitlines()): + try: + event = json.loads(line) if line.strip() else {} + except ValueError: + continue + if isinstance(event, dict) and isinstance(event.get("t"), (int, float)): + times[index] = float(event["t"]) + try: + candidates = candidates_from_recording(Path(recording_dir)) + except SystemExit: + candidates = [] + indices = [ + int(c.step_id.split("_")[1]) + for c in candidates + if int(c.step_id.split("_")[1]) in times + ] + if not indices: + return None, None + save = max(indices) + earlier = [times[i] for i in times if i < save] + return times[save], (max(earlier) if earlier else None) + + +def _accepted(line: Mapping[str, Any]) -> bool: + """A write the server accepted: 2xx, or a redirect after a form post.""" + + status = line.get("status") + return isinstance(status, int) and 200 <= status < 400 and not line.get("failed") + + +def _ok(line: Mapping[str, Any]) -> bool: + status = line.get("status") + return isinstance(status, int) and 200 <= status < 300 and not line.get("failed") + + +def save_window( + recording_dir: Path, network: list[dict[str, Any]] +) -> tuple[Optional[float], Optional[float], list[dict[str, Any]]]: + """``(first, last, writes)``: the save's accepted write requests. + + An event's ``t`` is stamped after the screen settles, so the save's own + requests come BEFORE the Save step's ``t``: the window runs from the + previous event to the Save step. A write stamped after the Save step (a + slow settle on the recorder's side) is taken when the window is empty. + With no write request, the Save step's ``t`` is the only time there is. + """ + + event_t, previous_t = _save_event_times(recording_dir) + writes = sorted( + ( + line + for line in network + if str(line.get("method") or "").upper() in _WRITE_METHODS + and _accepted(line) + ), + key=_t, + ) + if not writes: + return event_t, event_t, [] + low = previous_t if previous_t is not None else float("-inf") + high = event_t if event_t is not None else float("inf") + chosen = [w for w in writes if low < _t(w) <= high] + if not chosen: + later = [w for w in writes if _t(w) > low] + chosen = later[:1] if later else writes[-1:] + return _t(chosen[0]), _t(chosen[-1]), chosen + + +def _surface_from_listing( + line: Mapping[str, Any], + listing: _Listing, + lookup: Mapping[str, str], + *, + before_line: Optional[Mapping[str, Any]], + taken: set[str], +) -> tuple[Optional[Surface], Optional[str]]: + """A surface for one listing that holds the saved record, or why not.""" + + url = str(line.get("url") or "") + tpl = _templated(url, lookup) + if _SESSION_VALUE.search(url): + return None, f"GET {tpl.safe}: the URL carries a session token" + + best: Optional[tuple[dict[str, Any], dict[str, str], dict[str, str]]] = None + for resource in listing.resources: + exact, reference = _bind_fields(_flat(resource), lookup) + if not exact: + continue + if best is None or len(exact) > len(best[1]): + best = (resource, exact, reference) + if best is None: + return None, None + resource, exact, reference = best + origin = _origin(url) + source = f"GET {tpl.safe} (t={line.get('t')})" + + if listing.kind == "fhir": + resource_type = listing.resource_type or str(resource.get("resourceType") or "") + marker = "/" + resource_type + if not resource_type or marker not in tpl.path: + return None, f"GET {tpl.safe}: FHIR resource type not in the URL" + base = origin + tpl.path[: tpl.path.rindex(marker)] + if tpl.path_params: + return None, ( + f"GET {tpl.safe}: the FHIR base path holds a recorded value; " + "the FHIR check takes a fixed base and search parameters" + ) + field_paths = {"id": "id"} + names: set[str] = {"id"} + bound: dict[str, str] = {} + for dotted, param in sorted(exact.items()): + if dotted == "id": + continue + name = _field_name(dotted, names) + field_paths[name] = dotted + bound[name] = param + config: dict[str, Any] = { + "kind": "fhir", + "base_url": base, + "resource_type": resource_type, + "field_paths": field_paths, + } + if tpl.literal_query: + config["search_params"] = dict(tpl.literal_query) + if tpl.bound_query: + config["search_param_exprs"] = { + key: {"param": name} for key, name in tpl.bound_query.items() + } + from openadapt_flow.runtime.effects.fhir import extract_path + + def shape(res: dict[str, Any]) -> dict[str, Any]: + return {key: extract_path(res, p) for key, p in field_paths.items()} + + after = [shape(res) for res in listing.resources] + target = shape(resource) + before_records, recorded = _before_records( + before_line, listing_kind="fhir", records_key=None, shape=shape + ) + surface = Surface( + name=_unique_name(f"fhir_{resource_type}", taken), + config=config, + after=after, + before=before_records if recorded else _minus(after, target), + before_recorded=recorded, + source=source, + target=target, + bound=bound, + scope=sorted(set(tpl.bound_query.values())), + ) + unscoped = sorted( + set(reference.values()) + - set(tpl.bound_query.values()) + - set(bound.values()) + ) + if unscoped: + surface.config["_unscoped_references"] = unscoped + if tpl.pinned: + surface.config["_pinned"] = tpl.pinned + return surface, None + + bound = {path_: param for path_, param in exact.items() if "." not in path_} + if not bound: + return None, ( + f"GET {tpl.safe}: the recorded values sit in nested fields; " + "the REST check reads top-level fields only" + ) + records_path = tpl.path + placeholders = set(tpl.path_params) + if tpl.literal_query or tpl.bound_query: + query = [ + f"{quote(k, safe='')}={quote(v, safe='')}" for k, v in tpl.literal_query + ] + query += [ + f"{quote(k, safe='')}={{{name}}}" for k, name in tpl.bound_query.items() + ] + if tpl.bound_query and not tpl.path_params: + # The query makes the path a template: double literal braces. + records_path = records_path.replace("{", "{{").replace("}", "}}") + records_path += "?" + "&".join(query) + placeholders.update(tpl.bound_query.values()) + config = { + "kind": "rest", + "base_url": origin, + "records_path": records_path, + "records_key": listing.records_key, + } + if placeholders: + config["path_params"] = {key: {"param": key} for key in sorted(placeholders)} + if tpl.pinned: + config["_pinned"] = tpl.pinned + after = [dict(r) for r in listing.resources] + target = dict(resource) + before_records, recorded = _before_records( + before_line, listing_kind="rest", records_key=listing.records_key, shape=dict + ) + surface = Surface( + name=_unique_name(tpl.safe.replace("*", "x").strip("/") or "records", taken), + config=config, + after=after, + before=before_records if recorded else _minus(after, target), + before_recorded=recorded, + source=source, + target=target, + bound=bound, + scope=sorted(placeholders), + ) + return surface, None + + +def _before_records( + line: Optional[Mapping[str, Any]], + *, + listing_kind: str, + records_key: Optional[str], + shape: Any, +) -> tuple[list[dict[str, Any]], bool]: + if line is None: + return [], False + body = _json_body(line) + records: Any = None + if listing_kind == "fhir": + if isinstance(body, dict) and body.get("resourceType") == "Bundle": + records = [ + entry["resource"] + for entry in body.get("entry") or [] + if isinstance(entry, dict) and isinstance(entry.get("resource"), dict) + ] + elif records_key is None: + records = body if isinstance(body, list) else None + elif isinstance(body, dict): + records = body.get(records_key) + if not isinstance(records, list): + return [], False + return [shape(r) for r in records if isinstance(r, dict)], True + + +def _minus( + records: list[dict[str, Any]], target: Mapping[str, Any] +) -> list[dict[str, Any]]: + """``records`` without one copy of ``target``.""" + + key = stable_id(dict(target)) + out: list[dict[str, Any]] = [] + removed = False + for record in records: + if not removed and stable_id(dict(record)) == key: + removed = True + continue + out.append(dict(record)) + return out + + +def reads_from_network( + recording_dir: Path, params: Mapping[str, str], search: ReadSearch +) -> None: + """Add the GET that returns the saved record to ``search``.""" + + network = load_network(recording_dir) + if not network: + return + lookup, _ = param_lookup(params) + first, last, writes = save_window(recording_dir, network) + search.save_t = last + search.write_requests = writes + if not writes: + search.notes.append( + "the capture holds no accepted write request; reads are taken after " + "the Save step's time" + ) + gets = [ + line + for line in network + if str(line.get("method") or "").upper() == "GET" and _ok(line) + ] + after_gets = [line for line in gets if last is None or _t(line) > last] + candidates: list[tuple[tuple[Any, ...], Surface]] = [] + rejected: list[str] = [] + seen_urls: set[str] = set() + names: set[str] = set(search.taken) + for line in after_gets: + url = str(line.get("url") or "") + if url in seen_urls: + continue + seen_urls.add(url) + before_line = None + if first is not None: + for earlier in gets: + if earlier.get("url") == url and _t(earlier) < first: + before_line = earlier + for listing in _listings(_json_body(line)): + surface, why = _surface_from_listing( + line, listing, lookup, before_line=before_line, taken=names + ) + if why: + rejected.append(why) + if surface is None: + continue + rank = ( + len(surface.bound), + "id" in (surface.target or {}), + surface.before_recorded, + -_t(line), + ) + candidates.append((rank, surface)) + if not candidates: + if after_gets: + search.notes.append( + f"{len(after_gets)} read(s) after the save, none returning a " + "record with the recorded values" + ) + search.notes.extend(rejected) + return + candidates.sort(key=lambda item: item[0], reverse=True) + chosen = candidates[0][1] + search.taken.update({chosen.name, chosen.content_name}) + search.surfaces.append(chosen) + for _rank, other in candidates[1:]: + search.notes.append( + f"also returns the saved record: {other.source} (not used; the check " + f"reads {chosen.source})" + ) + read_back = {str((chosen.target or {}).get(fld)) for fld in chosen.bound} + for line in writes: + sent = set(_write_values(line).values()) + if not sent & read_back: + tpl = _templated(str(line.get("url") or ""), lookup) + search.uncovered.append( + f"{line.get('method')} {tpl.safe}: no read returns what this " + "request sent" + ) + + +# -- database read path -------------------------------------------------------- + + +def reads_from_sqlite( + db_path: Path, params: Mapping[str, str], search: ReadSearch +) -> None: + """Every table: the ones holding the saved row are read, the rest guarded. + + A table that can't be read (an irregular name or column, too many rows) + is listed in ``uncovered`` and the read is not complete. + """ + + lookup, _ = param_lookup(params) + uri = f"file:{Path(db_path).resolve()}?mode=ro" + try: + conn = sqlite3.connect(uri, uri=True) + except sqlite3.Error as exc: + raise SynthesisError(f"cannot open {db_path} read-only: {exc}") from exc + complete = True + try: + conn.row_factory = sqlite3.Row + tables = [ + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name" + ) + ] + for table in tables: + if not _PLAIN_NAME.match(table): + search.uncovered.append( + "a table with an irregular name: the check can't read it" + ) + complete = False + continue + count = conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + if count > MAX_SCAN_ROWS: + search.uncovered.append( + f"table {table}: {count} rows, too many to scan for the saved row" + ) + complete = False + continue + columns = [row[1] for row in conn.execute(f'PRAGMA table_info("{table}")')] + if not all(_PLAIN_NAME.match(str(c)) for c in columns): + search.uncovered.append( + f"table {table}: a column with an irregular name; the check " + "can't read it" + ) + complete = False + continue + rows = [dict(r) for r in conn.execute(f'SELECT * FROM "{table}"')] + best: Optional[tuple[dict[str, Any], dict[str, str]]] = None + for row in rows: + exact, _ = _bind_fields( + {k: v for k, v in row.items() if k != "id"}, lookup + ) + if exact and (best is None or len(exact) > len(best[1])): + best = (row, exact) + query = f'SELECT * FROM "{table}"' + config: dict[str, Any] = { + "kind": "sql", + "sqlite_database": str(db_path), + "sql_query": query, + } + name = _unique_name(table, search.taken) + if best is None: + search.surfaces.append( + Surface( + name=name, + config=config, + after=rows, + before=list(rows), + before_recorded=False, + source=f"table {table}", + ) + ) + continue + row, bound = best + scope = { + column: param + for column, param in bound.items() + if _identity_like(params.get(param, "")) + } + if scope: + slots = { + param: f"p{i}" + for i, param in enumerate(sorted(set(scope.values()))) + } + where = " AND ".join( + f'"{column}" = :{slots[param]}' + for column, param in sorted(scope.items()) + ) + config["sql_query"] = f"{query} WHERE {where}" + config["sql_query_params"] = { + slot: {"param": param} for param, slot in sorted(slots.items()) + } + visible = [ + r + for r in rows + if all(str(r.get(c)) == params[p] for c, p in scope.items()) + ] + else: + visible = rows + search.surfaces.append( + Surface( + name=name, + config=config, + after=visible, + before=_minus(visible, row), + before_recorded=False, + source=f"table {table}", + target=row, + bound=bound, + scope=sorted(set(scope.values())), + ) + ) + finally: + conn.close() + if complete: + search.closed_world = True + search.closed_world_basis = "database" + + +# -- an existing verifier section ---------------------------------------------- + + +def reads_from_verifier( + section: Mapping[str, Any], + params: Mapping[str, str], + search: ReadSearch, + *, + snapshots: Optional[Mapping[str, Any]] = None, +) -> None: + """Read once through a reviewed verifier section and find the saved record. + + ``snapshots`` is ``sor_snapshots.json`` from ``record --verify`` when the + recording has one; its ``before`` read becomes the baseline. + """ + + from openadapt_flow.deployment import EffectsConfig, build_effect_verifier + + cfg = EffectsConfig.model_validate(dict(section)) + verifier = build_effect_verifier(cfg, params) + if verifier is None: + raise SynthesisError("the verifier section builds no verifier") + state = verifier.capture_pre_state() + if not state.reachable: + raise SynthesisError("the verifier could not read the system of record") + lookup, _ = param_lookup(params) + best: Optional[tuple[dict[str, Any], dict[str, str]]] = None + for record in state.records: + exact, _ = _bind_fields( + { + k: v + for k, v in record.items() + if k != "id" and not isinstance(v, (dict, list)) + }, + lookup, + ) + if exact and (best is None or len(exact) > len(best[1])): + best = (record, exact) + if best is None: + search.notes.append( + f"the {cfg.kind} read returns {len(state.records)} record(s), none " + "holding the recorded values" + ) + return + before: list[dict[str, Any]] = [] + recorded = False + if snapshots and isinstance(snapshots.get("before"), dict): + prior = snapshots["before"].get("records") + if isinstance(prior, list): + before = [r for r in prior if isinstance(r, dict)] + recorded = True + record, bound = best + search.surfaces.append( + Surface( + name=_unique_name(f"{cfg.kind or 'records'}_read", search.taken), + config=dict(section), + after=list(state.records), + before=before if recorded else _minus(list(state.records), record), + before_recorded=recorded, + source=f"the {cfg.kind} verifier section", + target=record, + bound=bound, + ) + ) + + +def _apply_sent_constants(search: ReadSearch, params: Mapping[str, str]) -> None: + """Match also on fields the save sent with a fixed (non-parameter) value, + under the same field name: they tell the saved record apart from other + records of the same subject.""" + + lookup, _ = param_lookup(params) + sent = [_write_values(line) for line in search.write_requests] + for surface in search.surfaces: + if not surface.is_target: + continue + for fld, value in (surface.target or {}).items(): + if fld in surface.bound or fld == "id" or value is None: + continue + if str(value) in lookup or not str(value).strip(): + continue + if any(str(values.get(fld)) == str(value) for values in sent): + surface.constants[fld] = str(value) + + +# --------------------------------------------------------------------------- +# Effects +# --------------------------------------------------------------------------- + + +def _effect( + surface_name: str, + note: str, + *, + match: Mapping[str, ValueExpr], + expected_count: int, + forbid_collateral_loss: bool, + confirmed: bool, +) -> Effect: + return Effect( + kind=EffectKind.RECORD_WRITTEN, + match=dict(match), + expected_count=expected_count, + count_new_only=True, + forbid_collateral_loss=forbid_collateral_loss, + risk="irreversible", + probe=surface_probe(surface_name, note), + needs_operator_confirmation=not confirmed, + ) + + +def effects_for( + surfaces: Iterable[Surface], *, confirmed: bool = False +) -> list[Effect]: + """The effect contract for a set of surfaces (each surface in turn).""" + + effects: list[Effect] = [] + for surface in surfaces: + if surface.is_target: + match = { + fld: ValueExpr(param=p) for fld, p in sorted(surface.bound.items()) + } + match.update( + { + fld: ValueExpr(literal=v) + for fld, v in sorted(surface.constants.items()) + } + ) + effects.append( + _effect( + surface.name, + "exactly one new record with the run's values; no record " + "that was there before may disappear", + match=match, + expected_count=1, + forbid_collateral_loss=True, + confirmed=confirmed, + ) + ) + else: + effects.append( + _effect( + surface.name, + "the save adds no row here", + match={}, + expected_count=0, + forbid_collateral_loss=False, + confirmed=confirmed, + ) + ) + if not surface.has_ids: + effects.append( + _effect( + surface.name, + "no row here changes or disappears", + match=_NO_RECORD, + expected_count=0, + forbid_collateral_loss=True, + confirmed=confirmed, + ) + ) + if surface.has_ids: + effects.append( + _effect( + surface.content_name, + "no record that was there before changes or disappears " + "(records compared by content, not id)", + match=_NO_RECORD, + expected_count=0, + forbid_collateral_loss=True, + confirmed=confirmed, + ) + ) + return effects + + +def _clean(config: Mapping[str, Any]) -> dict[str, Any]: + return {k: v for k, v in config.items() if not k.startswith("_")} + + +def verifier_section(surfaces: list[Surface]) -> dict[str, Any]: + """The ``verifier:`` section: one read per surface, plus content twins.""" + + out: dict[str, Any] = {} + for surface in surfaces: + names = [surface.name] + ([surface.content_name] if surface.has_ids else []) + for name in names: + if name in out: + raise SynthesisError(f"two reads share the name {name!r}") + out[surface.name] = _clean(surface.config) + if surface.has_ids: + twin = _clean(surface.config) + twin["records_drop_fields"] = ["id"] + out[surface.content_name] = twin + return {"kind": "surfaces", "surfaces": out} + + +def _without_ids(records: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [{k: v for k, v in r.items() if k != "id"} for r in records] + + +def _signature(records: list[dict[str, Any]]) -> Counter: + return Counter(json.dumps(r, sort_keys=True, default=str) for r in records) + + +def check_reads( + surfaces: list[Surface], section: Mapping[str, Any], params: Mapping[str, str] +) -> list[str]: + """Build every generated read; run each database read and compare it with + what the scan saw. Returns the problems (no record values).""" + + from openadapt_flow.deployment import EffectsConfig, build_effect_verifier + + problems: list[str] = [] + configs = section.get("surfaces", {}) + for surface in surfaces: + if surface.config.get("_pinned"): + problems.append( + f"{surface.name}: the read's path names one fixed record; bind it " + "to a run input or confirm it is not the demonstration's record" + ) + for name in (surface.name, surface.content_name): + if name not in configs: + continue + try: + verifier = build_effect_verifier( + EffectsConfig.model_validate(configs[name]), params + ) + except Exception as exc: # noqa: BLE001 - any build failure is a problem + problems.append(f"{name}: the generated read does not build ({exc})") + continue + if configs[name].get("kind") != "sql" or verifier is None: + continue + state = verifier.capture_pre_state() + expected = surface.after + if name == surface.content_name: + expected = _without_ids(surface.after) + if not state.reachable: + problems.append(f"{name}: the generated query does not run") + elif _signature(state.records) != _signature(expected): + problems.append( + f"{name}: the generated query returns {len(state.records)} " + f"row(s), the scan saw {len(expected)}" + ) + return problems + + +# --------------------------------------------------------------------------- +# Proof against injected bad writes (recorded-state) +# --------------------------------------------------------------------------- + + +@dataclass +class FaultResult: + fault: str + detail: str + #: True when the save was correct (only the clean case). + correct: bool + #: True when the check passed every effect. + passed: bool + reasons: list[str] = field(default_factory=list) + #: True when the fault could not be built (the check is not credited). + untestable: bool = False + + @property + def ok(self) -> bool: + if self.untestable: + return False + return self.passed if self.correct else not self.passed + + def as_dict(self) -> dict[str, Any]: + return { + "fault": self.fault, + "detail": self.detail, + "correct_save": self.correct, + "check_passed": self.passed, + "result": ( + "untestable" + if self.untestable + else "clean pass" + if self.correct and self.passed + else "false stop" + if self.correct + else "caught" + if not self.passed + else "MISSED" + ), + "reasons": self.reasons, + } + + +def _fresh_id(records: list[dict[str, Any]]) -> Any: + ids = [int(r["id"]) for r in records if isinstance(r.get("id"), int)] + if ids: + return max(ids) + 1000 + return "fault-" + str(len(records) + 1000) + + +_State = tuple[list[dict[str, Any]], list[dict[str, Any]]] + + +def _judge( + effects: list[Effect], + surfaces: list[Surface], + states: Mapping[str, _State], + params: Mapping[str, str], +) -> tuple[bool, list[str]]: + from openadapt_flow.runtime.effects.surfaces import surface_of + + views: dict[str, _State] = {} + for surface in surfaces: + before, after = states[surface.name] + views[surface.name] = (before, after) + if surface.has_ids: + views[surface.content_name] = (_without_ids(before), _without_ids(after)) + reasons: list[str] = [] + for effect in effects: + name = surface_of(effect) or "" + if name not in views: + reasons.append(f"effect routes to unknown surface {name!r}") + continue + before, after = views[name] + verdict = judge_records( + effect.resolve(params), + EffectState(substrate="proof", reachable=True, records=before), + after, + substrate="proof", + ) + if verdict.verdict is not Verdict.CONFIRMED: + reasons.append(f"{name}: {verdict.reason}") + return not reasons, reasons + + +def _changed(record: dict[str, Any]) -> dict[str, Any]: + """``record`` with one non-id field changed (or one added).""" + + out = dict(record) + for key, value in out.items(): + if key != "id" and isinstance(value, str): + out[key] = value + " (changed)" + return out + out["_changed"] = "1" + return out + + +def prove_against_faults( + surfaces: list[Surface], + effects: list[Effect], + params: Mapping[str, str], + *, + closed_world: bool, +) -> list[FaultResult]: + """Judge the check on the clean save and on each injected bad write. + + The bad writes are the persistence faults of ``benchmark/effect_e2e``, + built from the records the recording saw, plus another record changed or + deleted. Transport faults (a rejected write, an expired session, a + timeout) are handled by the actuator before any check runs, so they are + not repeated here. + """ + + base: dict[str, _State] = { + s.name: (list(s.before), list(s.after)) for s in surfaces + } + results: list[FaultResult] = [] + + def run( + fault: str, detail: str, states: Mapping[str, _State], *, correct: bool = False + ) -> None: + passed, reasons = _judge(effects, surfaces, states, params) + results.append(FaultResult(fault, detail, correct, passed, reasons)) + + def miss(fault: str, detail: str, reason: str) -> None: + results.append(FaultResult(fault, detail, False, True, [reason])) + + def other_records( + surface: Surface, + ) -> tuple[list[dict[str, Any]], dict[str, Any], bool]: + """A record that was there before (injected when there was none).""" + + before, after = base[surface.name] + target_key = stable_id(surface.target) if surface.target else None + for record in before: + if stable_id(record) != target_key and record in after: + return before, record, False + seed = dict(surface.target or (after[-1] if after else {})) + for fld in list(surface.bound) + list(surface.constants): + seed[fld] = f"other {seed.get(fld)}" + if "id" in seed or surface.has_ids: + seed["id"] = _fresh_id(before + after) + else: + seed["_other"] = "1" + return before + [seed], seed, True + + run("ok", "the clean save as recorded", base, correct=True) + + for surface in surfaces: + before, after = base[surface.name] + if surface.is_target: + target = surface.target or {} + without = _minus(after, target) + run( + "no_persist", + f"nothing saved in {surface.name}", + {**base, surface.name: (before, without)}, + ) + for fld in sorted(surface.bound): + run( + "partial", + f"{surface.name}.{fld} dropped", + { + **base, + surface.name: (before, without + [dict(target, **{fld: ""})]), + }, + ) + copy = dict(target) + if "id" in copy: + copy["id"] = _fresh_id(after) + run( + "duplicate", + f"saved twice in {surface.name}", + {**base, surface.name: (before, after + [copy])}, + ) + + identity = [ + f for f, p in surface.bound.items() if _identity_like(params.get(p, "")) + ] + if not identity and not surface.scope: + miss( + "wrong_record", + f"{surface.name}: saved on another record", + "no recorded value identifies the record, and the read is " + "not scoped by one", + ) + for fld in sorted(identity): + moved = dict(target, **{fld: f"{target.get(fld)}-other"}) + run( + "wrong_record", + f"{surface.name}: saved on another {fld}", + {**base, surface.name: (before, without + [moved])}, + ) + for param in sorted( + set(surface.scope) - {surface.bound[f] for f in identity} + ): + run( + "wrong_record", + f"{surface.name}: saved under another {param}, which the read " + "is scoped by", + {**base, surface.name: (before, without)}, + ) + + telling = [f for f in surface.bound if f not in identity] + telling += sorted(surface.constants) + if not telling: + miss( + "stale", + f"{surface.name}: the save destroyed another record of the same " + "subject", + "every checked field identifies the subject, so another record " + "of the same subject looks like the saved one", + ) + else: + concurrent = dict(target) + if "id" in concurrent: + concurrent["id"] = _fresh_id(after + before) + for fld in telling: + concurrent[fld] = f"concurrent {concurrent.get(fld)}" + run( + "stale", + f"{surface.name}: the save destroyed another record of the same " + "subject", + {**base, surface.name: (before + [concurrent], after)}, + ) + else: + stray: dict[str, Any] = dict(after[-1]) if after else {} + if "id" in stray or surface.has_ids: + stray["id"] = _fresh_id(after + before) + else: + stray["_stray"] = "1" + run( + "collateral_unaudited", + f"a stray row written to {surface.name}", + {**base, surface.name: (before, after + [stray])}, + ) + + prior, other, injected = other_records(surface) + current = after + [dict(other)] if injected else list(after) + at = current.index(other) + changed = current[:at] + [_changed(other)] + current[at + 1 :] + run( + "changed", + f"{surface.name}: another record changed by the save", + {**base, surface.name: (prior, changed)}, + ) + run( + "deleted", + f"{surface.name}: another record deleted by the save", + {**base, surface.name: (prior, current[:at] + current[at + 1 :])}, + ) + + if not closed_world: + miss( + "collateral_unaudited", + "a stray row written to a table this check does not read", + "the check reads only " + + ", ".join(s.name for s in surfaces) + + "; a write anywhere else passes it (give a database read path with " + "--db, or more reads)", + ) + return results + + +# --------------------------------------------------------------------------- +# One call +# --------------------------------------------------------------------------- + + +@dataclass +class Proposal: + """The proposed check and everything the reviewer needs to judge it.""" + + params: dict[str, str] + search: ReadSearch + effects: list[Effect] + verifier: dict[str, Any] + faults: list[FaultResult] + live: Optional[dict[str, Any]] = None + identity: Optional[dict[str, Any]] = None + + @property + def misses(self) -> list[str]: + out = [ + f"{r.fault}: {r.detail}" + + (f" ({'; '.join(r.reasons)})" if r.reasons else "") + for r in self.faults + if not r.ok + ] + out += [f"not covered: {item}" for item in self.search.uncovered] + out += [f"read: {item}" for item in self.search.problems] + if self.live is not None: + out += [f"live {m}" for m in self.live.get("misses", [])] + return out + + @property + def working(self) -> bool: + if not self.effects or not any(s.is_target for s in self.search.surfaces): + return False + if self.live is not None and not self.live.get("passed"): + return False + return not self.misses + + def proof(self) -> dict[str, Any]: + targets = [s for s in self.search.surfaces if s.is_target] + reads = [] + for s in self.search.surfaces: + reads.append( + { + "surface": s.name, + "source": s.source, + "role": "saved record" if s.is_target else "no row added", + "checked_fields": sorted(s.bound) + sorted(s.constants), + "baseline": "recorded" if s.before_recorded else "derived", + } + ) + if s.has_ids: + reads.append( + { + "surface": s.content_name, + "source": s.source + ", compared by content", + "role": "no record changed or removed", + "checked_fields": [], + "baseline": "recorded" if s.before_recorded else "derived", + } + ) + return { + "schema": PROOF_SCHEMA, + "status": "working" if self.working else "draft", + "approved": False, + "reads": reads, + "not_read_back": sorted( + set(self.params) - {p for s in targets for p in s.bound.values()} + ), + "uncovered_write_targets": list(self.search.uncovered), + "read_problems": list(self.search.problems), + "closed_world": self.search.closed_world_basis, + "faults": [r.as_dict() for r in self.faults], + "live": self.live, + "misses": self.misses, + "notes": list(self.search.notes), + "identity": self.identity, + } + + +def synthesize( + recording_dir: Optional[Path], + *, + params: Optional[Mapping[str, str]] = None, + db_path: Optional[Path] = None, + verifier: Optional[Mapping[str, Any]] = None, + complete_read: bool = False, +) -> Proposal: + """Find the read, build the check, and run the recorded-state fault test. + + ``complete_read`` records a person's statement that the reads return + everything the save can change (one store, no other tables). It is not + proven: the proof says it was declared. + """ + + if params is None: + params = recorded_params(recording_dir) if recording_dir else {} + params = dict(params) + if not params: + raise SynthesisError( + "the recording holds no parameter values (meta.json params), so " + "there is nothing to find the saved record by" + ) + search = ReadSearch() + _, shared = param_lookup(params) + search.notes.extend(shared) + if recording_dir is not None: + reads_from_network(Path(recording_dir), params, search) + if verifier is not None: + snapshots = None + if recording_dir is not None: + snap = Path(recording_dir) / "sor_snapshots.json" + if snap.is_file(): + try: + snapshots = json.loads(snap.read_text("utf-8")) + except ValueError: + snapshots = None + reads_from_verifier(verifier, params, search, snapshots=snapshots) + if db_path is not None: + reads_from_sqlite(Path(db_path), params, search) + if not any(s.is_target for s in search.surfaces): + return Proposal(params, search, [], {}, []) + _apply_sent_constants(search, params) + if complete_read and not search.closed_world: + search.closed_world = True + search.closed_world_basis = "declared" + search.notes.append( + "a person declared that the reads return everything the save can " + "change; writes elsewhere are not tested" + ) + for surface in search.surfaces: + for name in surface.config.get("_unscoped_references", []): + search.notes.append( + f"{surface.name}: the record names parameter {name!r} only as a " + "reference the read is not scoped by; that link is not checked" + ) + effects = effects_for(search.surfaces) + section = verifier_section(search.surfaces) + search.problems.extend(check_reads(search.surfaces, section, params)) + faults = prove_against_faults( + search.surfaces, + effects_for(search.surfaces, confirmed=True), + params, + closed_world=search.closed_world, + ) + return Proposal(params, search, effects, section, faults) + + +# --------------------------------------------------------------------------- +# Output +# --------------------------------------------------------------------------- + +_HEADER_WORKING = """\ +# Record check built by `openadapt-flow scaffold-verifier` from the reads the +# app made after the save. It caught every injected bad write in its fault test +# and passed the clean save ({proof}). It is NOT approved: a person must +# - confirm each read is the system of record, with its own read-only +# credential (the recording's reads used the browser's sign-in); +# - check that each {{param: ...}} is the input that should fill that field; +# - delete every `needs_operator_confirmation: true` line. +# verify-run refuses this file, and qualify reports a problem, until then. +""" + +_HEADER_DRAFT = """\ +# DRAFT record check from `openadapt-flow scaffold-verifier`. It is NOT working: +# its fault test found misses (listed below and in {proof_file}). Fix the reads +# or add the missing ones, then run scaffold-verifier again. +# +{misses} +# +# verify-run refuses this file until every `needs_operator_confirmation: true` +# line is gone; do not remove them while misses remain. +""" + + +def render(proposal: Proposal) -> str: + import yaml + + doc = { + "schema_version": "openadapt.verify/v1", + "verifier": proposal.verifier, + "effects": [ + effect.model_dump(mode="json", exclude_defaults=True) + | {"needs_operator_confirmation": True} + for effect in proposal.effects + ], + } + proof = "recorded-state faults" + if proposal.live is not None: + proof += f" and the {proposal.live.get('fixture')} fault suite" + if proposal.working: + header = _HEADER_WORKING.format(proof=proof) + else: + misses = ( + "\n".join(f"# MISS: {m}" for m in proposal.misses) + or "# MISS: no read found" + ) + header = _HEADER_DRAFT.format(proof_file=PROOF_NAME, misses=misses) + return header + yaml.safe_dump(doc, sort_keys=False, allow_unicode=True) + + +def write(proposal: Proposal, out_dir: Path) -> tuple[Path, Path]: + """Write the proposed check and its proof; return both paths.""" + + out_dir.mkdir(parents=True, exist_ok=True) + check = out_dir / PROPOSED_NAME + proof = out_dir / PROOF_NAME + check.write_text(render(proposal), encoding="utf-8") + proof.write_text( + json.dumps(proposal.proof(), indent=2, sort_keys=True, default=str) + "\n", + encoding="utf-8", + ) + return check, proof + + +def summary(proposal: Proposal, check: Path, proof: Path) -> str: + """What the operator sees: status, reads, faults, misses, next step.""" + + lines: list[str] = [] + status = "WORKING (not approved)" if proposal.working else "DRAFT (not working)" + lines.append(f"Record check: {status}") + for item in proposal.proof()["reads"]: + lines.append(f" read {item['surface']}: {item['source']} -> {item['role']}") + caught = sum(1 for r in proposal.faults if r.ok and not r.correct) + bad = sum(1 for r in proposal.faults if not r.correct) + lines.append( + f"Fault test: {caught} of {bad} bad writes caught; clean save " + + ("passed" if all(r.ok for r in proposal.faults if r.correct) else "STOPPED") + ) + if proposal.live is not None: + lines.append( + f"Live fault suite ({proposal.live.get('fixture')}): " + + ("passed" if proposal.live.get("passed") else "FAILED") + ) + for miss in proposal.misses: + lines.append(f" MISS: {miss}") + if proposal.identity is not None: + ident = proposal.identity + lines.append( + f"Identity checks: {ident['armed_before']} of {ident['applicable']} " + f"clicks armed now; {ident['armed_after']} with the proposed evidence" + ) + lines.append(f"Wrote {check} and {proof}") + if proposal.working: + lines.append( + "Next: a person reviews it and deletes each " + "`needs_operator_confirmation: true` line, then " + "`openadapt-flow qualify --verify " + check.name + "`." + ) + else: + lines.append("Next: fix the misses above, then run scaffold-verifier again.") + return "\n".join(lines) diff --git a/packages/flow/openadapt_flow/runtime/effects/_common.py b/packages/flow/openadapt_flow/runtime/effects/_common.py index 0f59534d1..ef4c8adee 100644 --- a/packages/flow/openadapt_flow/runtime/effects/_common.py +++ b/packages/flow/openadapt_flow/runtime/effects/_common.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any, Iterable, Optional from openadapt_flow.runtime.effects.effect import ( Effect, @@ -24,6 +24,17 @@ ) +def drop_record_fields( + records: Optional[list[dict[str, Any]]], fields: Iterable[str] +) -> Optional[list[dict[str, Any]]]: + """``records`` without ``fields`` (``None`` stays ``None``: unreadable).""" + + names = set(fields) + if records is None or not names: + return records + return [{k: v for k, v in r.items() if k not in names} for r in records] + + def new_records_only( matched: list[dict[str, Any]], before: EffectState ) -> list[dict[str, Any]]: diff --git a/packages/flow/openadapt_flow/runtime/effects/fhir.py b/packages/flow/openadapt_flow/runtime/effects/fhir.py index 09685b920..912e340b6 100644 --- a/packages/flow/openadapt_flow/runtime/effects/fhir.py +++ b/packages/flow/openadapt_flow/runtime/effects/fhir.py @@ -47,7 +47,7 @@ import time from typing import Any, Optional, Protocol -from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects._common import drop_record_fields, judge_records from openadapt_flow.runtime.effects.adapter import VerifierAdapterBase from openadapt_flow.runtime.effects.effect import ( Effect, @@ -114,6 +114,8 @@ class FhirEffectVerifier(VerifierAdapterBase): timeout_s: Per-request timeout. verify_tls: Passed through to the HTTP client's ``verify``. poll_interval_s: Gap between polls while waiting for the write. + drop_fields: Record fields removed before judging (``["id"]`` judges + records by their whole content); see :class:`RestRecordVerifier`. """ substrate = "fhir" @@ -132,8 +134,10 @@ def __init__( timeout_s: float = 5.0, verify_tls: bool = True, poll_interval_s: float = 0.3, + drop_fields: tuple[str, ...] = (), ) -> None: self.base_url = base_url.rstrip("/") + self.drop_fields = tuple(drop_fields) self.resource_type = resource_type self.search_params = dict(search_params or {}) self.field_paths = dict(field_paths or DEFAULT_OBSERVATION_PATHS) @@ -249,7 +253,7 @@ def _search(self) -> Optional[list[dict[str, Any]]]: if not isinstance(resource, dict): return None out.append(self._flatten(resource)) - return out + return drop_record_fields(out, self.drop_fields) # -- EffectVerifier protocol -------------------------------------------- diff --git a/packages/flow/openadapt_flow/runtime/effects/rest.py b/packages/flow/openadapt_flow/runtime/effects/rest.py index 9127623ff..d9e9f608d 100644 --- a/packages/flow/openadapt_flow/runtime/effects/rest.py +++ b/packages/flow/openadapt_flow/runtime/effects/rest.py @@ -23,7 +23,7 @@ import time from typing import Any, Optional -from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects._common import drop_record_fields, judge_records from openadapt_flow.runtime.effects.adapter import VerifierAdapterBase from openadapt_flow.runtime.effects.effect import ( Effect, @@ -51,6 +51,10 @@ class RestRecordVerifier(VerifierAdapterBase): timeout_s: Per-request timeout in seconds. poll_interval_s: Gap between reachability retries while polling for the write to land within ``Effect.timeout_s``. + drop_fields: Record fields removed before judging (``["id"]`` judges + records by their whole content, so a change to another record + reads as that record removed and a new one added, and the + collateral-loss check catches it). Empty -> records as read. """ substrate = "rest" @@ -66,8 +70,10 @@ def __init__( headers: Optional[dict[str, str]] = None, timeout_s: float = 5.0, poll_interval_s: float = 0.2, + drop_fields: tuple[str, ...] = (), ) -> None: self.base_url = base_url.rstrip("/") + self.drop_fields = tuple(drop_fields) self.records_path = records_path self.records_key = records_key self.headers = dict(headers) if headers else None @@ -134,7 +140,9 @@ def _fetch_records(self) -> Optional[list[dict[str, Any]]]: return None if not isinstance(records, list): return None - return [r for r in records if isinstance(r, dict)] + return drop_record_fields( + [r for r in records if isinstance(r, dict)], self.drop_fields + ) # -- EffectVerifier protocol -------------------------------------------- diff --git a/packages/flow/openadapt_flow/runtime/effects/sql.py b/packages/flow/openadapt_flow/runtime/effects/sql.py index b3d2eaf82..2a6a305be 100644 --- a/packages/flow/openadapt_flow/runtime/effects/sql.py +++ b/packages/flow/openadapt_flow/runtime/effects/sql.py @@ -44,7 +44,7 @@ import time from typing import Any, Callable, Iterable, Mapping, Optional -from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects._common import drop_record_fields, judge_records from openadapt_flow.runtime.effects.adapter import VerifierAdapterBase from openadapt_flow.runtime.effects.effect import ( Effect, @@ -150,6 +150,8 @@ class SqlRecordVerifier(VerifierAdapterBase): bounds the polling loop, mirroring the REST verifier). poll_interval_s: Gap between polls while waiting for the write to land within ``Effect.timeout_s``. + drop_fields: Record fields removed before judging (``["id"]`` judges + records by their whole content); see :class:`~openadapt_flow.runtime.effects.RestRecordVerifier`. """ substrate = "sql" @@ -163,7 +165,9 @@ def __init__( query_params: Optional[Mapping[str, Any]] = None, timeout_s: float = 5.0, poll_interval_s: float = 0.2, + drop_fields: tuple[str, ...] = (), ) -> None: + self.drop_fields = tuple(drop_fields) self._connect = connect self.query = assert_read_only_sql(query) self.query_params = dict(query_params or {}) @@ -193,7 +197,9 @@ def _fetch_records(self) -> Optional[list[dict[str, Any]]]: return None # not a result-set query -> unusable columns = [str(col[0]) for col in description] rows = cursor.fetchall() - return [dict(zip(columns, row)) for row in rows] + return drop_record_fields( + [dict(zip(columns, row)) for row in rows], self.drop_fields + ) except Exception: # noqa: BLE001 - any read failure is unreadable return None finally: diff --git a/packages/flow/openadapt_flow/runtime/effects/surfaces.py b/packages/flow/openadapt_flow/runtime/effects/surfaces.py new file mode 100644 index 000000000..b2b91fdff --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/effects/surfaces.py @@ -0,0 +1,164 @@ +"""Route each effect to the read for the table or resource it concerns. + +One save can write more than one place: the encounter row, a billing row, an +audit row. A single read sees one of them. A bad write to another place passes +any check built on that read alone (``benchmark/effect_e2e/EFFECT_E2E.md``: one +REST read let 9 of 72 bad writes through because they went to a table it did +not read). + +:class:`SurfaceRoutedVerifier` holds one verifier per named surface and sends +each effect to the surface its ``probe`` names (``"surface="`` or +``"surface=|free text"``). An effect that names no surface, or names one +that is not configured, is INDETERMINATE: it halts instead of being judged +against the wrong read. + +The pre-state is captured once per surface and carried inside the returned +:class:`EffectState`, because the replayer captures one pre-state and passes +it to every ``verify`` call. + +The evidence tier of a routed effect is the tier of the verifier it routes +to; the verifier as a whole advertises its weakest surface. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Optional + +from openadapt_flow.runtime.effects.effect import ( + Effect, + EffectState, + EffectVerdict, + Verdict, +) +from openadapt_flow.verification import VerificationTier + +SURFACE_PREFIX = "surface=" + + +def surface_of(effect: Effect) -> Optional[str]: + """The surface an effect's ``probe`` names, or ``None``.""" + + probe = (effect.probe or "").strip() + if not probe.startswith(SURFACE_PREFIX): + return None + name = probe[len(SURFACE_PREFIX) :].split("|", 1)[0].strip() + return name or None + + +def surface_probe(name: str, note: str = "") -> str: + """The ``probe`` value that routes an effect to ``name``.""" + + return f"{SURFACE_PREFIX}{name}" + (f"|{note}" if note else "") + + +def _tier(verifier: Any) -> Optional[VerificationTier]: + value = getattr(verifier, "verification_tier", None) + if value is None or isinstance(value, bool): + return None + try: + return VerificationTier(value) + except (TypeError, ValueError): + return None + + +class SurfaceRoutedVerifier: + """One verifier per surface; each effect is judged by its own surface. + + Args: + verifiers: Surface name -> a verifier implementing the + :class:`~openadapt_flow.runtime.effects.effect.EffectVerifier` + protocol. At least one is required. + """ + + substrate = "surfaces" + + def __init__(self, verifiers: Mapping[str, Any]) -> None: + if not verifiers: + raise ValueError("a surface-routed verifier needs at least one surface") + self._verifiers = dict(verifiers) + tiers = [_tier(verifier) for verifier in self._verifiers.values()] + declared = [tier for tier in tiers if tier is not None] + if len(declared) != len(tiers): + raise ValueError("every surface verifier must declare a verification tier") + #: The weakest surface (highest number) sets the whole verifier's tier. + self.verification_tier: VerificationTier = max(declared) + self.independent_system_of_record = all( + bool(getattr(verifier, "independent_system_of_record", True)) + for verifier in self._verifiers.values() + ) + + @property + def surfaces(self) -> tuple[str, ...]: + return tuple(self._verifiers) + + @property + def verifiers(self) -> dict[str, Any]: + """Surface name -> its verifier (a copy of the mapping).""" + return dict(self._verifiers) + + def verification_tier_for(self, effect: Effect) -> VerificationTier: + """The tier of the surface this effect routes to (weakest if none).""" + + name = surface_of(effect) + verifier = self._verifiers.get(name or "") + tier = _tier(verifier) if verifier is not None else None + return tier if tier is not None else self.verification_tier + + def capture_pre_state(self, context: Any = None) -> EffectState: + detail: dict[str, Any] = {} + reachable = True + for name, verifier in self._verifiers.items(): + state = verifier.capture_pre_state(context) + detail[name] = { + "substrate": state.substrate, + "reachable": state.reachable, + "records": state.records, + } + reachable = reachable and state.reachable + return EffectState( + substrate=self.substrate, reachable=reachable, records=[], detail=detail + ) + + def capture_post_state(self, context: Any = None) -> EffectState: + return self.capture_pre_state(context) + + def test_connection(self, context: Any = None) -> Any: + from openadapt_flow.runtime.effects.adapter import ConnectionProbe + + state = self.capture_pre_state(context) + down = [ + name + for name, sub in state.detail.items() + if not (isinstance(sub, dict) and sub.get("reachable")) + ] + return ConnectionProbe( + ok=not down, + substrate=self.substrate, + reason="reachable" if not down else "unreachable: " + ", ".join(down), + ) + + def verify( + self, expected: Effect, before: EffectState, context: Any = None + ) -> EffectVerdict: + name = surface_of(expected) + verifier = self._verifiers.get(name or "") + if name is None or verifier is None: + return EffectVerdict( + verdict=Verdict.INDETERMINATE, + kind=expected.kind, + substrate=self.substrate, + reason=( + f"effect names surface {name!r}, which this check does not " + "read; HALT rather than judge it against another read" + if name + else "effect names no surface (probe 'surface='); " + "HALT rather than guess which read covers it" + ), + ) + sub = before.detail.get(name, {}) if isinstance(before.detail, dict) else {} + sub_before = EffectState( + substrate=str(sub.get("substrate") or getattr(verifier, "substrate", "")), + reachable=bool(sub.get("reachable", False)), + records=list(sub.get("records", [])), + ) + return verifier.verify(expected, sub_before, context) diff --git a/packages/flow/openadapt_flow/scaffold_verifier.py b/packages/flow/openadapt_flow/scaffold_verifier.py index 28d4e512f..d0411fd38 100644 --- a/packages/flow/openadapt_flow/scaffold_verifier.py +++ b/packages/flow/openadapt_flow/scaffold_verifier.py @@ -3,7 +3,16 @@ Two read-first operator helpers live here. ``flow scaffold-verifier `` - Reads the retained action/effect evidence in a recording directory or a + First tries to BUILD a working record check from the reads the app made + after the save (the recording's network capture, ``--db`` or + ``--verifier``), fault-tests it, and writes ``record_check.proposed.yaml`` + plus ``record_check.proof.json`` + (:mod:`openadapt_flow.record_check_synthesis`). A check that misses an + injected bad write is written as a draft with the misses listed. Nothing + is approved automatically. + + When no read returns the saved record, it falls back to the older path: + it reads the retained action/effect evidence in a recording directory or a compiled bundle and emits a DRAFT effect-oracle contract (``effect_contract.yaml``) pre-filled from the demonstration's write-shaped steps: a proposed system-of-record read, expected postcondition fields with @@ -46,6 +55,14 @@ class ScaffoldRefused(SystemExit): """The input has nothing to scaffold from; refuse with a clear message.""" +NO_WRITE_REFUSAL = ( + "scaffold-verifier REFUSED: the demonstration has no consequential " + "(write-shaped) step to verify. An effect oracle asserts that a " + "WRITE landed in a system of record; this input shows only " + "read/navigation actions, so there is nothing to scaffold." +) + + # --------------------------------------------------------------------------- # Target inspection (recording dir vs bundle dir) # --------------------------------------------------------------------------- @@ -442,12 +459,7 @@ def write_draft(source: Path, out_dir: Optional[Path] = None) -> tuple[Path, int candidates = candidates_from_recording(source) if not candidates: - raise ScaffoldRefused( - "scaffold-verifier REFUSED: the demonstration has no consequential " - "(write-shaped) step to verify. An effect oracle asserts that a " - "WRITE landed in a system of record; this input shows only " - "read/navigation actions, so there is nothing to scaffold." - ) + raise ScaffoldRefused(NO_WRITE_REFUSAL) text = render_draft_yaml( source=source, @@ -461,6 +473,123 @@ def write_draft(source: Path, out_dir: Optional[Path] = None) -> tuple[Path, int return destination, len(candidates) +# --------------------------------------------------------------------------- +# A working record check from the reads the app made +# --------------------------------------------------------------------------- + + +@dataclass +class RecordCheckResult: + """What :func:`build_record_check` wrote and said.""" + + working: bool + check_path: Path + proof_path: Path + summary: str + identity_path: Optional[Path] = None + + +def build_record_check( + source: Path, + out_dir: Optional[Path] = None, + *, + db: Optional[Path] = None, + verifier: Optional[Path] = None, + prove: Optional[str] = None, + recording: Optional[Path] = None, + bundle: Optional[Path] = None, + complete_read: bool = False, +) -> Optional[RecordCheckResult]: + """Build, fault-test and write a proposed record check. + + ``source`` is a recording or a bundle; the other one may be given with + ``recording`` / ``bundle`` (a bundle alone holds no reads, and identity + coverage needs both). Returns ``None`` when no read returns the saved + record, so the caller can fall back to the TODO draft. + """ + + import tempfile + + from openadapt_flow import record_check_synthesis as rcs + + kind = classify_target(source) + recording_dir = source if kind == "recording" else recording + bundle_dir = source if kind == "bundle" else bundle + if recording_dir is None: + return None + writes = ( + candidates_from_bundle(bundle_dir) + if kind == "bundle" and bundle_dir is not None + else candidates_from_recording(recording_dir) + ) + if not writes: + raise ScaffoldRefused(NO_WRITE_REFUSAL) + section = None + if verifier is not None: + import yaml + + try: + loaded = yaml.safe_load(Path(verifier).read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as exc: + raise ScaffoldRefused(f"scaffold-verifier: cannot read {verifier}: {exc}") + section = loaded.get("verifier") if isinstance(loaded, dict) else None + if not isinstance(section, dict): + raise ScaffoldRefused( + f"scaffold-verifier: {verifier} has no `verifier:` section" + ) + try: + proposal = rcs.synthesize( + recording_dir, db_path=db, verifier=section, complete_read=complete_read + ) + except rcs.SynthesisError as exc: + if db is None and section is None: + return None + raise ScaffoldRefused(f"scaffold-verifier: {exc}") from exc + if not proposal.effects: + if db is not None or section is not None: + raise ScaffoldRefused( + "scaffold-verifier: no read returns a record holding the " + "recorded values: " + "; ".join(proposal.search.notes or ["none"]) + ) + return None + if prove: + from openadapt_flow.record_check_proof import ProofUnavailable, prove_live + + try: + with tempfile.TemporaryDirectory(prefix="record-check-proof-") as tmp: + proposal.live = prove_live( + prove, + proposal.verifier, + proposal.effects, + proposal.params, + write_requests=proposal.search.write_requests, + work_dir=Path(tmp), + ) + except ProofUnavailable as exc: + raise ScaffoldRefused(f"scaffold-verifier --prove: {exc}") from exc + destination = out_dir or recording_dir + identity_path = None + if bundle_dir is not None: + from openadapt_flow import record_check_identity + + report, proposals = record_check_identity.identity_coverage( + bundle_dir, recording_dir, proposal.params + ) + proposal.identity = report + destination.mkdir(parents=True, exist_ok=True) + identity_path = record_check_identity.write_identity_proposals( + proposals, destination + ) + check, proof = rcs.write(proposal, destination) + return RecordCheckResult( + working=proposal.working, + check_path=check, + proof_path=proof, + summary=rcs.summary(proposal, check, proof), + identity_path=identity_path, + ) + + # --------------------------------------------------------------------------- # flow explain: read-only plain-language run summary # --------------------------------------------------------------------------- diff --git a/packages/flow/openadapt_flow/verify_run.py b/packages/flow/openadapt_flow/verify_run.py index 69b26dc1a..480a46125 100644 --- a/packages/flow/openadapt_flow/verify_run.py +++ b/packages/flow/openadapt_flow/verify_run.py @@ -252,12 +252,14 @@ def parse_verify_config(data: Any, *, source: str = "verify config") -> VerifyCo cfg = VerifyConfig.model_validate(data) except Exception as exc: raise VerifyRunError(f"invalid {source}: {exc}") from exc - if cfg.verifier.kind in _NOT_INDEPENDENT: - raise VerifyRunError( - f"verifier kind {cfg.verifier.kind!r} does not read independently of " - "the executor; verify-run needs a system-of-record or " - "separate-session verifier" - ) + kinds = [cfg.verifier.kind] + [s.kind for s in cfg.verifier.surfaces.values()] + for kind in kinds: + if kind in _NOT_INDEPENDENT: + raise VerifyRunError( + f"verifier kind {kind!r} does not read independently of " + "the executor; verify-run needs a system-of-record or " + "separate-session verifier" + ) if cfg.verifier.candidates: raise VerifyRunError("verify-run does not support verifier candidates yet") for effect in cfg.effects: @@ -364,6 +366,11 @@ def _tag_reads(verifier: Any, execution_id: str) -> None: across runs of one ``verify.yaml``. """ + routed = getattr(verifier, "verifiers", None) + if isinstance(routed, dict): + for sub in routed.values(): + _tag_reads(sub, execution_id) + return if not hasattr(verifier, "headers"): return trace = secrets.token_hex(16) diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index 094eafac3..5b1026254 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -629,7 +629,7 @@ }, { "path": "claims.yaml", - "sha256": "18bbd20323ebc03dbd324281abff0341665eb922f78683235872055b56a02124" + "sha256": "4b5e156e7ffdb0f8a704baea0440f24736a87439c1603af2108d964575436106" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1865,7 +1865,7 @@ }, { "path": "docs/verification.json", - "sha256": "3180937cfabea39634fabb5afc51f0e8f003a4042f076be94d65e495de4ce6c7" + "sha256": "38e06bea3b2eeda468e5f1a3ec2c42c99066f0d3a8735c715abd86c8c8cf200c" }, { "path": "examples/claude-code-demo/demo.gif", diff --git a/packages/flow/tests/_record_check_fixtures.py b/packages/flow/tests/_record_check_fixtures.py new file mode 100644 index 000000000..d774b10df --- /dev/null +++ b/packages/flow/tests/_record_check_fixtures.py @@ -0,0 +1,132 @@ +"""Synthetic recordings for the record-check synthesis tests. + +Each builder performs one save against a fixture system of record, then the +reads an app makes to show the saved record, and writes them as a recording: +``events.jsonl`` (the Save click), ``meta.json`` (the parameter values) and +``network.jsonl`` in the ``openadapt.network-capture/1`` format that +``record --capture-network`` writes. Made-up patients only. + +A test helper (leading underscore: pytest does not collect it). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Optional + +import requests + +SCHEMA = "openadapt.network-capture/1" +NOTE = "Synthetic triage note: follow up in two weeks" + + +class Capture: + """Records requests the way the recorder's network capture lays them out.""" + + def __init__(self) -> None: + self.lines: list[dict[str, Any]] = [] + + def _line( + self, method: str, url: str, resp: requests.Response, t: float, **extra: Any + ) -> None: + line: dict[str, Any] = { + "method": method, + "url": url, + "resource_type": "xhr", + "frame_url": url, + "status": resp.status_code, + "response_type": (resp.headers.get("content-type") or "").split(";")[0], + "response_text": resp.text, + "seq": len(self.lines), + "t": t, + } + line.update(extra) + self.lines.append(line) + + def get( + self, url: str, t: float, headers: Optional[dict] = None + ) -> requests.Response: + resp = requests.get(url, timeout=5, headers=headers or {}) + self._line("GET", url, resp, t) + return resp + + def post_json(self, url: str, body: dict, t: float) -> requests.Response: + resp = requests.post(url, json=body, timeout=5) + self._line( + "POST", url, resp, t, request_type="application/json", body=json.dumps(body) + ) + return resp + + def write(self, recording: Path) -> None: + path = recording / "network.jsonl" + with path.open("w", encoding="utf-8") as fh: + fh.write( + json.dumps({"schema": SCHEMA, "exchanges": len(self.lines)}) + "\n" + ) + for line in self.lines: + fh.write(json.dumps(line, sort_keys=True) + "\n") + + +def write_recording( + recording: Path, + params: dict[str, str], + *, + save_t: float = 2.0, + extra_events: Optional[list[dict]] = None, +) -> Path: + recording.mkdir(parents=True, exist_ok=True) + events = list(extra_events or []) + [ + { + "i": len(extra_events or []), + "kind": "click", + "x": 300, + "y": 400, + "t": save_t, + "structural": {"role": "button", "name": "Save encounter"}, + } + ] + (recording / "events.jsonl").write_text( + "".join(json.dumps(e) + "\n" for e in events), encoding="utf-8" + ) + (recording / "meta.json").write_text( + json.dumps({"params": params}), encoding="utf-8" + ) + return recording + + +def effect_e2e_recording(recording: Path, base_url: str) -> Path: + """Save one encounter on the effect_e2e service; the app then shows the + chart (GET /api/records) and its banner (GET /api/ui/last-save).""" + + requests.post(f"{base_url}/api/reset", json={}, timeout=5) + cap = Capture() + cap.get(f"{base_url}/api/records", 1.0) + cap.post_json( + f"{base_url}/api/encounter", + {"patient_id": "p1", "type": "Triage", "note": NOTE}, + 2.1, + ) + cap.get(f"{base_url}/api/ui/last-save", 2.4) + cap.get(f"{base_url}/api/records", 2.6) + write_recording(recording, {"patient_id": "p1", "note": NOTE}) + cap.write(recording) + return recording + + +def mockmed_recording(recording: Path, base_url: str, db: Any) -> Path: + """Save one encounter on the MockMed fault server; the app reads /api/db.""" + + db.reset() + cap = Capture() + base = base_url.rstrip("/") + cap.get(f"{base}/api/db", 1.0) + cap.post_json( + f"{base}/api/encounter", + {"patient_id": "p1", "type": "Triage", "note": NOTE}, + 2.1, + ) + cap.get(f"{base}/api/db", 2.5) + write_recording(recording, {"patient_id": "p1", "note": NOTE}) + cap.write(recording) + return recording diff --git a/packages/flow/tests/test_effect_plugin_options.py b/packages/flow/tests/test_effect_plugin_options.py index 24d6dd611..554bbf45f 100644 --- a/packages/flow/tests/test_effect_plugin_options.py +++ b/packages/flow/tests/test_effect_plugin_options.py @@ -253,14 +253,16 @@ def test_identity_of_configs_without_plugin_fields_is_unchanged(): cfg = EffectsConfig(kind="rest", base_url="http://127.0.0.1:9") verifier = build_effect_verifier(cfg) legacy_config = cfg.model_dump(mode="json") - # Neither the plugin fields, ui_readback nor smart_backend existed when - # identities were first issued. + # Neither the plugin fields, ui_readback, smart_backend, surfaces nor + # records_drop_fields existed when identities were first issued. for name in ( "options", "option_params", "secret_env", "ui_readback", "smart_backend", + "surfaces", + "records_drop_fields", ): legacy_config.pop(name, None) legacy_config["candidates"] = [] diff --git a/packages/flow/tests/test_record_check_synthesis.py b/packages/flow/tests/test_record_check_synthesis.py new file mode 100644 index 000000000..0033b0344 --- /dev/null +++ b/packages/flow/tests/test_record_check_synthesis.py @@ -0,0 +1,986 @@ +"""scaffold-verifier builds a working record check and tries to break it. + +The load-bearing contracts: + +* the read is found in the recording's network capture (the + ``openadapt.network-capture/1`` format), a database read path or a verifier + section, by the recorded parameter values; +* the check binds those values to run parameters and covers every table the + save touched; tables nothing reads are reported; +* the check is proved against injected bad writes, on the recorded records and + end to end through the replayer (MockMed and the effect_e2e record service). + One REST read misses the stray billing write (EFFECT_E2E.md's 9 of 72): that + check is written as a draft with the miss listed, never as working; +* nothing is approved: verify-run refuses the proposed file until a person + removes every ``needs_operator_confirmation`` line. + +Synthetic data only, localhost only, no model calls. +""" + +from __future__ import annotations + +import json +import os +import stat +from pathlib import Path + +import pytest +import yaml + +from openadapt_flow.__main__ import main +from openadapt_flow.deployment import EffectsConfig, build_effect_verifier +from openadapt_flow.record_check_proof import prove_live +from openadapt_flow.record_check_synthesis import ( + PROOF_NAME, + PROPOSED_NAME, + SynthesisError, + render, + synthesize, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, Verdict +from openadapt_flow.runtime.effects.effect import EffectState +from openadapt_flow.runtime.effects.surfaces import ( + SurfaceRoutedVerifier, + surface_probe, +) +from openadapt_flow.verify_run import VerifyRunError, parse_verify_config +from tests._record_check_fixtures import ( + NOTE, + Capture, + effect_e2e_recording, + mockmed_recording, + write_recording, +) + + +@pytest.fixture +def e2e_service(tmp_path): + from benchmark.effect_e2e.record_service import serve + + handle = serve(tmp_path / "record.db") + yield handle + handle.stop() + + +@pytest.fixture +def mockmed(): + from openadapt_flow.mockmed.fault_server import serve + + url, db, stop = serve() + yield url, db + stop() + + +def _results(proposal) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + for fault in proposal.faults: + out.setdefault(fault.fault, []).append(fault.as_dict()["result"]) + return out + + +# --------------------------------------------------------------------------- +# effect_e2e: one REST read vs a database read path +# --------------------------------------------------------------------------- + + +def test_rest_read_found_by_recorded_values_and_values_bound(tmp_path, e2e_service): + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + proposal = synthesize(rec) + + [surface] = proposal.search.surfaces + assert surface.config["kind"] == "rest" + assert surface.config["records_path"] == "/api/records" + assert surface.config["records_key"] == "records" + # The banner read also returns the saved values; the persisted row wins. + assert any("/api/ui/last-save" in note for note in proposal.search.notes) + effect, content = proposal.effects + assert content.probe.startswith("surface=api_records.content") + assert proposal.verifier["surfaces"]["api_records.content"][ + "records_drop_fields" + ] == ["id"] + assert effect.match["note"].param == "note" + assert effect.match["patient_id"].param == "patient_id" + # The save sent type=Triage as a fixed value: it tells this record apart. + assert effect.match["type"].literal == "Triage" + assert effect.count_new_only and effect.forbid_collateral_loss + assert effect.needs_operator_confirmation + + +def test_one_rest_read_misses_the_billing_write_and_stays_a_draft( + tmp_path, e2e_service +): + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + proposal = synthesize(rec) + + results = _results(proposal) + assert results["ok"] == ["clean pass"] + for fault in ("no_persist", "partial", "duplicate", "wrong_record", "stale"): + assert set(results[fault]) == {"caught"}, fault + assert results["collateral_unaudited"] == ["MISSED"] + + proposal.live = prove_live( + "effect-e2e", + proposal.verifier, + proposal.effects, + proposal.params, + write_requests=proposal.search.write_requests, + work_dir=tmp_path / "live", + ) + rows = {row["scenario"]: row["result"] for row in proposal.live["runs"]} + assert rows["ok"] == "clean pass" + assert rows["collateral_unaudited"] == "MISSED" + assert all( + rows[name].startswith("caught") + for name in ("no_persist", "partial", "duplicate", "wrong_record", "stale") + ) + assert not proposal.live["passed"] + assert not proposal.working + text = render(proposal) + assert text.startswith("# DRAFT record check") + assert "MISS: collateral_unaudited" in text + + +def test_database_read_path_covers_every_table_and_passes_the_suite( + tmp_path, e2e_service +): + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + proposal = synthesize(rec, db_path=e2e_service.db_path) + + roles = {s.name: s.is_target for s in proposal.search.surfaces} + assert roles["encounters"] and roles["api_records"] + assert roles["billing"] is False # guarded: the save adds no row there + assert proposal.search.closed_world_basis == "database" + assert proposal.search.uncovered == [] + assert all(fault.ok for fault in proposal.faults) + + proposal.live = prove_live( + "effect-e2e", + proposal.verifier, + proposal.effects, + proposal.params, + write_requests=proposal.search.write_requests, + work_dir=tmp_path / "live", + ) + rows = {row["scenario"]: row for row in proposal.live["runs"]} + assert rows["ok"]["result"] == "clean pass" + assert rows["collateral_unaudited"]["result"] == "caught by the check" + # Every wrong save stopped; the timeout committed but the actuator could + # not know, which is a safe stop the check is not blamed for. + for name, row in rows.items(): + if row["ground_truth"] != "correct": + assert not row["reported_done"], name + assert rows["timeout"]["result"] == "stopped by the actuator (outcome unknown)" + assert proposal.live["passed"], proposal.live["misses"] + assert proposal.working + + +def test_frozen_demo_value_fails_the_live_clean_save(tmp_path, e2e_service): + """The live proof uses new note values, so a literal demo note can't pass.""" + + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + proposal = synthesize(rec, db_path=e2e_service.db_path) + frozen = [] + for effect in proposal.effects: + match = dict(effect.match) + if "note" in match: + from openadapt_flow.runtime.effects import ValueExpr + + match["note"] = ValueExpr(literal=NOTE) + frozen.append(effect.model_copy(update={"match": match})) + live = prove_live( + "effect-e2e", + proposal.verifier, + frozen, + proposal.params, + write_requests=proposal.search.write_requests, + work_dir=tmp_path / "live", + ) + assert live["runs"][0]["result"] == "false stop" + assert not live["passed"] + + +# --------------------------------------------------------------------------- +# MockMed (the --break-it fixture) +# --------------------------------------------------------------------------- + + +def test_mockmed_needs_a_declared_complete_read(tmp_path, mockmed): + url, db = mockmed + rec = mockmed_recording(tmp_path / "rec", url, db) + + plain = synthesize(rec) + assert plain.search.surfaces[0].config["records_path"] == "/api/db" + assert not plain.working + assert any("collateral_unaudited" in miss for miss in plain.misses) + + declared = synthesize(rec, complete_read=True) + assert declared.search.closed_world_basis == "declared" + assert all(fault.ok for fault in declared.faults) + declared.live = prove_live( + "mockmed", + declared.verifier, + declared.effects, + declared.params, + write_requests=declared.search.write_requests, + work_dir=tmp_path / "live", + ) + rows = {row["scenario"]: row["result"] for row in declared.live["runs"]} + assert rows["ok"] == "clean pass" + for name in ( + "no_persist", + "partial", + "wrong_record", + "stale", + "optimistic", + "session", + ): + assert rows[name].startswith("caught"), (name, rows[name]) + assert declared.live["passed"] + assert declared.working + assert declared.proof()["closed_world"] == "declared" + + +# --------------------------------------------------------------------------- +# OpenEMR shapes: FHIR R4 Observation search and the Standard API patient list +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fhir(): + from tests._fhir_fake import serve + + base, store, stop = serve() + yield base, store + stop() + + +def _fhir_recording(tmp_path: Path, base: str, store) -> Path: + cap = Capture() + cap.get(f"{base}/Observation?patient=p1", 1.0) + store.add_observation(patient="p1", note=NOTE) + cap.get(f"{base}/Observation?patient=p1", 2.5) + rec = write_recording(tmp_path / "rec", {"patient_id": "p1", "note": NOTE}) + cap.write(rec) + return rec + + +def test_openemr_fhir_read_is_scoped_by_the_run_patient(tmp_path, fhir): + base, store = fhir + store.add_observation(patient="p1", note="An older synthetic note", source="other") + rec = _fhir_recording(tmp_path, base, store) + proposal = synthesize(rec) + + [surface] = proposal.search.surfaces + assert surface.config["kind"] == "fhir" + assert surface.config["base_url"] == base + assert surface.config["resource_type"] == "Observation" + assert surface.config["search_param_exprs"] == {"patient": {"param": "patient_id"}} + assert surface.config["field_paths"]["valueString"] == "valueString" + assert surface.before_recorded # the chart was read before the save too + results = _results(proposal) + assert results["ok"] == ["clean pass"] + assert "caught" in results["wrong_record"] # the read is scoped by patient + assert set(results["stale"]) == {"caught"} + + +def test_openemr_fhir_check_refutes_store_level_faults(tmp_path, fhir): + """Each fault is injected in the FHIR store; the check built from the + recording must refute it and confirm the clean save.""" + + base, store = fhir + rec = _fhir_recording(tmp_path, base, store) + proposal = synthesize(rec) + params = {"patient_id": "p1", "note": "Second synthetic note"} + effects = [ + e.model_copy(update={"needs_operator_confirmation": False, "timeout_s": 0.2}) + for e in proposal.effects + ] + + def verdicts(fault: str) -> set[Verdict]: + store.resources.clear() + store.add_observation( + patient="p1", note="Concurrent synthetic note", source="other" + ) + verifier = build_effect_verifier( + EffectsConfig.model_validate(proposal.verifier), params + ) + before = verifier.capture_pre_state() + if fault == "ok": + store.add_observation(patient="p1", note=params["note"]) + elif fault == "partial": + store.add_observation(patient="p1", note=None) + elif fault == "duplicate": + store.add_observation(patient="p1", note=params["note"]) + store.add_observation(patient="p1", note=params["note"]) + elif fault == "wrong_record": + store.add_observation(patient="p2", note=params["note"]) + elif fault == "stale": + store.delete_where(patient="p1", source="other") + store.add_observation(patient="p1", note=params["note"]) + return { + verifier.verify(effect.resolve(params), before).verdict + for effect in effects + } + + assert verdicts("ok") == {Verdict.CONFIRMED} + for fault in ("no_persist", "partial", "duplicate", "wrong_record", "stale"): + assert Verdict.REFUTED in verdicts(fault), fault + + +def test_openemr_patient_list_matches_the_hand_written_check(tmp_path): + """The OpenEMR Standard API patient list (``{"data": [...]}``) yields the + same read and fields the hand-written benchmark/openemr_local check uses.""" + + params = { + "fname": "Avery", + "lname": "Quill", + "DOB": "1984-03-09", + "email": "avery.quill@example.test", + "city": "Springfield", + } + before = {"data": [{"pid": 7, "uuid": "u-7", "fname": "Rowan", "lname": "Ash"}]} + after = { + "data": before["data"] + + [{"pid": 8, "uuid": "u-8", **params, "sex": "Female", "title": ""}] + } + url = "https://127.0.0.1:9300/apis/default/api/patient?_limit=1000" + lines = [ + { + "method": "GET", + "url": url, + "status": 200, + "t": 1.0, + "response_type": "application/json", + "response_text": json.dumps(before), + }, + { + "method": "POST", + "url": "https://127.0.0.1:9300/interface/new/new_comprehensive_save.php", + "status": 302, + "t": 2.0, + "form": [ + ["form_fname", "Avery"], + ["csrf_token_form", ""], + ], + }, + { + "method": "GET", + "url": url, + "status": 200, + "t": 3.0, + "response_type": "application/json", + "response_text": json.dumps(after), + }, + ] + rec = write_recording(tmp_path / "rec", params) + with (rec / "network.jsonl").open("w") as fh: + fh.write( + json.dumps({"schema": "openadapt.network-capture/1", "exchanges": 3}) + "\n" + ) + for line in lines: + fh.write(json.dumps(line) + "\n") + + proposal = synthesize(rec) + [surface] = proposal.search.surfaces + assert surface.config["records_path"] == "/apis/default/api/patient?_limit=1000" + assert surface.config["records_key"] == "data" + assert surface.before_recorded + effect = proposal.effects[0] + assert {k: v.param for k, v in effect.match.items()} == {k: k for k in params} + # The form save answered with a redirect; its sent value is read back. + assert len(proposal.search.write_requests) == 1 + assert proposal.search.uncovered == [] + # Every checked field identifies the patient, so a lost update to another + # chart of the same patient can't be told apart: reported, not credited. + missed = {f.fault for f in proposal.faults if not f.ok} + assert missed == {"stale", "collateral_unaudited"} + results = _results(proposal) + assert set(results["changed"]) == set(results["deleted"]) == {"caught"} + + +def test_session_token_reads_and_nested_fields_are_reported(tmp_path): + params = {"note": NOTE} + lines = [ + { + "method": "POST", + "url": "http://app.test/save", + "status": 200, + "t": 2.0, + "body": json.dumps({"note": NOTE}), + }, + { + "method": "GET", + "url": "http://app.test/api/x?token=", + "status": 200, + "t": 3.0, + "response_type": "application/json", + "response_text": json.dumps({"records": [{"id": 1, "note": NOTE}]}), + }, + { + "method": "GET", + "url": "http://app.test/api/nested", + "status": 200, + "t": 3.1, + "response_type": "application/json", + "response_text": json.dumps( + {"records": [{"id": 1, "body": {"note": NOTE}}]} + ), + }, + ] + rec = write_recording(tmp_path / "rec", params) + with (rec / "network.jsonl").open("w") as fh: + fh.write(json.dumps({"schema": "openadapt.network-capture/1"}) + "\n") + for line in lines: + fh.write(json.dumps(line) + "\n") + proposal = synthesize(rec) + assert proposal.effects == [] + notes = " ".join(proposal.search.notes) + assert "session token" in notes + assert "nested fields" in notes + + +def test_no_params_is_refused(tmp_path): + rec = write_recording(tmp_path / "rec", {}) + with pytest.raises(SynthesisError): + synthesize(rec) + + +# --------------------------------------------------------------------------- +# Nothing is approved +# --------------------------------------------------------------------------- + + +def test_verify_run_refuses_until_a_person_removes_the_flags(tmp_path, e2e_service): + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + proposal = synthesize(rec, db_path=e2e_service.db_path) + doc = yaml.safe_load(render(proposal)) + with pytest.raises(VerifyRunError, match="unconfirmed placeholder"): + parse_verify_config(doc) + for effect in doc["effects"]: + effect.pop("needs_operator_confirmation") + cfg = parse_verify_config(doc) + verifier = build_effect_verifier(cfg.verifier, proposal.params) + assert isinstance(verifier, SurfaceRoutedVerifier) + assert set(verifier.surfaces) == set(proposal.verifier["surfaces"]) + assert {s.name for s in proposal.search.surfaces} <= set(verifier.surfaces) + + from openadapt_flow.verify_run import _tag_reads + + _tag_reads(verifier, "verify-test-run-0001") + rest = verifier.verifiers["api_records"] + assert rest.headers["X-OpenAdapt-Execution-Id"] == "verify-test-run-0001" + + screen = dict( + doc, verifier={"kind": "surfaces", "surfaces": {"a": {"kind": "onscreen"}}} + ) + with pytest.raises(VerifyRunError, match="does not read independently"): + parse_verify_config(screen) + + +# --------------------------------------------------------------------------- +# The routed verifier +# --------------------------------------------------------------------------- + + +class _Fixed: + substrate = "fixed" + verification_tier = 1 + + def __init__(self, records): + self.records = records + + def capture_pre_state(self, context=None): + return EffectState( + substrate="fixed", reachable=True, records=list(self.records) + ) + + def verify(self, expected, before, context=None): + from openadapt_flow.runtime.effects._common import judge_records + + return judge_records(expected, before, self.records, substrate="fixed") + + +def test_surface_routed_verifier_judges_each_effect_on_its_own_read(): + a = _Fixed([{"id": 1, "note": "x"}]) + b = _Fixed([]) + verifier = SurfaceRoutedVerifier({"a": a, "b": b}) + before = verifier.capture_pre_state() + a.records = a.records + [{"id": 2, "note": "y"}] + b.records = [{"id": 9}] + wrote = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"note": "y"}, + count_new_only=True, + probe=surface_probe("a"), + ) + guard = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={}, + expected_count=0, + count_new_only=True, + probe=surface_probe("b", "no new rows"), + ) + assert verifier.verify(wrote, before).verdict is Verdict.CONFIRMED + assert verifier.verify(guard, before).verdict is Verdict.REFUTED + for probe in (None, "surface=c"): + lost = wrote.model_copy(update={"probe": probe}) + assert verifier.verify(lost, before).verdict is Verdict.INDETERMINATE + + +def test_surfaces_config_validation_and_identity_unchanged(): + with pytest.raises(ValueError): + EffectsConfig.model_validate({"kind": "surfaces"}) + with pytest.raises(ValueError): + EffectsConfig.model_validate( + { + "kind": "rest", + "base_url": "http://x", + "surfaces": {"a": {"kind": "rest"}}, + } + ) + with pytest.raises(ValueError): + EffectsConfig.model_validate( + {"kind": "surfaces", "surfaces": {"a": {"kind": "surfaces"}}} + ) + # A config that sets no surfaces keeps the identity it had before. + from openadapt_flow.deployment import _identity_config_dump + + dumped = _identity_config_dump( + EffectsConfig.model_validate({"kind": "rest", "base_url": "http://x"}) + ) + assert "surfaces" not in dumped + + +# --------------------------------------------------------------------------- +# Identity coverage +# --------------------------------------------------------------------------- + + +def _identity_bundle(tmp_path: Path) -> tuple[Path, Path]: + from openadapt_flow.ir import ActionKind, Anchor, Step, Workflow + + def anchor(**extra): + return Anchor( + template="templates/x.png", + region=(0, 0, 10, 10), + click_point=(5, 5), + **extra, + ) + + steps = [ + Step( + id="step_000", + intent="open row", + action=ActionKind.CLICK, + anchor=anchor(context_text="Quill Avery 1984-03-09"), + ), + Step( + id="step_001", + intent="open chart", + action=ActionKind.CLICK, + anchor=anchor(), + identity_armed=False, + ), + Step( + id="step_002", + intent="pick patient", + action=ActionKind.CLICK, + anchor=anchor(), + identity_armed=False, + ), + Step( + id="step_003", + intent="save", + action=ActionKind.CLICK, + anchor=anchor(), + identity_armed=False, + identity_unarmed_reason="no readable band text", + ), + ] + bundle = tmp_path / "bundle" + Workflow(name="identity demo", steps=steps).save(bundle) + events = [ + {"kind": "click", "t": 0.1}, + { + "kind": "click", + "t": 0.2, + "structured_identity": "Quill, Avery 1984-03-09 MRN 42", + }, + { + "kind": "click", + "t": 0.3, + "structural": {"role": "row", "name": "Quill Avery"}, + }, + { + "kind": "click", + "t": 0.4, + "structural": {"role": "button", "name": "Save"}, + "title_before": "Quill Avery - Chart", + }, + ] + rec = tmp_path / "rec" + rec.mkdir() + (rec / "events.jsonl").write_text("".join(json.dumps(e) + "\n" for e in events)) + return bundle, rec + + +def test_identity_coverage_before_and_after(tmp_path): + from openadapt_flow.record_check_identity import ( + identity_coverage, + write_identity_proposals, + ) + + bundle, rec = _identity_bundle(tmp_path) + report, proposals = identity_coverage(bundle, rec, {"lname": "Quill"}) + assert report["applicable"] == 4 + assert report["armed_before"] == 1 + assert report["armed_after"] == 3 + assert {p["step_id"] for p in proposals} == {"step_001", "step_002"} + assert report["proposed"][0]["contains_params"] == ["lname"] + assert "value" not in report["proposed"][0] # record text stays private + [seen] = report["seen_not_checkable"] + assert seen["step_id"] == "step_003" + path = write_identity_proposals(proposals, tmp_path) + assert stat.S_IMODE(os.stat(path).st_mode) == 0o600 + + +def test_identity_not_proposed_where_the_runtime_cannot_check_it(tmp_path): + from openadapt_flow.ir import Workflow + from openadapt_flow.record_check_identity import identity_coverage + + bundle, rec = _identity_bundle(tmp_path) + workflow = Workflow.load(bundle) + Workflow.model_validate(workflow.model_dump(mode="json") | {"surface": "rdp"}).save( + bundle + ) + report, proposals = identity_coverage(bundle, rec, {"lname": "Quill"}) + assert proposals == [] + assert report["armed_after"] == report["armed_before"] == 1 + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def test_cli_writes_a_working_check_and_its_proof(tmp_path, e2e_service, capsys): + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + code = main( + [ + "scaffold-verifier", + str(rec), + "--db", + str(e2e_service.db_path), + "--prove", + "effect-e2e", + "--out", + str(tmp_path / "out"), + ] + ) + out = capsys.readouterr().out + assert code == 0, out + assert "Record check: WORKING (not approved)" in out + proof = json.loads((tmp_path / "out" / PROOF_NAME).read_text()) + assert proof["status"] == "working" and proof["approved"] is False + assert proof["live"]["passed"] + assert (tmp_path / "out" / PROPOSED_NAME).is_file() + + +def test_cli_exits_nonzero_for_a_draft(tmp_path, e2e_service, capsys): + rec = effect_e2e_recording(tmp_path / "rec", e2e_service.base_url) + code = main(["scaffold-verifier", str(rec), "--prove", "effect-e2e"]) + out = capsys.readouterr().out + assert code == 1 + assert "DRAFT (not working)" in out + assert "MISS: collateral_unaudited" in out + + +def test_cli_falls_back_to_the_todo_draft_without_reads(tmp_path, capsys): + rec = write_recording(tmp_path / "rec", {"note": NOTE}) + code = main(["scaffold-verifier", str(rec)]) + out = capsys.readouterr().out + assert code == 0 + assert "TODO draft" in out + assert (rec / "effect_contract.yaml").is_file() + + +# --------------------------------------------------------------------------- +# Review findings: each would have let a broken check be called working +# --------------------------------------------------------------------------- + + +def _capture(rec: Path, lines: list[dict]) -> None: + with (rec / "network.jsonl").open("w") as fh: + fh.write(json.dumps({"schema": "openadapt.network-capture/1"}) + "\n") + for line in lines: + fh.write(json.dumps(line) + "\n") + + +def _db(path: Path, rows: dict[str, list[tuple]]) -> Path: + import sqlite3 + + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE encounters (id INTEGER PRIMARY KEY, patient_id TEXT, note TEXT)" + ) + conn.execute( + "CREATE TABLE billing (id INTEGER PRIMARY KEY, patient_id TEXT, amount TEXT)" + ) + for table, values in rows.items(): + for value in values: + marks = ",".join("?" * len(value)) + conn.execute(f"INSERT INTO {table} VALUES ({marks})", value) + conn.commit() + conn.close() + return path + + +def test_a_table_too_big_to_scan_keeps_the_check_a_draft(tmp_path, monkeypatch): + import openadapt_flow.record_check_synthesis as rcs + + db = _db( + tmp_path / "app.db", + { + "encounters": [(1, "p1", NOTE)], + "billing": [(i, "p9", "1.00") for i in range(1, 6)], + }, + ) + monkeypatch.setattr(rcs, "MAX_SCAN_ROWS", 3) + rec = write_recording(tmp_path / "rec", {"patient_id": "p1", "note": NOTE}) + proposal = synthesize(rec, db_path=db) + assert proposal.search.closed_world_basis is None + assert any("billing" in item for item in proposal.search.uncovered) + assert not proposal.working + + +def test_stale_is_a_miss_when_no_field_tells_records_apart(tmp_path): + db = _db(tmp_path / "app.db", {"encounters": [(1, "p1", None)]}) + rec = write_recording(tmp_path / "rec", {"patient_id": "p1"}) + proposal = synthesize(rec, db_path=db) + stale = [f for f in proposal.faults if f.fault == "stale"] + assert stale and all(f.as_dict()["result"] == "MISSED" for f in stale) + assert not proposal.working + + +def test_guards_catch_a_changed_or_deleted_row_through_the_real_read(tmp_path): + import sqlite3 + + db = _db( + tmp_path / "app.db", + {"encounters": [(1, "p1", NOTE)], "billing": [(1, "p2", "10.00")]}, + ) + rec = write_recording(tmp_path / "rec", {"patient_id": "p1", "note": NOTE}) + proposal = synthesize(rec, db_path=db) + results = _results(proposal) + assert set(results["changed"]) == {"caught"} + assert set(results["deleted"]) == {"caught"} + assert proposal.working, proposal.misses + + effects = [ + e.model_copy(update={"needs_operator_confirmation": False, "timeout_s": 0.05}) + for e in proposal.effects + ] + + def verdicts(mutate) -> set[Verdict]: + verifier = build_effect_verifier( + EffectsConfig.model_validate(proposal.verifier), proposal.params + ) + conn = sqlite3.connect(db) + conn.execute("DELETE FROM encounters") + conn.commit() + before = verifier.capture_pre_state() + conn.execute("INSERT INTO encounters VALUES (7, 'p1', ?)", (NOTE,)) + mutate(conn) + conn.commit() + conn.close() + return { + verifier.verify(e.resolve(proposal.params), before).verdict for e in effects + } + + assert verdicts(lambda c: None) == {Verdict.CONFIRMED} + changed = verdicts(lambda c: c.execute("UPDATE billing SET amount = '99.00'")) + assert Verdict.REFUTED in changed + deleted = verdicts(lambda c: c.execute("DELETE FROM billing")) + assert Verdict.REFUTED in deleted + + +def test_names_are_unique_across_sources(tmp_path): + db = _db(tmp_path / "app.db", {"encounters": [(1, "p1", NOTE)]}) + rec = write_recording(tmp_path / "rec", {"patient_id": "p1", "note": NOTE}) + _capture( + rec, + [ + {"method": "POST", "url": "http://app.test/save", "status": 200, "t": 2.1}, + { + "method": "GET", + "url": "http://app.test/encounters", + "status": 200, + "t": 2.5, + "response_text": json.dumps( + {"records": [{"id": 1, "patient_id": "p1", "note": NOTE}]} + ), + }, + ], + ) + proposal = synthesize(rec, db_path=db) + names = [s.name for s in proposal.search.surfaces] + assert len(names) == len(set(names)) + assert len(proposal.verifier["surfaces"]) == len(names) + sum( + s.has_ids for s in proposal.search.surfaces + ) + + +def test_the_save_is_timed_from_its_write_request(tmp_path): + """An event's t is stamped after the screen settles, so the save's POST + and the app's read-back come before the Save step's t.""" + + before = {"records": []} + after = {"records": [{"id": 1, "patient_id": "p1", "note": NOTE}]} + rec = write_recording( + tmp_path / "rec", {"patient_id": "p1", "note": NOTE}, save_t=3.0 + ) + _capture( + rec, + [ + { + "method": "GET", + "url": "http://app.test/api/records", + "status": 200, + "t": 1.0, + "response_text": json.dumps(before), + }, + { + "method": "POST", + "url": "http://app.test/api/encounter", + "status": 200, + "t": 2.1, + "body": json.dumps({"patient_id": "p1", "note": NOTE}), + }, + { + "method": "GET", + "url": "http://app.test/api/records", + "status": 200, + "t": 2.5, + "response_text": json.dumps(after), + }, + { + "method": "GET", + "url": "http://app.test/api/records", + "status": 200, + "t": 6.0, + "response_text": json.dumps(after), + }, + ], + ) + proposal = synthesize(rec, complete_read=True) + [surface] = proposal.search.surfaces + assert surface.before_recorded and surface.before == [] + assert proposal.search.save_t == 2.1 + assert _results(proposal)["ok"] == ["clean pass"] + + +def test_record_ids_in_a_url_stay_out_of_names_and_block_working(tmp_path): + body = {"records": [{"id": 1, "patient_id": "p1", "note": NOTE}]} + rec = write_recording(tmp_path / "rec", {"note": NOTE}) + _capture( + rec, + [ + {"method": "POST", "url": "http://app.test/save", "status": 200, "t": 2.1}, + { + "method": "GET", + "url": "http://app.test/api/patients/MRN4471/encounters", + "status": 200, + "t": 2.5, + "response_text": json.dumps(body), + }, + ], + ) + proposal = synthesize(rec, complete_read=True) + text = json.dumps(proposal.proof()) + render(proposal).split("schema_version")[0] + assert "MRN4471" not in json.dumps(proposal.proof()) + assert all("MRN4471" not in s.name for s in proposal.search.surfaces) + assert any("fixed record" in p for p in proposal.search.problems) + assert not proposal.working + assert "MRN4471" not in text + + +def test_generated_reads_are_built_and_run(tmp_path): + import sqlite3 + + path = tmp_path / "app.db" + conn = sqlite3.connect(path) + conn.execute( + 'CREATE TABLE visits (id INTEGER PRIMARY KEY, "patient_id" TEXT, note TEXT)' + ) + conn.execute("INSERT INTO visits VALUES (1, 'p1', ?)", (NOTE,)) + conn.commit() + conn.close() + rec = write_recording(tmp_path / "rec", {"patient-id": "p1", "note": NOTE}) + proposal = synthesize(rec, db_path=path) + config = proposal.verifier["surfaces"]["visits"] + assert config["sql_query_params"] == {"p0": {"param": "patient-id"}} + assert proposal.search.problems == [] + assert proposal.working, proposal.misses + + +def test_braces_in_a_literal_path_are_not_doubled(tmp_path): + body = {"records": [{"id": 1, "note": NOTE}]} + rec = write_recording(tmp_path / "rec", {"note": NOTE}) + _capture( + rec, + [ + {"method": "POST", "url": "http://app.test/save", "status": 200, "t": 2.1}, + { + "method": "GET", + "url": "http://app.test/api/{v}/records", + "status": 200, + "t": 2.5, + "response_text": json.dumps(body), + }, + ], + ) + proposal = synthesize(rec) + [surface] = proposal.search.surfaces + assert surface.config["records_path"] == "/api/{v}/records" + assert "path_params" not in surface.config + + +def test_a_truncated_capture_falls_back_to_the_todo_draft(tmp_path, capsys): + rec = write_recording(tmp_path / "rec", {"note": NOTE}) + (rec / "network.jsonl").write_text( + '{"schema": "openadapt.network-capture/1"}\n{"meth' + ) + code = main(["scaffold-verifier", str(rec)]) + assert code == 0 + assert "TODO draft" in capsys.readouterr().out + + +def test_a_demonstration_with_no_write_is_refused(tmp_path, capsys): + rec = tmp_path / "rec" + rec.mkdir() + (rec / "events.jsonl").write_text( + json.dumps( + { + "kind": "click", + "t": 1.0, + "structural": {"role": "link", "name": "Open chart"}, + } + ) + + "\n" + ) + (rec / "meta.json").write_text(json.dumps({"params": {"note": NOTE}})) + _capture( + rec, + [ + { + "method": "GET", + "url": "http://app.test/api/records", + "status": 200, + "t": 2.0, + "response_text": json.dumps({"records": [{"id": 1, "note": NOTE}]}), + }, + ], + ) + with pytest.raises(SystemExit) as exc: + main(["scaffold-verifier", str(rec), "--complete-read"]) + assert "REFUSED" in str(exc.value.code if exc.value.code is not None else exc.value)