diff --git a/packages/flow/README.md b/packages/flow/README.md index 5b9bf43ab..80817823a 100644 --- a/packages/flow/README.md +++ b/packages/flow/README.md @@ -380,6 +380,10 @@ The full list, capability by capability: [docs/LIMITS.md](docs/LIMITS.md). ## More documentation +- [Qualify a workflow](docs/QUALIFY_A_WORKFLOW.md): `openadapt-flow qualify` + runs a bundle against the normal case and deliberate faults, reads the write + back from the system of record, and certifies it under a policy. Terms are + defined in [docs/QUALIFICATION_TERMS.md](docs/QUALIFICATION_TERMS.md). - [Production setup](docs/PRODUCTION.md): qualifying a bundle from its recording, policies, multi-step processes, signed results, and release status. diff --git a/packages/flow/benchmark/openemr_local/qualify_hooks.py b/packages/flow/benchmark/openemr_local/qualify_hooks.py new file mode 100644 index 000000000..469e53556 --- /dev/null +++ b/packages/flow/benchmark/openemr_local/qualify_hooks.py @@ -0,0 +1,123 @@ +"""Sign-in, reset and read-back session for the local OpenEMR fixture. + +``benchmark/openemr_local/verify.yaml`` names these three functions. They are +environment setup for a made-up-patient test system, not workflow logic: +OpenEMR's sign-in opens the new-patient form through its tab shell, the +fixture resets by restoring a database snapshot, and the read-back client +uses an OAuth password grant that a static header cannot express. + +Environment (all optional): + +- ``OPENEMR_FIXTURE_STATE``: fixture state directory + (default ``benchmark/openemr_local/state``); +- ``OPENEMR_FIXTURE_LOCK``: a different environment lock file; +- ``OPENEMR_UI_URL`` / ``OPENEMR_API_URL``: non-default ports. +""" + +from __future__ import annotations + +import atexit +import os +import time +from pathlib import Path +from typing import Any, Callable + +HERE = Path(__file__).resolve().parent +_fixture: Any = None +_browser: Any = None + + +def fixture() -> Any: + """The fixture named by the environment, created once per process.""" + + global _fixture + if _fixture is not None: + return _fixture + import benchmark.openemr_local.fixture as fx + + lock = os.environ.get("OPENEMR_FIXTURE_LOCK") + if lock: + fx.LOCK_PATH = Path(lock) + ui = os.environ.get("OPENEMR_UI_URL") + api = os.environ.get("OPENEMR_API_URL") + + class _Fixture(fx.OpenEMRFixture): + @property + def ui_base_url(self) -> str: + return ui or super().ui_base_url + + @property + def api_base_url(self) -> str: + return api or super().api_base_url + + def _validate_runtime_env(self) -> None: + if ui or api: # non-default ports: the runtime env names them + self.runtime_env.chmod(0o600) + return + super()._validate_runtime_env() + + _fixture = _Fixture(Path(os.environ.get("OPENEMR_FIXTURE_STATE", HERE / "state"))) + return _fixture + + +def _shared_browser() -> Any: + global _browser + if _browser is None: + from playwright.sync_api import sync_playwright + + from openadapt_flow._browser_setup import ensure_chromium_installed + + ensure_chromium_installed() + pw = sync_playwright().start() + browser = pw.chromium.launch(headless=True) + _browser = (pw, browser) + + def _stop() -> None: + try: + browser.close() + finally: + pw.stop() + + atexit.register(_stop) + return _browser[1] + + +def open_session(config: Any) -> tuple[Any, Callable[[], None]]: + """A fresh signed-in context at the new-patient form.""" + + from scripts.openemr_local_demo import ( + OpenEMRPlaywrightBackend, + _preauthenticate_browser, + ) + + fx = fixture() + width, height = config.app.viewport + context = _shared_browser().new_context( + viewport={"width": width, "height": height}, device_scale_factor=1 + ) + page = context.new_page() + page.goto(fx.ui_base_url) + backend = OpenEMRPlaywrightBackend(page) + _preauthenticate_browser(backend, fx) + return backend, context.close + + +def reset() -> None: + """Restore the verified baseline database.""" + + fixture().reset() + + +def oracle_session(config: Any, attempts: int = 5) -> Any: + """A read-only OAuth client; retries transport errors right after a reset.""" + + import httpx + + for attempt in range(1, attempts + 1): + try: + return fixture().token_session("oracle") + except httpx.TransportError: + if attempt == attempts: + raise + time.sleep(2.0 * attempt) + raise AssertionError("unreachable") diff --git a/packages/flow/benchmark/openemr_local/verify.yaml b/packages/flow/benchmark/openemr_local/verify.yaml new file mode 100644 index 000000000..685c9a123 --- /dev/null +++ b/packages/flow/benchmark/openemr_local/verify.yaml @@ -0,0 +1,60 @@ +# Qualify the OpenEMR patient-registration workflow on the local fixture. +# +# PYTHONPATH=. openadapt-flow qualify path/to/bundle \ +# --verify benchmark/openemr_local/verify.yaml --out runs/openemr-qual \ +# --certify clinical-write +# +# Made-up patients only. See docs/QUALIFY_A_WORKFLOW.md. +schema_version: openadapt.verify/v1 +environment_id: openemr-local-fixture-7c96c8eefe46 +minimum_effect_strength: independent_system_of_record + +app: + target: web + name: OpenEMR (local fixture, made-up patients) + url: ${OPENEMR_UI_URL:-http://127.0.0.1:9301} + version: openemr@7c96c8eefe46 (benchmark/openemr_local lock) + # OpenEMR's sign-in opens the form through its tab shell, and the fixture + # resets by restoring a database snapshot; both are small functions. + session: benchmark.openemr_local.qualify_hooks:open_session + reset: + python: benchmark.openemr_local.qualify_hooks:reset + +# Read the patient back through OpenEMR's REST API with a separate, +# read-only OAuth client. This is the existing RestRecordVerifier. +verifier: + kind: rest + base_url: ${OPENEMR_API_URL:-https://127.0.0.1:9300} + records_path: /apis/default/api/patient?_limit=1000 + records_key: data + session: benchmark.openemr_local.qualify_hooks:oracle_session + +# What the save writes; expands to the effect contract list (one +# record_written plus a field_equals per field). A bare value is a parameter. +record: + name: patient + match: [email, lname] + duplicate_key: email # OpenEMR has no idempotency column; one chart per email + fields: + email: email + lname: lname + title: title + fname: fname + DOB: DOB + sex: sex + street: street + city: city + state: {literal: MA} # the select shows "Massachusetts"; the record stores MA + postal_code: postal_code + phone_home: phone_home + country_code: country_code + +save_step: auto + +inputs: + source: recorded + +campaign: + id: openemr-local-patient-registration-v2 + trials: 3 + policy: clinical-write diff --git a/packages/flow/docs/QUALIFICATION_TERMS.md b/packages/flow/docs/QUALIFICATION_TERMS.md new file mode 100644 index 000000000..c0aab89d7 --- /dev/null +++ b/packages/flow/docs/QUALIFICATION_TERMS.md @@ -0,0 +1,126 @@ +# Qualification terms + +This page defines the words `openadapt-flow qualify` uses. Other pages link +here instead of repeating them. + +## Qualify + +Qualifying a workflow means running the compiled bundle against the normal +case and a set of deliberate faults before it may run unattended on a real +system. It passes only when: + +- the normal case completes, and the write is confirmed by reading it back + from the system of record (the application's API or database), not from the + screen; +- every fault case halts before anything is written. + +The faults are: + +| Case | What changes | Must | +|---|---|---| +| `representative` | nothing | complete, verified by read-back | +| `fault-ambiguity` | a second copy of the save button appears | halt | +| `fault-wrong-identity` | the save button is relabeled to a different action | halt | +| `fault-stale-identity` | the screen goes blank at the save | halt | +| `fault-weak-effect` | the write check is swapped for one that only reads the screen | halt | +| `fault-missing-effect` | the write check is removed | halt | + +Each case runs several times (three by default). After each trial an +independent oracle reads the system of record and checks that the normal case +wrote exactly one correct record and that no fault case wrote anything. + +**When you need it:** before a workflow makes consequential or irreversible +writes (creating a patient, submitting a claim, sending a payment) without a +person watching. It is optional for read-only tasks. + +**What you get:** a signed report (`qualification/report.json`) and a bundle +whose `workflow.json` carries the qualification project and its signed case +results. With certification (below), the runner admits that bundle for +unattended runs. + +**What it does not prove:** anything beyond the environment it was qualified +in. The report records the application version, browser, viewport and +runtime. A different application version, a changed screen, or a different +system of record needs a new qualification. A pass also does not show the +workflow handles inputs or screens that the cases did not exercise. + +## Certify + +Certifying adds one step to qualifying: the result is checked against a +certification policy and signed under it. `openadapt-flow qualify ... +--certify clinical-write` refuses to certify unless all six cases ran and +passed and the bundle satisfies every rule of the policy. Nothing is marked +certified when it refuses; the refusal reasons are in +`qualification/verdict.txt`. + +What is signed is `qualification/result.json` +(`openadapt.qualification-result/v1`, RFC-0002): the coverage report without +free text, the digest of the read-back contract it was judged against, and +one read-back verdict per trial. Its digest goes into the existing +qualification admission envelope; there is no separate signing scheme. + +The keys `qualify` signs with are local test keys. Case results are signed +with runner and fault-driver keys made for one campaign and held in memory. +The admission envelope is signed with Flow's well-known local-dev +qualification key, which every production trust map refuses. None of them is +a production trust root; a production deployment admits results signed by its +own authority (see [PRODUCTION.md](PRODUCTION.md)). + +## Certification policy + +A certification policy is a named set of rules a bundle must meet before it is +certified. The rules live in a YAML file and are enforced by +`openadapt_flow/policy.py`. They cover: + +- **identity**: which steps must check that the target on screen is the right + one before acting (`require_identity_for`, `prohibit_unarmed_clicks`); +- **effect verification**: which steps must confirm their write in the system + of record, not only on screen (`require_system_effects_for`, + `require_effects_for_irreversible`, `require_effect_verification_for`), and + which need an idempotency key so a retry cannot write twice + (`require_idempotency_key_for`); +- **screen checks**: which steps must also check the screen afterwards + (`require_screen_postconditions_for`, `prohibit_vacuous_postconditions`, + `max_unverified_steps`); +- **operator review**: effect bindings the compiler could not derive must be + confirmed by a person (`prohibit_unconfirmed_effect_bindings`), and steps + found on thin evidence need sign-off + (`require_human_approval_below_confidence`). + +Which faults must halt, the minimum effect tier and the trial count are not +policy rules today. `qualify` always requires all six cases for +certification, sets the minimum effect tier to the tier of the read-back you +configure (a REST or SQL read-back is the strongest, "independent system"), +and runs `campaign.trials` trials per case (three by default). + +### Built-in policies + +| Policy | Requires | Use it for | +|---|---|---| +| `clinical-write` | identity on every navigation and write step; no unchecked clicks; a system-of-record check and an idempotency key on every write; screen checks on write, save, submit, create, update, delete, confirm and send steps; no step without a check; no unconfirmed effect bindings; sign-off for steps found with confidence below 0.8 | unattended writes to a system of record: EHR, claims, billing, anything where a wrong or duplicate write is costly | +| `permissive` | only that write steps carry some check of their outcome | development, CI smoke tests, read-only tasks | + +The `qualify propose` and `qualify from-demo` commands also take a +`--policy-pack` that bundles a policy with a minimum effect tier and signer: + +| Pack | Policy | Minimum effect tier | Signer | +|---|---|---|---| +| `community` | `permissive` | persisted-state reacquisition | local dev signer allowed | +| `cloud` | `clinical-write` | independent session | production signer | +| `regulated` | `clinical-write` | independent system | production signer | + +### How to choose + +Use `clinical-write` whenever the workflow writes to a system of record and +will run without a person approving each write. Use `permissive` while you are +building the workflow, or for tasks that only read. + +### Your own policy + +You can write your own. `--certify` takes a built-in name or a path to a YAML +file with the same keys; unknown keys are rejected. Start from a copy of +`openadapt_flow/policies/clinical-write.yaml`, and tighten rather than loosen +it for production writes. + +See also: [Qualify a workflow](QUALIFY_A_WORKFLOW.md), +[QUALIFICATION_PROJECT.md](QUALIFICATION_PROJECT.md). diff --git a/packages/flow/docs/QUALIFY_A_WORKFLOW.md b/packages/flow/docs/QUALIFY_A_WORKFLOW.md new file mode 100644 index 000000000..d5f48b7c9 --- /dev/null +++ b/packages/flow/docs/QUALIFY_A_WORKFLOW.md @@ -0,0 +1,239 @@ +# Qualify a workflow + +Before a workflow writes to a real system without a person watching, qualify +it: run it against the normal case and deliberate faults, and check that it +completes with the write confirmed by reading back the system of record, and +halts on every fault. The terms (qualify, certify, certification policy) are +defined in [QUALIFICATION_TERMS.md](QUALIFICATION_TERMS.md). + +You need three things: + +1. A compiled bundle. A task can be learned from one or more demonstrations: + `openadapt-flow record`, then `openadapt-flow compile` (or `induce` for + several recordings). +2. A test copy of the application you can reset between trials. Qualifying + performs real writes. +3. A `verify.yaml` that says how to read the result back. + +Then: + +```bash +openadapt-flow qualify path/to/bundle --verify verify.yaml \ + --out runs/qual-1 --certify clinical-write +``` + +`qualify` checks `verify.yaml` and the bundle before it runs anything, and +lists every problem it finds (no reset, no read-back, a save step without an +identity check, inputs the record needs but the bundle does not have). It then +runs the six cases, stops at the first case that does not behave as required, +and, with `--certify`, signs the result under the policy. + +## verify.yaml + +This is the file for the local OpenEMR fixture +(`benchmark/openemr_local/verify.yaml`), shortened: + +```yaml +schema_version: openadapt.verify/v1 +environment_id: openemr-local-fixture-7c96c8eefe46 # optional +minimum_effect_strength: independent_system_of_record + +app: + name: OpenEMR (local benchmark fixture) + url: ${OPENEMR_UI_URL:-http://127.0.0.1:9301} + version: "openemr@7c96c8eefe46" + # Sign-in and fixture reset are app-specific; name a Python function. + session: benchmark.openemr_local.qualify_hooks:open_session + reset: + python: benchmark.openemr_local.qualify_hooks:reset + +# Where the write is read back from. Not the screen. +verifier: + kind: rest + base_url: ${OPENEMR_API_URL:-https://127.0.0.1:9300} + records_path: /apis/default/api/patient?_limit=1000 + records_key: data + session: benchmark.openemr_local.qualify_hooks:oracle_session + +# The record the save must create, and which input fills each field. +# Shorthand for the effects list (see below). +record: + name: patient + match: [email, lname] # how to find the new record + duplicate_key: email # an existing record with this value means "already done" + fields: + email: email # field: input parameter name + lname: lname + fname: fname + DOB: DOB + state: {literal: MA} # a fixed value + +save_step: auto # or a step id, e.g. s27 + +inputs: + source: recorded # or: file, with file: inputs.json + +campaign: + id: openemr-local-patient-registration-v2 + trials: 3 + policy: clinical-write +``` + +The first block is the read-back contract from RFC-0002 +(`openadapt.verify/v1`): `verifier`, `effects` (or the `record` shorthand) and +`minimum_effect_strength`. The same contract can judge a run by any executor. +`app`, `save_step`, `inputs` and `campaign` are what a qualification campaign +adds. + +What each part does: + +- **app**: where the application is, its version (recorded in the report), + how to start a signed-in session, and how to reset it. `reset` takes exactly + one of `command` (a shell command), `http` (`method`, `url`, `body`) or + `python` (`module:function`). For a web app without sign-in you can leave + out `session`; `qualify` opens a fresh browser at `url`. +- **verifier**: the read-back. `kind: rest` reads `records_path` under + `base_url` and takes the list at `records_key`. Authentication is + `auth: {bearer_env: NAME}`, `{header: X-Api-Key, value_env: NAME}` or + `{basic_env: NAME}`, with the secret in the named environment variable. + `kind: sql`, `fhir` and `graphql` take the same keys as the effects section + of a deployment file ([EFFECT_KIT.md](EFFECT_KIT.md)). An on-screen check is + refused: it cannot tell a real write from a screen that only looks right. +- **effects**: the save's checks as an effect contract list, the same + entries an `effect_contract.yaml` holds: + + ```yaml + effects: + - kind: record_written + match: {email: {param: email}, lname: {param: lname}} + expected_count: 1 + idempotency_key: {param: email} + key_field: email + - kind: field_equals + match: {email: {param: email}} + field: DOB + value: {param: DOB} + ``` + +- **record**: a shorter way to write the same list. `match` finds the record, + `fields` says which input each field must equal, and `duplicate_key` is the + field that makes a second write a duplicate. It expands to one + `record_written` plus a `field_equals` per other field. Give `effects` or + `record`, not both; leave both out when the compiler already bound the + save's checks from the demonstration. +- **minimum_effect_strength**: `independent_system_of_record` (default) or + `independent_session`. `qualify` refuses a verifier that reads weaker. + Certification needs more than the minimum, though: see below. +- **save_step**: `auto` picks the single irreversible step. If there are none + or several, `qualify` says so and asks you to name one. +- **inputs**: the test inputs. `recorded` reuses the demonstration's values; + `file` loads a JSON object; `params` overrides single values; `unique` + lists inputs to suffix per campaign, so reruns against a database that was + not reset do not collide. +- **campaign**: the id written into the report, trials per case, the cases + (all six by default) and the policy `--certify` uses when given no name. + +`${NAME}` and `${NAME:-default}` read environment variables. A missing +variable without a default is an error. + +## What you get + +``` +runs/qual-1/ + bundle/ the bundle, with the qualification project + cases//trial-NN/ run report, screenshots, oracle.json + qualification/result.json openadapt.qualification-result/v1 + qualification/admission.local.json the result bound into an admission + envelope (with --certify, 3+ trials per case) + qualification/report.json the project's coverage report + qualification/verdict.txt the result, or why certification refused + campaign-summary.json one row per trial + timing.json wall-clock time per trial and in total + QUALIFICATION.md a summary to read +``` + +`result.json` has one entry per trial: what the case expected the system of +record to show (`CONFIRMED` for the normal case, `REFUTED` for a fault), what +the read-back showed, how it was read, and digests of the trial's run report +and read-back. It counts silent incorrect successes (must be 0) and +over-halts, and reports model calls in two numbers: `executor_model_calls` +(calls the replay made; qualification requires 0) and `verifier_model_calls` +(calls made while judging; a read-back makes none). It holds no record +values, workflow names or reason text. + +A fault case passes only when both hold: the read-back shows nothing landed +(`REFUTED`, outcome `halted_before_effect`), and the engine halted. A store +that shows nothing while the engine reported success counts as a silent +incorrect success. + +Keep `result.json` inside the customer's systems when the campaign ran +against real records. For now, each case's `receipt_sha256` is a plain, +unsalted SHA-256 of that trial's `report.json`, and `evidence_digest` is the +same over its read-back result. Those files hold the inputs and record +details the run used, so someone with a guess at the values could confirm it +against the digest. Effect receipts (RFC-0002), which bind identity values in +a form that does not allow this, will replace these digests. A campaign on +made-up test data, like the OpenEMR fixture or MockMed, has nothing to +protect. + +How each read is labelled (`verification_tier`): + +- `independent_readback`: the system of record read without going through + the UI: `rest`, `sql`, `fhir`, `graphql`, or a file export. +- `screen_readback`: anything that reads through the application's UI, + including `ui_readback`, which signs in to a separate UI session. +- `unchecked`: no read happened. + +To certify, the normal case must reach `verified` at `independent_readback`, +and every fault case must show `REFUTED` (nothing landed) at +`independent_readback`. A workflow whose only read-back is `ui_readback` can +run the cases but cannot be certified. + +`verify_config_sha256` identifies the contract the cases were judged +against. It is SHA-256 over the verifier's identity and the effective effects +list bound to the save step for the campaign: the `effects` you wrote, the +expansion of `record`, or the checks the compiler mined from the +demonstration, after the duplicate key is applied and before input values are +filled in. It is not a hash of the file, so the same contract written two +ways (an `effects` list or the equivalent `record`) gets the same digest. + +With `--certify`, its digest is bound into the existing +`openadapt.qualification-admission/v1` envelope (`campaign.outcomes_sha256`; +`campaign.oracle_contract_sha256` is the verify-config digest) and signed with +the local-dev qualification key. Production trust maps refuse that key; a +production admission is signed by the production authority from the same +result. The envelope needs at least three trials per case. + +With `--run-params inputs.json` it also resets the application and runs the +certified bundle once, as the runner would, into `runs/qual-1/certified-run/`. + +## Limits + +- Web only today. The ambiguity, wrong-identity and stale-identity faults are + staged in the browser, on the save button the bundle's own selector finds. + A fault driver for pixel surfaces (Windows, RDP, Citrix) is the next piece; + until it exists, `qualify` refuses `app.target` other than `web`. The two + effect faults do not depend on the surface. +- Sign-in, reset and API sessions are often app-specific. They are Python + hooks named in `verify.yaml`, not configuration. +- The keys are local test keys. See + [QUALIFICATION_TERMS.md](QUALIFICATION_TERMS.md#certify). +- A pass covers the environment it ran in and nothing more. + +## Compiler fix-ups + +`compile` now applies four fix-ups by default and lists each change in the +bundle's `compile_notes.json`: + +- Enter after typing an option into a dropdown is dropped (web only; the + typed text already selected the option). +- Ctrl+A on an empty field that changed no pixel is dropped. +- A click into a text field gets a check that the field is still where it was. +- A button with no label evidence gets a crop of itself as its identity + check, but only when the bundle will run where that crop can be verified: + pass `compile --pixel-verify-enabled` when the deployment sets + `runtime.pixel_verify_enabled`. Without it, the pixel check can only reject + a crop, never accept one, so a crop-only check would stop an irreversible + step on every run. The button then compiles as it did before. + +These used to be hand edits in a per-workflow script. diff --git a/packages/flow/openadapt_flow/__main__.py b/packages/flow/openadapt_flow/__main__.py index a654d5fc6..f0d58a0ea 100644 --- a/packages/flow/openadapt_flow/__main__.py +++ b/packages/flow/openadapt_flow/__main__.py @@ -1617,7 +1617,12 @@ def _cmd_compile(args: argparse.Namespace) -> int: from openadapt_flow.compiler import compile_recording from openadapt_flow.compiler import param_confirm as pc - workflow = compile_recording(Path(args.recording), Path(args.out), name=args.name) + workflow = compile_recording( + Path(args.recording), + Path(args.out), + name=args.name, + pixel_verify_enabled=bool(getattr(args, "pixel_verify_enabled", False)), + ) print( f"Compiled {len(workflow.steps)} steps into {args.out} " f"(workflow: {workflow.name!r})" @@ -2307,7 +2312,7 @@ def _attach_cloud_seal(args: argparse.Namespace) -> None: ) -def _cmd_seal(args: argparse.Namespace) -> int: +def _cmd_attach_seal(args: argparse.Namespace) -> int: """``attach-seal``: fetch Cloud's signed receipt into a VERIFIED run report.""" from openadapt_flow.cloud_seal import attach_from_cloud @@ -6290,6 +6295,110 @@ def _package_version() -> str: return __version__ +QUALIFY_DEFINITION = ( + "Qualify: before a compiled workflow may run unattended on a real system, " + "run it against the normal case and deliberate faults, and confirm it is " + "verified by reading back the system of record and halts on every fault." +) +QUALIFY_HELP = """The faults are a duplicated save button, a relabeled save button, a blanked +screen, and a weak or missing check that the write happened. Certifying +(--certify) signs the result under a policy such as clinical-write; you get a +signed report and a certified bundle the runner will admit. A certificate +covers only the environment the workflow was qualified in. + +Definitions of qualify and certify: + docs/QUALIFICATION_TERMS.md +Certification policies (built-in clinical-write and permissive, or your own +YAML file passed to --certify): + docs/QUALIFICATION_TERMS.md#certification-policy +How to write verify.yaml: + docs/QUALIFY_A_WORKFLOW.md +""" + + +def _definition_first(parser: argparse.ArgumentParser) -> None: + """Make ``--help`` open with the one-line definition, before usage.""" + + original = parser.format_help + + def format_help() -> str: + return QUALIFY_DEFINITION + "\n\n" + original() + + parser.format_help = format_help # type: ignore[method-assign] + + +def _cmd_qualify_campaign(args: argparse.Namespace) -> int: + import json + import os + import time + from pathlib import Path + + from openadapt_flow.qualify import ( + QualifyConfigError, + QualifyError, + load_verify_config, + qualify, + ) + + # verify.yaml may name module:function hooks importable from here. + if os.getcwd() not in sys.path: + sys.path.insert(0, os.getcwd()) + try: + config = load_verify_config(args.verify) + except QualifyConfigError as exc: + print(str(exc), file=sys.stderr) + return 2 + certify = ( + config.campaign.policy if args.certify == "__from_verify__" else args.certify + ) + run_params = None + if args.run_params: + run_params = json.loads(Path(args.run_params).read_text()) + if not isinstance(run_params, dict): + print("--run-params must hold a JSON object", file=sys.stderr) + return 2 + bundle = Path(args.bundle) + out = ( + Path(args.out) + if args.out + else Path("runs") + / f"qualify-{bundle.resolve().name}-{time.strftime('%Y%m%d-%H%M%S')}" + ) + try: + result = qualify( + bundle, + config, + out=out, + cases=args.cases, + trials=args.trials, + certify=certify, + run_params=run_params, + campaign_id=args.campaign_id, + ) + except QualifyConfigError as exc: + print(str(exc), file=sys.stderr) + return 2 + except QualifyError as exc: + print(f"qualify REFUSED: {exc}\nEvidence so far: {out}", file=sys.stderr) + return 1 + if not result.passed and result.partial: + print( + f"\nqualify PARTIAL: {len(result.rows)} trial(s) behaved as required in " + f"{result.timing['campaign_wall_s']} s; qualifying needs all six cases." + ) + return 0 + verdict = "PASS" if result.passed else "FAIL" + print( + f"\nqualify {verdict}: {len(result.rows)} trial(s) in {result.timing['campaign_wall_s']} s" + ) + if result.certified: + print(f"Certified under {result.policy}. Bundle: {result.bundle_dir}") + elif certify is None: + print("Not certified (pass --certify to sign the result under a policy).") + print(f"Report: {result.out / 'QUALIFICATION.md'}") + return 0 if result.passed else 1 + + def build_parser() -> argparse.ArgumentParser: """Build the top-level argument parser.""" from openadapt_flow.cli_hints import command @@ -6630,6 +6739,17 @@ def build_parser() -> argparse.ArgumentParser: "proposals stay demonstrated constants." ), ) + p.add_argument( + "--pixel-verify-enabled", + action="store_true", + help=( + "The runtime that will run this bundle sets " + "runtime.pixel_verify_enabled. Lets the compiler give an unlabeled " + "button a pixel identifier crop as its identity check. Without it " + "the compiler doesn't, because a crop that can't be verified halts " + "an irreversible step on every run." + ), + ) p.set_defaults(func=_cmd_compile) p = sub.add_parser( @@ -7164,7 +7284,7 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="Cloud base URL (or OPENADAPT_CLOUD_URL; default https://app.openadapt.ai)", ) - sa.set_defaults(func=_cmd_seal) + sa.set_defaults(func=_cmd_attach_seal) p = sub.add_parser( "resume", @@ -7567,12 +7687,78 @@ def build_parser() -> argparse.ArgumentParser: p = sub.add_parser( "qualify", + help=QUALIFY_DEFINITION + + " Also: create, edit, test, explain and certify the qualification " + "project sealed into a bundle.", + description=QUALIFY_HELP, + usage=( + "%(prog)s --verify verify.yaml [--cases ...] [--out DIR] " + "[--certify [POLICY]] [--run-params FILE]\n" + " %(prog)s ..." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + _definition_first(p) + qsub = p.add_subparsers(dest="qualify_cmd", required=True) + + q = qsub.add_parser( + "campaign", help=( - "Create, edit, test, explain, and certify the versioned " - "qualification project sealed into a workflow bundle" + "Qualify a bundle from verify.yaml (also the default form: " + "`qualify --verify verify.yaml`)" ), + description=QUALIFY_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, ) - qsub = p.add_subparsers(dest="qualify_cmd", required=True) + _definition_first(q) + q.add_argument("bundle", help="Compiled workflow bundle directory") + q.add_argument( + "--verify", + required=True, + metavar="YAML", + help="verify.yaml: the per-workflow read-back, field mapping, save step and inputs", + ) + q.add_argument( + "--cases", + nargs="+", + default=None, + metavar="CASE", + help=( + "Run only these cases (representative, fault-ambiguity, " + "fault-wrong-identity, fault-stale-identity, fault-weak-effect, " + "fault-missing-effect). Certification needs all six." + ), + ) + q.add_argument( + "--trials", + type=int, + default=None, + help="Trials per case (default: verify.yaml, 3)", + ) + q.add_argument("--out", default=None, metavar="DIR", help="New output directory") + q.add_argument( + "--certify", + nargs="?", + const="__from_verify__", + default=None, + metavar="POLICY", + help=( + "Certify under this policy (default: campaign.policy in verify.yaml, " + "usually clinical-write). See docs/QUALIFICATION_TERMS.md#policies." + ), + ) + q.add_argument( + "--run-params", + default=None, + metavar="JSON", + help="After certifying, run the certified bundle once with these inputs", + ) + q.add_argument( + "--campaign-id", + default=None, + help="Campaign id (default: verify.yaml or generated)", + ) + q.set_defaults(func=_cmd_qualify_campaign) q = qsub.add_parser("schema", help="Print the qualification-project JSON Schema") q.set_defaults(func=_cmd_qualify) @@ -9583,10 +9769,27 @@ def _cmd_business_decisions_serve(args: argparse.Namespace) -> int: return 0 +def _qualify_default_form(argv: list[str]) -> list[str]: + """``qualify --verify ...`` means ``qualify campaign ...``.""" + + if len(argv) >= 2 and argv[0] == "qualify" and not argv[1].startswith("-"): + verbs = set() + for action in build_parser()._subparsers._group_actions: # type: ignore[union-attr] + qualify_parser = getattr(action, "choices", {}).get("qualify") + if qualify_parser is not None: + for sub_action in qualify_parser._subparsers._group_actions: # type: ignore[union-attr] + verbs |= set(getattr(sub_action, "choices", {}) or {}) + if argv[1] not in verbs: + return [argv[0], "campaign", *argv[1:]] + return argv + + def main(argv: Sequence[str] | None = None) -> int: """CLI entry point. Returns a process exit code.""" from openadapt_flow._browser_setup import BrowserSupportMissing + argv = list(sys.argv[1:] if argv is None else argv) + argv = _qualify_default_form(argv) args = build_parser().parse_args(argv) try: return args.func(args) diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index 015ff4f5d..a12271c9e 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -1647,6 +1647,7 @@ def compile_recording( annotate: bool = False, annotator: Optional["StepAnnotator"] = None, target_surface: Optional[ExecutionTargetKind] = None, + pixel_verify_enabled: bool = False, ) -> Workflow: """Compile a recording directory into a workflow bundle. @@ -1764,6 +1765,11 @@ def compile_recording( target_surface: Explicit surface binding for a legacy recording whose ``meta.json`` predates surface stamps. This may only fill missing metadata; contradicting a recorded surface fails compilation. + pixel_verify_enabled: The target runtime's + ``runtime.pixel_verify_enabled``. The default fix-up passes attach + a pixel identifier crop as a step's only identity evidence only + when that runtime can verify it (see + :func:`openadapt_flow.runtime.identity.verifiable_identity_evidence`). Returns: The compiled :class:`Workflow` (also saved to the bundle). @@ -2651,6 +2657,27 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: len(decisions), ) + # Default fix-up passes (see compiler.fixup_passes): replace a demonstrated + # gesture that asserts nothing with an equivalent checked one, using only + # the recording's own frames. Every change is recorded in + # ``compile_notes.json`` so the bundle stays auditable. + from openadapt_flow.compiler.fixup_passes import ( + apply_fixup_passes, + write_compile_notes, + ) + + steps, compile_notes = apply_fixup_passes( + steps, + {step.id: (sb, sa) for step, sb, sa, _event in pending}, + bundle, + crop_png=_crop_png, + phash_png=phash_png, + surface=surface, + pixel_verify_enabled=pixel_verify_enabled, + ) + for note in compile_notes: + logger.info("compile-pass %s: %s", note["step"], note["change"]) + workflow = Workflow( name=name, recording_id=meta.get("id"), @@ -2901,6 +2928,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ) workflow.encrypted = False + write_compile_notes(bundle, compile_notes) workflow.save(bundle) (bundle / "workflow.py").write_text(render_workflow_py(workflow)) return workflow diff --git a/packages/flow/openadapt_flow/compiler/fixup_passes.py b/packages/flow/openadapt_flow/compiler/fixup_passes.py new file mode 100644 index 000000000..abca4a5ab --- /dev/null +++ b/packages/flow/openadapt_flow/compiler/fixup_passes.py @@ -0,0 +1,349 @@ +"""Default compiler passes that turn demonstrated gestures into checked steps. + +A demonstration keeps every literal gesture. A few of them leave a step that +asserts nothing, and a strict certification policy (``clinical-write``) then +refuses the bundle. These passes replace such a gesture with an equivalent one +that carries its own check, using only the recording's own frames. They used +to live in one script per workflow (``scripts/openemr_qualification.py``); +they now run in every compile. + +Each pass is evidence-gated and records every change it makes, so the result +stays auditable: :func:`apply_fixup_passes` returns the notes, and the +compiler writes them to ``compile_notes.json`` in the bundle. + +================================= ========================================== +Pass What it does, and the evidence it needs +================================= ========================================== +``drop_select_commit_enter`` Drops the Enter that followed typing an + option label into a browser ``. + for i, step in enumerate(steps): + structural = _structural(step) + if ( + web_structure + and step.action is ActionKind.CLICK + and structural is not None + and structural.role == "combobox" + and i + 2 < len(steps) + and steps[i + 1].action is ActionKind.TYPE + and steps[i + 1].text + and steps[i + 2].action is ActionKind.KEY + and (steps[i + 2].key or "").lower() in _COMMIT_KEYS + ): + dropped.add(steps[i + 2].id) + notes.append( + { + "change": "drop_select_commit_enter", + "step": steps[i + 2].id, + "evidence": "structural role combobox, then TYPE, then Enter", + "selector": structural.selector, + } + ) + + # Pass 2: Ctrl+A on a freshly clicked, empty field before TYPE. + for i, step in enumerate(steps): + if ( + step.id in dropped + or not is_select_all(step) + or i == 0 + or i + 1 >= len(steps) + or steps[i - 1].action is not ActionKind.CLICK + or steps[i + 1].action is not ActionKind.TYPE + or steps[i - 1].anchor is None + ): + continue + before, after = frames.get(step.id, (None, None)) + region = tuple(steps[i - 1].anchor.region) # type: ignore[union-attr] + if region_unchanged(before, after, region): # type: ignore[arg-type] + dropped.add(step.id) + notes.append( + { + "change": "drop_select_all_on_empty_field", + "step": step.id, + "evidence": "Ctrl+A changed no pixel in the clicked field box", + "region": list(region), + } + ) + + kept = [step for step in steps if step.id not in dropped] + out: list[Step] = [] + for i, step in enumerate(kept): + changed = False + new = step + # Pass 3: REGION_STABLE for a postcondition-less field click. + if not step.expect and _is_field_click(kept, i): + _before, after = frames.get(step.id, (None, None)) + if after is not None: + region = tuple(int(v) for v in step.anchor.region) # type: ignore[union-attr] + new = new.model_copy(deep=True) + changed = True + rel = f"templates/{step.id}_expect.png" + (bundle_dir / rel).parent.mkdir(parents=True, exist_ok=True) + (bundle_dir / rel).write_bytes(crop_png(after, region)) # type: ignore[arg-type] + new.expect = [ + Postcondition( + kind=PostconditionKind.REGION_STABLE, + region=region, # type: ignore[arg-type] + phash=phash_png(after, region=region), + phash_tolerance=8, + template=rel, + ) + ] + notes.append( + { + "change": "focused_field_region_stable", + "step": step.id, + "evidence": ( + "structural field role" + if _structural(step) is not None + else "click followed by TYPE (no structural info)" + ), + "region": list(region), + } + ) + # Pass 4: identifier crop for an unlabeled static button. + structural = _structural(step) + if ( + step.action is ActionKind.CLICK + and step.anchor is not None + and structural is not None + and structural.role == "button" + and not _has_identity_evidence(step) + and "identifier_crop" in checkable(step) + ): + before, _after = frames.get(step.id, (None, None)) + if before is not None: + if not changed: + new = new.model_copy(deep=True) + changed = True + region = tuple(int(v) for v in step.anchor.region) + rel = f"templates/identifiers/{step.id}.png" + (bundle_dir / rel).parent.mkdir(parents=True, exist_ok=True) + (bundle_dir / rel).write_bytes(crop_png(before, region)) # type: ignore[arg-type] + assert new.anchor is not None + new.anchor.identifier_crop = rel + new.anchor.identifier_region = region # type: ignore[assignment] + new.identity_armed = True + new.identity_unarmed_reason = None + new.identifier_crop_missing_reason = None + notes.append( + { + "change": "static_control_identifier_crop", + "step": step.id, + "evidence": "structural role button with no identity evidence", + "region": list(region), + } + ) + out.append(new) + return out, notes + + +def write_compile_notes(bundle_dir: Path, notes: list[dict[str, Any]]) -> None: + """Write ``compile_notes.json``; remove a stale one when nothing changed.""" + + import json + + path = Path(bundle_dir) / COMPILE_NOTES_FILE + if not notes: + path.unlink(missing_ok=True) + return + path.write_text( + json.dumps( + { + "schema": COMPILE_NOTES_SCHEMA, + "note": ( + "Changes the default compiler passes made to demonstrated " + "gestures. Each replaces a gesture that asserted nothing " + "with an equivalent one that carries its own check." + ), + "changes": notes, + }, + indent=2, + ) + + "\n" + ) + + +__all__ = [ + "COMPILE_NOTES_FILE", + "apply_fixup_passes", + "is_select_all", + "region_unchanged", + "write_compile_notes", +] diff --git a/packages/flow/openadapt_flow/qualify/__init__.py b/packages/flow/openadapt_flow/qualify/__init__.py new file mode 100644 index 000000000..e702c7c36 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/__init__.py @@ -0,0 +1,39 @@ +"""Qualify a compiled workflow from a small ``verify.yaml``; no new code. + +To qualify a workflow is to run it, before it may run unattended on a real +system, against the normal case plus deliberate faults, and confirm it +completes and is verified by reading back the system of record, and halts on +every fault instead of acting wrongly. See ``docs/QUALIFICATION_TERMS.md``. +""" + +from openadapt_flow.qualify.campaign import ( + CASES, + QualifyError, + QualifyHooks, + QualifyResult, + TrialContext, + qualify, +) +from openadapt_flow.qualify.config import ( + QualifyConfigError, + VerifyConfig, + load_verify_config, + parse_verify_config, +) +from openadapt_flow.qualify.environment import WebEnvironmentObserver +from openadapt_flow.qualify.faults import WebFaultDriver + +__all__ = [ + "CASES", + "QualifyConfigError", + "QualifyError", + "QualifyHooks", + "QualifyResult", + "TrialContext", + "VerifyConfig", + "WebEnvironmentObserver", + "WebFaultDriver", + "load_verify_config", + "parse_verify_config", + "qualify", +] diff --git a/packages/flow/openadapt_flow/qualify/campaign.py b/packages/flow/openadapt_flow/qualify/campaign.py new file mode 100644 index 000000000..b43ea12c2 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/campaign.py @@ -0,0 +1,1458 @@ +"""Qualify a compiled workflow from ``verify.yaml``, and optionally certify it. + +Qualifying runs the compiled workflow against test cases before it may run +unattended: the normal case plus five deliberate faults. It passes only when +the normal case completes and is verified by reading back the system of +record, and every fault halts before the save instead of acting wrongly. +Certifying signs that result under a policy (see +``docs/QUALIFICATION_TERMS.md``). + +This module is the generic orchestration that used to be copied into one +script per workflow. Per workflow, only ``verify.yaml`` changes. + +Flow of :func:`qualify`:: + + copy bundle -> apply verify.yaml (save step, record checks, duplicate key) + -> check everything needed is there (fail with every problem at once) + -> open the qualification project (risk classification, identity and + effect policies, the six cases, case scope, per-campaign local keys) + -> for each case x trial: reset, run in the Standard profile, read the + record back independently, sign the case result + -> record results, certify under the policy (optional) + -> optionally run the certified bundle once on new inputs + +The keys are per-campaign local test keys (:mod:`openadapt_flow.qualify.keys`). +No production trust root is created or used. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Mapping, Optional + +from openadapt_flow import __version__ as FLOW_VERSION +from openadapt_flow.ir import RunReport, Workflow +from openadapt_flow.qualify.config import CASE_IDS, QualifyConfigError, VerifyConfig +from openadapt_flow.qualify.environment import ( + WEB_OBSERVER_CONTRACT_SHA256, + WEB_OBSERVER_ID, + WebEnvironmentObserver, + web_environment_digest, + web_environment_payload, +) +from openadapt_flow.qualify.faults import WebFaultDriver +from openadapt_flow.qualify.keys import LocalTestKey, new_local_test_key +from openadapt_flow.verification import VerificationTier + +#: (case id, expected outcome, oracle kind), in run order. +CASES: tuple[tuple[str, str, str], ...] = ( + ("representative", "VERIFIED", "exact_record"), + ("fault-ambiguity", "HALTED", "no_mutation"), + ("fault-wrong-identity", "HALTED", "no_mutation"), + ("fault-stale-identity", "HALTED", "no_mutation"), + ("fault-weak-effect", "HALTED", "no_mutation"), + ("fault-missing-effect", "HALTED", "no_mutation"), +) +assert tuple(case for case, _e, _o in CASES) == CASE_IDS + + +class QualifyError(RuntimeError): + """The campaign refused. Nothing it produced may be called certified.""" + + +# --------------------------------------------------------------------------- +# small helpers +# --------------------------------------------------------------------------- + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_json(path: Path, value: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True, default=str) + "\n") + return path + + +def report_summary(report: RunReport) -> dict[str, Any]: + failed = next((item for item in report.results if not item.ok), None) + return { + "execution_outcome": report.execution_outcome, + "transaction_outcome": report.transaction_outcome, + "execution_profile": report.execution_profile, + "production_eligible": report.production_eligible, + "success": report.success, + "model_calls": report.model_calls, + "total_ms": report.total_ms, + "failed_step_id": failed.step_id if failed is not None else None, + "failed_error": failed.error if failed is not None else None, + } + + +# --------------------------------------------------------------------------- +# applying verify.yaml to the bundle +# --------------------------------------------------------------------------- + + +def resolve_save_step(workflow: Workflow, config: VerifyConfig) -> Any: + """The single irreversible step the campaign qualifies.""" + + if config.save_step != "auto": + found = [s for s in workflow.steps if s.id == config.save_step] + if not found: + raise QualifyConfigError( + f"save_step {config.save_step!r} is not a step of this bundle" + ) + return found[0] + with_effects = [s for s in workflow.steps if s.risk == "irreversible" and s.effects] + if len(with_effects) == 1: + return with_effects[0] + irreversible = [s for s in workflow.steps if s.risk == "irreversible"] + if not with_effects and len(irreversible) == 1: + return irreversible[0] + found_ids = [s.id for s in (with_effects or irreversible)] + raise QualifyConfigError( + "save_step: auto found " + + ( + f"{len(found_ids)} candidate saves {found_ids}" + if found_ids + else "no irreversible step" + ) + + "; name the save in verify.yaml (save_step: step_NNN)" + ) + + +def record_effects(config: VerifyConfig) -> list[Any]: + """The save's system-of-record checks, from ``record`` in verify.yaml.""" + + from openadapt_flow.runtime.effects import Effect, EffectKind + + record = config.record + selector = {name: record.expr(name) for name in record.match} + written = Effect( + kind=EffectKind.RECORD_WRITTEN, + match=selector, + expected_count=record.expected_count, + forbid_collateral_loss=True, + risk="reversible", + probe=f"exactly {record.expected_count} {record.name} record(s) written", + timeout_s=record.timeout_s, + ) + if record.duplicate_key is not None: + written.idempotency_key = record.expr(record.duplicate_key) + written.key_field = record.duplicate_key + effects = [written] + for name in record.fields: + if name in record.match: + continue + effects.append( + Effect( + kind=EffectKind.FIELD_EQUALS, + match=selector, + field=name, + value=record.expr(name), + risk="reversible", + probe=f"{record.name} field {name!r} read-back", + timeout_s=record.timeout_s, + ) + ) + return effects + + +def apply_verify_config( + workflow: Workflow, config: VerifyConfig +) -> tuple[Any, list[dict[str, Any]]]: + """Bind the save step and its record checks; return (save step, notes).""" + + from openadapt_flow.runtime.effects import ValueExpr + + notes: list[dict[str, Any]] = [] + save = resolve_save_step(workflow, config) + if save.risk != "irreversible": + save.risk = "irreversible" + save.risk_explanation = "named as the save in verify.yaml" + save.risk_review_required = False + notes.append({"change": "save_step_irreversible", "step": save.id}) + if config.effects is not None: + save.effects = [effect.model_copy(deep=True) for effect in config.effects] + notes.append( + { + "change": "effects_from_verify_yaml", + "step": save.id, + "effects": len(save.effects), + } + ) + elif config.record.fields: + save.effects = record_effects(config) + notes.append( + { + "change": "record_checks_from_verify_yaml", + "step": save.id, + "effects": len(save.effects), + } + ) + elif config.record.duplicate_key is not None: + for effect in save.effects: + if effect.kind.value == "record_written" and effect.idempotency_key is None: + effect.idempotency_key = ValueExpr(param=config.record.duplicate_key) + effect.key_field = config.record.duplicate_key + notes.append({"change": "duplicate_key", "step": save.id}) + return save, notes + + +def resolve_inputs( + workflow: Workflow, config: VerifyConfig, *, campaign_id: str +) -> dict[str, str]: + inputs = config.inputs + params = {k: str(v) for k, v in (workflow.params or {}).items()} + if inputs.source == "file": + path = config.resolve_path(inputs.file or "") + if not path.is_file(): + raise QualifyConfigError(f"inputs.file not found: {path}") + loaded = json.loads(path.read_text()) + if not isinstance(loaded, dict): + raise QualifyConfigError(f"inputs.file must hold a JSON object: {path}") + params.update({k: str(v) for k, v in loaded.items()}) + params.update(inputs.params) + suffix = hashlib.sha256(campaign_id.encode()).hexdigest()[:6] + for name in inputs.unique: + value = params.get(name) + if value is None: + continue + if "@" in value: + local, _, domain = value.partition("@") + params[name] = f"{local}+q{suffix}@{domain}" + else: + params[name] = f"{value}q{suffix}" + return params + + +def check_ready( + workflow: Workflow, config: VerifyConfig, save: Any, params: Mapping[str, str] +) -> list[str]: + """Every problem that would stop the campaign, found before it runs.""" + + problems: list[str] = [] + if config.app.target != "web": + problems.append( + f"app.target {config.app.target!r}: qualify runs web campaigns today; " + "the pixel-surface fault driver for Windows, RDP and Citrix is not built" + ) + if ( + config.app.reset is None + and config.campaign.trials * len(config.campaign.cases) > 1 + ): + problems.append( + "app.reset is missing: every trial must start from the same baseline " + "(command, http or python)" + ) + if not params: + problems.append( + "no test inputs: the bundle recorded none and inputs gives none" + ) + specs = getattr(workflow, "param_specs", None) or {} + for name, spec in specs.items(): + if getattr(spec, "required", True) and name not in params: + problems.append(f"inputs: no value for required parameter {name!r}") + for name in config.inputs.unique: + if name not in params: + problems.append(f"inputs.unique names unknown parameter {name!r}") + for name, value in config.record.fields.items(): + expr = config.record.expr(name) + if expr.param is not None and expr.param not in params: + problems.append( + f"record.fields.{name} uses parameter {expr.param!r}, which has no input" + ) + if save.anchor is None or not save.identity_armed: + problems.append(f"save step {save.id} has no armed identity check") + if not save.effects: + problems.append( + f"save step {save.id} has no record check: add effects (or " + "record.fields and record.match) to verify.yaml" + ) + if any(effect.needs_operator_confirmation for effect in save.effects): + problems.append( + f"save step {save.id} has an unconfirmed placeholder record check" + ) + if save.anchor is not None and ( + save.anchor.structural is None or not save.anchor.structural.selector + ): + problems.append( + f"save step {save.id} has no structural selector: the web fault " + "driver needs it for the ambiguity and identity faults" + ) + return problems + + +# --------------------------------------------------------------------------- +# the qualification project +# --------------------------------------------------------------------------- + + +@dataclass +class CampaignSetup: + runner_key: LocalTestKey + driver: WebFaultDriver + observer: WebEnvironmentObserver + environment: dict[str, Any] + + +def configure_project( + workflow: Workflow, + config: VerifyConfig, + *, + save: Any, + params: Mapping[str, str], + environment: dict[str, Any], + campaign_id: str, + minimum_effect_tier: Any, +) -> CampaignSetup: + """Open the project: classify, set policies, add and scope the six cases.""" + + from openadapt_flow.qualification import ( + ActionRiskClass, + ActionRiskClassification, + EnvironmentBoundary, + IdentityEnforcement, + IdentityPolicy, + QualificationActionTarget, + QualificationCase, + QualificationCaseKind, + QualificationOutcome, + add_case, + init_project, + set_action_classification, + set_case_scope, + set_effect_policy, + set_identity_policy, + set_trusted_fault_driver_key, + set_trusted_runner_key, + ) + from openadapt_flow.runtime.authorization import runtime_inputs_digest + + app = config.app + capabilities = sorted( + { + "headless_chromium" if app.headless else "headed_chromium", + "independent_system_of_record", + "playwright_dom", + } + ) + boundary = EnvironmentBoundary( + target_kind="web", + application=app.name, + application_identity=app.origin, + application_version=app.version, + environment_observer_id=WEB_OBSERVER_ID, + environment_observer_contract_sha256=WEB_OBSERVER_CONTRACT_SHA256, + environment_digest=web_environment_digest(environment), + runtime_version=FLOW_VERSION, + required_capabilities=capabilities, + ) + project = init_project( + workflow, environment=boundary, minimum_effect_tier=minimum_effect_tier + ) + record = config.record.name + for step in workflow.steps: + irreversible = step.risk == "irreversible" + set_action_classification( + workflow, + ActionRiskClassification( + step_id=step.id, + classification=( + ActionRiskClass.IRREVERSIBLE + if irreversible + else ActionRiskClass.READ_ONLY + ), + explanation=( + f"writes the {record} in the system of record (verify.yaml save)" + if step.id == save.id + else "compiled as an irreversible write" + if irreversible + else f"fills or navigates; writes no {record}" + ), + operator_confirmed=True, + ), + ) + set_identity_policy( + workflow, + IdentityPolicy( + step_id=save.id, enforcement=IdentityEnforcement.CANONICAL_LADDER + ), + ) + for index, _effect in enumerate(save.effects): + set_effect_policy( + workflow, step_id=save.id, effect_index=index, tier=minimum_effect_tier + ) + add_case( + workflow, + QualificationCase( + id="representative", + kind=QualificationCaseKind.REPRESENTATIVE, + expected_outcome=QualificationOutcome.VERIFIED, + description=f"Recorded workflow on {app.name} under its qualified boundary", + ), + ) + runner_key = new_local_test_key(f"{campaign_id}-runner") + set_trusted_runner_key( + workflow, + key_id=runner_key.key_id, + public_key_base64=runner_key.public_key_base64, + ) + driver = WebFaultDriver(workflow, key_id=f"{campaign_id}-fault-driver") + set_trusted_fault_driver_key( + workflow, + key_id=driver.attestation_key_id, + public_key_base64=driver.public_key_base64, + ) + input_sha256 = runtime_inputs_digest(workflow, dict(params), None) + target = QualificationActionTarget(step_id=save.id, actuation_path="gui") + for case in list(project.cases): + set_case_scope( + workflow, + case_id=case.id, + runtime_input_sha256=input_sha256, + action_targets=[target], + fault_target=None + if case.kind is QualificationCaseKind.REPRESENTATIVE + else target, + ) + return CampaignSetup( + runner_key=runner_key, + driver=driver, + observer=WebEnvironmentObserver( + application_version=app.version, headless=app.headless + ), + environment=environment, + ) + + +# --------------------------------------------------------------------------- +# the independent read-back after each trial +# --------------------------------------------------------------------------- + + +def _records_reader( + verifier: Any, +) -> Optional[Callable[[], Optional[list[dict[str, Any]]]]]: + for candidate in ( + verifier, + getattr(verifier, "inner", None), + getattr(verifier, "_inner", None), + ): + fetch = getattr(candidate, "_fetch_records", None) + if callable(fetch): + return fetch + return None + + +def _canon(record: Mapping[str, Any]) -> str: + return json.dumps(record, sort_keys=True, default=str) + + +class RecordReadback: + """Read the system of record before and after a trial; judge the trial. + + ``exact_record``: exactly the expected number of records match the save's + selector, every declared field equals its expected value, and no record + that existed before changed. ``no_mutation``: nothing matches and no + record changed. Uses the configured verifier's own read path (REST today); + other kinds fall back to the run report and say so. + """ + + def __init__(self, verifier: Any, save: Any, params: Mapping[str, str]) -> None: + self._read = _records_reader(verifier) + self._effects = [effect.resolve(params) for effect in save.effects] + self.before: Optional[list[dict[str, Any]]] = None + + @property + def independent(self) -> bool: + return self._read is not None + + def snapshot(self) -> None: + if self._read is not None: + self.before = self._read() + + def judge(self, oracle_kind: str, report: RunReport) -> dict[str, Any]: + base = { + "schema_version": "openadapt.qualify-readback/v1", + "oracle_kind": oracle_kind, + } + if self._read is None or self.before is None: + if oracle_kind == "exact_record": + passed = report.execution_outcome == "VERIFIED" + else: + passed = report.execution_outcome == "HALTED" and not any( + r.delivery_attempted for r in report.results if r.step_id + ) + return { + **base, + "read_path": "run report only (this verifier kind has no records read)", + "independent": False, + "passed": passed, + "observed": oracle_kind if passed else f"not_{oracle_kind}", + "silent_incorrect_success": bool(not passed and report.success), + "effect_verdict": "INDETERMINATE", + } + after = self._read() or [] + written = next( + (e for e in self._effects if e.kind.value == "record_written"), None + ) + selector = { + k: v.literal if hasattr(v, "literal") else v + for k, v in ((written.match if written else {}) or {}).items() + } + + def matches(rec: Mapping[str, Any], sel: Mapping[str, Any]) -> bool: + return all(str(rec.get(k)) == str(v) for k, v in sel.items()) + + matched = [rec for rec in after if selector and matches(rec, selector)] + after_set = {_canon(rec) for rec in after} + changed_existing = [rec for rec in self.before if _canon(rec) not in after_set] + field_checks = [] + for effect in self._effects: + if effect.kind.value != "field_equals" or effect.field is None: + continue + want = effect.value.literal if effect.value is not None else None + ok = len(matched) == 1 and str(matched[0].get(effect.field)) == str(want) + field_checks.append({"field": effect.field, "match": ok}) + expected = written.expected_count if written else 1 + exact = ( + len(matched) == expected + and not changed_existing + and all(item["match"] for item in field_checks) + ) + nothing = ( + not matched and not changed_existing and len(after) == len(self.before) + ) + if oracle_kind == "exact_record": + passed = exact + observed = "exact_record" if passed else "record_mismatch" + elif oracle_kind == "no_mutation": + passed = ( + not matched and not changed_existing and len(after) == len(self.before) + ) + observed = "no_mutation" if passed else "unexpected_mutation" + else: + raise QualifyError(f"unknown oracle kind {oracle_kind!r}") + return { + **base, + "read_path": "system_of_record verifier records read", + "independent": True, + "passed": passed, + "observed": observed, + # RFC-0002 verdict of the read itself, whatever the case expected. + "effect_verdict": "CONFIRMED" if exact else "REFUTED", + "landed": "exact" if exact else ("none" if nothing else "some"), + "records_before": len(self.before), + "records_after": len(after), + "matching_records": len(matched), + "existing_records_changed": len(changed_existing), + "fields_matched": sum(item["match"] for item in field_checks), + "fields_expected": len(field_checks), + "field_check": field_checks, + "silent_incorrect_success": bool(not passed and report.success), + } + + +# --------------------------------------------------------------------------- +# hooks for callers that need more than verify.yaml (demos, packs) +# --------------------------------------------------------------------------- + + +@dataclass +class TrialContext: + case_id: str + expected: str + oracle_kind: str + trial: int + trial_dir: Path + run_dir: Path + run_id: str + params: dict[str, str] + certified_run: bool = False + state: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class QualifyHooks: + """Optional extension points. Each defaults to the verify.yaml behaviour. + + - ``open_session`` / ``reset`` / ``make_verifier`` replace the + declarative ones; + - ``before_trial(ctx)`` runs after reset, before the browser opens; + - ``replayer_kwargs(ctx)`` adds Replayer options (presentation overlays); + - ``after_run(ctx, backend, report)`` runs before the browser closes; + - ``after_close(ctx, report)`` runs once the run report is rendered; + - ``oracle(ctx, report)`` replaces the generic read-back judgement; + - ``extra_checks`` each return ``(name, result)``; ``result["passed"]`` + must be true, and the result is written next to the trial as + ``.json`` (a demo's second oracle, for example). + - ``open_session_for(ctx)`` replaces ``open_session`` when the session + depends on the trial (video for trial one, for example). + """ + + open_session: Optional[Callable[[], tuple[Any, Callable[[], None]]]] = None + open_session_for: Optional[ + Callable[[TrialContext], tuple[Any, Callable[[], None]]] + ] = None + reset: Optional[Callable[[], Any]] = None + make_verifier: Optional[Callable[[Mapping[str, object]], Any]] = None + before_trial: Optional[Callable[[TrialContext], None]] = None + replayer_kwargs: Optional[Callable[[TrialContext], dict[str, Any]]] = None + after_run: Optional[Callable[[TrialContext, Any, RunReport], None]] = None + after_close: Optional[Callable[[TrialContext, RunReport], None]] = None + oracle: Optional[Callable[[TrialContext, RunReport], dict[str, Any]]] = None + extra_checks: list[ + Callable[[TrialContext, RunReport], tuple[str, dict[str, Any]]] + ] = field(default_factory=list) + settle_s: float = 1.0 + runner_id: Optional[str] = None + + +# --------------------------------------------------------------------------- +# one trial, one case result +# --------------------------------------------------------------------------- + + +def _run( + *, + workflow: Workflow, + bundle_dir: Path, + config: VerifyConfig, + hooks: QualifyHooks, + ctx: TrialContext, + setup: CampaignSetup, + authorize: Callable[[Any, dict[str, Any]], Any], + fault_driver: Any, + marks: dict[str, float], +) -> tuple[RunReport, Any]: + from openadapt_flow.deployment import DeploymentConfig, PolicySection + from openadapt_flow.execution_profiles import ( + ExecutionProfile, + execution_profile_contract, + ) + from openadapt_flow.runtime import Replayer + + opener = ( + (lambda: hooks.open_session_for(ctx)) # type: ignore[misc] + if hooks.open_session_for is not None + else hooks.open_session + ) + assert opener is not None and hooks.make_verifier is not None + backend, close = opener() + marks["browser_ready"] = time.time() + try: + verifier = hooks.make_verifier(ctx.params) + gate_kwargs = dict( + bundle_dir=bundle_dir, + deployment=DeploymentConfig( + policy=PolicySection( + policy=ctx.state.get("policy", config.campaign.policy) + ) + ), + effect_verifier=verifier, + profile_contract=execution_profile_contract(ExecutionProfile.STANDARD), + effective_durable=True, + effective_require_settled=True, + ) + authorization = authorize(verifier, gate_kwargs) + marks["admitted"] = time.time() + kwargs: dict[str, Any] = dict( + effect_verifier=verifier, + governed_authorization=authorization, + qualification_fault_driver=fault_driver, + qualification_environment_observer=setup.observer, + durable=True, + require_settled=True, + pixel_verify_enabled=any( + s.anchor is not None and s.anchor.identifier_crop + for s in workflow.steps + ), + ) + if hooks.replayer_kwargs is not None: + kwargs.update(hooks.replayer_kwargs(ctx)) + marks["task_start"] = time.time() + report = Replayer(backend, **kwargs).run( + workflow.model_copy(deep=True), + params=dict(ctx.params), + bundle_dir=bundle_dir, + run_dir=ctx.run_dir, + execution_target_kind="web", + execution_origin=config.app.origin, + execution_entry_url=backend.page.url, + run_id=ctx.run_id, + ) + marks["task_end"] = time.time() + if hooks.after_run is not None: + hooks.after_run(ctx, backend, report) + return report, verifier + finally: + close() + marks["browser_closed"] = time.time() + + +def _judge_trial( + *, + hooks: QualifyHooks, + ctx: TrialContext, + report: RunReport, + readback: RecordReadback, +) -> dict[str, Any]: + from openadapt_flow.report import render_run_report + + time.sleep(hooks.settle_s) + render_run_report(ctx.run_dir) + if hooks.after_close is not None: + hooks.after_close(ctx, report) + oracle = ( + hooks.oracle(ctx, report) + if hooks.oracle is not None + else readback.judge(ctx.oracle_kind, report) + ) + write_json(ctx.trial_dir / "oracle.json", oracle) + extra: dict[str, Any] = {} + for check in hooks.extra_checks: + name, result = check(ctx, report) + write_json(ctx.trial_dir / f"{name}.json", result) + extra[name] = result + return {"oracle": oracle, "extra": extra} + + +def _evidence_ref(root: Path, path: Path, kind: Any) -> Any: + from openadapt_flow.qualification import EvidenceRef + + return EvidenceRef( + kind=kind, sha256=_sha256(path), relative_path=path.relative_to(root).as_posix() + ) + + +def case_result( + *, + root: Path, + workflow: Workflow, + case_id: str, + observed_outcome: str, + run_id: str, + trial_dir: Path, + runner_key: LocalTestKey, + runner_id: str, + case_params: Mapping[str, str], + campaign_id: str, +) -> Any: + """The signed result of one case trial, bound to every retained file.""" + + from openadapt_flow.qualification import ( + QualificationCaseResult, + QualificationOutcome, + qualification_campaign_id_sha256, + qualification_run_id_sha256, + sign_case_result, + workflow_contract_sha256, + ) + from openadapt_flow.runtime.authorization import runtime_inputs_digest + + project = workflow.qualification + if project is None: + raise QualifyError("the workflow's qualification project disappeared") + run_dir = trial_dir / "run" + run_artifacts = [ + _evidence_ref(root, path, "other") + for path in sorted(run_dir.rglob("*")) + if path.is_file() and path.name != "report.json" + ] + faults = ( + [ + _evidence_ref(root, trial_dir / "fault-receipt.json", "fault_receipt"), + _evidence_ref(root, trial_dir / "fault-mutation.bin", "fault_mutation"), + ] + if (trial_dir / "fault-receipt.json").is_file() + else [] + ) + result = QualificationCaseResult( + case_id=case_id, + project_id=project.project_id, + project_revision=project.revision, + project_contract_sha256=project.contract_sha256(), + workflow_contract_sha256=workflow_contract_sha256(workflow), + environment_contract_sha256=project.environment.contract_sha256(), + environment_digest=project.environment.environment_digest, + runtime_version=project.environment.runtime_version, + runner_id=runner_id, + runner_capabilities=list(project.environment.required_capabilities), + status="passed", + observed_outcome=QualificationOutcome(observed_outcome.lower()), + campaign_id_sha256=qualification_campaign_id_sha256(campaign_id), + case_input_sha256=runtime_inputs_digest(workflow, dict(case_params), None), + run_id_sha256=qualification_run_id_sha256(run_id), + evidence=[ + _evidence_ref(root, run_dir / "report.json", "run_report"), + _evidence_ref(root, trial_dir / "case-input.json", "case_input"), + _evidence_ref(root, trial_dir / "oracle.json", "effect"), + *run_artifacts, + *faults, + ], + detail_code=f"{case_id}.{run_id.rsplit('-', 1)[-1]}", + attestation_key_id=runner_key.key_id, + ) + return sign_case_result(result, private_key=runner_key.private_key) + + +# --------------------------------------------------------------------------- +# the campaign +# --------------------------------------------------------------------------- + + +@dataclass +class QualifyResult: + passed: bool + certified: bool + policy: Optional[str] + out: Path + bundle_dir: Path + report: Any + rows: list[dict[str, Any]] + timing: dict[str, Any] + notes: list[dict[str, Any]] + certified_run: Optional[dict[str, Any]] = None + #: Every case's run reports, in trial order. + reports: dict[str, list[RunReport]] = field(default_factory=dict) + #: The environment observer, for runs after certification. + observer: Any = None + environment_digest: Optional[str] = None + #: ``openadapt.qualification-result/v1``. + result: Optional[dict[str, Any]] = None + #: Why no admission envelope was written, when certify did not write one. + admission_note: Optional[str] = None + + @property + def partial(self) -> bool: + """True when only some of the six cases ran (each behaved as required). + + A partial run is a development check, never a qualification. + """ + + return len({row["case_id"] for row in self.rows}) < len(CASES) + + +def _default_hooks(config: VerifyConfig, hooks: Optional[QualifyHooks]) -> QualifyHooks: + from openadapt_flow.qualify.session import ( + reset_runner, + session_opener, + verifier_factory, + ) + + hooks = hooks or QualifyHooks() + if hooks.open_session is None and hooks.open_session_for is None: + hooks.open_session = session_opener(config) + if hooks.reset is None: + hooks.reset = reset_runner(config) + if hooks.make_verifier is None: + hooks.make_verifier = verifier_factory(config) + return hooks + + +def qualify( + bundle_src: Path | str, + config: VerifyConfig, + *, + out: Path | str, + cases: Optional[list[str]] = None, + trials: Optional[int] = None, + certify: Optional[str] = None, + run_params: Optional[Mapping[str, str]] = None, + hooks: Optional[QualifyHooks] = None, + campaign_id: Optional[str] = None, + run_id_prefix: Optional[str] = None, + evidence_root: Optional[Path | str] = None, + in_place: bool = False, + write_campaign_files: bool = True, + log: Callable[[str], None] = print, +) -> QualifyResult: + """Run the campaign; certify under ``certify`` (a policy name) if given. + + Raises :class:`QualifyConfigError` before running anything when + verify.yaml or the bundle is missing something, and :class:`QualifyError` + when a case does not behave as required or certification refuses. + + Layout: ``out/bundle`` (a copy, or ``bundle_src`` itself with + ``in_place``), ``out/cases//trial-NN`` and ``out/qualification``. + ``evidence_root`` (default ``out``) is the root case evidence is relative + to. ``write_campaign_files=False`` leaves out the campaign summary, + timing, notes and ``QUALIFICATION.md`` (for packs that list every file). + """ + + from openadapt_flow.policy import load_policy + from openadapt_flow.qualification import ( + QualificationCaseKind, + certify_project, + evaluate_qualification, + record_case_results, + save_qualified_workflow, + ) + from openadapt_flow.run_gate import ( + build_qualification_case_authorization, + evaluate_run_gate, + ) + from openadapt_flow.runtime.authorization import runtime_inputs_bytes + + t0 = time.time() + bundle_src = Path(bundle_src) + out = Path(out) + root = Path(evidence_root) if evidence_root is not None else out + if (out / "cases").exists() or (out.exists() and not in_place): + raise QualifyError(f"output already exists: {out}") + selected = list(cases or config.campaign.cases) + unknown = [c for c in selected if c not in CASE_IDS] + if unknown: + raise QualifyConfigError(f"unknown case(s) {unknown}; known: {list(CASE_IDS)}") + n_trials = trials or config.campaign.trials + if certify is not None: + load_policy(certify) # fail before any run on an unknown policy + if set(selected) != set(CASE_IDS): + raise QualifyConfigError( + "--certify needs all six cases; certification refuses a partial campaign" + ) + campaign_id = ( + campaign_id or config.campaign.id or f"qualify-{bundle_src.name}-{int(t0)}" + ) + + workflow = Workflow.load(bundle_src) + save, notes = apply_verify_config(workflow, config) + params = resolve_inputs(workflow, config, campaign_id=campaign_id) + problems = check_ready(workflow, config, save, params) + hooks = _default_hooks(config, hooks) + if config.app.reset is None and hooks.reset is not None: + problems = [p for p in problems if not p.startswith("app.reset")] + if problems: + raise QualifyConfigError(problems) + + if in_place: + bundle_dir = bundle_src + out.mkdir(parents=True, exist_ok=True) + else: + bundle_dir = out / "bundle" + out.mkdir(parents=True) + shutil.copytree(bundle_src, bundle_dir) + if notes: + save_qualified_workflow(workflow, bundle_dir) + workflow = Workflow.load(bundle_dir) + save = next(s for s in workflow.steps if s.id == save.id) + if write_campaign_files: + write_json(out / "qualify-notes.json", notes) + write_json(out / "verify.resolved.json", config.model_dump(mode="json")) + + # The environment boundary is read from a live session. + assert hooks.make_verifier is not None + probe_verifier = hooks.make_verifier(params) + verifier_tier = getattr(probe_verifier, "verification_tier", None) + if verifier_tier is None: + raise QualifyConfigError("verifier declares no verification tier") + verifier_tier = VerificationTier(verifier_tier) + tier = ( + VerificationTier.INDEPENDENT_SYSTEM + if config.minimum_effect_strength == "independent_system_of_record" + else VerificationTier.INDEPENDENT_SESSION + ) + if int(verifier_tier) > int(tier): + raise QualifyConfigError( + f"verifier reads at {verifier_tier.name.lower()}, weaker than " + f"minimum_effect_strength {config.minimum_effect_strength}" + ) + from openadapt_flow.qualify.result import ( + build_result, + case_entry, + local_admission, + result_sha256, + sha256_hex, + verifier_identity_of, + verify_config_sha256, + ) + + verifier_id = verifier_identity_of(config.verifier.verifier, probe_verifier) + verify_sha = verify_config_sha256(verifier_id, list(save.effects)) + if hooks.reset is not None: + hooks.reset() + opener = hooks.open_session or ( + lambda: hooks.open_session_for( # type: ignore[misc] + TrialContext( + "environment", "", "", 0, out, out, "environment", dict(params) + ) + ) + ) + backend, close = opener() + try: + environment = web_environment_payload( + backend, + application_version=config.app.version, + headless=config.app.headless, + ) + finally: + close() + setup = configure_project( + workflow, + config, + save=save, + params=params, + environment=environment, + campaign_id=campaign_id, + minimum_effect_tier=tier, + ) + save_qualified_workflow(workflow, bundle_dir) + workflow = Workflow.load(bundle_dir) + save = next(s for s in workflow.steps if s.id == save.id) + runner_id = ( + hooks.runner_id + or f"openadapt-flow/qualify-web-{'headless' if config.app.headless else 'headed'}" + ) + policy_name = certify or config.campaign.policy + + rows: list[dict[str, Any]] = [] + case_entries: list[dict[str, Any]] = [] + reports: dict[str, list[RunReport]] = {} + results: list[Any] = [] + trial_times: list[float] = [] + reset_times: list[float] = [] + project_cases = { + c.id: c + for c in (workflow.qualification.cases if workflow.qualification else []) + } + for case_id, expected, oracle_kind in CASES: + if case_id not in selected: + continue + qcase = project_cases[case_id] + for trial in range(1, n_trials + 1): + trial_dir = out / "cases" / case_id / f"trial-{trial:02d}" + ctx = TrialContext( + case_id=case_id, + expected=expected, + oracle_kind=oracle_kind, + trial=trial, + trial_dir=trial_dir, + run_dir=trial_dir / "run", + run_id=f"{run_id_prefix or campaign_id}-{case_id}-trial-{trial:02d}", + params=dict(params), + state={"policy": policy_name}, + ) + trial_dir.mkdir(parents=True) + t_trial = time.time() + if hooks.reset is not None: + hooks.reset() + reset_times.append(time.time() - t_trial) + readback = RecordReadback(hooks.make_verifier(params), save, params) + readback.snapshot() + if hooks.before_trial is not None: + hooks.before_trial(ctx) + + def authorize( + _verifier: Any, + gate_kwargs: dict[str, Any], + _q: Any = qcase, + _c: Any = ctx, + ) -> Any: + gate = evaluate_run_gate( + workflow, qualification_evidence_only=True, **gate_kwargs + ) + if not gate.passed: + raise QualifyError(gate.render()) + return build_qualification_case_authorization( + workflow, + gate, + case_id=_q.id, + params=dict(params), + worklists=None, + campaign_id=campaign_id, + run_id=_c.run_id, + fault_driver=( + None + if _q.kind is QualificationCaseKind.REPRESENTATIVE + else setup.driver + ), + ) + + marks: dict[str, float] = {"start": time.time()} + report, _verifier = _run( + workflow=workflow, + bundle_dir=bundle_dir, + config=config, + hooks=hooks, + ctx=ctx, + setup=setup, + authorize=authorize, + fault_driver=setup.driver, + marks=marks, + ) + reports.setdefault(case_id, []).append(report) + judged = _judge_trial( + hooks=hooks, ctx=ctx, report=report, readback=readback + ) + oracle = judged["oracle"] + (trial_dir / "case-input.json").write_bytes( + runtime_inputs_bytes(workflow, dict(params), None) + ) + if report.qualification_fault_mutations: + if len(report.qualification_fault_mutations) != 1: + raise QualifyError("a fault case retained several fault receipts") + receipt = report.qualification_fault_mutations[0] + (trial_dir / "fault-receipt.json").write_bytes(receipt.artifact_bytes()) + (trial_dir / "fault-mutation.bin").write_bytes( + setup.driver.mutation_bytes(receipt.fault_kind) + ) + trial_s = time.time() - t_trial + trial_times.append(trial_s) + row = { + "case_id": case_id, + "trial": trial, + "expected": expected, + **report_summary(report), + "oracle": oracle.get("observed"), + "trial_wall_s": round(trial_s, 1), + "fault_declined": setup.driver.declined, + **{ + f"{name}_passed": bool(res.get("passed")) + for name, res in judged["extra"].items() + }, + } + case_entries.append( + case_entry( + case_id=case_id, + trial=trial, + report=report, + oracle=oracle, + verifier_tier=verifier_tier, + report_path=ctx.run_dir / "report.json", + verifier_substrate=getattr(probe_verifier, "substrate", None), + ) + ) + rows.append(row) + if write_campaign_files: + write_json(out / "campaign-summary.json", rows) + log(json.dumps(row, default=str)) + failure = None + if report.execution_outcome != expected: + failure = f"observed {report.execution_outcome}, expected {expected}" + if report.halt is not None: + failure += f" ({report.halt.reason})" + if setup.driver.declined: + failure += f"; the fault driver declined: {setup.driver.declined}" + elif report.model_calls != 0: + failure = "a qualification run called a model" + elif report.screenshots_may_leave_box: + failure = "a qualification run let screenshots leave this computer" + elif not oracle.get("passed") or oracle.get("silent_incorrect_success"): + failure = f"read-back {oracle.get('observed')}" + elif not case_entries[-1]["passed"]: + entry = case_entries[-1] + failure = ( + f"read-back verdict {entry['observed_verdict']} at " + f"{entry['verification_tier']}, expected " + f"{entry['expected_verdict']} at independent_readback" + ) + else: + bad = [n for n, r in judged["extra"].items() if not r.get("passed")] + if bad: + failure = f"extra check(s) failed: {bad}" + if failure is not None: + write_json( + out / "qualification" / "result.json", + build_result( + report=evaluate_qualification(workflow, evidence_root=root), + verify_sha256=verify_sha, + verifier_identity=verifier_id, + fault_driver_version=setup.driver.contract_sha256[:16], + cases=case_entries, + ), + ) + raise QualifyError(f"{case_id} trial {trial}: {failure}") + results.append( + case_result( + root=root, + workflow=workflow, + case_id=case_id, + observed_outcome=expected, + run_id=ctx.run_id, + trial_dir=trial_dir, + runner_key=setup.runner_key, + runner_id=runner_id, + case_params=params, + campaign_id=campaign_id, + ) + ) + for result in results: + try: + record_case_results(workflow, [result], evidence_root=root) + except Exception as exc: + raise QualifyError( + f"case result refused for {result.case_id} ({result.detail_code}): {exc}" + ) from exc + if certify is not None: + qreport = certify_project( + workflow, policy=load_policy(certify), evidence_root=root + ) + else: + qreport = evaluate_qualification(workflow, evidence_root=root) + write_json(out / "qualification" / "report.json", qreport.model_dump(mode="json")) + qresult = build_result( + report=qreport, + verify_sha256=verify_sha, + verifier_identity=verifier_id, + fault_driver_version=setup.driver.contract_sha256[:16], + cases=case_entries, + ) + result_path = write_json(out / "qualification" / "result.json", qresult) + if write_campaign_files: + (out / "qualification" / "verdict.txt").write_text(qreport.render() + "\n") + log(qreport.render()) + log( + f"qualification-result/v1 {result_sha256(qresult)[:16]}: passed=" + f"{qresult['passed']} silent_incorrect_success=" + f"{qresult['silent_incorrect_success_count']} over_halt=" + f"{qresult['over_halt_count']} executor_model_calls=" + f"{qresult['executor_model_calls']} verifier_model_calls=" + f"{qresult['verifier_model_calls']}" + ) + admission_note: Optional[str] = None + if certify is not None and qreport.passed and qresult["passed"]: + from openadapt_flow.admitted_composition import live_bundle_content_digest + + save_qualified_workflow(workflow, bundle_dir) + try: + envelope = local_admission( + result=qresult, + result_bytes=result_path.read_bytes(), + oracle_id=config.verifier.verifier.kind, + campaign_id=campaign_id, + trials_required=n_trials, + bundle_content_digest=live_bundle_content_digest( + Workflow.load(bundle_dir), bundle_dir + ), + effect_contract_sha256=sha256_hex( + [effect.model_dump(mode="json") for effect in save.effects] + ), + identity_contract_sha256=None, + ) + except ValueError as exc: + first = str(exc).splitlines() + admission_note = "no admission envelope: " + ( + first[0] if len(first) == 1 else f"{first[0]} {first[-2].strip()}" + ) + log(admission_note) + else: + write_json( + out / "qualification" / "admission.local.json", + envelope.model_dump(mode="json"), + ) + t_campaign = time.time() + timing: dict[str, Any] = { + "campaign_wall_s": round(t_campaign - t0, 1), + "trials": len(trial_times), + "trial_wall_s_mean": round(sum(trial_times) / len(trial_times), 1) + if trial_times + else None, + "reset_s_total": round(sum(reset_times), 1), + "engine_ms_by_case": { + row["case_id"]: [ + r["total_ms"] for r in rows if r["case_id"] == row["case_id"] + ] + for row in rows + }, + } + if certify is not None and qreport.passed and not qresult["passed"]: + raise QualifyError( + "certification refused: the qualification result did not pass " + f"(silent incorrect success {qresult['silent_incorrect_success_count']}, " + "or a case verdict not from an independent read-back)" + ) + if certify is not None and not qreport.passed: + if write_campaign_files: + write_json(out / "timing.json", timing) + raise QualifyError(f"certification refused:\n{qreport.render()}") + save_qualified_workflow(workflow, bundle_dir) + write_json( + out / "qualification" / "project.json", + workflow.qualification.model_dump(mode="json") + if workflow.qualification + else {}, + ) + result = QualifyResult( + passed=bool(qreport.passed) and bool(qresult["passed"]), + certified=certify is not None and bool(qreport.passed) and qresult["passed"], + policy=certify, + out=out, + bundle_dir=bundle_dir, + report=qreport, + rows=rows, + timing=timing, + notes=notes, + reports=reports, + observer=setup.observer, + environment_digest=web_environment_digest(environment), + result=qresult, + admission_note=admission_note, + ) + if run_params is not None: + if not result.certified: + raise QualifyError("--run-params needs a certified bundle (pass --certify)") + result.certified_run = certified_run( + Workflow.load(bundle_dir), + bundle_dir, + config, + out=out / "certified-run", + params={k: str(v) for k, v in run_params.items()}, + hooks=hooks, + setup=setup, + policy=certify or config.campaign.policy, + run_id=f"{campaign_id}-certified-run", + ) + timing["certified_run"] = result.certified_run["timing"] + if write_campaign_files: + write_json(out / "timing.json", timing) + write_summary_markdown(result) + return result + + +def certified_run( + workflow: Workflow, + bundle_dir: Path, + config: VerifyConfig, + *, + out: Path, + params: dict[str, str], + hooks: QualifyHooks, + setup: CampaignSetup, + policy: str, + run_id: str, +) -> dict[str, Any]: + """One ordinary Standard-profile run of the certified bundle.""" + + from openadapt_flow.run_gate import build_runtime_authorization, evaluate_run_gate + + out.mkdir(parents=True) + t0 = time.time() + if hooks.reset is not None: + hooks.reset() + t_reset = time.time() + save = next(s for s in workflow.steps if s.risk == "irreversible" and s.effects) + readback = RecordReadback(hooks.make_verifier(params), save, params) # type: ignore[misc] + readback.snapshot() + ctx = TrialContext( + case_id="certified-run", + expected="VERIFIED", + oracle_kind="exact_record", + trial=1, + trial_dir=out, + run_dir=out / "run", + run_id=run_id, + params=params, + certified_run=True, + state={"policy": policy}, + ) + if hooks.before_trial is not None: + hooks.before_trial(ctx) + + def authorize(_verifier: Any, gate_kwargs: dict[str, Any]) -> Any: + gate = evaluate_run_gate(workflow, **gate_kwargs) + write_json(out / "run-gate.json", gate.model_dump(mode="json")) + if not gate.passed: + raise QualifyError(gate.render()) + return build_runtime_authorization(workflow, gate, params=params) + + marks: dict[str, float] = {"start": time.time()} + report, _ = _run( + workflow=workflow, + bundle_dir=bundle_dir, + config=config, + hooks=hooks, + ctx=ctx, + setup=setup, + authorize=authorize, + fault_driver=None, + marks=marks, + ) + judged = _judge_trial(hooks=hooks, ctx=ctx, report=report, readback=readback) + marks["checked"] = time.time() + timing = { + "reset_s": round(t_reset - t0, 1), + "run_wall_s": round(marks["task_end"] - marks["start"], 3), + "replay_task_s": round(marks["task_end"] - marks["task_start"], 3), + "engine_total_ms": report.total_ms, + "session_open_s": round(marks["browser_ready"] - marks["start"], 3), + } + summary = { + **report_summary(report), + "oracle": judged["oracle"].get("observed"), + **{f"{n}_passed": bool(r.get("passed")) for n, r in judged["extra"].items()}, + "timing": timing, + } + write_json(out / "summary.json", summary) + write_json(out / "timing.json", timing) + return summary + + +def write_summary_markdown(result: QualifyResult) -> Path: + lines = [ + "# Qualification result", + "", + f"- Verdict: {'PASS' if result.passed else 'FAIL'}" + + (f", certified under `{result.policy}`" if result.certified else ""), + f"- Campaign time: {result.timing.get('campaign_wall_s')} s for " + f"{result.timing.get('trials')} trials", + f"- Certified bundle: `{result.bundle_dir.name}/`" + if result.certified + else "- Not certified", + "", + "| Case | Trial | Expected | Observed | Read-back | Model calls | Engine ms |", + "|---|---|---|---|---|---|---|", + ] + for row in result.rows: + lines.append( + f"| {row['case_id']} | {row['trial']} | {row['expected']} | " + f"{row['execution_outcome']} | {row['oracle']} | {row['model_calls']} | " + f"{round(row['total_ms'] or 0)} |" + ) + if result.certified_run is not None: + run = result.certified_run + lines += [ + "", + "## Certified run", + "", + f"- Outcome: {run['execution_outcome']}, read-back {run['oracle']}, " + f"model calls {run['model_calls']}", + f"- Replay: {run['timing']['replay_task_s']} s", + ] + lines += [ + "", + "The case results are signed with per-campaign local test keys. They " + "prove this campaign in this environment, nothing more.", + "", + ] + path = result.out / "QUALIFICATION.md" + path.write_text("\n".join(lines)) + return path + + +__all__ = [ + "CASES", + "QualifyError", + "QualifyHooks", + "QualifyResult", + "RecordReadback", + "TrialContext", + "apply_verify_config", + "case_result", + "certified_run", + "check_ready", + "configure_project", + "qualify", + "record_effects", + "report_summary", + "resolve_inputs", + "resolve_save_step", + "write_json", +] diff --git a/packages/flow/openadapt_flow/qualify/config.py b/packages/flow/openadapt_flow/qualify/config.py new file mode 100644 index 000000000..53338b489 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/config.py @@ -0,0 +1,420 @@ +"""The per-workflow part of qualification: ``verify.yaml``. + +Everything Flow cannot learn from the demonstration lives here, and nothing +else does. A workflow is qualified with:: + + openadapt-flow qualify --verify verify.yaml + +The file is ``schema_version: openadapt.verify/v1`` (RFC-0002). Its core is +the read-back contract, the same shape ``verify-run`` reads: + +- ``verifier``: how to read the written record back, independently of the + screen. Exactly the deployment ``effects`` section (``kind: rest``, ``sql``, + ``fhir``, ``graphql`` or a plugin kind). ``none`` and ``onscreen`` are + refused. ``session`` (``module:function``) may add an authenticated client + for ``kind: rest``; it is not part of the verifier's identity; +- ``effects``: the effect contract list (``effect_contract.yaml`` + ``effects:``), bound to the save step. Or ``record``, a shorthand that + expands to that list (record fields from inputs, the selecting fields, the + duplicate key). With neither, the effects the compiler mined from the + demonstration are used; +- ``minimum_effect_strength``: ``independent_system_of_record`` (default) or + ``independent_session``; +- ``qualification_id`` / ``environment_id``: optional opaque ids. + +Qualify adds what a campaign needs on top: + +- ``app``: the system the workflow runs against (address, version, how a + trial starts, how the test system is reset between trials); +- ``save_step``: the irreversible step, or ``auto`` to detect it; +- ``inputs``: the test inputs for the cases; +- ``campaign``: trials per case, cases, and the default certification policy. + +``${NAME}`` in any string is replaced by the environment variable ``NAME`` +when the file is loaded (``${NAME:-default}`` gives a default); an unset +variable without a default is an error. Secrets are never +written in the file: the read-back's ``auth`` names environment variables. + +See ``docs/QUALIFY_A_WORKFLOW.md``. +""" + +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any, Literal, Mapping, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from openadapt_flow.deployment import EffectsConfig +from openadapt_flow.runtime.effects import Effect, ValueExpr + +SCHEMA_VERSION = "openadapt.verify/v1" +EffectStrength = Literal["independent_system_of_record", "independent_session"] +_OPAQUE = r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$" +TargetKind = Literal["web", "windows", "macos", "linux", "rdp", "citrix"] +CASE_IDS: tuple[str, ...] = ( + "representative", + "fault-ambiguity", + "fault-wrong-identity", + "fault-stale-identity", + "fault-weak-effect", + "fault-missing-effect", +) + + +class QualifyConfigError(ValueError): + """``verify.yaml`` is missing something, or does not fit the bundle.""" + + def __init__(self, problems: list[str] | str) -> None: + self.problems = [problems] if isinstance(problems, str) else list(problems) + super().__init__( + "verify.yaml cannot qualify this bundle:\n - " + + "\n - ".join(self.problems) + ) + + +_HOOK = re.compile(r"^[A-Za-z_][\w.]*:[A-Za-z_]\w*$") + + +def _check_hook(value: Optional[str], where: str) -> Optional[str]: + if value is not None and not _HOOK.match(value): + raise ValueError(f"{where} must look like 'package.module:function'") + return value + + +class HttpReset(BaseModel): + model_config = ConfigDict(extra="forbid") + + method: Literal["GET", "POST", "PUT", "DELETE"] = "POST" + url: str + body: Optional[Any] = None + + +class ResetConfig(BaseModel): + """How the test system returns to its baseline before every trial. + + Exactly one of ``command`` (a shell command), ``http`` (one request) or + ``python`` (``module:function``, called with no arguments). + """ + + model_config = ConfigDict(extra="forbid") + + command: Optional[str] = None + http: Optional[HttpReset] = None + python: Optional[str] = None + timeout_s: float = 600.0 + + @model_validator(mode="after") + def _one(self) -> "ResetConfig": + if sum(x is not None for x in (self.command, self.http, self.python)) != 1: + raise ValueError("app.reset needs exactly one of command, http, python") + _check_hook(self.python, "app.reset.python") + return self + + +class AppConfig(BaseModel): + """The system under test and how each trial reaches the start screen.""" + + model_config = ConfigDict(extra="forbid") + + target: TargetKind = "web" + name: str = Field(min_length=1, max_length=256) + #: The application's address. Its origin is the identity the runtime + #: checks before every input. + url: str + #: Where a trial starts (default: ``url``). + entry_url: Optional[str] = None + #: The application version you are qualifying. The page cannot report it; + #: it is bound into the environment digest. + version: str = Field(min_length=1, max_length=128) + headless: bool = True + viewport: tuple[int, int] = (1280, 800) + #: A Playwright storage-state file for a signed-in session (web only). + storage_state: Optional[str] = None + #: ``module:function`` returning ``(backend, close)``, for applications + #: whose sign-in cannot be expressed by ``storage_state``. + session: Optional[str] = None + reset: Optional[ResetConfig] = None + + @field_validator("session") + @classmethod + def _session_hook(cls, value: Optional[str]) -> Optional[str]: + return _check_hook(value, "app.session") + + @property + def origin(self) -> str: + from urllib.parse import urlsplit + + parts = urlsplit(self.url) + if parts.scheme and parts.netloc: + return f"{parts.scheme}://{parts.netloc}" + return self.url.rstrip("/") + + +class SystemOfRecordConfig(BaseModel): + """The independent read-back, as a deployment ``effects`` section. + + ``session`` (``module:function``) may supply an authenticated HTTP client + for ``kind: rest`` when ``auth`` cannot (for example an OAuth flow). + """ + + model_config = ConfigDict(extra="forbid") + + session: Optional[str] = None + verifier: EffectsConfig + + @model_validator(mode="before") + @classmethod + def _split(cls, data: Any) -> Any: + if isinstance(data, dict) and "verifier" not in data: + data = dict(data) + session = data.pop("session", None) + return {"session": session, "verifier": data} + return data + + @model_validator(mode="after") + def _check(self) -> "SystemOfRecordConfig": + _check_hook(self.session, "verifier.session") + if self.verifier.kind in ("none", "onscreen"): + raise ValueError( + "verifier.kind must read the system of record " + "(rest, sql, fhir, graphql, ui_readback or a plugin), not " + f"{self.verifier.kind!r}" + ) + if self.session is not None and self.verifier.kind != "rest": + raise ValueError("verifier.session is supported for kind: rest") + return self + + +FieldValue = Union[str, ValueExpr] + + +class RecordConfig(BaseModel): + """What the save writes, mapped onto the run's inputs. + + ``fields`` maps a record field to an input: a bare string is a parameter + name, ``{literal: "MA"}`` a fixed value, ``{param: x}`` a parameter. + """ + + model_config = ConfigDict(extra="forbid") + + #: A plain name for the record, used in check descriptions. + name: str = "record" + #: Record fields whose values select the written record. + match: list[str] = Field(default_factory=list) + fields: dict[str, FieldValue] = Field(default_factory=dict) + #: The record field that must be unique per write (idempotency key). + duplicate_key: Optional[str] = None + expected_count: int = Field(default=1, ge=1) + timeout_s: float = Field(default=5.0, gt=0) + + def expr(self, field: str) -> ValueExpr: + value = self.fields[field] + return ValueExpr(param=value) if isinstance(value, str) else value + + +class InputsConfig(BaseModel): + """Test inputs. Every case and trial uses the same inputs. + + ``source: recorded`` uses the values the demonstration typed; + ``source: file`` reads a JSON object. ``params`` overrides single values. + ``unique`` lists parameters that get a per-campaign suffix, so a campaign + never collides with a record left by an earlier one. + """ + + model_config = ConfigDict(extra="forbid") + + source: Literal["recorded", "file"] = "recorded" + file: Optional[str] = None + params: dict[str, str] = Field(default_factory=dict) + unique: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _file(self) -> "InputsConfig": + if (self.source == "file") != (self.file is not None): + raise ValueError("inputs.file is required exactly when source: file") + return self + + +class CampaignConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: Optional[str] = None + trials: int = Field(default=3, ge=1, le=50) + cases: list[str] = Field(default_factory=lambda: list(CASE_IDS)) + #: Policy ``--certify`` uses when it is given without a value. + policy: str = "clinical-write" + + @field_validator("cases") + @classmethod + def _cases(cls, value: list[str]) -> list[str]: + unknown = [case for case in value if case not in CASE_IDS] + if unknown: + raise ValueError(f"unknown case(s) {unknown}; known: {list(CASE_IDS)}") + if not value: + raise ValueError("campaign.cases is empty") + return value + + +class VerifyConfig(BaseModel): + """``verify.yaml`` (``openadapt.verify/v1``): the per-workflow input.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal["openadapt.verify/v1"] + qualification_id: Optional[str] = Field(default=None, pattern=_OPAQUE) + environment_id: Optional[str] = Field(default=None, pattern=_OPAQUE) + minimum_effect_strength: EffectStrength = "independent_system_of_record" + verifier: SystemOfRecordConfig + effects: Optional[list[Effect]] = None + app: AppConfig + record: RecordConfig = Field(default_factory=RecordConfig) + save_step: str = "auto" + inputs: InputsConfig = Field(default_factory=InputsConfig) + campaign: CampaignConfig = Field(default_factory=CampaignConfig) + #: Where relative paths in the file resolve (set by the loader). + base_dir: Optional[str] = Field(default=None, exclude=True) + + @model_validator(mode="before") + @classmethod + def _migrate_hint(cls, data: Any) -> Any: + if ( + isinstance(data, dict) + and "schema_version" not in data + and ("version" in data or "system_of_record" in data) + ): + raise ValueError( + "this is the pre-RFC verify.yaml shape: replace `version: 1` with " + "`schema_version: openadapt.verify/v1` and rename " + "`system_of_record` to `verifier`" + ) + return data + + @property + def system_of_record(self) -> SystemOfRecordConfig: + return self.verifier + + @model_validator(mode="after") + def _record(self) -> "VerifyConfig": + record = self.record + if self.effects is not None and (record.fields or record.duplicate_key): + raise ValueError( + "give the save's checks as either effects or record, not both" + ) + if self.effects is not None and not self.effects: + raise ValueError("effects is empty") + missing = [f for f in record.match if f not in record.fields] + if record.fields and missing: + raise ValueError( + f"record.match names fields not in record.fields: {missing}" + ) + if record.fields and not record.match: + raise ValueError("record.match must name the fields that select the record") + if ( + record.duplicate_key is not None + and record.fields + and record.duplicate_key not in record.fields + ): + raise ValueError("record.duplicate_key must be one of record.fields") + return self + + def resolve_path(self, value: str) -> Path: + path = Path(value).expanduser() + if not path.is_absolute() and self.base_dir: + path = Path(self.base_dir) / path + return path + + +_ENV = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-([^}]*))?\}") + + +def _expand( + value: Any, missing: list[str], env: Optional[Mapping[str, str]] = None +) -> Any: + source = os.environ if env is None else env + if isinstance(value, str): + + def sub(match: re.Match[str]) -> str: + name, default = match.group(1), match.group(2) + if name in source: + return source[name] + if default is not None: + return default + missing.append(name) + return match.group(0) + + return _ENV.sub(sub, value) + if isinstance(value, list): + return [_expand(item, missing, env) for item in value] + if isinstance(value, dict): + return {key: _expand(item, missing, env) for key, item in value.items()} + return value + + +def parse_verify_config( + data: Any, + *, + base_dir: Optional[Path] = None, + env: Optional[Mapping[str, str]] = None, +) -> VerifyConfig: + """Validate a parsed verify.yaml. ``env`` replaces ``os.environ`` for ``${}``.""" + + if not isinstance(data, dict): + raise QualifyConfigError("verify.yaml must be a YAML mapping") + missing: list[str] = [] + data = _expand(data, missing, env) + if missing: + raise QualifyConfigError( + [f"environment variable {name} is not set" for name in sorted(set(missing))] + ) + try: + config = VerifyConfig.model_validate(data) + except Exception as exc: # pydantic ValidationError -> plain problems + errors = getattr(exc, "errors", None) + if callable(errors): + problems = [ + f"{'.'.join(str(p) for p in item['loc']) or '(top level)'}: {item['msg']}" + for item in errors() + ] + raise QualifyConfigError(problems) from exc + raise QualifyConfigError(str(exc)) from exc + if base_dir is not None: + config.base_dir = str(base_dir) + return config + + +def load_verify_config( + path: Path | str, *, env: Optional[Mapping[str, str]] = None +) -> VerifyConfig: + import yaml + + path = Path(path) + if not path.is_file(): + raise QualifyConfigError(f"verify file not found: {path}") + try: + data = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + raise QualifyConfigError(f"could not parse {path}: {exc}") from exc + return parse_verify_config(data, base_dir=path.resolve().parent, env=env) + + +def verify_config_schema() -> dict[str, Any]: + return VerifyConfig.model_json_schema() + + +__all__ = [ + "CASE_IDS", + "AppConfig", + "CampaignConfig", + "InputsConfig", + "QualifyConfigError", + "RecordConfig", + "ResetConfig", + "SystemOfRecordConfig", + "VerifyConfig", + "load_verify_config", + "parse_verify_config", + "verify_config_schema", +] diff --git a/packages/flow/openadapt_flow/qualify/environment.py b/packages/flow/openadapt_flow/qualify/environment.py new file mode 100644 index 000000000..b4299b028 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/environment.py @@ -0,0 +1,109 @@ +"""A generic environment observer for a web application. + +The runtime re-observes the environment at every input edge and compares it +with the qualification project. This observer reads what any web page has: +its origin (the application identity), a digest of the browser session, and +a digest of the browser, viewport, platform, runtime and the application +version you declared (the page cannot report it). +""" + +from __future__ import annotations + +import hashlib +import json +import platform +from typing import Any, Literal + +from openadapt_flow import __version__ as FLOW_VERSION +from openadapt_flow.qualification_environment import ( + QualificationEnvironmentObservation, +) + +WEB_OBSERVER_ID = "openadapt.qualify.web-environment" +WEB_OBSERVER_CONTRACT_SHA256 = hashlib.sha256( + b"openadapt.qualify.web-environment/v1\n" + b"application_identity: page origin\n" + b"session_identity: sha256(origin, timeOrigin, userAgent)\n" + b"environment: sha256(browser, viewport, platform, runtime, declared version)\n" +).hexdigest() + + +def _canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def web_environment_payload( + backend: Any, *, application_version: str, headless: bool +) -> dict[str, Any]: + """The PHI-free browser and application boundary read from the live page.""" + + page = backend.page + browser = page.context.browser + if browser is None: + raise ValueError("the qualification page has no owning browser") + return { + "browser": "chromium", + "browser_version": browser.version, + "user_agent": page.evaluate("navigator.userAgent"), + "viewport": list(backend.viewport), + "device_scale_factor": page.evaluate("window.devicePixelRatio"), + "headless": bool(headless), + "platform": platform.platform(), + "python": platform.python_version(), + "application": application_version, + "runtime_version": FLOW_VERSION, + "target_kind": "web", + } + + +def web_environment_digest(payload: dict[str, Any]) -> str: + return hashlib.sha256(_canonical(payload)).hexdigest() + + +class WebEnvironmentObserver: + """Observe the exact live browser boundary for qualification and runs.""" + + observer_id = WEB_OBSERVER_ID + contract_sha256 = WEB_OBSERVER_CONTRACT_SHA256 + + def __init__(self, *, application_version: str, headless: bool = True) -> None: + self._application_version = application_version + self._headless = headless + + def observe( + self, + backend: Any, + target_kind: Literal["web", "windows", "macos", "linux", "rdp", "citrix"], + ) -> QualificationEnvironmentObservation: + if target_kind != "web" or getattr(backend, "page", None) is None: + raise ValueError("the web environment observer needs a browser page") + page = backend.page + origin = page.evaluate("location.origin") + session_material = page.evaluate( + """() => JSON.stringify({origin: location.origin, + timeOrigin: performance.timeOrigin, + userAgent: navigator.userAgent})""" + ) + payload = web_environment_payload( + backend, + application_version=self._application_version, + headless=self._headless, + ) + return QualificationEnvironmentObservation( + target_kind="web", + application_identity=origin, + application_version=self._application_version, + session_identity_sha256=hashlib.sha256( + session_material.encode("utf-8") + ).hexdigest(), + environment_digest=web_environment_digest(payload), + ) + + +__all__ = [ + "WEB_OBSERVER_CONTRACT_SHA256", + "WEB_OBSERVER_ID", + "WebEnvironmentObserver", + "web_environment_digest", + "web_environment_payload", +] diff --git a/packages/flow/openadapt_flow/qualify/faults.py b/packages/flow/openadapt_flow/qualify/faults.py new file mode 100644 index 000000000..47ec24dd8 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/faults.py @@ -0,0 +1,302 @@ +"""Generic fault drivers for the five qualification fault cases. + +A fault driver changes the real input at the gate the runtime selects, so +that the engine's ordinary check must refuse it. It never returns a verdict: +the engine takes its own frame after the change, refuses a mutation that did +not change the input, and the case passes only when the detector refused that +exact input before any input was delivered. + +:class:`WebFaultDriver` needs nothing from the application beyond the save +step's own structural selector, recorded in the bundle: + +================== ===================== =================================== +Case Gate What changes +================== ===================== =================================== +ambiguity target_resolution the save control is duplicated in + place, so two candidates match +wrong_identity identity_verification the save control is relabeled to a + different action and, when the + page shows plain record text around + it, that text is replaced by a + different record of the same shape +stale_identity actuation_revalidation the document holding the save + control is blanked after the + identity check +weak_effect effect_strength the record check is replaced by one + that only reads the screen +missing_effect effect_verifier the record check is removed +================== ===================== =================================== + +``weak_effect`` and ``missing_effect`` change only the verifier, so they work +on every surface. The three screen faults need a browser page. A pixel-surface +driver for Windows, RDP and Citrix (changing what the remote frame shows at +the save) is not built yet; on those surfaces the driver declines the screen +faults and the case does not pass. See ``docs/QUALIFY_A_WORKFLOW.md``. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Optional + +from openadapt_flow.qualification_faults import ( + FaultMutationReceipt, + QualificationFaultContext, + QualificationFaultMutation, + effect_verifier_input_sha256, + sha256_bytes, + sign_fault_mutation_receipt, +) +from openadapt_flow.qualify.keys import LocalTestKey, new_local_test_key +from openadapt_flow.verification import VerificationTier + +WEB_FAULT_DRIVER_ID = "openadapt.qualify.web-faults" +_CONTRACT = ( + b"openadapt.qualify.web-faults/v1\n" + b"ambiguity@target_resolution: clone the save control in place\n" + b"wrong_identity@identity_verification: relabel the save control; replace " + b"plain record text around it\n" + b"stale_identity@actuation_revalidation: blank the save control's document\n" + b"weak_effect@effect_strength: screen-only record check\n" + b"missing_effect@effect_verifier: no record check\n" +) +WEB_FAULT_DRIVER_CONTRACT_SHA256 = hashlib.sha256(_CONTRACT).hexdigest() +#: The label a wrong-identity save control is given. +WRONG_ACTION_LABEL = "Merge Into Existing Record" +SCREEN_FAULTS = frozenset({"ambiguity", "wrong_identity", "stale_identity"}) + +_CLONE_JS = ( + """(el) => { el.parentNode.insertBefore(el.cloneNode(true), el); return 1; }""" +) +_RELABEL_JS = """(el, label) => { + if (el.tagName === 'INPUT') { el.value = label; } + else { el.textContent = label; } + for (const name of ['aria-label', 'title', 'value']) { + if (el.hasAttribute(name)) { el.setAttribute(name, label); } + } + return 1; +}""" +_BLANK_JS = """(el) => { + const doc = el.ownerDocument; + doc.documentElement.style.background = '#ffffff'; + doc.body.replaceChildren(); + doc.body.style.background = '#ffffff'; + return 1; +}""" + + +class ScreenOnlyRecordCheck: + """The weak-effect stand-in: a verifier that only declares a screen read. + + It has no other method; the engine must refuse it on its declared tier + before it reads or writes anything. + """ + + verification_tier = VerificationTier.IMMEDIATE_SCREEN + + +def _rotate(value: str) -> str: + out = [] + for char in value: + if "a" <= char <= "z": + out.append(chr((ord(char) - 97 + 13) % 26 + 97)) + elif "A" <= char <= "Z": + out.append(chr((ord(char) - 65 + 13) % 26 + 65)) + elif "0" <= char <= "9": + out.append(chr((ord(char) - 48 + 5) % 10 + 48)) + else: + out.append(char) + return "".join(out) + + +class WebFaultDriver: + """Apply the five faults at the save step, using the bundle's own selector.""" + + driver_id = WEB_FAULT_DRIVER_ID + contract_sha256 = WEB_FAULT_DRIVER_CONTRACT_SHA256 + + def __init__( + self, + workflow: Any, + *, + key: Optional[LocalTestKey] = None, + key_id: str = "qualify-web-fault-driver", + ) -> None: + from openadapt_flow.traversal import iter_workflow_steps + + self._key = key or new_local_test_key(key_id) + self.attestation_key_id = self._key.key_id + self.public_key_base64 = self._key.public_key_base64 + self._steps = {step.id: step for step in iter_workflow_steps(workflow)} + #: Why the last screen fault could not be staged, as a closed code. + self.declined: Optional[str] = None + + @staticmethod + def mutation_bytes(kind: str) -> bytes: + return f"{WEB_FAULT_DRIVER_ID}/{kind}/v1\n".encode("ascii") + + # -- page helpers --------------------------------------------------------- + def _save_target(self, context: QualificationFaultContext) -> Any: + step = self._steps.get(context.step_id) + locator = ( + step.anchor.structural + if step is not None and step.anchor is not None + else None + ) + if locator is None or not locator.selector: + return None + scoped = getattr(context.backend, "_locator_with_scope", None) + if callable(scoped): + try: + resolved = scoped(locator) + if resolved is not None: + _scope, target = resolved + if target.count() == 1: + return target.first + except Exception: # noqa: BLE001 - fall back to a frame search + pass + page = getattr(context.backend, "page", None) + found = [] + for frame in list(getattr(page, "frames", None) or []): + try: + loc = frame.locator(locator.selector) + if loc.count() == 1: + found.append(loc.first) + except Exception: # noqa: BLE001 - detached frames are skipped + continue + return found[0] if len(found) == 1 else None + + def _record_text_pairs( + self, context: QualificationFaultContext, target: Any + ) -> list: + reader = getattr(context.backend, "structured_text_at", None) + if not callable(reader): + return [] + try: + box = target.bounding_box() + if not box: + return [] + text = reader( + int(round(box["x"] + box["width"] / 2)), + int(round(box["y"] + box["height"] / 2)), + ) + except Exception: # noqa: BLE001 + return [] + if not text: + return [] + try: + json.loads(text) + return [] # a structural identity document, not text on the page + except ValueError: + pass + pairs: dict[str, str] = {} + for word in re.findall(r"[0-9A-Za-z\u00C0-\u024F]{2,}", text): + altered = _rotate(word) + if altered != word: + pairs[word] = altered + return sorted(pairs.items(), key=lambda item: -len(item[0])) + + def _settled_frame_sha256( + self, context: QualificationFaultContext + ) -> Optional[str]: + from openadapt_flow.qualification_drills import ( + _await_paint, + _settled_screenshot, + ) + + _await_paint(context.backend.page) + capture = context.capture_frame or context.backend.screenshot + frame = _settled_screenshot(capture) + return None if frame is None else sha256_bytes(frame) + + def _screen_fault(self, context: QualificationFaultContext) -> Optional[str]: + page = getattr(context.backend, "page", None) + if context.actuation_path != "gui" or page is None: + self.declined = "screen_fault_needs_a_browser" + return None + target = self._save_target(context) + if target is None: + self.declined = "save_target_not_found" + return None + kind = context.fault_kind + try: + if kind == "ambiguity": + target.evaluate(_CLONE_JS) + elif kind == "wrong_identity": + pairs = self._record_text_pairs(context, target) + target.evaluate(_RELABEL_JS, WRONG_ACTION_LABEL) + if pairs: + from openadapt_flow.qualification_drills import _SUBSTITUTE_JS + + for frame in list(page.frames): + try: + frame.evaluate(_SUBSTITUTE_JS, pairs) + except Exception: # noqa: BLE001 + continue + else: + target.evaluate(_BLANK_JS) + except Exception: # noqa: BLE001 - a fault that cannot stage declines + self.declined = "page_change_failed" + return None + after = self._settled_frame_sha256(context) + if after is None: + self.declined = "page_did_not_settle" + return after + + # -- QualificationFaultDriver ------------------------------------------ + def mutate( + self, context: QualificationFaultContext + ) -> Optional[QualificationFaultMutation]: + kind = context.fault_kind + replacement: Any = context.effect_verifier + replace_effect_verifier = False + if kind in {"weak_effect", "missing_effect"}: + replace_effect_verifier = True + replacement = ScreenOnlyRecordCheck() if kind == "weak_effect" else None + after = effect_verifier_input_sha256(replacement, context.effects) + elif kind in SCREEN_FAULTS: + screen_after = self._screen_fault(context) + if screen_after is None: + return None + after = screen_after + else: + return None + receipt = FaultMutationReceipt( + project_id=context.project_id, + project_revision=context.project_revision, + project_contract_sha256=context.project_contract_sha256, + campaign_id_sha256=context.campaign_id_sha256, + case_id_sha256=context.case_id_sha256, + case_input_sha256=context.case_input_sha256, + run_id_sha256=context.run_id_sha256, + step_id_sha256=sha256_bytes(context.step_id.encode("utf-8")), + actuation_path=context.actuation_path, + fault_kind=kind, + gate=context.gate, + driver_id=self.driver_id, + driver_contract_sha256=self.contract_sha256, + before_input_sha256=context.before_input_sha256, + after_input_sha256=after, + mutation_artifact_sha256=sha256_bytes(self.mutation_bytes(kind)), + attestation_key_id=self.attestation_key_id, + ) + self.declined = None + return QualificationFaultMutation( + receipt=sign_fault_mutation_receipt( + receipt, private_key=self._key.private_key + ), + replace_effect_verifier=replace_effect_verifier, + effect_verifier=replacement, + ) + + +__all__ = [ + "SCREEN_FAULTS", + "WEB_FAULT_DRIVER_CONTRACT_SHA256", + "WEB_FAULT_DRIVER_ID", + "WRONG_ACTION_LABEL", + "ScreenOnlyRecordCheck", + "WebFaultDriver", +] diff --git a/packages/flow/openadapt_flow/qualify/keys.py b/packages/flow/openadapt_flow/qualify/keys.py new file mode 100644 index 000000000..13fa0e7fa --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/keys.py @@ -0,0 +1,45 @@ +"""Per-campaign local test keys. + +A campaign signs its case results and fault receipts with Ed25519 keys made +for that one campaign. The private halves stay in memory and are never +written. The public halves are trusted only by the qualification project the +campaign writes. They are local test keys: nothing here creates, reads or +uses a production trust root, and a bundle certified with them proves the +local campaign and nothing more. +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class LocalTestKey: + key_id: str + public_key_base64: str + private_key: bytes = field(repr=False) + + +def new_local_test_key(key_id: str) -> LocalTestKey: + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + key = Ed25519PrivateKey.generate() + private_raw = key.private_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PrivateFormat.Raw, + encryption_algorithm=serialization.NoEncryption(), + ) + public_raw = key.public_key().public_bytes( + encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw, + ) + return LocalTestKey( + key_id=key_id, + public_key_base64=base64.b64encode(public_raw).decode("ascii"), + private_key=private_raw, + ) + + +__all__ = ["LocalTestKey", "new_local_test_key"] diff --git a/packages/flow/openadapt_flow/qualify/result.py b/packages/flow/openadapt_flow/qualify/result.py new file mode 100644 index 000000000..bbad8f243 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/result.py @@ -0,0 +1,445 @@ +"""``openadapt.qualification-result/v1`` and how ``--certify`` binds it. + +RFC-0002 ("What certify signs"). The result is the ``QualificationReport`` +without ``workflow_name`` (free text), with refusal codes only, plus what the +campaign itself observed: + +- ``verify_config_sha256``: the read-back contract the cases were judged + against, SHA-256 over ``{"verifier_identity", "effects"}``. ``effects`` is + the effective list bound to the save step for the campaign: the + ``effects`` from verify.yaml, the expansion of ``record``, or the effects + the compiler mined, after the duplicate key is applied and before input + values are substituted. It is the same form as an effect receipt's + ``workflow_digest``; +- ``verifier_identity``: ``openadapt.effect-verifier-identity/v1`` digest; +- ``fault_driver``: which driver staged the faults; +- ``cases``: one entry per trial, each verdict taken from the independent + read-back, never from the run report; +- ``silent_incorrect_success_count`` (must be 0), ``over_halt_count`` and + ``executor_model_calls`` and ``verifier_model_calls``. + +Signing adds no scheme. ``certify`` binds the result's digest into the +existing ``openadapt.qualification-admission/v1`` envelope +(``campaign.outcomes_sha256``, ``campaign.oracle_contract_sha256``, +``campaign.oracle_id``) and signs that envelope with the well-known local-dev +qualification key, which every production trust map refuses. A production +admission is signed by the production authority from the same result. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Literal, Mapping, Optional +from uuid import NAMESPACE_URL, uuid5 + +from openadapt_flow.verification import VerificationTier + +RESULT_SCHEMA = "openadapt.qualification-result/v1" +FAULT_DRIVER_KIND = "web-generic" + +Verdict = Literal["CONFIRMED", "REFUTED", "INDETERMINATE"] + +#: What each case must observe in the system of record. +EXPECTED_VERDICT: Mapping[str, Verdict] = { + "representative": "CONFIRMED", + "fault-ambiguity": "REFUTED", + "fault-wrong-identity": "REFUTED", + "fault-stale-identity": "REFUTED", + "fault-weak-effect": "REFUTED", + "fault-missing-effect": "REFUTED", +} +FAULT_CLASS: Mapping[str, Optional[str]] = { + "representative": None, + "fault-ambiguity": "ambiguity", + "fault-wrong-identity": "wrong_identity", + "fault-stale-identity": "stale_identity", + "fault-weak-effect": "weak_effect", + "fault-missing-effect": "missing_effect", +} + + +def canonical(value: Any) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + + +def sha256_hex(value: Any) -> str: + return hashlib.sha256(canonical(value)).hexdigest() + + +# -- the read-back contract ---------------------------------------------------- +def verifier_identity_of(cfg: Any, verifier: Any) -> str: + """``openadapt.effect-verifier-identity/v1`` of the configured read-back.""" + + from openadapt_flow.deployment import _effect_verifier_config_identity + from openadapt_flow.runtime.effects.adapter import verifier_identity + + try: + existing = verifier_identity(verifier) + except Exception: # noqa: BLE001 - not set on a hook-built client + existing = None + return existing or _effect_verifier_config_identity(cfg, verifier) + + +def verify_config_sha256(verifier_identity: str, effects: list[Any]) -> str: + """``sha256:`` over ``{"verifier_identity", "effects": [Effect JSON]}``.""" + + payload = { + "verifier_identity": verifier_identity, + "effects": [effect.model_dump(mode="json") for effect in effects], + } + return f"sha256:{sha256_hex(payload)}" + + +#: Verifier substrates that read through the application's UI. A separate +#: signed-in UI session (``ui_readback``) is still the UI. +UI_SUBSTRATES = frozenset({"ui-readback", "ui_readback", "onscreen"}) + + +def readback_tier( + tier: Optional[VerificationTier], + independent: bool, + substrate: Optional[str] = None, +) -> str: + """RFC-0002 ``verification_tier`` for one trial's read. + + ``independent_readback`` only for a read of the system of record that does + not go through the UI (REST, SQL, FHIR, GraphQL, a file export): Flow tier + 1 from a non-UI substrate. Any other read, including a separate UI + session, is ``screen_readback``. No read is ``unchecked``. + """ + + if tier is None or not independent: + return "unchecked" + if ( + tier is VerificationTier.INDEPENDENT_SYSTEM + and (substrate or "").lower() not in UI_SUBSTRATES + ): + return "independent_readback" + return "screen_readback" + + +# -- one trial ----------------------------------------------------------------- +def observed_verdict(oracle: Mapping[str, Any]) -> Verdict: + """The verdict the independent read gave, from the trial's oracle.""" + + if oracle.get("independent") is False: + return "INDETERMINATE" + verdict = oracle.get("effect_verdict") + if verdict in ("CONFIRMED", "REFUTED", "INDETERMINATE"): + return verdict # type: ignore[return-value] + observed = oracle.get("observed") + if observed == "exact_record": + return "CONFIRMED" + if observed in ("no_mutation", "record_mismatch", "unexpected_mutation"): + return "REFUTED" + return "INDETERMINATE" + + +def _outcome(verdict: Verdict, tier: str, landed: str) -> str: + if verdict == "CONFIRMED": + return ( + "verified" if tier == "independent_readback" else "reconciliation_required" + ) + if verdict == "REFUTED" and landed == "none": + return "halted_before_effect" + return "reconciliation_required" + + +def case_entry( + *, + case_id: str, + trial: int, + report: Any, + oracle: Mapping[str, Any], + verifier_tier: Optional[VerificationTier], + report_path: Path, + verifier_substrate: Optional[str] = None, +) -> dict[str, Any]: + """One ``cases[]`` entry. The verdict never reads the run report.""" + + expected = EXPECTED_VERDICT[case_id] + observed = observed_verdict(oracle) + tier = readback_tier( + verifier_tier, + oracle.get("independent", True) is not False, + verifier_substrate, + ) + landed = oracle.get("landed") or ( + "none" if oracle.get("observed") == "no_mutation" else "some" + ) + entry = { + "case_id": case_id, + "trial": trial, + "fault_class": FAULT_CLASS[case_id], + "executor_kind": "flow_replay", + "expected_verdict": expected, + "observed_verdict": observed, + "verification_tier": tier, + "outcome": _outcome(observed, tier, landed), + # The run report is the record this trial produced today. An + # openadapt.effect-receipt/v1 replaces it once openadapt-types ships it. + "receipt_sha256": hashlib.sha256(report_path.read_bytes()).hexdigest(), + "evidence_digest": f"sha256:{sha256_hex(dict(oracle))}", + "engine_outcome": report.execution_outcome, + "executor_model_calls": int(report.model_calls or 0), + } + # The normal case must reach ``verified``; a fault must match its verdict. + # Both only at independent_readback. + if expected == "CONFIRMED": + # The normal case: the record landed, read at independent_readback. + entry["passed"] = bool( + observed == "CONFIRMED" + and tier == "independent_readback" + and entry["outcome"] == "verified" + ) + else: + # A fault (RFC-0002 decision 4): the read shows nothing landed AND + # the engine halted. Either alone is not enough. + entry["passed"] = bool( + observed == "REFUTED" + and landed == "none" + and tier == "independent_readback" + and report.execution_outcome == "HALTED" + ) + # A write landed where nothing should have, or the engine reported a + # success that the system of record does not confirm. + entry["silent_incorrect_success"] = bool( + (expected != "CONFIRMED" and observed == "CONFIRMED") + or (report.success and observed != "CONFIRMED") + ) + entry["over_halt"] = bool( + expected == "CONFIRMED" and report.execution_outcome == "HALTED" + ) + return entry + + +# -- the result ---------------------------------------------------------------- +#: A refusal path is dot-separated identifiers: ``qualification.cases.``, +#: ``steps..identity_armed``. Nothing with spaces, ``@``, ``/``, ``:`` or +#: quotes, so an input value or a sentence cannot pass for one. +_PATH_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z0-9_-]{1,128}){0,8}$") +_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,128}$") + + +def closed_id(value: Optional[str]) -> Optional[str]: + """A step or case id, or ``None`` when it is not a plain id.""" + + if value is None: + return None + return value if _ID_RE.fullmatch(value) else "redacted" + + +def closed_path(value: Optional[str]) -> Optional[str]: + """A refusal field path, cut back to its fixed prefix if a segment is not an id.""" + + if value is None: + return None + if _PATH_RE.fullmatch(value): + return value + head = value.split(".", 1)[0] + return ( + f"{head}.redacted" + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", head) + else "redacted" + ) + + +def build_result( + *, + report: Any, + verify_sha256: str, + verifier_identity: str, + fault_driver_version: str, + cases: list[dict[str, Any]], + verifier_model_calls: int = 0, +) -> dict[str, Any]: + """``openadapt.qualification-result/v1`` from the project's report.""" + + base = dict(report.model_dump(mode="json")) + base.pop("workflow_name", None) # free text stays in the local report + base.pop("schema_version", None) + base["refusals"] = sorted( + { + json.dumps( + { + k: v + for k, v in { + "code": r["code"], + "path": closed_path(r.get("path")), + "step_id": closed_id(r.get("step_id")), + "case_id": closed_id(r.get("case_id")), + }.items() + if v is not None + }, + sort_keys=True, + ) + for r in base.get("refusals", []) + } + ) + base["refusals"] = [json.loads(item) for item in base["refusals"]] + silent = sum(1 for c in cases if c["silent_incorrect_success"]) + over_halt = sum(1 for c in cases if c["over_halt"]) + passed = bool( + base["passed"] and silent == 0 and cases and all(c["passed"] for c in cases) + ) + return { + "schema_version": RESULT_SCHEMA, + **base, + "passed": passed, + "verify_config_sha256": verify_sha256, + "verifier_identity": verifier_identity, + "fault_driver": {"kind": FAULT_DRIVER_KIND, "version": fault_driver_version}, + "cases": cases, + "silent_incorrect_success_count": silent, + "over_halt_count": over_halt, + "executor_model_calls": sum( + int(c.get("executor_model_calls", 0)) for c in cases + ), + # Calls made while judging. A read-back verifier makes none. + "verifier_model_calls": verifier_model_calls, + } + + +def result_sha256(result: Mapping[str, Any]) -> str: + """Digest of the result without ``generated_at`` (unprefixed hex).""" + + return sha256_hex({k: v for k, v in result.items() if k != "generated_at"}) + + +# -- binding through the admission envelope ------------------------------------- +def _utc(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _hex(value: Optional[str], label: str) -> str: + """An unprefixed digest, or a labelled placeholder for an unbound field.""" + + if value: + return value.split(":", 1)[-1] + return hashlib.sha256(f"openadapt.qualify.unbound.{label}".encode()).hexdigest() + + +def local_admission( + *, + result: Mapping[str, Any], + result_bytes: bytes, + oracle_id: str, + campaign_id: str, + trials_required: int, + bundle_content_digest: str, + effect_contract_sha256: str, + identity_contract_sha256: Optional[str], + now: Optional[datetime] = None, +) -> Any: + """Bind ``result`` into a ``qualification-admission/v1`` envelope. + + Signed with the local-dev qualification key. Fields that only a managed + deployment knows (tenant, workflow version, runtime validation, + authorization template, input/action/network/operator policy) are + deterministic placeholders. Raises ``ValueError`` when the campaign does + not meet the admission's own floor (three trials per case). + """ + + from openadapt_flow.qualification_admission import QualificationAdmissionPayload + from openadapt_flow.qualification_dev_signer import ( + LOCAL_DEV_ISSUER_WORKFLOW, + LOCAL_DEV_KEY_ID, + LOCAL_DEV_REF_PREFIX, + sign_production_shaped_local_admission, + ) + + clock = (now or datetime.now(timezone.utc)).astimezone(timezone.utc) + observed: dict[str, int] = {} + for case in result["cases"]: + observed[case["case_id"]] = observed.get(case["case_id"], 0) + 1 + fewest = min(observed.values(), default=0) + if trials_required < 3 or fewest < 3: + raise ValueError( + "the qualification-admission/v1 envelope needs at least 3 trials " + f"per case; this campaign ran {fewest}" + ) + + def uid(label: str) -> str: + return str(uuid5(NAMESPACE_URL, f"openadapt-qualify:{campaign_id}:{label}")) + + payload = QualificationAdmissionPayload.model_validate( + { + "admission_id": uid(f"admission:{result_sha256(result)}"), + "tenant_id": uid("tenant"), + "workflow_id": uid("workflow"), + "workflow_version_id": uid( + f"workflow-version:{result['workflow_contract_sha256']}" + ), + "bundle_version_id": uid(f"bundle:{bundle_content_digest}"), + "runtime_validation_id": uid("runtime-validation"), + "bundle_artifact_sha256": _hex(bundle_content_digest, "bundle-artifact"), + "bundle_content_digest": _hex(bundle_content_digest, "bundle"), + "governed_authorization_template_sha256": _hex( + None, "authorization-template" + ), + "application_contract_sha256": _hex(None, "application"), + "substrate_contract_sha256": _hex(None, "substrate"), + "environment_contract_sha256": _hex( + result.get("environment_contract_sha256"), "environment" + ), + "runtime_environment_sha256": _hex(None, "runtime-environment"), + "runtime_contract_sha256": _hex(None, "runtime"), + "input_policy_sha256": _hex(None, "input-policy"), + "action_policy_sha256": _hex(None, "action-policy"), + "network_policy_sha256": _hex(None, "network-policy"), + "identity_contract_sha256": _hex(identity_contract_sha256, "identity"), + "effect_contract_sha256": _hex(effect_contract_sha256, "effect"), + "operator_contract_sha256": _hex(None, "operator"), + "campaign": { + "artifact_sha256": hashlib.sha256(result_bytes).hexdigest(), + "contract_sha256": _hex(result["workflow_contract_sha256"], "workflow"), + "outcomes_sha256": result_sha256(result), + "oracle_id": oracle_id, + "oracle_contract_sha256": _hex( + result["verify_config_sha256"], "verify" + ), + "tasks": [ + { + "task": campaign_id, + "condition": case_id, + "required_trials": trials_required, + "observed_trials": count, + } + for case_id, count in sorted(observed.items()) + ], + "failure_taxonomy": sorted( + {"over_halt", "silent_incorrect_success"} + | {c["fault_class"] for c in result["cases"] if c["fault_class"]} + ), + "decision": "admitted", + }, + "issuer": { + "key_id": LOCAL_DEV_KEY_ID, + "workflow": LOCAL_DEV_ISSUER_WORKFLOW, + "ref": f"{LOCAL_DEV_REF_PREFIX}qualify", + }, + "issued_at": _utc(clock), + "not_before": _utc(clock), + "expires_at": _utc(clock + timedelta(days=7)), + } + ) + return sign_production_shaped_local_admission(payload) + + +__all__ = [ + "EXPECTED_VERDICT", + "RESULT_SCHEMA", + "build_result", + "case_entry", + "local_admission", + "observed_verdict", + "result_sha256", + "verifier_identity_of", + "verify_config_sha256", +] diff --git a/packages/flow/openadapt_flow/qualify/session.py b/packages/flow/openadapt_flow/qualify/session.py new file mode 100644 index 000000000..1417c25e2 --- /dev/null +++ b/packages/flow/openadapt_flow/qualify/session.py @@ -0,0 +1,171 @@ +"""Turn ``verify.yaml`` into the callables a campaign needs. + +- a session opener: a browser at the trial's start screen; +- a reset: the test system back at its baseline; +- a verifier factory: the independent read-back for one run's inputs. + +Each has a declarative default and a ``module:function`` escape hatch for +applications whose sign-in or reset cannot be written as data. +""" + +from __future__ import annotations + +import importlib +import json +import subprocess +from typing import Any, Callable, Mapping, Optional + +from openadapt_flow.qualify.config import AppConfig, QualifyConfigError, VerifyConfig + +SessionOpener = Callable[[], tuple[Any, Callable[[], None]]] + + +def load_hook(reference: str) -> Callable[..., Any]: + """Import ``package.module:function``.""" + + module_name, _, attr = reference.partition(":") + try: + module = importlib.import_module(module_name) + except ImportError as exc: + raise QualifyConfigError( + f"cannot import {module_name!r} for {reference!r}: {exc}" + ) from exc + hook = getattr(module, attr, None) + if not callable(hook): + raise QualifyConfigError(f"{reference!r} is not a callable") + return hook + + +def playwright_session(config: VerifyConfig) -> SessionOpener: + """Open a fresh browser context at ``entry_url`` for each trial.""" + + app: AppConfig = config.app + storage_state = ( + str(config.resolve_path(app.storage_state)) if app.storage_state else None + ) + + def open_session() -> tuple[Any, Callable[[], None]]: + from openadapt_flow._browser_setup import ensure_chromium_installed + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + ensure_chromium_installed() + from playwright.sync_api import sync_playwright + + pw = sync_playwright().start() + try: + browser = pw.chromium.launch(headless=app.headless) + context = browser.new_context( + viewport={"width": app.viewport[0], "height": app.viewport[1]}, + device_scale_factor=1, + storage_state=storage_state, + ) + page = context.new_page() + page.goto(app.entry_url or app.url) + except Exception: + pw.stop() + raise + + def close() -> None: + try: + browser.close() + finally: + pw.stop() + + return PlaywrightBackend(page), close + + return open_session + + +def session_opener(config: VerifyConfig) -> SessionOpener: + if config.app.session: + hook = load_hook(config.app.session) + return lambda: hook(config) + if config.app.target != "web": + raise QualifyConfigError( + f"app.target {config.app.target!r} needs app.session (module:function " + "returning (backend, close)); only web has a built-in session" + ) + return playwright_session(config) + + +def reset_runner(config: VerifyConfig) -> Optional[Callable[[], None]]: + reset = config.app.reset + if reset is None: + return None + if reset.python: + return load_hook(reset.python) + if reset.command: + command = reset.command + + def run_command() -> None: + done = subprocess.run( + command, shell=True, timeout=reset.timeout_s, capture_output=True + ) + if done.returncode != 0: + raise RuntimeError( + f"app.reset command exited {done.returncode}: " + f"{done.stderr.decode(errors='replace')[-400:]}" + ) + + return run_command + http = reset.http + assert http is not None + + def run_http() -> None: + import httpx + + body = None if http.body is None else json.dumps(http.body) + response = httpx.request( + http.method, + http.url, + content=body, + headers={"Content-Type": "application/json"} if body is not None else None, + timeout=reset.timeout_s, + ) + if response.status_code >= 300: + raise RuntimeError(f"app.reset HTTP {response.status_code}") + + return run_http + + +def verifier_factory(config: VerifyConfig) -> Callable[[Mapping[str, object]], Any]: + """Build the read-back verifier for one run's inputs.""" + + sor = config.system_of_record + cfg = sor.verifier + if sor.session: + hook = load_hook(sor.session) + from openadapt_flow.runtime.effects import RestRecordVerifier + + if not cfg.base_url: + raise QualifyConfigError("verifier.base_url is required for kind: rest") + + def build_rest(params: Mapping[str, object]) -> Any: + return RestRecordVerifier( + cfg.base_url.rstrip("/"), # type: ignore[union-attr] + records_path=cfg.records_path, + records_key=cfg.records_key, + session=hook(config), + timeout_s=10.0, + poll_interval_s=0.2, + ) + + return build_rest + from openadapt_flow.deployment import build_effect_verifier + + def build(params: Mapping[str, object]) -> Any: + verifier = build_effect_verifier(cfg, params) + if verifier is None: + raise QualifyConfigError("verifier built no verifier") + return verifier + + return build + + +__all__ = [ + "load_hook", + "playwright_session", + "reset_runner", + "session_opener", + "verifier_factory", +] diff --git a/packages/flow/openadapt_flow/runtime/identity.py b/packages/flow/openadapt_flow/runtime/identity.py index fe2335d3a..5394e8737 100644 --- a/packages/flow/openadapt_flow/runtime/identity.py +++ b/packages/flow/openadapt_flow/runtime/identity.py @@ -1768,6 +1768,56 @@ def run_identity_ladder( PIXEL_VERIFY_ENABLED = False +# --------------------------------------------------------------------------- +# Which identity evidence a runtime can turn into a VERIFIED verdict +# --------------------------------------------------------------------------- +# +# A step that carries identity evidence is checked by the ladder above, and on +# an irreversible step a check that cannot reach ``verified`` halts. So +# evidence a runtime can only abstain on is not neutral: it turns a step that +# ran into one that always stops. Anything that attaches identity evidence +# (a compiler pass, a qualification hook) asks this function first and +# attaches only evidence the surface that runs the step can verify. + +#: Surfaces whose backends expose ``structured_text_at`` (the browser DOM, or +#: a native accessibility tree). RDP and Citrix are pixels only. +STRUCTURED_TEXT_SURFACES: frozenset[str] = frozenset( + {"web", "windows", "macos", "linux"} +) + + +def verifiable_identity_evidence( + surface_kind: Optional[str], + *, + pixel_verify_enabled: Optional[bool] = None, +) -> frozenset[str]: + """Identity evidence kinds the ladder can VERIFY on ``surface_kind``. + + Returns a subset of ``{"structured_text", "context_band", + "identifier_crop"}``: + + - ``structured_text`` (tier 1): the surface exposes structured text. + - ``context_band`` (the OCR tier): every surface. + - ``identifier_crop`` (tier 2): only when the runtime's pixel VERIFY path + is on. With it off, the pixel tier can still MISMATCH (and halt) but + never verify, so a crop alone can't pass an irreversible step. + + ``surface_kind`` None means a legacy bundle with no recorded surface, + which runs in a browser. ``pixel_verify_enabled`` None reads the module + default :data:`PIXEL_VERIFY_ENABLED`; pass the target runtime's + ``runtime.pixel_verify_enabled`` when it is known. + """ + if pixel_verify_enabled is None: + pixel_verify_enabled = PIXEL_VERIFY_ENABLED + kind = surface_kind or "web" + kinds = {"context_band"} + if kind in STRUCTURED_TEXT_SURFACES: + kinds.add("structured_text") + if pixel_verify_enabled: + kinds.add("identifier_crop") + return frozenset(kinds) + + def pixel_identity_evaluator_contract_sha256() -> str: """Return the exact PHI-free contract digest for the pixel evaluator.""" diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index cc44a63e7..629501879 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -355,6 +355,10 @@ "path": "benchmark/openemr_local/environment.lock.json", "sha256": "026dab7a2059332f8a62a7e5a24ed22ed94c14dc37a42c73eec7b1a10b4b6a19" }, + { + "path": "benchmark/openemr_local/verify.yaml", + "sha256": "c8b8cfc97f7ba654d3b016798a5ec442570fb3361b717b53b7363c6e0dd8d6ad" + }, { "path": "benchmark/openimis_claims/deployment.eligibility.yaml", "sha256": "03f46f4762bad3e465b65217f6080f203e8fe0b167288f65a7a76edfe469d538" @@ -5763,6 +5767,10 @@ "path": "public-demo/evidence-packs/mockmed-triage-v3/manifest.json", "sha256": "a4c531de62d875355a359dce4655fe80963ab1be421c99c1be317d82cdccd49d" }, + { + "path": "public-demo/mockmed-verify.yaml", + "sha256": "27257a28ed148fcbb19fe98791be4c100d45bb0ebff0739cb6be410d49daf311" + }, { "path": "registry/bundles/openemr/new-referral/benchmark-contract.json", "sha256": "7f0854cc6e462ab1e2527b5d14cecb0610dcb917bd84bb7611c75c3208de5c3e" diff --git a/packages/flow/public-demo/mockmed-verify.yaml b/packages/flow/public-demo/mockmed-verify.yaml new file mode 100644 index 000000000..b68652f47 --- /dev/null +++ b/packages/flow/public-demo/mockmed-verify.yaml @@ -0,0 +1,27 @@ +# Qualify the MockMed triage workflow (public demo evidence packs). +# The exporter fills MOCKMED_BASE_URL (a fresh loopback server) and +# MOCKMED_APP_VERSION (the digest of openadapt_flow/mockmed). +schema_version: openadapt.verify/v1 +minimum_effect_strength: independent_system_of_record +app: + target: web + name: OpenAdapt MockMed synthetic reference + url: ${MOCKMED_BASE_URL} + entry_url: ${MOCKMED_BASE_URL}?fault=ok&idempotency=demo#tasks + version: sha256:${MOCKMED_APP_VERSION} + reset: + http: {method: POST, url: "${MOCKMED_BASE_URL}api/reset", body: {}} +verifier: + kind: rest + base_url: ${MOCKMED_BASE_URL} + records_path: /api/db + records_key: records +# No effects: the compiler mined the save's record checks from the +# demonstration, and those are the contract. +save_step: auto +inputs: + source: recorded +campaign: + id: public-demo-mockmed-qualified-campaign-v1 + trials: 3 + policy: clinical-write diff --git a/packages/flow/scripts/export_public_demo_evidence.py b/packages/flow/scripts/export_public_demo_evidence.py index 7c2bb7aa6..af87246f1 100644 --- a/packages/flow/scripts/export_public_demo_evidence.py +++ b/packages/flow/scripts/export_public_demo_evidence.py @@ -26,7 +26,6 @@ from __future__ import annotations import argparse -import base64 import hashlib import ipaddress import json @@ -41,7 +40,7 @@ from datetime import datetime, timezone from decimal import Decimal from pathlib import Path, PurePosixPath -from typing import Any, Callable, Literal, Optional +from typing import Any, Callable, Optional from urllib.parse import urlsplit from urllib.request import Request, urlopen @@ -59,58 +58,27 @@ ) from openadapt_flow.ir import GovernedAuthorizationTemplate, RunReport, Workflow from openadapt_flow.mockmed.fault_server import serve -from openadapt_flow.policy import has_structured_identity, load_policy +from openadapt_flow.policy import has_structured_identity from openadapt_flow.qualification import ( - ActionRiskClass, - ActionRiskClassification, - EnvironmentBoundary, - EvidenceRef, - IdentityEnforcement, - IdentityPolicy, - QualificationActionTarget, - QualificationCase, - QualificationCaseKind, - QualificationCaseResult, - QualificationOutcome, - add_case, - certify_project, - init_project, - qualification_campaign_id_sha256, - qualification_run_id_sha256, - record_case_results, - save_qualified_workflow, - set_action_classification, - set_case_scope, - set_effect_policy, - set_identity_policy, - set_trusted_fault_driver_key, - set_trusted_runner_key, - sign_case_result, workflow_contract_sha256, ) -from openadapt_flow.qualification_environment import ( - QualificationEnvironmentObservation, -) -from openadapt_flow.qualification_faults import ( - FaultMutationReceipt, - QualificationFaultContext, - QualificationFaultMutation, - effect_verifier_input_sha256, - sha256_bytes, - sign_fault_mutation_receipt, +from openadapt_flow.qualify import ( + CASES, + QualifyConfigError, + QualifyError, + QualifyHooks, + TrialContext, + VerifyConfig, + load_verify_config, + qualify, ) from openadapt_flow.recorder import Recorder from openadapt_flow.report import render_run_report from openadapt_flow.run_gate import ( - build_qualification_case_authorization, build_runtime_authorization, evaluate_run_gate, ) from openadapt_flow.runtime import Replayer -from openadapt_flow.runtime.authorization import ( - runtime_inputs_bytes, - runtime_inputs_digest, -) from openadapt_flow.runtime.effects import RestRecordVerifier from openadapt_flow.transaction import IdempotencyLedger from openadapt_flow.verification import VerificationTier @@ -890,7 +858,9 @@ def _record( return environment -def _save_step(workflow: Workflow) -> Any: +def _require_structured_save(workflow: Workflow) -> Any: + """The public demo requires the save to carry retained structured identity.""" + candidates = [ step for step in workflow.steps if step.risk == "irreversible" and step.effects ] @@ -909,335 +879,9 @@ def _save_step(workflow: Workflow) -> Any: "consequential save did not compile with retained structured " "identity evidence" ) - if any(effect.needs_operator_confirmation for effect in step.effects): - raise EvidencePackError("compiler emitted an unbound placeholder effect") return step -def _environment_payload( - *, - backend: PlaywrightBackend, - app_digest: str, -) -> dict[str, Any]: - """Read the PHI-free browser and application boundary from the live page.""" - - page = backend.page - browser = page.context.browser - if browser is None: - raise EvidencePackError("qualification page has no owning browser") - return { - "browser": "chromium", - "browser_version": browser.version, - "user_agent": page.evaluate("navigator.userAgent"), - "viewport": list(backend.viewport), - "device_scale_factor": page.evaluate("window.devicePixelRatio"), - "headless": True, - "platform": platform.platform(), - "python": platform.python_version(), - "application_sha256": app_digest, - "runtime_version": FLOW_VERSION, - "target_kind": "web", - } - - -class _MockMedEnvironmentObserver: - """Observe the exact live MockMed browser boundary for qualification.""" - - observer_id = ENVIRONMENT_OBSERVER_ID - contract_sha256 = ENVIRONMENT_OBSERVER_CONTRACT_SHA256 - - def __init__(self, *, app_digest: str) -> None: - self._app_digest = app_digest - - def observe( - self, - backend: Any, - target_kind: Literal["web", "windows", "macos", "linux", "rdp", "citrix"], - ) -> QualificationEnvironmentObservation: - if target_kind != "web" or not isinstance(backend, PlaywrightBackend): - raise ValueError("MockMed qualification requires its Playwright boundary") - page = backend.page - origin = page.evaluate("location.origin") - session_material = page.evaluate( - """() => JSON.stringify({ - origin: location.origin, - timeOrigin: performance.timeOrigin, - userAgent: navigator.userAgent, - })""" - ) - payload = _environment_payload(backend=backend, app_digest=self._app_digest) - return QualificationEnvironmentObservation( - target_kind="web", - application_identity=origin, - application_version=f"sha256:{self._app_digest}", - session_identity_sha256=_sha256_bytes(session_material.encode("utf-8")), - environment_digest=_sha256_bytes(_canonical_json(payload)), - ) - - -def _configure_qualification( - workflow: Workflow, - *, - base_url: str, - environment: dict[str, Any], - app_digest: str, -) -> tuple[ - bytes, - str, - _MockMedQualificationFaultDriver, - _MockMedEnvironmentObserver, -]: - environment_payload = { - **environment, - "application_sha256": app_digest, - "runtime_version": FLOW_VERSION, - "target_kind": "web", - } - environment_digest = _sha256_bytes(_canonical_json(environment_payload)) - boundary = EnvironmentBoundary( - target_kind="web", - application="OpenAdapt MockMed synthetic reference", - application_identity=_origin(base_url), - application_version=f"sha256:{app_digest}", - environment_observer_id=ENVIRONMENT_OBSERVER_ID, - environment_observer_contract_sha256=( - ENVIRONMENT_OBSERVER_CONTRACT_SHA256 - ), - environment_digest=environment_digest, - runtime_version=FLOW_VERSION, - required_capabilities=[ - "headless_chromium", - "independent_system_of_record", - "playwright_dom", - ], - ) - project = init_project( - workflow, - environment=boundary, - minimum_effect_tier=VerificationTier.INDEPENDENT_SYSTEM, - ) - - for step in workflow.steps: - classification = ( - ActionRiskClass.IRREVERSIBLE - if step.risk == "irreversible" - else ActionRiskClass.READ_ONLY - ) - explanation = ( - "compiler classified this as an irreversible system-of-record write" - if classification is ActionRiskClass.IRREVERSIBLE - else "reviewed as workflow preparation/navigation with no business-record effect" - ) - set_action_classification( - workflow, - ActionRiskClassification( - step_id=step.id, - classification=classification, - explanation=explanation, - operator_confirmed=True, - ), - ) - - save = _save_step(workflow) - set_identity_policy( - workflow, - IdentityPolicy( - step_id=save.id, - enforcement=IdentityEnforcement.CANONICAL_LADDER, - ), - ) - for index, _effect in enumerate(save.effects): - set_effect_policy( - workflow, - step_id=save.id, - effect_index=index, - tier=VerificationTier.INDEPENDENT_SYSTEM, - ) - add_case( - workflow, - QualificationCase( - id="representative", - kind=QualificationCaseKind.REPRESENTATIVE, - expected_outcome=QualificationOutcome.VERIFIED, - description="Recorded workflow under its qualified application boundary", - ), - ) - # This key is pack-local, generated for this immutable campaign, and never - # reused as a production trust root. - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey - - private_key = Ed25519PrivateKey.generate() - private_raw = private_key.private_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PrivateFormat.Raw, - encryption_algorithm=serialization.NoEncryption(), - ) - public_raw = private_key.public_key().public_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PublicFormat.Raw, - ) - set_trusted_runner_key( - workflow, - key_id=RUNNER_KEY_ID, - public_key_base64=base64.b64encode(public_raw).decode("ascii"), - ) - fault_driver = _MockMedQualificationFaultDriver() - set_trusted_fault_driver_key( - workflow, - key_id=fault_driver.attestation_key_id, - public_key_base64=fault_driver.public_key_base64, - ) - - input_sha256 = runtime_inputs_digest(workflow, {"note": NOTE}, None) - save_target = QualificationActionTarget(step_id=save.id, actuation_path="gui") - for case in list(project.cases): - fault_target = ( - None - if case.kind is QualificationCaseKind.REPRESENTATIVE - else save_target - ) - set_case_scope( - workflow, - case_id=case.id, - runtime_input_sha256=input_sha256, - action_targets=[save_target], - fault_target=fault_target, - ) - assert workflow.qualification is project - return ( - private_raw, - environment_digest, - fault_driver, - _MockMedEnvironmentObserver(app_digest=app_digest), - ) - - -class _WeakEffectVerifier: - """A real configured verifier whose evidence is too weak for Standard.""" - - verification_tier = VerificationTier.IMMEDIATE_SCREEN - - -class _MockMedQualificationFaultDriver: - """Fixture-owned mutations for the first-party synthetic MockMed app. - - The driver changes the real detector input at the runtime-selected gate. - The ordinary resolver, identity check, or effect gate must then refuse it. - It does not return a detector verdict. - """ - - driver_id = "openadapt.mockmed.public-demo-faults" - attestation_key_id = FAULT_DRIVER_KEY_ID - contract_sha256 = hashlib.sha256( - b"openadapt.mockmed.public-demo-faults/v1" - ).hexdigest() - - def __init__(self) -> None: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric.ed25519 import ( - Ed25519PrivateKey, - ) - - key = Ed25519PrivateKey.generate() - self.private_key = key.private_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PrivateFormat.Raw, - encryption_algorithm=serialization.NoEncryption(), - ) - self.public_key_base64 = base64.b64encode( - key.public_key().public_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PublicFormat.Raw, - ) - ).decode("ascii") - - @staticmethod - def mutation_bytes(kind: str) -> bytes: - return f"openadapt.mockmed.synthetic-fault/{kind}/v1\n".encode("ascii") - - def mutate( - self, context: QualificationFaultContext - ) -> QualificationFaultMutation | None: - replacement = context.effect_verifier - replace_effect_verifier = False - if context.fault_kind == "ambiguity": - context.backend.page.evaluate( - """() => { - const button = document.querySelector('#save-encounter'); - if (!button) throw new Error('synthetic save target missing'); - button.parentNode.insertBefore(button.cloneNode(true), button); - }""" - ) - after_sha256 = sha256_bytes(context.backend.screenshot()) - elif context.fault_kind == "wrong_identity": - context.backend.page.evaluate( - """() => { - const record = document.querySelector('#encounter-record'); - const banner = document.querySelector('#patient-banner'); - if (!record || !banner) { - throw new Error('synthetic record identity missing'); - } - const changed = 'Alex Testcase — MRN P2 — DOB 1975-05-05'; - record.setAttribute('aria-label', changed); - banner.textContent = changed; - }""" - ) - after_sha256 = sha256_bytes(context.backend.screenshot()) - elif context.fault_kind == "stale_identity": - context.backend.page.evaluate( - """() => { - document.documentElement.style.background = '#ffffff'; - document.body.replaceChildren(); - document.body.style.background = '#ffffff'; - }""" - ) - mutated_png = context.backend.screenshot() - after_sha256 = sha256_bytes(mutated_png) - elif context.fault_kind in {"weak_effect", "missing_effect"}: - replace_effect_verifier = True - replacement = ( - _WeakEffectVerifier() - if context.fault_kind == "weak_effect" - else None - ) - after_sha256 = effect_verifier_input_sha256( - replacement, - context.effects, - ) - else: - return None - - mutation = self.mutation_bytes(context.fault_kind) - receipt = FaultMutationReceipt( - project_id=context.project_id, - project_revision=context.project_revision, - project_contract_sha256=context.project_contract_sha256, - campaign_id_sha256=context.campaign_id_sha256, - case_id_sha256=context.case_id_sha256, - case_input_sha256=context.case_input_sha256, - run_id_sha256=context.run_id_sha256, - step_id_sha256=sha256_bytes(context.step_id.encode("utf-8")), - actuation_path=context.actuation_path, - fault_kind=context.fault_kind, - gate=context.gate, - driver_id=self.driver_id, - driver_contract_sha256=self.contract_sha256, - before_input_sha256=context.before_input_sha256, - after_input_sha256=after_sha256, - mutation_artifact_sha256=sha256_bytes(mutation), - attestation_key_id=self.attestation_key_id, - ) - return QualificationFaultMutation( - receipt=sign_fault_mutation_receipt( - receipt, - private_key=self.private_key, - ), - replace_effect_verifier=replace_effect_verifier, - effect_verifier=replacement, - ) - - def _origin(url: str) -> str: return url.rstrip("/").split("?", 1)[0] @@ -1293,55 +937,6 @@ def _browser_network_observation(page: Any, *, base_url: str) -> dict[str, Any]: } -def _case_plan() -> list[dict[str, Any]]: - """Machine conditions keyed to the canonical qualification case ids.""" - - return [ - { - "case_id": "representative", - "query": "?fault=ok&idempotency=demo", - "expected": "VERIFIED", - "oracle": "exact_record", - "use_structural": True, - }, - { - "case_id": "fault-ambiguity", - "query": "?fault=ok&idempotency=demo", - "expected": "HALTED", - "oracle": "no_mutation", - "use_structural": True, - }, - { - "case_id": "fault-wrong-identity", - "query": "?fault=ok&idempotency=demo", - "expected": "HALTED", - "oracle": "no_mutation", - "use_structural": True, - }, - { - "case_id": "fault-stale-identity", - "query": "?fault=ok&idempotency=demo", - "expected": "HALTED", - "oracle": "no_mutation", - "use_structural": True, - }, - { - "case_id": "fault-weak-effect", - "query": "?fault=ok&idempotency=demo", - "expected": "HALTED", - "oracle": "no_mutation", - "use_structural": True, - }, - { - "case_id": "fault-missing-effect", - "query": "?fault=ok&idempotency=demo", - "expected": "HALTED", - "oracle": "no_mutation", - "use_structural": True, - }, - ] - - def _oracle_snapshot( base_url: str, *, @@ -1472,195 +1067,124 @@ def _reject_run_authority(relative: str) -> None: ) -def _run_case_trial( - *, - base_url: str, - workflow: Workflow, - bundle_dir: Path, - case: dict[str, Any], - trial: int, - case_dir: Path, - presentation_dir: Path, - pack_id: str, - fault_driver: _MockMedQualificationFaultDriver, - environment_observer: _MockMedEnvironmentObserver, -) -> tuple[RunReport, dict[str, Any], dict[str, Any]]: - _http_json(f"{base_url}api/reset", method="POST", body={}) - trial_dir = case_dir / f"trial-{trial:02d}" - run_dir = trial_dir / "run" - media_tmp = trial_dir / ".video" - record_video = trial == 1 - if record_video: - media_tmp.mkdir(parents=True) - entry_url = f"{base_url.rstrip('/')}/{case['query']}#tasks" - backend, close = PlaywrightBackend.launch( - entry_url, - headless=True, - record_video_dir=str(media_tmp) if record_video else None, - ) - active_backend: PlaywrightBackend = backend - clip_id = ( - "verified" - if trial == 1 and case["case_id"] == "representative" - else "halted" - if trial == 1 and case["case_id"] == "fault-ambiguity" - else None - ) - presentation = ( - _PresentationCapture(mode="governed") if clip_id is not None else None - ) - network_observation: dict[str, Any] | None = None - try: - verifier = RestRecordVerifier( - base_url, - records_path="/api/db", - records_key="records", - timeout_s=1.0, - poll_interval_s=0.05, +class _PackTrialHooks: + """What a public pack adds to each qualification trial. + + Qualification itself (cases, faults, keys, case results, certification) + is :func:`openadapt_flow.qualify.qualify`, driven by + ``public-demo/mockmed-verify.yaml``. A pack also keeps a video of each + case's first trial, the presentation clips, the browser network + observation, MockMed's own database oracle, the run events and an outcome + envelope, and strips live run authority before the evidence is hashed. + """ + + def __init__( + self, *, base_url: str, entry_url: str, presentation_dir: Path, pack_id: str + ) -> None: + self.base_url = base_url + self.entry_url = entry_url + self.presentation_dir = presentation_dir + self.pack_id = pack_id + + @staticmethod + def _clip_id(ctx: TrialContext) -> Optional[str]: + if ctx.trial != 1: + return None + return {"representative": "verified", "fault-ambiguity": "halted"}.get( + ctx.case_id ) - gate = evaluate_run_gate( - workflow, - bundle_dir=bundle_dir, - deployment=DeploymentConfig(policy=PolicySection(policy="clinical-write")), - effect_verifier=verifier, - profile_contract=execution_profile_contract(ExecutionProfile.STANDARD), - effective_durable=True, - effective_require_settled=True, - qualification_evidence_only=True, - ) - if not gate.passed: - raise EvidencePackError(gate.render()) - qualification_case = next( - item - for item in (workflow.qualification.cases if workflow.qualification else []) - if item.id == case["case_id"] - ) - run_id = f"{pack_id}-{case['case_id']}-trial-{trial:02d}" - authorization = build_qualification_case_authorization( - workflow, - gate, - case_id=qualification_case.id, - params={"note": NOTE}, - worklists=None, - campaign_id=CAMPAIGN_ID, - run_id=run_id, - fault_driver=( - None - if qualification_case.kind is QualificationCaseKind.REPRESENTATIVE - else fault_driver - ), + + def open_session_for(self, ctx: TrialContext) -> tuple[Any, Callable[[], None]]: + media_tmp = ctx.trial_dir / ".video" + record_video = ctx.trial == 1 and not ctx.certified_run + if record_video: + media_tmp.mkdir(parents=True) + ctx.state["record_video"] = record_video + clip_id = self._clip_id(ctx) + ctx.state["clip_id"] = clip_id + ctx.state["presentation"] = ( + _PresentationCapture(mode="governed") if clip_id is not None else None ) - report = Replayer( - active_backend, - effect_verifier=verifier, - governed_authorization=authorization, - qualification_fault_driver=fault_driver, - qualification_environment_observer=environment_observer, - durable=True, - require_settled=True, - use_structural=bool(case["use_structural"]), - control_overlay=( - presentation.emitter if presentation is not None else None - ), - ).run( - workflow.model_copy(deep=True), - params={"note": NOTE}, - bundle_dir=bundle_dir, - run_dir=run_dir, - execution_target_kind="web", - execution_origin=_origin(entry_url), - execution_entry_url=entry_url, - run_id=run_id, + return PlaywrightBackend.launch( + self.entry_url, + headless=True, + record_video_dir=str(media_tmp) if record_video else None, ) + + def replayer_kwargs(self, ctx: TrialContext) -> dict[str, Any]: + presentation = ctx.state.get("presentation") + return { + "use_structural": True, + "control_overlay": presentation.emitter + if presentation is not None + else None, + } + + def after_run(self, ctx: TrialContext, backend: Any, report: RunReport) -> None: backend.page.wait_for_timeout(200) - network_observation = _browser_network_observation( - backend.page, - base_url=base_url, - ) - finally: - close() - if network_observation is None: - raise EvidencePackError("run retained no browser network observation") - if record_video: - _finish_video( - media_tmp, - trial_dir - / ("replay.webm" if case["expected"] == "VERIFIED" else "halt.webm"), - ) - if presentation is not None and clip_id is not None: - _write_presentation_clip( - capture=presentation, - pack_id=pack_id, - clip_id=clip_id, - presentation_dir=presentation_dir, - ) + ctx.state["network"] = _browser_network_observation( + backend.page, base_url=self.base_url + ) + + def after_close(self, ctx: TrialContext, report: RunReport) -> None: + if ctx.state.get("network") is None: + raise EvidencePackError("run retained no browser network observation") + if ctx.state.get("record_video"): + _finish_video( + ctx.trial_dir / ".video", + ctx.trial_dir + / ("replay.webm" if ctx.expected == "VERIFIED" else "halt.webm"), + ) + presentation = ctx.state.get("presentation") + if presentation is not None: + _write_presentation_clip( + capture=presentation, + pack_id=self.pack_id, + clip_id=ctx.state["clip_id"], + presentation_dir=self.presentation_dir, + ) + _strip_run_authority(ctx.run_dir) - report_path = run_dir / "report.json" - render_run_report(run_dir) - _strip_run_authority(run_dir) - oracle = _oracle_snapshot( - base_url, - oracle_kind=str(case["oracle"]), - report=report, - ) - _write_json(trial_dir / "oracle.json", oracle) - (trial_dir / "case-input.json").write_bytes( - runtime_inputs_bytes(workflow, {"note": NOTE}, None) - ) - if report.qualification_fault_mutations: - if len(report.qualification_fault_mutations) != 1: - raise EvidencePackError("fault case retained multiple mutation receipts") - receipt = report.qualification_fault_mutations[0] - (trial_dir / "fault-receipt.json").write_bytes(receipt.artifact_bytes()) - (trial_dir / "fault-mutation.bin").write_bytes( - fault_driver.mutation_bytes(receipt.fault_kind) - ) - (trial_dir / "events.jsonl").write_text( - _report_events(report), - encoding="utf-8", - ) - envelope = _outcome_envelope( - case_id=str(case["case_id"]), - trial=trial, - expected_outcome=str(case["expected"]), - report=report, - report_path=report_path, - oracle=oracle, - network_observation=network_observation, - ) - _write_json(trial_dir / "outcome.json", envelope) - if report.execution_outcome != case["expected"]: - failures = [ - result.error for result in report.results if not result.ok and result.error - ] - result_contract = [ - { - "step_id": result.step_id, - "ok": result.ok, - "identity": ( - result.identity.status if result.identity is not None else None - ), - "postconditions_ok": result.postconditions_ok, - "effect_verified": result.effect_verified, - "effect_contracts": len(result.effect_contract_hashes), - "effect_evidence": len(result.effect_evidence), - } - for result in report.results - ] - raise EvidencePackError( - f"{case['case_id']} trial {trial} observed " - f"{report.execution_outcome}, expected {case['expected']}; " - f"failures={failures}; results={result_contract}" + def oracle(self, ctx: TrialContext, report: RunReport) -> dict[str, Any]: + """MockMed's own database oracle, plus the pack's events and envelope.""" + + oracle = _oracle_snapshot( + self.base_url, oracle_kind=ctx.oracle_kind, report=report ) - if report.model_calls != 0 or report.screenshots_may_leave_box: - raise EvidencePackError( - f"{case['case_id']} trial {trial} violated zero-model/local boundary" + (ctx.trial_dir / "events.jsonl").write_text( + _report_events(report), encoding="utf-8" ) - if not oracle["passed"] or oracle["silent_incorrect_success"]: - raise EvidencePackError( - f"{case['case_id']} trial {trial} failed independent oracle" + _write_json( + ctx.trial_dir / "outcome.json", + _outcome_envelope( + case_id=ctx.case_id, + trial=ctx.trial, + expected_outcome=ctx.expected, + report=report, + report_path=ctx.run_dir / "report.json", + oracle=oracle, + network_observation=ctx.state["network"], + ), + ) + return oracle + + def hooks(self) -> QualifyHooks: + return QualifyHooks( + open_session_for=self.open_session_for, + replayer_kwargs=self.replayer_kwargs, + after_run=self.after_run, + after_close=self.after_close, + oracle=self.oracle, + settle_s=0.0, + runner_id="openadapt-flow/public-demo-headless", ) - return report, oracle, envelope + + +def _verify_config(base_url: str, app_digest: str) -> VerifyConfig: + return load_verify_config( + REPO_ROOT / "public-demo" / "mockmed-verify.yaml", + env={"MOCKMED_BASE_URL": base_url, "MOCKMED_APP_VERSION": app_digest}, + ) def _run_idempotency_campaign( @@ -1669,7 +1193,7 @@ def _run_idempotency_campaign( base_url: str, workflow: Workflow, bundle_dir: Path, - environment_observer: _MockMedEnvironmentObserver, + environment_observer: Any, ) -> dict[str, Any]: """Prove at-most-once suppression after production certification. @@ -1802,9 +1326,7 @@ def _run_idempotency_campaign( "transaction_outcome": duplicate.transaction_outcome, "idempotent_replay": duplicate.idempotent_replay, "refusal_step_id": duplicate_results[0].step_id, - "delivery_attempted": bool( - duplicate_results[0].delivery_attempted - ), + "delivery_attempted": bool(duplicate_results[0].delivery_attempted), }, "independent_oracle": { "sha256": _sha256(campaign_dir / "oracle.json"), @@ -1858,9 +1380,7 @@ def _write_execute_acceptance_artifacts( template_path = qualification_dir / "governed-authorization-template.json" _write_json(template_path, template.model_dump(mode="json")) outcome_counts = Counter( - outcome["observed_outcome"] - for case in cases - for outcome in case["outcomes"] + outcome["observed_outcome"] for case in cases for outcome in case["outcomes"] ) acceptance = { "schema_version": "openadapt.execute-transport-acceptance/v1", @@ -1887,9 +1407,7 @@ def _write_execute_acceptance_artifacts( "protocol_value": int(project.minimum_effect_tier), "name": "INDEPENDENT_SYSTEM_OF_RECORD", }, - "qualification_report_sha256": _sha256( - qualification_dir / "report.json" - ), + "qualification_report_sha256": _sha256(qualification_dir / "report.json"), "case_evidence_contract_sha256": ( certification.case_evidence_contract_sha256 ), @@ -1907,9 +1425,7 @@ def _write_execute_acceptance_artifacts( } _write_json(qualification_dir / "execute-transport-acceptance.json", acceptance) - representative = next( - case for case in cases if case["case_id"] == "representative" - ) + representative = next(case for case in cases if case["case_id"] == "representative") representative_report = RunReport.model_validate_json( (root / representative["reports"][0]["path"]).read_text(encoding="utf-8") ) @@ -1946,94 +1462,6 @@ def _write_execute_acceptance_artifacts( _write_json(qualification_dir / "cloud-ingestion-inputs.json", cloud_inputs) -def _evidence_ref( - root: Path, - path: Path, - kind: Literal[ - "run_report", - "identity", - "effect", - "case_input", - "fault_receipt", - "fault_mutation", - "fault_campaign", - "other", - ], -) -> EvidenceRef: - return EvidenceRef( - kind=kind, - sha256=_sha256(path), - relative_path=path.relative_to(root).as_posix(), - ) - - -def _case_result( - *, - root: Path, - workflow: Workflow, - case_id: str, - observed_outcome: str, - run_id: str, - trial_dir: Path, - private_key: bytes, -) -> QualificationCaseResult: - project = workflow.qualification - if project is None: - raise EvidencePackError("workflow qualification project disappeared") - run_dir = trial_dir / "run" - run_artifacts = [ - _evidence_ref(root, path, "other") - for path in sorted(run_dir.rglob("*")) - if path.is_file() and path.name != "report.json" - ] - result = QualificationCaseResult( - case_id=case_id, - project_id=project.project_id, - project_revision=project.revision, - project_contract_sha256=project.contract_sha256(), - workflow_contract_sha256=workflow_contract_sha256(workflow), - environment_contract_sha256=project.environment.contract_sha256(), - environment_digest=project.environment.environment_digest, - runtime_version=project.environment.runtime_version, - runner_id="openadapt-flow/public-demo-headless", - runner_capabilities=list(project.environment.required_capabilities), - status="passed", - observed_outcome=QualificationOutcome(observed_outcome.lower()), - campaign_id_sha256=qualification_campaign_id_sha256(CAMPAIGN_ID), - case_input_sha256=runtime_inputs_digest( - workflow, - {"note": NOTE}, - None, - ), - run_id_sha256=qualification_run_id_sha256(run_id), - evidence=[ - _evidence_ref(root, trial_dir / "run" / "report.json", "run_report"), - _evidence_ref(root, trial_dir / "case-input.json", "case_input"), - _evidence_ref(root, trial_dir / "oracle.json", "effect"), - *run_artifacts, - *( - [ - _evidence_ref( - root, - trial_dir / "fault-receipt.json", - "fault_receipt", - ), - _evidence_ref( - root, - trial_dir / "fault-mutation.bin", - "fault_mutation", - ), - ] - if (trial_dir / "fault-receipt.json").is_file() - else [] - ), - ], - detail_code=f"{case_id}.{run_id.rsplit('-', 1)[-1]}", - attestation_key_id=RUNNER_KEY_ID, - ) - return sign_case_result(result, private_key=private_key) - - def _copy_binding( *, root: Path, @@ -2477,7 +1905,6 @@ def export_pack( bundle_dir = artifacts / "bundle" media_dir = artifacts / "media" presentation_dir = artifacts / "presentation" - qualification_dir = artifacts / "qualification" media_dir.mkdir(parents=True) base_url, _db, stop = serve() @@ -2494,7 +1921,7 @@ def export_pack( name=WORKFLOW_NAME, mine_effects=True, ) - save = _save_step(workflow) + save = _require_structured_save(workflow) if not any( effect.kind.value == "record_written" for effect in save.effects ) or not any(effect.kind.value == "field_equals" for effect in save.effects): @@ -2502,92 +1929,44 @@ def export_pack( "recording did not compile the required source-of-record effects" ) app_digest = _tree_digest(REPO_ROOT / "openadapt_flow" / "mockmed") - ( - private_key, - environment_digest, - fault_driver, - environment_observer, - ) = _configure_qualification( - workflow, + config = _verify_config(base_url, app_digest) + pack_hooks = _PackTrialHooks( base_url=base_url, - environment=environment, - app_digest=app_digest, + entry_url=config.app.entry_url or base_url, + presentation_dir=presentation_dir, + pack_id=pack_id, ) - save_qualified_workflow(workflow, bundle_dir) - workflow = Workflow.load(bundle_dir) - - case_manifests: list[dict[str, Any]] = [] - qualification_results: list[QualificationCaseResult] = [] - for case in _case_plan(): - case_dir = artifacts / "cases" / str(case["case_id"]) - reports: list[RunReport] = [] - for trial in range(1, trials + 1): - report, _oracle, _envelope = _run_case_trial( - base_url=base_url, - workflow=workflow, - bundle_dir=bundle_dir, - case=case, - trial=trial, - case_dir=case_dir, - presentation_dir=presentation_dir, - pack_id=pack_id, - fault_driver=fault_driver, - environment_observer=environment_observer, - ) - reports.append(report) - trial_dir = case_dir / f"trial-{trial:02d}" - run_id = f"{pack_id}-{case['case_id']}-trial-{trial:02d}" - qualification_results.append( - _case_result( - root=temp_root, - workflow=workflow, - case_id=str(case["case_id"]), - observed_outcome=str(case["expected"]), - run_id=run_id, - trial_dir=trial_dir, - private_key=private_key, - ) - ) - case_manifests.append( - _case_manifest( - root=temp_root, - workflow=workflow, - case_config=case, - reports=reports, - case_dir=case_dir, - ) + try: + result = qualify( + bundle_dir, + config, + out=artifacts, + trials=trials, + certify="clinical-write", + hooks=pack_hooks.hooks(), + campaign_id=CAMPAIGN_ID, + run_id_prefix=pack_id, + evidence_root=temp_root, + in_place=True, + write_campaign_files=False, + log=lambda _line: None, ) - - for qualification_result in qualification_results: - try: - record_case_results( - workflow, - [qualification_result], - evidence_root=temp_root, - ) - except Exception as exc: - raise EvidencePackError( - "qualification result refused for " - f"{qualification_result.case_id} " - f"({qualification_result.detail_code}): {exc}" - ) from exc - qualification_report_model = certify_project( - workflow, - policy=load_policy("clinical-write"), - evidence_root=temp_root, - ) - if not qualification_report_model.passed: - raise EvidencePackError(qualification_report_model.render()) - qualification_report = qualification_report_model.model_dump(mode="json") - _write_json( - qualification_dir / "project.json", - workflow.qualification.model_dump(mode="json") - if workflow.qualification - else {}, - ) - _write_json(qualification_dir / "report.json", qualification_report) - save_qualified_workflow(workflow, bundle_dir) + except (QualifyError, QualifyConfigError) as exc: + raise EvidencePackError(str(exc)) from exc workflow = Workflow.load(bundle_dir) + environment_digest = result.environment_digest or "" + environment_observer = result.observer + qualification_report = result.report.model_dump(mode="json") + case_manifests = [ + _case_manifest( + root=temp_root, + workflow=workflow, + case_config={"case_id": case_id, "expected": expected}, + reports=result.reports[case_id], + case_dir=artifacts / "cases" / case_id, + ) + for case_id, expected, _oracle in CASES + ] idempotency = _run_idempotency_campaign( root=temp_root, @@ -2939,14 +2318,10 @@ def _validate_execute_transport_artifacts( or oracle.get("passed") is not True or len(oracle.get("snapshot", {}).get("records", [])) != 1 or summary.get("first", {}).get("report_sha256") != _sha256(first_path) - or summary.get("duplicate", {}).get("report_sha256") - != _sha256(duplicate_path) - or summary.get("independent_oracle", {}).get("sha256") - != _sha256(oracle_path) + or summary.get("duplicate", {}).get("report_sha256") != _sha256(duplicate_path) + or summary.get("independent_oracle", {}).get("sha256") != _sha256(oracle_path) ): - raise EvidencePackError( - "post-certification idempotency evidence is not exact" - ) + raise EvidencePackError("post-certification idempotency evidence is not exact") template_path = _safe_file( root, @@ -2969,8 +2344,7 @@ def _validate_execute_transport_artifacts( ) if ( acceptance.get("result") != "PASS" - or acceptance.get("scope", {}).get("bundle_content_digest") - != compiled_digest + or acceptance.get("scope", {}).get("bundle_content_digest") != compiled_digest or acceptance.get("qualification", {}).get( "governed_authorization_template_sha256" ) diff --git a/packages/flow/scripts/openemr_qualification.py b/packages/flow/scripts/openemr_qualification.py index c568260ff..5b692cd52 100644 --- a/packages/flow/scripts/openemr_qualification.py +++ b/packages/flow/scripts/openemr_qualification.py @@ -1,29 +1,19 @@ -"""Qualify, certify and run a compiled OpenEMR patient-registration bundle. +"""Qualify, certify and run the OpenEMR patient-registration bundle. -This is the OpenEMR counterpart of :mod:`scripts.export_public_demo_evidence` -(MockMed). It drives the local, made-up-patient OpenEMR fixture in -``benchmark/openemr_local`` and never calls a model: +A thin wrapper over ``openadapt-flow qualify``. Everything app-specific is in +``benchmark/openemr_local/verify.yaml`` (read-back, record fields, duplicate +key) and ``benchmark/openemr_local/qualify_hooks.py`` (sign-in, fixture reset, +OAuth session). The same campaign runs without this script:: -1. copies a compiled patient-registration bundle (one irreversible - ``#confirmCreate`` save carrying ``record_written``/``field_equals`` - effects) and opens a qualification project bound to the live OpenEMR - browser boundary, with the independent system-of-record read-back - (OpenEMR's REST ``patient`` API under a separate read-only OAuth client) as - the required effect tier; -2. runs the six canonical qualification cases (representative plus the - ambiguity, wrong-identity, stale-identity, weak-effect and missing-effect - faults) through the real Standard-profile :class:`Replayer`, each trial - checked by a second oracle (direct SQL on the fixture database); -3. records the signed case results and certifies the project with the - ``clinical-write`` policy; nothing is written when certification fails; -4. optionally runs the certified bundle once in the Standard profile on a new - made-up patient and writes the run report and timing. + openadapt-flow qualify path/to/bundle \ + --verify benchmark/openemr_local/verify.yaml \ + --out runs/openemr-qual-1 --certify clinical-write --run-params patient.json -Keys. The runner and fault-driver Ed25519 keys are generated for this one -campaign, held in memory, and trusted only by the qualification project they -are written into (exactly as the MockMed exporter does). They are local test -keys, never a production trust root: a bundle certified here proves the local -fixture campaign, nothing more. +What this script adds is demo-only: a second oracle that reads +``openemr.patient_data`` directly over SQL on the local fixture database +(``--sql-oracle``, on by default), and flags for the fixture's state, lock and +ports. It never calls a model. Keys are local test keys generated per +campaign, never a production trust root. Usage (fixture already up; see benchmark/openemr_local/README.md):: @@ -31,632 +21,37 @@ --bundle-src path/to/compiled/bundle --out runs/openemr-qual-1 \ --fixture-state path/to/state [--lock path/to/environment.lock.json] \ [--ui-url http://127.0.0.1:9301 --api-url https://127.0.0.1:9300] \ - [--run-params patient.json] + [--run-params patient.json] [--no-sql-oracle] """ from __future__ import annotations -import base64 -import hashlib +import argparse import json -import platform -import shutil +import os import sys -import time from pathlib import Path -from typing import Any, Callable, Literal, Optional +from typing import Any, Optional REPO_ROOT = Path(__file__).resolve().parents[1] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from openadapt_flow import __version__ as FLOW_VERSION # noqa: E402 -from openadapt_flow.deployment import DeploymentConfig, PolicySection # noqa: E402 -from openadapt_flow.execution_profiles import ( # noqa: E402 - ExecutionProfile, - execution_profile_contract, +from openadapt_flow.ir import RunReport # noqa: E402 +from openadapt_flow.qualify import ( # noqa: E402 + QualifyConfigError, + QualifyError, + QualifyHooks, + TrialContext, + load_verify_config, + qualify, ) -from openadapt_flow.ir import RunReport, Workflow # noqa: E402 -from openadapt_flow.policy import load_policy # noqa: E402 -from openadapt_flow.qualification import ( # noqa: E402 - ActionRiskClass, - ActionRiskClassification, - EnvironmentBoundary, - EvidenceRef, - IdentityEnforcement, - IdentityPolicy, - QualificationActionTarget, - QualificationCase, - QualificationCaseKind, - QualificationCaseResult, - QualificationOutcome, - add_case, - certify_project, - init_project, - qualification_campaign_id_sha256, - qualification_run_id_sha256, - record_case_results, - save_qualified_workflow, - set_action_classification, - set_case_scope, - set_effect_policy, - set_identity_policy, - set_trusted_fault_driver_key, - set_trusted_runner_key, - sign_case_result, - workflow_contract_sha256, -) -from openadapt_flow.qualification_environment import ( # noqa: E402 - QualificationEnvironmentObservation, -) -from openadapt_flow.qualification_faults import ( # noqa: E402 - FaultMutationReceipt, - QualificationFaultContext, - QualificationFaultMutation, - effect_verifier_input_sha256, - sha256_bytes, - sign_fault_mutation_receipt, -) -from openadapt_flow.report import render_run_report # noqa: E402 -from openadapt_flow.run_gate import ( # noqa: E402 - build_qualification_case_authorization, - build_runtime_authorization, - evaluate_run_gate, -) -from openadapt_flow.runtime import Replayer # noqa: E402 -from openadapt_flow.runtime.authorization import ( # noqa: E402 - runtime_inputs_bytes, - runtime_inputs_digest, -) -from openadapt_flow.runtime.effects import RestRecordVerifier # noqa: E402 -from openadapt_flow.verification import VerificationTier # noqa: E402 -RUNNER_KEY_ID = "openemr-local-qualification-runner" -FAULT_DRIVER_KEY_ID = "openemr-local-qualification-fault-driver" -DEFAULT_CAMPAIGN_ID = "openemr-local-patient-registration-v1" -ENVIRONMENT_OBSERVER_ID = "openadapt.openemr.playwright-environment" -ENVIRONMENT_OBSERVER_CONTRACT_SHA256 = hashlib.sha256( - b"openadapt.openemr.playwright-environment/v1" -).hexdigest() -SAVE_SELECTOR = "#confirmCreate" -PATIENT_API_PATH = "/apis/default/api/patient?_limit=1000" +VERIFY_YAML = REPO_ROOT / "benchmark" / "openemr_local" / "verify.yaml" SQL_FIELDS = ( - "id", - "pid", - "title", - "fname", - "lname", - "DOB", - "sex", - "street", - "city", - "state", - "postal_code", - "phone_home", - "email", - "country_code", -) -CASES: tuple[dict[str, str], ...] = ( - {"case_id": "representative", "expected": "VERIFIED", "oracle": "exact_record"}, - {"case_id": "fault-ambiguity", "expected": "HALTED", "oracle": "no_mutation"}, - {"case_id": "fault-wrong-identity", "expected": "HALTED", "oracle": "no_mutation"}, - {"case_id": "fault-stale-identity", "expected": "HALTED", "oracle": "no_mutation"}, - {"case_id": "fault-weak-effect", "expected": "HALTED", "oracle": "no_mutation"}, - {"case_id": "fault-missing-effect", "expected": "HALTED", "oracle": "no_mutation"}, -) - - -class QualificationError(RuntimeError): - """The campaign refused; nothing it produced may be called certified.""" - - -def _sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _canonical_json(value: Any) -> bytes: - return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") - - -def _write_json(path: Path, value: Any) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") - return path - - -def _ed25519_keypair() -> tuple[bytes, str]: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey - - key = Ed25519PrivateKey.generate() - private_raw = key.private_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PrivateFormat.Raw, - encryption_algorithm=serialization.NoEncryption(), - ) - public_raw = key.public_key().public_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PublicFormat.Raw, - ) - return private_raw, base64.b64encode(public_raw).decode("ascii") - - -def patient_record_verifier(api_base_url: str, session: Any) -> RestRecordVerifier: - """The independent read-back: OpenEMR's REST patient list, read-only client. - - The query is input-independent so the verifier is the same object shape - for every patient; the typed effects select the record by ``lname`` and - ``email`` and compare every declared field. - """ - - return RestRecordVerifier( - api_base_url.rstrip("/"), - records_path=PATIENT_API_PATH, - records_key="data", - session=session, - timeout_s=10.0, - poll_interval_s=0.2, - ) - - -def _is_ctrl_a(step: Any) -> bool: - from openadapt_flow.ir import ActionKind - - if step.action is ActionKind.HOTKEY: - return (step.key or "").lower() == "a" and [ - m.lower() for m in step.modifiers - ] in (["control"], ["ctrl"]) - return step.action is ActionKind.KEY and (step.key or "").lower() in { - "control+a", - "ctrl+a", - } - - -def prepare_registration_bundle( - src: Path, dst: Path, recording_dir: Path -) -> tuple[Workflow, list[dict[str, Any]]]: - """Author the compiled registration program up to ``clinical-write``. - - The compiler keeps the demonstration's literal gestures. Four of them leave - steps that assert nothing, which ``clinical-write`` refuses. Each change - below is listed in the returned notes and replaces a gesture with an - equivalent one that carries its own check, using only the recording's own - frames: - - - the Enter after typing an option label into a browser `` takes the typed option label directly; the - # TYPE step's read-back then verifies the focused select's - # selected option label equals the typed value. The Enter that - # followed in the demonstration asserts nothing and is dropped. - key = steps[i + 2] - dropped_enter.add(key.id) - notes.append( - { - "change": "drop_select_commit_enter", - "step": key.id, - "selector": structural.selector, - } - ) - if step.id in dropped_enter: - i += 1 - continue - if _is_ctrl_a(step) and i > 0 and steps[i - 1].action is ActionKind.CLICK and ( - i + 1 < len(steps) and steps[i + 1].action is ActionKind.TYPE - ): - notes.append({"change": "drop_select_all_on_empty_field", "step": step.id}) - i += 1 - continue - step = step.model_copy(deep=True) - if ( - step.action is ActionKind.CLICK - and not step.expect - and structural is not None - and structural.role in {"textbox", "combobox"} - and step.anchor is not None - ): - after = frame(step, "after") - region = tuple(step.anchor.region) - rel = f"templates/{step.id}_expect.png" - (dst / rel).write_bytes(_crop_png(after, region)) - step.expect = [ - Postcondition( - kind=PostconditionKind.REGION_STABLE, - region=region, - phash=phash_png(after, region=region), - phash_tolerance=8, - template=rel, - ) - ] - notes.append({"change": "focused_field_region_stable", "step": step.id}) - if ( - step.action is ActionKind.CLICK - and step.anchor is not None - and not ( - step.anchor.context_text - or step.anchor.structured_identity - or step.anchor.identity_template - or step.anchor.identifier_crop - ) - and structural is not None - and structural.role == "button" - ): - before = frame(step, "before") - region = tuple(step.anchor.region) - rel = f"templates/identifiers/{step.id}.png" - (dst / rel).parent.mkdir(parents=True, exist_ok=True) - (dst / rel).write_bytes(_crop_png(before, region)) - step.anchor.identifier_crop = rel - step.anchor.identifier_region = region - step.identity_armed = True - step.identity_unarmed_reason = None - step.identifier_crop_missing_reason = None - notes.append( - {"change": "static_control_identifier_crop", "step": step.id} - ) - out.append(step) - i += 1 - # OpenEMR has no idempotency column. The record_written read-back counts - # charts bearing this run's email (a made-up, per-patient address) and - # requires exactly one, so a second chart for the same person is refuted - # by the read-back. This detects a duplicate; OpenEMR itself does not - # collapse one. - from openadapt_flow.runtime.effects import ValueExpr - - for step in out: - for effect in step.effects: - if effect.kind.value == "record_written" and effect.idempotency_key is None: - effect.idempotency_key = ValueExpr(param="email") - effect.key_field = "email" - notes.append({"change": "email_as_duplicate_key", "step": step.id}) - workflow.steps = out - save_qualified_workflow(workflow, dst) - return Workflow.load(dst), notes - - -def save_step(workflow: Workflow) -> Any: - """The single irreversible, effect-bound, identity-armed save step.""" - - candidates = [s for s in workflow.steps if s.risk == "irreversible" and s.effects] - if len(candidates) != 1: - raise QualificationError( - "expected exactly one irreversible effect-bound save, found " - f"{[s.id for s in candidates]}" - ) - step = candidates[0] - # Identity may rest on structured (DOM) text or on the retained identity - # template; the run gate and certification decide whether it is enough. - if step.anchor is None or not step.identity_armed: - raise QualificationError("the save step has no armed identity") - if any(effect.needs_operator_confirmation for effect in step.effects): - raise QualificationError("the save step has an unbound placeholder effect") - kinds = {effect.kind.value for effect in step.effects} - if not {"record_written", "field_equals"} <= kinds: - raise QualificationError("the save step lacks record_written/field_equals") - return step - - -def _modal_frame(page: Any) -> Any: - """The frame that holds the duplicate-check dialog's confirm button.""" - - for frame in page.frames: - try: - if frame.locator(SAVE_SELECTOR).count() == 1: - return frame - except Exception: # noqa: BLE001 - detached frames are skipped - continue - raise QualificationError("OpenEMR confirm control not found for the fault") - - -def _environment_payload(backend: Any, application_identity: str) -> dict[str, Any]: - page = backend.page - browser = page.context.browser - if browser is None: - raise QualificationError("qualification page has no owning browser") - return { - "browser": "chromium", - "browser_version": browser.version, - "user_agent": page.evaluate("navigator.userAgent"), - "viewport": list(backend.viewport), - "device_scale_factor": page.evaluate("window.devicePixelRatio"), - "headless": True, - "platform": platform.platform(), - "python": platform.python_version(), - "application": application_identity, - "runtime_version": FLOW_VERSION, - "target_kind": "web", - } - - -class OpenEMREnvironmentObserver: - """Observe the exact live OpenEMR browser boundary for qualification.""" - - observer_id = ENVIRONMENT_OBSERVER_ID - contract_sha256 = ENVIRONMENT_OBSERVER_CONTRACT_SHA256 - - def __init__(self, *, application_version: str) -> None: - self._application_version = application_version - - def observe( - self, - backend: Any, - target_kind: Literal["web", "windows", "macos", "linux", "rdp", "citrix"], - ) -> QualificationEnvironmentObservation: - if target_kind != "web": - raise ValueError("OpenEMR qualification requires its web boundary") - page = backend.page - origin = page.evaluate("location.origin") - session_material = page.evaluate( - """() => JSON.stringify({origin: location.origin, - timeOrigin: performance.timeOrigin, - userAgent: navigator.userAgent})""" - ) - payload = _environment_payload(backend, self._application_version) - return QualificationEnvironmentObservation( - target_kind="web", - application_identity=origin, - application_version=self._application_version, - session_identity_sha256=sha256_bytes(session_material.encode("utf-8")), - environment_digest=sha256_bytes(_canonical_json(payload)), - ) - - -class _WeakEffectVerifier: - """A configured verifier whose evidence is too weak for Standard.""" - - verification_tier = VerificationTier.IMMEDIATE_SCREEN - - -class OpenEMRFaultDriver: - """Fixture-owned mutations of the live OpenEMR duplicate-check dialog. - - Each mutation changes the real detector input at the gate the runtime - selects (the ``#confirmCreate`` save); the ordinary resolver, identity - check or effect gate must then refuse. It never returns a verdict. - """ - - driver_id = "openadapt.openemr.local-qualification-faults" - attestation_key_id = FAULT_DRIVER_KEY_ID - contract_sha256 = hashlib.sha256( - b"openadapt.openemr.local-qualification-faults/v1" - ).hexdigest() - - def __init__(self) -> None: - self.private_key, self.public_key_base64 = _ed25519_keypair() - - @staticmethod - def _settled_frame_sha256(context: QualificationFaultContext) -> str: - """Hash the frame the detector reads, once the mutation has rendered. - - Uses the detector's own capture path; a frame taken mid-repaint would - name bytes the detector never sees and the receipt would not bind. - """ - - from openadapt_flow.qualification_drills import _settled_screenshot - - capture = context.capture_frame or context.backend.screenshot - frame = _settled_screenshot(capture) - if frame is None: - raise QualificationError("fault mutation never settled on screen") - return sha256_bytes(frame) - - @staticmethod - def mutation_bytes(kind: str) -> bytes: - return f"openadapt.openemr.synthetic-fault/{kind}/v1\n".encode("ascii") - - def mutate( - self, context: QualificationFaultContext - ) -> QualificationFaultMutation | None: - replacement = context.effect_verifier - replace_effect_verifier = False - page = context.backend.page - if context.fault_kind == "ambiguity": - _modal_frame(page).evaluate( - """(sel) => { const b = document.querySelector(sel); - b.parentNode.insertBefore(b.cloneNode(true), b); }""", - SAVE_SELECTOR, - ) - after_sha256 = self._settled_frame_sha256(context) - elif context.fault_kind == "wrong_identity": - # Relabel the confirm control itself to a different action. The - # structural selector still resolves the same unique element, so - # only the identity check (which binds the control's own label) - # can refuse it. - _modal_frame(page).evaluate( - """(sel) => { const b = document.querySelector(sel); - if (b.tagName === 'INPUT') b.value = 'Merge Into Existing Chart'; - else b.textContent = 'Merge Into Existing Chart'; }""", - SAVE_SELECTOR, - ) - after_sha256 = self._settled_frame_sha256(context) - elif context.fault_kind == "stale_identity": - frame = _modal_frame(page) - frame.evaluate( - """() => { document.documentElement.style.background = '#ffffff'; - document.body.replaceChildren(); - document.body.style.background = '#ffffff'; }""" - ) - after_sha256 = self._settled_frame_sha256(context) - elif context.fault_kind in {"weak_effect", "missing_effect"}: - replace_effect_verifier = True - replacement = ( - _WeakEffectVerifier() if context.fault_kind == "weak_effect" else None - ) - after_sha256 = effect_verifier_input_sha256(replacement, context.effects) - else: - return None - mutation = self.mutation_bytes(context.fault_kind) - receipt = FaultMutationReceipt( - project_id=context.project_id, - project_revision=context.project_revision, - project_contract_sha256=context.project_contract_sha256, - campaign_id_sha256=context.campaign_id_sha256, - case_id_sha256=context.case_id_sha256, - case_input_sha256=context.case_input_sha256, - run_id_sha256=context.run_id_sha256, - step_id_sha256=sha256_bytes(context.step_id.encode("utf-8")), - actuation_path=context.actuation_path, - fault_kind=context.fault_kind, - gate=context.gate, - driver_id=self.driver_id, - driver_contract_sha256=self.contract_sha256, - before_input_sha256=context.before_input_sha256, - after_input_sha256=after_sha256, - mutation_artifact_sha256=sha256_bytes(mutation), - attestation_key_id=self.attestation_key_id, - ) - return QualificationFaultMutation( - receipt=sign_fault_mutation_receipt(receipt, private_key=self.private_key), - replace_effect_verifier=replace_effect_verifier, - effect_verifier=replacement, - ) - - -def configure_qualification( - workflow: Workflow, - *, - ui_base_url: str, - application_version: str, - environment: dict[str, Any], - case_params: dict[str, Any], -) -> tuple[bytes, OpenEMRFaultDriver, OpenEMREnvironmentObserver]: - """Open the project, classify every step and bind the six cases.""" - - environment_digest = sha256_bytes(_canonical_json(environment)) - boundary = EnvironmentBoundary( - target_kind="web", - application="OpenEMR (local fixture, made-up patients)", - application_identity=ui_base_url.rstrip("/"), - application_version=application_version, - environment_observer_id=ENVIRONMENT_OBSERVER_ID, - environment_observer_contract_sha256=ENVIRONMENT_OBSERVER_CONTRACT_SHA256, - environment_digest=environment_digest, - runtime_version=FLOW_VERSION, - required_capabilities=[ - "headless_chromium", - "independent_system_of_record", - "playwright_dom", - ], - ) - project = init_project( - workflow, - environment=boundary, - minimum_effect_tier=VerificationTier.INDEPENDENT_SYSTEM, - ) - for step in workflow.steps: - irreversible = step.risk == "irreversible" - set_action_classification( - workflow, - ActionRiskClassification( - step_id=step.id, - classification=( - ActionRiskClass.IRREVERSIBLE - if irreversible - else ActionRiskClass.READ_ONLY - ), - explanation=( - "confirms the new patient chart: a system-of-record write" - if irreversible - else "fills or navigates the new-patient form; no record effect" - ), - operator_confirmed=True, - ), - ) - save = save_step(workflow) - set_identity_policy( - workflow, - IdentityPolicy(step_id=save.id, enforcement=IdentityEnforcement.CANONICAL_LADDER), - ) - for index, _effect in enumerate(save.effects): - set_effect_policy( - workflow, - step_id=save.id, - effect_index=index, - tier=VerificationTier.INDEPENDENT_SYSTEM, - ) - add_case( - workflow, - QualificationCase( - id="representative", - kind=QualificationCaseKind.REPRESENTATIVE, - expected_outcome=QualificationOutcome.VERIFIED, - description="Recorded registration under its qualified OpenEMR boundary", - ), - ) - private_raw, public_b64 = _ed25519_keypair() - set_trusted_runner_key(workflow, key_id=RUNNER_KEY_ID, public_key_base64=public_b64) - driver = OpenEMRFaultDriver() - set_trusted_fault_driver_key( - workflow, - key_id=driver.attestation_key_id, - public_key_base64=driver.public_key_base64, - ) - input_sha256 = runtime_inputs_digest(workflow, case_params, None) - target = QualificationActionTarget(step_id=save.id, actuation_path="gui") - for case in list(project.cases): - set_case_scope( - workflow, - case_id=case.id, - runtime_input_sha256=input_sha256, - action_targets=[target], - fault_target=( - None if case.kind is QualificationCaseKind.REPRESENTATIVE else target - ), - ) - return ( - private_raw, - driver, - OpenEMREnvironmentObserver(application_version=application_version), - ) + "id", "pid", "title", "fname", "lname", "DOB", "sex", "street", "city", + "state", "postal_code", "phone_home", "email", "country_code", +) # fmt: skip def sql_patients(fixture: Any) -> list[dict[str, str]]: @@ -672,462 +67,129 @@ def sql_patients(fixture: Any) -> list[dict[str, str]]: ] -def sql_oracle( - fixture: Any, - *, - before: list[dict[str, str]], - params: dict[str, Any], - expected_fields: dict[str, str], - oracle_kind: str, - report: RunReport, -) -> dict[str, Any]: - """Second, direct-SQL oracle; independent of the run report and the REST read.""" - - after = sql_patients(fixture) - before_ids = {row["id"] for row in before} - new_rows = [row for row in after if row["id"] not in before_ids] - changed_existing = [ - row for row in after if row["id"] in before_ids and row not in before - ] - fields = [] - row = new_rows[0] if len(new_rows) == 1 else None - for name, want in expected_fields.items(): - got = None if row is None else row.get(name) - fields.append({"field": name, "match": row is not None and got == want}) - if oracle_kind == "exact_record": - passed = ( - len(new_rows) == 1 - and not changed_existing - and all(item["match"] for item in fields) - ) - observed = "exact_record" if passed else "record_mismatch" - elif oracle_kind == "no_mutation": - passed = not new_rows and not changed_existing - observed = "no_mutation" if passed else "unexpected_mutation" - else: - raise QualificationError(f"unknown oracle kind {oracle_kind!r}") - return { - "schema_version": "openadapt.openemr-qualification-oracle/v1", - "oracle_kind": oracle_kind, - "read_path": "direct SQL on openemr.patient_data", - "read_boundary": "independent from browser pixels, the REST read-back and RunReport", - "passed": passed, - "observed": observed, - "patients_before": len(before), - "patients_after": len(after), - "new_charts": len(new_rows), - "existing_charts_changed": len(changed_existing), - "fields_matched": sum(item["match"] for item in fields), - "fields_expected": len(fields), - "field_check": fields, - "silent_incorrect_success": bool(not passed and report.success), - } - - -def expected_sql_fields(params: dict[str, Any], state_code: Optional[str]) -> dict[str, str]: - """The patient_data columns the registration must write, from the inputs.""" - - fields = { - "title": params["title"], - "fname": params["fname"], - "lname": params["lname"], - "DOB": params["DOB"], - "sex": params["sex"], - "street": params["street"], - "city": params["city"], - "postal_code": params["postal_code"], - "phone_home": params["phone_home"], - "email": params["email"], - "country_code": params["country_code"], - } - if state_code: - fields["state"] = state_code - return {k: str(v) for k, v in fields.items()} - - -BrowserFactory = Callable[[], tuple[Any, Callable[[], None]]] - - -def _token_session_with_retry(fixture: Any, attempts: int = 5) -> Any: - """The read-only OAuth session; retries only transport errors. - - Right after a fixture reset the API container can drop the first - connection. A refused credential is never retried. - """ +def expected_sql_fields(params: dict[str, Any]) -> dict[str, str]: + """The patient_data columns the registration must write.""" - import httpx + names = ("title", "fname", "lname", "DOB", "sex", "street", "city", + "postal_code", "phone_home", "email", "country_code") # fmt: skip + fields = {name: str(params[name]) for name in names} + fields["state"] = "MA" # the recorded state selection; see verify.yaml + return fields - for attempt in range(1, attempts + 1): - try: - return fixture.token_session("oracle") - except httpx.TransportError: - if attempt == attempts: - raise - time.sleep(2.0 * attempt) - raise AssertionError("unreachable") +class SqlOracle: + """Demo-only second oracle: direct SQL, independent of REST and the report.""" -def run_once( - *, - fixture: Any, - workflow: Workflow, - bundle_dir: Path, - run_dir: Path, - params: dict[str, Any], - open_browser: BrowserFactory, - authorization_for: Callable[[Any], Any], - fault_driver: Optional[OpenEMRFaultDriver], - environment_observer: OpenEMREnvironmentObserver, - run_id: str, - on_ready: Optional[Callable[[Any], None]] = None, - marks: Optional[dict[str, float]] = None, -) -> RunReport: - """One Standard-profile replay with the REST read-back as the verifier.""" - - marks = marks if marks is not None else {} - backend, close = open_browser() - marks["browser_ready"] = time.time() - try: - verifier = patient_record_verifier( - fixture.api_base_url, _token_session_with_retry(fixture) - ) - gate_kwargs = dict( - bundle_dir=bundle_dir, - deployment=DeploymentConfig(policy=PolicySection(policy="clinical-write")), - effect_verifier=verifier, - profile_contract=execution_profile_contract(ExecutionProfile.STANDARD), - effective_durable=True, - effective_require_settled=True, - ) - authorization = authorization_for((verifier, gate_kwargs)) - marks["admitted"] = time.time() - if on_ready is not None: - on_ready(backend) - marks["task_start"] = time.time() - report = Replayer( - backend, - effect_verifier=verifier, - governed_authorization=authorization, - qualification_fault_driver=fault_driver, - qualification_environment_observer=environment_observer, - durable=True, - require_settled=True, - # Two static form controls (the Contact section header and the - # Create New Patient button) carry a pixel identifier crop as - # their identity evidence; see prepare_registration_bundle. - pixel_verify_enabled=True, - ).run( - workflow.model_copy(deep=True), - params=params, - bundle_dir=bundle_dir, - run_dir=run_dir, - execution_target_kind="web", - execution_origin=fixture.ui_base_url.rstrip("/"), - execution_entry_url=backend.page.url, - run_id=run_id, - ) - marks["task_end"] = time.time() - return report - finally: - close() - marks["browser_closed"] = time.time() - + def __init__(self) -> None: + from benchmark.openemr_local.qualify_hooks import fixture -def _evidence_ref(root: Path, path: Path, kind: Any) -> EvidenceRef: - return EvidenceRef( - kind=kind, sha256=_sha256(path), relative_path=path.relative_to(root).as_posix() - ) + self.fixture = fixture() + def before_trial(self, ctx: TrialContext) -> None: + ctx.state["sql_before"] = sql_patients(self.fixture) -def case_result( - *, - root: Path, - workflow: Workflow, - case_id: str, - observed_outcome: str, - run_id: str, - trial_dir: Path, - private_key: bytes, - case_params: dict[str, Any], - campaign_id: str, -) -> QualificationCaseResult: - project = workflow.qualification - if project is None: - raise QualificationError("workflow qualification project disappeared") - run_dir = trial_dir / "run" - run_artifacts = [ - _evidence_ref(root, path, "other") - for path in sorted(run_dir.rglob("*")) - if path.is_file() and path.name != "report.json" - ] - faults = ( - [ - _evidence_ref(root, trial_dir / "fault-receipt.json", "fault_receipt"), - _evidence_ref(root, trial_dir / "fault-mutation.bin", "fault_mutation"), + def check(self, ctx: TrialContext, report: RunReport) -> tuple[str, dict[str, Any]]: + before = ctx.state["sql_before"] + after = sql_patients(self.fixture) + before_ids = {row["id"] for row in before} + new_rows = [row for row in after if row["id"] not in before_ids] + changed = [ + row for row in after if row["id"] in before_ids and row not in before ] - if (trial_dir / "fault-receipt.json").is_file() - else [] - ) - result = QualificationCaseResult( - case_id=case_id, - project_id=project.project_id, - project_revision=project.revision, - project_contract_sha256=project.contract_sha256(), - workflow_contract_sha256=workflow_contract_sha256(workflow), - environment_contract_sha256=project.environment.contract_sha256(), - environment_digest=project.environment.environment_digest, - runtime_version=project.environment.runtime_version, - runner_id="openadapt-flow/openemr-local-headless", - runner_capabilities=list(project.environment.required_capabilities), - status="passed", - observed_outcome=QualificationOutcome(observed_outcome.lower()), - campaign_id_sha256=qualification_campaign_id_sha256(campaign_id), - case_input_sha256=runtime_inputs_digest(workflow, case_params, None), - run_id_sha256=qualification_run_id_sha256(run_id), - evidence=[ - _evidence_ref(root, run_dir / "report.json", "run_report"), - _evidence_ref(root, trial_dir / "case-input.json", "case_input"), - _evidence_ref(root, trial_dir / "oracle.json", "effect"), - *run_artifacts, - *faults, - ], - detail_code=f"{case_id}.{run_id.rsplit('-', 1)[-1]}", - attestation_key_id=RUNNER_KEY_ID, - ) - return sign_case_result(result, private_key=private_key) - - -def _summary(report: RunReport) -> dict[str, Any]: - failed = next((item for item in report.results if not item.ok), None) - return { - "execution_outcome": report.execution_outcome, - "transaction_outcome": report.transaction_outcome, - "execution_profile": report.execution_profile, - "production_eligible": report.production_eligible, - "success": report.success, - "model_calls": report.model_calls, - "total_ms": report.total_ms, - "failed_step_id": failed.step_id if failed is not None else None, - "failed_error": failed.error if failed is not None else None, - "halt": report.halt.model_dump(mode="json") if report.halt else None, - } - - -def qualify( - *, - fixture: Any, - bundle_src: Path, - out: Path, - open_browser: BrowserFactory, - trials: int, - state_code: Optional[str], - campaign_id: str = DEFAULT_CAMPAIGN_ID, - log: Callable[[str], None] = print, -) -> tuple[Workflow, Path, OpenEMREnvironmentObserver]: - """Run the six-case campaign and certify. Raises on any refusal.""" - - if trials < 1: - raise QualificationError("at least one trial per case") - if out.exists(): - raise QualificationError(f"output already exists: {out}") - bundle_dir = out / "bundle" - shutil.copytree(bundle_src, bundle_dir) - workflow = Workflow.load(bundle_dir) - save_step(workflow) - case_params = {k: str(v) for k, v in (workflow.params or {}).items()} - if not case_params: - raise QualificationError("bundle has no recorded inputs for the cases") - application_version = "openemr:" + str( - getattr(fixture, "source_identity", lambda: {})().get("openemr", "unknown") - ) - backend, close = open_browser() + row = new_rows[0] if len(new_rows) == 1 else None + fields = [ + {"field": name, "match": row is not None and row.get(name) == want} + for name, want in expected_sql_fields(ctx.params).items() + ] + if ctx.oracle_kind == "exact_record": + passed = ( + len(new_rows) == 1 and not changed and all(f["match"] for f in fields) + ) + observed = "exact_record" if passed else "record_mismatch" + else: + passed = not new_rows and not changed + observed = "no_mutation" if passed else "unexpected_mutation" + return "sql-oracle", { + "schema_version": "openadapt.openemr-qualification-oracle/v1", + "oracle_kind": ctx.oracle_kind, + "read_path": "direct SQL on openemr.patient_data", + "passed": passed, + "observed": observed, + "patients_before": len(before), + "patients_after": len(after), + "new_charts": len(new_rows), + "existing_charts_changed": len(changed), + "fields_matched": sum(f["match"] for f in fields), + "fields_expected": len(fields), + "field_check": fields, + "silent_incorrect_success": bool(not passed and report.success), + } + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--bundle-src", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--fixture-state", type=Path, required=True) + parser.add_argument("--lock", type=Path) + parser.add_argument("--ui-url") + parser.add_argument("--api-url") + parser.add_argument( + "--run-params", type=Path, help="JSON inputs for a certified run" + ) + parser.add_argument("--trials", type=int) + parser.add_argument("--cases", help="comma-separated case ids (default: all six)") + parser.add_argument("--no-certify", action="store_true") + parser.add_argument( + "--no-sql-oracle", action="store_true", help="skip the demo-only SQL check" + ) + args = parser.parse_args(argv) + + os.environ["OPENEMR_FIXTURE_STATE"] = str(args.fixture_state) + for name, value in ( + ("OPENEMR_FIXTURE_LOCK", args.lock), + ("OPENEMR_UI_URL", args.ui_url), + ("OPENEMR_API_URL", args.api_url), + ): + if value: + os.environ[name] = str(value) + + hooks = QualifyHooks() + if not args.no_sql_oracle: + sql = SqlOracle() + hooks.before_trial = sql.before_trial + hooks.extra_checks.append(sql.check) try: - environment = _environment_payload(backend, application_version) - finally: - close() - private_key, driver, observer = configure_qualification( - workflow, - ui_base_url=fixture.ui_base_url, - application_version=application_version, - environment=environment, - case_params=case_params, - ) - save_qualified_workflow(workflow, bundle_dir) - workflow = Workflow.load(bundle_dir) - results: list[QualificationCaseResult] = [] - summary: list[dict[str, Any]] = [] - for case in CASES: - qcase = next( - item - for item in (workflow.qualification.cases if workflow.qualification else []) - if item.id == case["case_id"] + config = load_verify_config(VERIFY_YAML) + result = qualify( + args.bundle_src, + config, + out=args.out, + cases=args.cases.split(",") if args.cases else None, + trials=args.trials, + certify=None if args.no_certify else config.campaign.policy, + run_params=json.loads(args.run_params.read_text()) + if args.run_params + else None, + hooks=hooks, ) - for trial in range(1, trials + 1): - # Every trial starts from the verified baseline, so no trial can - # see another trial's chart (the EMR's own duplicate check). - fixture.reset() - trial_dir = out / "cases" / case["case_id"] / f"trial-{trial:02d}" - run_dir = trial_dir / "run" - run_id = f"{campaign_id}-{case['case_id']}-trial-{trial:02d}" - before = sql_patients(fixture) - - def authorize(arg: Any, _qcase: Any = qcase, _run_id: str = run_id) -> Any: - _verifier, gate_kwargs = arg - gate = evaluate_run_gate( - workflow, qualification_evidence_only=True, **gate_kwargs - ) - if not gate.passed: - raise QualificationError(gate.render()) - return build_qualification_case_authorization( - workflow, - gate, - case_id=_qcase.id, - params=case_params, - worklists=None, - campaign_id=campaign_id, - run_id=_run_id, - fault_driver=( - None - if _qcase.kind is QualificationCaseKind.REPRESENTATIVE - else driver - ), - ) - - report = run_once( - fixture=fixture, - workflow=workflow, - bundle_dir=bundle_dir, - run_dir=run_dir, - params=case_params, - open_browser=open_browser, - authorization_for=authorize, - fault_driver=driver, - environment_observer=observer, - run_id=run_id, - ) - time.sleep(1.0) - render_run_report(run_dir) - oracle = sql_oracle( - fixture, - before=before, - params=case_params, - expected_fields=expected_sql_fields(case_params, state_code), - oracle_kind=case["oracle"], - report=report, - ) - _write_json(trial_dir / "oracle.json", oracle) - (trial_dir / "case-input.json").write_bytes( - runtime_inputs_bytes(workflow, case_params, None) - ) - if report.qualification_fault_mutations: - if len(report.qualification_fault_mutations) != 1: - raise QualificationError("fault case retained several receipts") - receipt = report.qualification_fault_mutations[0] - (trial_dir / "fault-receipt.json").write_bytes(receipt.artifact_bytes()) - (trial_dir / "fault-mutation.bin").write_bytes( - driver.mutation_bytes(receipt.fault_kind) - ) - row = { - "case_id": case["case_id"], - "trial": trial, - "expected": case["expected"], - **_summary(report), - "oracle": oracle["observed"], - } - summary.append(row) - _write_json(out / "campaign-summary.json", summary) - log(json.dumps(row, default=str)) - if report.execution_outcome != case["expected"]: - raise QualificationError( - f"{case['case_id']} trial {trial}: observed " - f"{report.execution_outcome}, expected {case['expected']}" - ) - if report.model_calls != 0: - raise QualificationError("a qualification run called a model") - if not oracle["passed"] or oracle["silent_incorrect_success"]: - raise QualificationError( - f"{case['case_id']} trial {trial}: SQL oracle {oracle['observed']}" - ) - results.append( - case_result( - root=out, - workflow=workflow, - case_id=case["case_id"], - observed_outcome=case["expected"], - run_id=run_id, - trial_dir=trial_dir, - private_key=private_key, - case_params=case_params, - campaign_id=campaign_id, - ) - ) - for result in results: - record_case_results(workflow, [result], evidence_root=out) - qualification = certify_project( - workflow, policy=load_policy("clinical-write"), evidence_root=out - ) - _write_json(out / "qualification" / "report.json", qualification.model_dump(mode="json")) - (out / "qualification" / "verdict.txt").write_text(qualification.render() + "\n") - log(qualification.render()) - if not qualification.passed: - raise QualificationError("certification refused; see qualification/verdict.txt") - save_qualified_workflow(workflow, bundle_dir) - _write_json( - out / "qualification" / "project.json", - workflow.qualification.model_dump(mode="json") if workflow.qualification else {}, + except QualifyConfigError as exc: + print(exc, file=sys.stderr) + return 2 + except QualifyError as exc: + print(f"QUALIFICATION FAILED: {exc}", file=sys.stderr) + return 1 + if not result.passed and result.partial: + print( + f"PARTIAL: every case run behaved as required; qualifying needs all six: {args.out}" + ) + return 0 + print( + f"{'CERTIFIED' if result.certified else 'QUALIFIED' if result.passed else 'FAILED'}" + f" under {result.policy or 'no policy'}: {args.out}" ) - return Workflow.load(bundle_dir), bundle_dir, observer - + return 0 if result.passed else 1 -def certified_run( - *, - fixture: Any, - workflow: Workflow, - bundle_dir: Path, - out: Path, - params: dict[str, Any], - open_browser: BrowserFactory, - observer: OpenEMREnvironmentObserver, - state_code: Optional[str], - run_id: str, - on_ready: Optional[Callable[[Any], None]] = None, -) -> tuple[RunReport, dict[str, Any], dict[str, float]]: - """One ordinary Standard-profile run of the certified bundle.""" - before = sql_patients(fixture) - marks: dict[str, float] = {"start": time.time()} - - def authorize(arg: Any) -> Any: - _verifier, gate_kwargs = arg - gate = evaluate_run_gate(workflow, **gate_kwargs) - _write_json(out / "run-gate.json", gate.model_dump(mode="json")) - if not gate.passed: - raise QualificationError(gate.render()) - return build_runtime_authorization(workflow, gate, params=params) - - report = run_once( - fixture=fixture, - workflow=workflow, - bundle_dir=bundle_dir, - run_dir=out / "run", - params=params, - open_browser=open_browser, - authorization_for=authorize, - fault_driver=None, - environment_observer=observer, - run_id=run_id, - on_ready=on_ready, - marks=marks, - ) - time.sleep(1.0) - oracle = sql_oracle( - fixture, - before=before, - params=params, - expected_fields=expected_sql_fields(params, state_code), - oracle_kind="exact_record", - report=report, - ) - marks["oracle_done"] = time.time() - render_run_report(out / "run") - return report, oracle, marks +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/tests/test_cli_attach_seal.py b/packages/flow/tests/test_cli_attach_seal.py new file mode 100644 index 000000000..bdcbda09e --- /dev/null +++ b/packages/flow/tests/test_cli_attach_seal.py @@ -0,0 +1,36 @@ +"""``attach-seal`` reaches its own handler, not the bundle ``seal`` command. + +``__main__`` used to define ``_cmd_seal`` twice; the bundle ``seal`` handler +won and ``attach-seal`` died with ``AttributeError: 'Namespace' object has no +attribute 'source'``. +""" + +from __future__ import annotations + +from pathlib import Path + +import openadapt_flow.__main__ as cli +from openadapt_flow import cloud_seal + + +def test_attach_seal_calls_attach_from_cloud(tmp_path, monkeypatch, capsys): + calls = [] + + def fake_attach(run_dir: Path, *, execution_id: str, cloud_url=None): + calls.append((run_dir, execution_id, cloud_url)) + return cloud_seal.AttachResult( + attached=True, code="attached", execution_id=execution_id + ) + + monkeypatch.setattr(cloud_seal, "attach_from_cloud", fake_attach) + code = cli.main(["attach-seal", str(tmp_path), "--execution-id", "exec-123"]) + assert code == 0 + assert calls == [(tmp_path, "exec-123", None)] + assert "exec-123" in capsys.readouterr().out + + +def test_attach_seal_and_seal_have_distinct_handlers(): + parser = cli.build_parser() + attach = parser.parse_args(["attach-seal", "run", "--execution-id", "e"]) + assert attach.func is cli._cmd_attach_seal + assert attach.func is not cli._cmd_seal diff --git a/packages/flow/tests/test_desktop_check_it_cli_mockmed.py b/packages/flow/tests/test_desktop_check_it_cli_mockmed.py index 0629fab90..67d1908f2 100644 --- a/packages/flow/tests/test_desktop_check_it_cli_mockmed.py +++ b/packages/flow/tests/test_desktop_check_it_cli_mockmed.py @@ -44,6 +44,13 @@ from openadapt_flow.compiler import compile_recording # noqa: E402 from openadapt_flow.ir import RunReport, Workflow # noqa: E402 from openadapt_flow.mockmed.fault_server import serve # noqa: E402 +from openadapt_flow.qualify.environment import ( # noqa: E402 + WEB_OBSERVER_CONTRACT_SHA256, + WEB_OBSERVER_ID, + WebEnvironmentObserver, + web_environment_digest, + web_environment_payload, +) from openadapt_flow.qualification import ( # noqa: E402 ActionRiskClass, ActionRiskClassification, @@ -197,10 +204,13 @@ def check_it(tmp_path_factory: pytest.TempPathFactory): ) save = consequential_step(workflow) app_digest = demo._tree_digest(REPO_ROOT / "openadapt_flow" / "mockmed") + app_version = f"sha256:{app_digest}" entry_url = f"{base_url.rstrip('/')}/{TUTORIAL_ENTRY_QUERY}" backend, close = PlaywrightBackend.launch(entry_url, headless=True) try: - payload = demo._environment_payload(backend=backend, app_digest=app_digest) + payload = web_environment_payload( + backend, application_version=app_version, headless=True + ) finally: close() init_project( @@ -209,12 +219,10 @@ def check_it(tmp_path_factory: pytest.TempPathFactory): target_kind="web", application="OpenAdapt MockMed synthetic reference", application_identity=demo._origin(base_url), - application_version=f"sha256:{app_digest}", - environment_observer_id=demo.ENVIRONMENT_OBSERVER_ID, - environment_observer_contract_sha256=( - demo.ENVIRONMENT_OBSERVER_CONTRACT_SHA256 - ), - environment_digest=demo._sha256_bytes(demo._canonical_json(payload)), + application_version=app_version, + environment_observer_id=WEB_OBSERVER_ID, + environment_observer_contract_sha256=WEB_OBSERVER_CONTRACT_SHA256, + environment_digest=web_environment_digest(payload), runtime_version=demo.FLOW_VERSION, required_capabilities=[ "headless_chromium", @@ -307,11 +315,11 @@ def check_it(tmp_path_factory: pytest.TempPathFactory): assert drills.returncode == 0, drills.stdout + drills.stderr campaign = json.loads(drills.stdout) - # MockMed's own environment observer stands in for the generic one. + # The declarative web observer stands in for the generic one. observer_patch.setattr( "openadapt_flow.qualification_environment." "BackendQualificationEnvironmentObserver", - lambda _backend: demo._MockMedEnvironmentObserver(app_digest=app_digest), + lambda _backend: WebEnvironmentObserver(application_version=app_version), ) def run_case(case_id: str) -> SimpleNamespace: diff --git a/packages/flow/tests/test_fixup_passes.py b/packages/flow/tests/test_fixup_passes.py new file mode 100644 index 000000000..0a1c0c513 --- /dev/null +++ b/packages/flow/tests/test_fixup_passes.py @@ -0,0 +1,190 @@ +"""The four default compiler fix-up passes, on web and pixel surfaces.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +from PIL import Image, ImageDraw + +from openadapt_flow.compiler.fixup_passes import ( + COMPILE_NOTES_FILE, + apply_fixup_passes, + region_unchanged, + write_compile_notes, +) +from openadapt_flow.ir import ( + ActionKind, + Anchor, + PostconditionKind, + Step, + StructuralLocator, +) + + +def _png(highlight: tuple[int, int, int, int] | None = None) -> bytes: + image = Image.new("RGB", (200, 100), "white") + if highlight is not None: + ImageDraw.Draw(image).rectangle(highlight, fill=(30, 90, 200)) + buf = io.BytesIO() + image.save(buf, format="PNG") + return buf.getvalue() + + +def _anchor(role: str | None = None, region=(10, 10, 80, 20)) -> Anchor: + return Anchor( + template="templates/x.png", + region=region, + click_point=(region[0] + 5, region[1] + 5), + structural=StructuralLocator(selector=f"#{role}", role=role) if role else None, + ) + + +def _click(sid: str, role: str | None = None, **kw) -> Step: + return Step(id=sid, intent=sid, action=ActionKind.CLICK, anchor=_anchor(role), **kw) + + +def _type(sid: str, text: str = "Mr.") -> Step: + return Step(id=sid, intent=sid, action=ActionKind.TYPE, text=text) + + +def _key(sid: str, key: str) -> Step: + return Step(id=sid, intent=sid, action=ActionKind.KEY, key=key) + + +def _run(steps, frames, tmp_path: Path, surface=None, **kw): + return apply_fixup_passes( + steps, + frames, + tmp_path, + crop_png=lambda png, region: png, + phash_png=lambda png, region=None: "0" * 16, + surface=surface, + **kw, + ) + + +def _changes(notes): + return [(n["change"], n["step"]) for n in notes] + + +def test_select_commit_enter_dropped_on_web_only(tmp_path): + steps = [_click("s1", "combobox"), _type("s2"), _key("s3", "Enter")] + out, notes = _run(steps, {}, tmp_path) + assert [s.id for s in out] == ["s1", "s2"] + assert ("drop_select_commit_enter", "s3") in _changes(notes) + + out, notes = _run(steps, {}, tmp_path, surface="rdp") + assert [s.id for s in out] == ["s1", "s2", "s3"] + assert "drop_select_commit_enter" not in {c for c, _ in _changes(notes)} + + +def test_select_all_on_empty_field_needs_unchanged_pixels(tmp_path): + steps = [_click("s1", "textbox"), _key("s2", "ctrl+a"), _type("s3")] + blank = _png() + out, notes = _run(steps, {"s2": (blank, blank), "s1": (blank, blank)}, tmp_path) + assert [s.id for s in out] == ["s1", "s3"] + assert ("drop_select_all_on_empty_field", "s2") in _changes(notes) + + # The field had text: Ctrl+A highlighted it, so it is kept. + selected = _png((12, 12, 85, 28)) + out, _ = _run(steps, {"s2": (blank, selected)}, tmp_path) + assert [s.id for s in out] == ["s1", "s2", "s3"] + + # No frames, no evidence: kept. + out, _ = _run(steps, {}, tmp_path) + assert [s.id for s in out] == ["s1", "s2", "s3"] + + +def test_region_unchanged_tolerates_a_caret(): + before = _png() + caret = _png((40, 12, 40, 26)) + assert region_unchanged(before, caret, (10, 5, 180, 30)) + assert not region_unchanged(before, _png((10, 10, 90, 30)), (10, 10, 80, 20)) + assert not region_unchanged(None, before, (10, 10, 80, 20)) + + +def test_field_click_gets_region_stable(tmp_path): + frame = _png() + steps = [_click("s1", "textbox"), _type("s2")] + out, notes = _run(steps, {"s1": (frame, frame)}, tmp_path) + assert out[0].expect[0].kind is PostconditionKind.REGION_STABLE + assert out[0].expect[0].template == "templates/s1_expect.png" + assert (tmp_path / "templates" / "s1_expect.png").is_file() + assert steps[0].expect == [] # input not modified + assert ("focused_field_region_stable", "s1") in _changes(notes) + + +def test_pixel_surface_field_click_detected_by_following_type(tmp_path): + frame = _png() + steps = [_click("s1"), _type("s2"), _click("s3")] + out, notes = _run( + steps, {"s1": (frame, frame), "s3": (frame, frame)}, tmp_path, "rdp" + ) + assert out[0].expect and not out[2].expect + assert _changes(notes) == [("focused_field_region_stable", "s1")] + + +def test_unlabeled_button_gets_identifier_crop(tmp_path): + frame = _png() + steps = [_click("s1", "button")] + out, notes = _run( + steps, {"s1": (frame, frame)}, tmp_path, pixel_verify_enabled=True + ) + assert out[0].anchor.identifier_crop == "templates/identifiers/s1.png" + assert out[0].identity_armed + assert ("static_control_identifier_crop", "s1") in _changes(notes) + + labeled = [_click("s1", "button")] + labeled[0].anchor.context_text = "Create New Patient" + out, notes = _run( + labeled, {"s1": (frame, frame)}, tmp_path, pixel_verify_enabled=True + ) + assert out[0].anchor.identifier_crop is None and notes == [] + + +def test_button_crop_only_where_the_runtime_can_verify_it(tmp_path): + # With the runtime's pixel VERIFY off (the default), the pixel tier can + # reject a crop but never accept one, so a crop-only identity check would + # halt an irreversible step on every run. The pass leaves the step as + # compiled instead, on every surface. + frame = _png() + for surface in (None, "web", "windows", "rdp", "citrix"): + steps = [_click("s1", "button")] + out, notes = _run(steps, {"s1": (frame, frame)}, tmp_path, surface=surface) + assert out[0].anchor.identifier_crop is None, surface + assert not out[0].identity_armed, surface + assert notes == [], surface + + +def test_button_crop_follows_each_steps_own_surface(tmp_path): + # A bundle whose steps run on several surfaces asks per step. The + # capability here depends only on the runtime flag, so both steps agree; + # the hook is what a multi-surface compile passes. + frame = _png() + steps = [_click("s1", "button"), _click("s2", "button")] + kinds = {"s1": "web", "s2": "windows"} + seen: list[str] = [] + + def kind_for(step): + seen.append(step.id) + return kinds[step.id] + + out, _notes = _run( + steps, + {"s1": (frame, frame), "s2": (frame, frame)}, + tmp_path, + pixel_verify_enabled=True, + surface_kind_for=kind_for, + ) + assert seen == ["s1", "s2"] + assert all(step.identity_armed for step in out) + + +def test_compile_notes_written_and_cleared(tmp_path): + write_compile_notes(tmp_path, [{"change": "x", "step": "s1"}]) + data = json.loads((tmp_path / COMPILE_NOTES_FILE).read_text()) + assert data["changes"][0]["change"] == "x" + write_compile_notes(tmp_path, []) + assert not (tmp_path / COMPILE_NOTES_FILE).exists() diff --git a/packages/flow/tests/test_identity_evidence_capability.py b/packages/flow/tests/test_identity_evidence_capability.py new file mode 100644 index 000000000..1ce373b76 --- /dev/null +++ b/packages/flow/tests/test_identity_evidence_capability.py @@ -0,0 +1,142 @@ +"""Identity evidence must be verifiable by the runtime that runs the step. + +An irreversible step whose identity check can't reach ``verified`` halts. These +tests pin the capability table the compiler consults, and the runtime verdicts +for a step whose only identity evidence is a pixel crop: no surrounding text, +so neither the structured tier nor the OCR tier can judge it. +""" + +from __future__ import annotations + +from pathlib import Path + +import cv2 +import numpy as np + +from openadapt_flow.ir import ( + ActionKind, + Anchor, + Resolution, + Step, + StepResult, + Workflow, +) +from openadapt_flow.runtime import identity as I +from openadapt_flow.runtime.replayer import Replayer + +_H, _W = I.PIXEL_CANON + + +def _png(arr: np.ndarray) -> bytes: + ok, buf = cv2.imencode(".png", arr) + assert ok + return buf.tobytes() + + +def _button(val: int = 255) -> np.ndarray: + """A plain control render with one dark bar standing in for its glyphs.""" + img = np.full((_H, _W), val, np.uint8) + img[:, 60:66] = 0 + return img + + +def _with_extra_bar(img: np.ndarray) -> np.ndarray: + other = img.copy() + other[:, 140:145] = 0 + return other + + +class _Backend: + viewport = (_W, _H) + + def screenshot(self): + return _png(_button()) + + def click(self, x, y, *, double=False): ... + + def wait_settled(self, **kw): + return self.screenshot() + + +def _crop_only_step(tmp_path: Path, recorded: np.ndarray) -> tuple[Step, Resolution]: + (tmp_path / "templates").mkdir(parents=True, exist_ok=True) + (tmp_path / "templates" / "id.png").write_bytes(_png(recorded)) + (tmp_path / "templates" / "t.png").write_bytes(_png(recorded)) + step = Step( + id="s_open", + intent="click 'Open next referral'", + action=ActionKind.CLICK, + risk="irreversible", + identity_armed=True, + anchor=Anchor( + template="templates/t.png", + region=(0, 0, _W, _H), + click_point=(0, 0), + identifier_crop="templates/id.png", + identifier_region=(0, 0, _W, _H), + ), + ) + assert not step.anchor.context_text and not step.anchor.structured_identity + res = Resolution(rung="structural", point=(0, 0), confidence=1.0, elapsed_ms=1.0) + return step, res + + +def _gate(tmp_path: Path, step: Step, res: Resolution, live: np.ndarray, **kw): + replayer = Replayer(_Backend(), vision=object(), poll_interval_s=0.01, **kw) + result = StepResult( + step_id=step.id, intent=step.intent, ok=False, risk="irreversible" + ) + error = replayer._identity_gate_error( + step, res, _png(live), {}, Workflow(name="wf"), tmp_path, result + ) + return error, result.identity + + +def test_capability_table() -> None: + default = I.verifiable_identity_evidence("web") + assert "identifier_crop" not in default + assert default == {"structured_text", "context_band"} + assert I.verifiable_identity_evidence(None) == default + assert I.verifiable_identity_evidence("rdp") == {"context_band"} + assert I.verifiable_identity_evidence("citrix") == {"context_band"} + for kind in ("web", "windows", "macos", "linux", "rdp", "citrix"): + assert "identifier_crop" in I.verifiable_identity_evidence( + kind, pixel_verify_enabled=True + ) + + +def test_no_surrounding_text_valid_crop_verifies_where_crops_are_verifiable( + tmp_path: Path, +) -> None: + step, res = _crop_only_step(tmp_path, _button()) + error, check = _gate(tmp_path, step, res, _button(), pixel_verify_enabled=True) + assert error is None + assert check is not None + assert check.status == "verified" and check.mode == "pixel" + + +def test_crop_that_does_not_match_halts(tmp_path: Path) -> None: + # A different control under the point: the pixel tier MISMATCHES whether + # or not its VERIFY path is on, and the step halts before acting. + step, res = _crop_only_step(tmp_path, _button()) + for enabled in (True, False): + error, check = _gate( + tmp_path, + step, + res, + _with_extra_bar(_button()), + pixel_verify_enabled=enabled, + ) + assert error is not None and "refusing to act" in error, enabled + assert check is not None and check.status == "mismatch", enabled + + +def test_crop_the_runtime_cannot_verify_still_halts(tmp_path: Path) -> None: + # The runtime never accepts without evidence it can verify: with pixel + # VERIFY off, a matching crop abstains, nothing else can judge the + # target, and the irreversible step halts. This is why the compiler + # doesn't attach such a crop. + step, res = _crop_only_step(tmp_path, _button()) + error, check = _gate(tmp_path, step, res, _button()) + assert error is not None + assert check is not None and check.status == "unreadable" diff --git a/packages/flow/tests/test_qualification_result.py b/packages/flow/tests/test_qualification_result.py new file mode 100644 index 000000000..8027b8a09 --- /dev/null +++ b/packages/flow/tests/test_qualification_result.py @@ -0,0 +1,390 @@ +"""openadapt.verify/v1 parsing and openadapt.qualification-result/v1.""" + +from __future__ import annotations + +import json +from base64 import b64decode +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openadapt_flow.qualification_admission import ( + SIGNATURE_DOMAIN, + QualificationAdmissionEnvelope, + QualificationAdmissionError, + expected_from_payload, + verify_qualification_admission, +) +from openadapt_flow.qualification_dev_signer import ( + LOCAL_DEV_PRIVATE_KEY, + local_dev_signer_trust, +) +from openadapt_flow.qualify import QualifyConfigError, parse_verify_config +from openadapt_flow.qualify.result import ( + build_result, + case_entry, + local_admission, + result_sha256, + verify_config_sha256, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr +from openadapt_flow.verification import VerificationTier + +H = "b" * 64 +BASE = { + "schema_version": "openadapt.verify/v1", + "app": {"name": "Demo", "version": "1", "url": "http://127.0.0.1:1/"}, + "verifier": { + "kind": "rest", + "base_url": "http://127.0.0.1:2", + "records_path": "/r", + }, +} + + +def test_pre_rfc_shape_gets_a_migration_hint(): + with pytest.raises(QualifyConfigError, match="schema_version: openadapt.verify/v1"): + parse_verify_config( + {"version": 1, "app": BASE["app"], "system_of_record": BASE["verifier"]}, + env={}, + ) + + +def test_effects_list_is_the_effect_contract(): + config = parse_verify_config( + { + **BASE, + "minimum_effect_strength": "independent_session", + "effects": [ + { + "kind": "record_written", + "match": {"note": {"param": "note"}}, + "expected_count": 1, + "count_new_only": True, + } + ], + }, + env={}, + ) + assert config.effects[0].kind is EffectKind.RECORD_WRITTEN + assert config.minimum_effect_strength == "independent_session" + + +def test_effects_and_record_are_exclusive(): + with pytest.raises(QualifyConfigError, match="either effects or record"): + parse_verify_config( + { + **BASE, + "effects": [{"kind": "record_written", "match": {"a": {"param": "a"}}}], + "record": {"match": ["a"], "fields": {"a": "a"}}, + }, + env={}, + ) + + +def test_verify_config_sha256_binds_identity_and_effects(): + effect = Effect(kind=EffectKind.RECORD_WRITTEN, match={"a": ValueExpr(param="a")}) + one = verify_config_sha256("sha256:" + H, [effect]) + assert one.startswith("sha256:") and len(one) == 71 + assert one != verify_config_sha256("sha256:" + "c" * 64, [effect]) + other = effect.model_copy(update={"expected_count": 2}) + assert one != verify_config_sha256("sha256:" + H, [other]) + + +def _report(outcome: str, success: bool) -> SimpleNamespace: + return SimpleNamespace(execution_outcome=outcome, success=success, model_calls=0) + + +def _entry( + tmp_path: Path, + case_id: str, + outcome: str, + oracle: dict, + trial=1, + tier=VerificationTier.INDEPENDENT_SYSTEM, + substrate="rest", +): + path = tmp_path / f"{case_id}-{trial}.json" + path.write_text("{}") + return case_entry( + case_id=case_id, + trial=trial, + report=_report(outcome, outcome == "VERIFIED"), + oracle=oracle, + verifier_tier=tier, + report_path=path, + verifier_substrate=substrate, + ) + + +EXACT = { + "observed": "exact_record", + "independent": True, + "effect_verdict": "CONFIRMED", + "landed": "exact", +} +NONE = { + "observed": "no_mutation", + "independent": True, + "effect_verdict": "REFUTED", + "landed": "none", +} + + +def test_case_entries_map_read_back_to_verdicts(tmp_path): + good = _entry(tmp_path, "representative", "VERIFIED", EXACT) + assert good["observed_verdict"] == "CONFIRMED" and good["outcome"] == "verified" + assert good["passed"] and not good["silent_incorrect_success"] + + halted = _entry(tmp_path, "fault-ambiguity", "HALTED", NONE) + assert halted["expected_verdict"] == halted["observed_verdict"] == "REFUTED" + assert halted["outcome"] == "halted_before_effect" and halted["passed"] + + # A fault the engine let through: the record landed. + leaked = _entry(tmp_path, "fault-wrong-identity", "VERIFIED", EXACT) + assert leaked["silent_incorrect_success"] and not leaked["passed"] + + over = _entry(tmp_path, "representative", "HALTED", NONE) + assert over["over_halt"] and not over["passed"] + + screen = _entry( + tmp_path, "representative", "VERIFIED", {**EXACT, "independent": False} + ) + assert screen["verification_tier"] == "unchecked" and not screen["passed"] + + +def _qreport(passed=True): + return SimpleNamespace( + model_dump=lambda **_: { + "schema_version": "openadapt.qualification-report/v1", + "workflow_name": "Register Jane Doe", + "workflow_contract_sha256": H, + "environment_contract_sha256": H, + "project_id": "p", + "project_revision": 2, + "policy_name": "clinical-write", + "passed": passed, + "case_count": 6, + "passed_case_count": 6, + "refusals": [ + {"code": "case_not_passed", "path": "cases", "message": "free text"} + ], + "generated_at": "2026-10-17T05:00:00Z", + } + ) + + +def _result(tmp_path, trials=3): + cases = [] + for trial in range(1, trials + 1): + cases.append(_entry(tmp_path, "representative", "VERIFIED", EXACT, trial)) + for case_id in ( + "fault-ambiguity", + "fault-wrong-identity", + "fault-stale-identity", + "fault-weak-effect", + "fault-missing-effect", + ): + cases.append(_entry(tmp_path, case_id, "HALTED", NONE, trial)) + return build_result( + report=_qreport(), + verify_sha256="sha256:" + H, + verifier_identity="sha256:" + H, + fault_driver_version="abc", + cases=cases, + ) + + +def test_result_drops_free_text_and_counts(tmp_path): + result = _result(tmp_path) + assert result["schema_version"] == "openadapt.qualification-result/v1" + assert "workflow_name" not in result + assert result["refusals"] == [{"code": "case_not_passed", "path": "cases"}] + assert result["passed"] + assert result["silent_incorrect_success_count"] == 0 + assert result["over_halt_count"] == 0 + assert result["executor_model_calls"] == 0 + assert result["verifier_model_calls"] == 0 + assert "model_calls" not in result + assert len(result["cases"]) == 18 + changed = {**result, "generated_at": "2030-01-01T00:00:00Z"} + assert result_sha256(changed) == result_sha256(result) + + +def test_admission_binds_the_result_digest(tmp_path): + result = _result(tmp_path) + envelope = local_admission( + result=result, + result_bytes=b"{}", + oracle_id="rest", + campaign_id="demo-campaign-v1", + trials_required=3, + bundle_content_digest=H, + effect_contract_sha256=H, + identity_contract_sha256=None, + ) + assert isinstance(envelope, QualificationAdmissionEnvelope) + campaign = envelope.payload.campaign + assert campaign.outcomes_sha256 == result_sha256(result) + assert campaign.oracle_contract_sha256 == H + assert campaign.oracle_id == "rest" + assert {"over_halt", "silent_incorrect_success"} <= set(campaign.failure_taxonomy) + # Signed with the local-dev key over the existing admission domain ... + LOCAL_DEV_PRIVATE_KEY.public_key().verify( + b64decode(envelope.signature), + SIGNATURE_DOMAIN + envelope.payload.canonical_bytes(), + ) + # ... which no production trust map accepts. + with pytest.raises(QualificationAdmissionError, match="local-dev"): + verify_qualification_admission( + envelope, + trusted_signers=local_dev_signer_trust(), + expected=expected_from_payload(envelope.payload), + ) + + +def test_admission_refuses_fewer_than_three_trials(tmp_path): + with pytest.raises(ValueError, match="at least 3 trials per case"): + local_admission( + result=_result(tmp_path, trials=1), + result_bytes=b"{}", + oracle_id="rest", + campaign_id="demo-campaign-v1", + trials_required=1, + bundle_content_digest=H, + effect_contract_sha256=H, + identity_contract_sha256=None, + ) + + +@pytest.mark.parametrize("substrate", ["rest", "sql", "fhir", "graphql", "document"]) +def test_system_of_record_reads_are_independent(tmp_path, substrate): + entry = _entry(tmp_path, "representative", "VERIFIED", EXACT, substrate=substrate) + assert entry["verification_tier"] == "independent_readback" + assert entry["outcome"] == "verified" and entry["passed"] + + +def test_a_separate_ui_session_is_screen_readback(tmp_path): + """Flow #492 ui_readback reads through the UI: never independent_readback.""" + + entry = _entry( + tmp_path, + "representative", + "VERIFIED", + EXACT, + tier=VerificationTier.INDEPENDENT_SESSION, + substrate="ui-readback", + ) + assert entry["verification_tier"] == "screen_readback" + assert entry["outcome"] == "reconciliation_required" + assert not entry["passed"] + # A plugin that claims tier 1 but reads through the UI is still the screen. + plugin = _entry( + tmp_path, "representative", "VERIFIED", EXACT, substrate="ui_readback" + ) + assert plugin["verification_tier"] == "screen_readback" and not plugin["passed"] + + +def test_certification_needs_the_normal_case_verified_at_independent_readback(tmp_path): + cases = [ + _entry( + tmp_path, + "representative", + "VERIFIED", + EXACT, + tier=VerificationTier.INDEPENDENT_SESSION, + substrate="ui-readback", + ) + ] + [ + _entry(tmp_path, case_id, "HALTED", NONE) + for case_id in ( + "fault-ambiguity", + "fault-wrong-identity", + "fault-stale-identity", + "fault-weak-effect", + "fault-missing-effect", + ) + ] + result = build_result( + report=_qreport(passed=True), + verify_sha256="sha256:" + H, + verifier_identity="sha256:" + H, + fault_driver_version="abc", + cases=cases, + ) + assert not result["passed"] + + +def test_digest_is_over_the_effective_effects_not_the_file(): + """A record shorthand and its expanded effects list are one contract.""" + + from openadapt_flow.qualify.campaign import record_effects + + record = parse_verify_config( + { + **BASE, + "record": { + "match": ["email"], + "duplicate_key": "email", + "fields": {"email": "email", "lname": "lname"}, + }, + }, + env={}, + ) + expanded = record_effects(record) + listed = parse_verify_config( + {**BASE, "effects": [e.model_dump(mode="json") for e in expanded]}, env={} + ) + identity = "sha256:" + H + assert verify_config_sha256(identity, expanded) == verify_config_sha256( + identity, listed.effects + ) + + +def test_a_fault_passes_only_when_nothing_landed_and_the_engine_halted(tmp_path): + ok = _entry(tmp_path, "fault-ambiguity", "HALTED", NONE) + assert ok["passed"] + # The store shows nothing, but the engine did not halt (it reported success). + not_halted = _entry(tmp_path, "fault-ambiguity", "VERIFIED", NONE) + assert not not_halted["passed"] and not_halted["silent_incorrect_success"] + # Not halted and not verified either (the run failed some other way). + failed = _entry(tmp_path, "fault-weak-effect", "FAILED", NONE) + assert not failed["passed"] + # The engine halted, but part of a write landed: REFUTED, not "nothing". + partial = {**NONE, "observed": "unexpected_mutation", "landed": "some"} + landed = _entry(tmp_path, "fault-stale-identity", "HALTED", partial) + assert landed["observed_verdict"] == "REFUTED" + assert landed["outcome"] == "reconciliation_required" + assert not landed["passed"] + + +def test_refusal_paths_carry_only_ids(tmp_path): + report = _qreport() + dumped = report.model_dump() + dumped["refusals"] = [ + {"code": "step_identity_unarmed", "path": "steps.step_040.identity_armed", + "step_id": "step_040", "message": "free text"}, + {"code": "case_not_passed", "path": "qualification.cases.fault-ambiguity.results", + "case_id": "fault-ambiguity", "message": "free text"}, + {"code": "policy_violation", "path": "steps.jane.doe@example.invalid", + "step_id": "jane.doe@example.invalid", "message": "free text"}, + {"code": "policy_violation", "path": "workflow note: Jane Doe, DOB 1971", + "message": "free text"}, + ] # fmt: skip + report = SimpleNamespace(model_dump=lambda **_: dumped) + result = build_result( + report=report, + verify_sha256="sha256:" + H, + verifier_identity="sha256:" + H, + fault_driver_version="abc", + cases=[], + ) + paths = {r["path"] for r in result["refusals"]} + assert "steps.step_040.identity_armed" in paths + assert "qualification.cases.fault-ambiguity.results" in paths + assert "steps.redacted" in paths and "redacted" in paths + text = json.dumps(result["refusals"]) + assert ( + "example.invalid" not in text and "Jane" not in text and "free text" not in text + ) diff --git a/packages/flow/tests/test_qualify_declarative.py b/packages/flow/tests/test_qualify_declarative.py new file mode 100644 index 000000000..b5c13c533 --- /dev/null +++ b/packages/flow/tests/test_qualify_declarative.py @@ -0,0 +1,268 @@ +"""verify.yaml, readiness checks, the fault driver's offline paths and the CLI.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openadapt_flow.__main__ import QUALIFY_DEFINITION, _qualify_default_form +from openadapt_flow.ir import ActionKind, Anchor, Step, StructuralLocator, Workflow +from openadapt_flow.qualify import ( + QualifyConfigError, + QualifyError, + WebFaultDriver, + parse_verify_config, + qualify, +) +from openadapt_flow.qualify.campaign import ( + apply_verify_config, + check_ready, + record_effects, + resolve_save_step, +) +from openadapt_flow.qualify.faults import ScreenOnlyRecordCheck +from openadapt_flow.verification import VerificationTier + +H = "a" * 64 +APP = {"name": "Demo", "version": "1", "url": "http://127.0.0.1:9301/"} + + +def _config(**over): + data = { + "schema_version": "openadapt.verify/v1", + "app": {**APP, "reset": {"command": "true"}}, + "verifier": { + "kind": "rest", + "base_url": "http://127.0.0.1:9300", + "records_path": "/api/patient", + "records_key": "data", + }, + "record": { + "name": "patient", + "match": ["email"], + "duplicate_key": "email", + "fields": {"email": "email", "lname": "lname", "state": {"literal": "MA"}}, + }, + } + data.update(over) + return parse_verify_config(data, env={}) + + +def _workflow(*, armed=True, risk="irreversible") -> Workflow: + save = Step( + id="s3", + intent="Create patient", + action=ActionKind.CLICK, + anchor=Anchor( + template="templates/s3.png", + region=(10, 10, 80, 20), + click_point=(20, 20), + structural=StructuralLocator(selector="#create", role="button"), + context_text="Create New Patient", + ), + identity_armed=armed, + risk=risk, + ) + return Workflow( + name="register", + steps=[ + Step(id="s1", intent="email", action=ActionKind.TYPE, param="email"), + Step(id="s2", intent="name", action=ActionKind.TYPE, param="lname"), + save, + ], + params={"email": "a@example.test", "lname": "Doe"}, + ) + + +# -- verify.yaml -------------------------------------------------------------- +def test_minimal_config_parses_with_defaults(): + config = _config() + assert config.campaign.trials == 3 + assert config.campaign.policy == "clinical-write" + assert len(config.campaign.cases) == 6 + assert config.save_step == "auto" + + +def test_env_expansion_and_missing_variables(): + data = { + "schema_version": "openadapt.verify/v1", + "app": { + **APP, + "url": "${UI:-http://127.0.0.1:1/}", + "reset": {"command": "true"}, + }, + "verifier": { + "kind": "rest", + "base_url": "${API}", + "records_path": "/r", + }, + } + config = parse_verify_config(data, env={"API": "http://127.0.0.1:2"}) + assert config.app.url == "http://127.0.0.1:1/" + with pytest.raises(QualifyConfigError, match="API"): + parse_verify_config(data, env={}) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + ({"verifier": {"kind": "onscreen"}}, "verifier"), + ( + {"record": {"name": "p", "match": ["email"], "fields": {"lname": "lname"}}}, + "match", + ), + ( + { + "record": { + "name": "p", + "match": ["email"], + "duplicate_key": "mrn", + "fields": {"email": "email"}, + } + }, + "duplicate_key", + ), + ({"campaign": {"cases": ["representative", "fault-typo"]}}, "fault-typo"), + ({"app": {**APP, "reset": {"command": "a", "python": "m:f"}}}, "reset"), + ], +) +def test_config_errors_name_the_problem(change, message): + with pytest.raises(QualifyConfigError, match=message): + _config(**change) + + +# -- binding and readiness ---------------------------------------------------- +def test_record_effects_bind_selector_fields_and_duplicate_key(): + effects = record_effects(_config()) + kinds = [e.kind.value for e in effects] + assert kinds[0] == "record_written" + assert set(effects[0].match) == {"email"} + assert effects[0].idempotency_key is not None + assert kinds.count("field_equals") == 2 # lname, state + + +def test_auto_save_step_and_ready_campaign(): + workflow = _workflow() + config = _config() + save, notes = apply_verify_config(workflow, config) + assert save.id == "s3" and save.effects + problems = check_ready(workflow, config, save, dict(workflow.params)) + assert problems == [] + + +def test_check_ready_lists_every_problem(): + workflow = _workflow(armed=False) + config = _config(app=APP) # no reset + save, _ = apply_verify_config(workflow, config) + problems = " | ".join(check_ready(workflow, config, save, {"email": "a@x"})) + assert "reset" in problems + assert "lname" in problems + assert "identity" in problems + + +def test_no_irreversible_step_is_a_clear_error(): + workflow = _workflow(risk="reversible") + workflow.steps[2].risk = "reversible" + with pytest.raises(QualifyConfigError, match="save_step"): + resolve_save_step(workflow, _config()) + + +def test_qualify_refuses_non_web_target_before_running(tmp_path): + bundle = tmp_path / "bundle" + _workflow().save(bundle) + config = _config(app={**APP, "target": "rdp", "reset": {"command": "true"}}) + with pytest.raises(QualifyConfigError, match="rdp"): + qualify(bundle, config, out=tmp_path / "out", log=lambda _l: None) + + +def test_qualify_refuses_existing_output(tmp_path): + bundle = tmp_path / "bundle" + _workflow().save(bundle) + (tmp_path / "out").mkdir() + with pytest.raises(QualifyError, match="already exists"): + qualify(bundle, _config(), out=tmp_path / "out", log=lambda _l: None) + + +# -- fault driver ------------------------------------------------------------- +GATES = {"weak_effect": "effect_strength", "missing_effect": "effect_verifier"} + + +def _context(kind, backend=None): + return SimpleNamespace( + fault_kind=kind, + backend=backend, + actuation_path="gui", + effect_verifier=object(), + effects=[], + project_id="p", + project_revision=1, + project_contract_sha256=H, + campaign_id_sha256=H, + case_id_sha256=H, + case_input_sha256=H, + run_id_sha256=H, + step_id="s3", + gate=GATES.get(kind, "identity_verification"), + before_input_sha256=H, + ) + + +def test_screen_faults_decline_without_a_browser(): + driver = WebFaultDriver(_workflow()) + for kind in ("ambiguity", "wrong_identity", "stale_identity"): + assert driver.mutate(_context(kind, backend=SimpleNamespace())) is None + assert driver.declined == "screen_fault_needs_a_browser" + + +def test_effect_faults_replace_the_verifier(): + driver = WebFaultDriver(_workflow()) + weak = driver.mutate(_context("weak_effect")) + assert weak.replace_effect_verifier + assert isinstance(weak.effect_verifier, ScreenOnlyRecordCheck) + assert weak.effect_verifier.verification_tier is VerificationTier.IMMEDIATE_SCREEN + missing = driver.mutate(_context("missing_effect")) + assert missing.replace_effect_verifier and missing.effect_verifier is None + assert missing.receipt is not None + + +# -- CLI ---------------------------------------------------------------------- +def test_default_form_routes_bundle_to_campaign(): + assert _qualify_default_form(["qualify", "b", "--verify", "v.yaml"])[:3] == [ + "qualify", + "campaign", + "b", + ] + assert _qualify_default_form(["qualify", "run", "x"]) == ["qualify", "run", "x"] + assert _qualify_default_form(["run", "b"]) == ["run", "b"] + + +def test_qualify_help_opens_with_the_definition(): + out = subprocess.run( + [sys.executable, "-m", "openadapt_flow", "qualify", "--help"], + capture_output=True, + text=True, + check=True, + ).stdout + assert out.splitlines()[0] == QUALIFY_DEFINITION + assert "QUALIFICATION_TERMS.md" in out + + +def test_cli_reports_config_errors(tmp_path): + bundle = tmp_path / "bundle" + _workflow().save(bundle) + (tmp_path / "verify.yaml").write_text( + "schema_version: openadapt.verify/v1\napp: {name: d, version: '1', url: 'http://x/'}\n" + ) + result = subprocess.run( + [sys.executable, "-m", "openadapt_flow", "qualify", str(bundle), + "--verify", str(tmp_path / "verify.yaml"), "--out", str(tmp_path / "o")], + capture_output=True, + text=True, + cwd=Path(__file__).resolve().parents[1], + ) # fmt: skip + assert result.returncode == 2 + assert "verifier" in result.stderr