From e387c81cd46f4ea96d79de075567a4fac2ccdfe9 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 15:22:15 -0400 Subject: [PATCH 1/3] flow: keep the server's answer to a save, labelled same_screen When the browser sees the server answer a save request (POST, PUT, PATCH, DELETE), the replayer keeps the answer on the save step (StepResult.server_responses) and the receipt counts it (server_responses: observed, 2xx, check_level same_screen). Kept: method, status, a digest of the endpoint (no query string), a digest and size of the body, the media type. Never a body, URL, header or value. The answer comes from the same transaction as the save, so on its own it never confirms an effect and never makes a run VERIFIED; the configured verifier still decides. Results and receipts without answers keep their exact bytes. Co-Authored-By: Claude Opus 5.5 --- packages/flow/claims.yaml | 22 ++ .../flow/docs/APPROVAL_AND_VERIFICATION.md | 7 + packages/flow/docs/EFFECT_KIT.md | 17 + packages/flow/docs/VERIFICATION.md | 19 +- packages/flow/docs/verification.json | 30 +- .../backends/_server_responses.py | 132 ++++++++ .../backends/playwright_backend.py | 12 + packages/flow/openadapt_flow/ir.py | 34 ++ packages/flow/openadapt_flow/receipt.py | 49 +++ .../flow/openadapt_flow/runtime/replayer.py | 36 +++ packages/flow/tests/test_run_receipt.py | 3 + packages/flow/tests/test_server_responses.py | 293 ++++++++++++++++++ 12 files changed, 651 insertions(+), 3 deletions(-) create mode 100644 packages/flow/openadapt_flow/backends/_server_responses.py create mode 100644 packages/flow/tests/test_server_responses.py diff --git a/packages/flow/claims.yaml b/packages/flow/claims.yaml index 4aa65fc2e1..f0097c70bc 100644 --- a/packages/flow/claims.yaml +++ b/packages/flow/claims.yaml @@ -415,6 +415,28 @@ claims: prove where keyboard focus is; it checks the search box shows the query before it submits. + - id: server-response-evidence + claim: >- + In a browser, the server's answer to a save request is kept on the step + and counted on the receipt at check level same_screen, with no body, + URL or value. It comes from the same transaction as the save, so on its + own it never confirms an effect or makes a run VERIFIED. + surfaces: [docs] + tier: supported + evidence: + - path: tests/test_server_responses.py + proves: >- + Only answers to write requests are kept, as method, status and + digests with no URL, query or body text; a real Chromium save + produces one; a 201 answer with an unconfirmed read-back still ends + RECONCILIATION_REQUIRED and not billable; the receipt counts answers + without changing its evidence tier; results without answers + serialize as before. + caveats: + - >- + Only the browser backend sees server answers. An answer that arrives + after the step's checks finish is not kept. + # --------------------------------------------------- identity gate / halt - id: identity-gate-halt-armed claim: >- diff --git a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md index a6dae48ca3..09e355769f 100644 --- a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md +++ b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md @@ -196,6 +196,13 @@ older scales keep their published numbers and map onto it: | 1 `same_screen` | Only the acting screen shows the value | 4 `IMMEDIATE_SCREEN` | 0 `VISUAL` | `screen_readback` | No | | 0 `unchecked` | No read, unreachable or stale store, no tier | (no confirmed tier) | (no Seal) | `unchecked` | No | +The server's own answer to a save, when the browser sees it, is kept too +(`StepResult.server_responses`: method, status, digests, never a body or a +URL; counted on the receipt as `server_responses`). It ranks `same_screen`: +it comes from the same transaction as the save, so on its own it never +confirms a save, never makes a run `VERIFIED`, and never makes a save clean. +A 500 behind a "Saved" banner is still worth seeing. + `VerificationTier` counts the other way (lower is stronger), so compare checks by `.check_level`, not by the tier number. A Seal at `VISUAL` reads as `same_screen`, because the Seal does not record whether the screen was diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 9d05d42976..4a5359f2d2 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -696,6 +696,23 @@ live Windows VM or the Docker RDP fixture. - Citrix is a separate Workspace window named in `session`. Opening a new Citrix session for each check is the operator's launcher, not Flow's. +## The server's own answer (kept, labelled `same_screen`) + +In a browser, a save usually sends a write request (POST, PUT, PATCH, +DELETE) and the server answers it. Flow keeps that answer on the save step +(`StepResult.server_responses`) and counts it on the receipt +(`server_responses`: observed, 2xx, `check_level: same_screen`). It keeps the +method, the status, a digest of the endpoint (origin and path, no query +string), a digest and size of the body, and the media type. It never keeps a +body, a URL, a header or a value. + +The answer comes from the same transaction as the save. A server can say +`201 Created` and store the wrong value, or store it under another record, or +roll it back afterwards. So the answer is evidence at check level +`same_screen`, never a check: on its own it never confirms an effect and +never makes a run `VERIFIED`. The configured verifier still decides. Only the +browser backend sees answers; desktop, RDP and Citrix runs have none. + ## Configuration reference (`deployment.yaml` → `effects:`) Complete commented example: [`docs/deployment.example.yaml`](./deployment.example.yaml). diff --git a/packages/flow/docs/VERIFICATION.md b/packages/flow/docs/VERIFICATION.md index 41dee8727c..2306fba41c 100644 --- a/packages/flow/docs/VERIFICATION.md +++ b/packages/flow/docs/VERIFICATION.md @@ -2,7 +2,7 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims.yaml`. Do not edit by hand — edit the registry and regenerate. -- Generated at: **2026-10-18T13:20:41-04:00 (git HEAD commit date)** +- Generated at: **2026-10-18T15:02:08-04:00 (git HEAD commit date)** - JUnit pass check: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) - Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). - Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. @@ -12,7 +12,7 @@ ## What is bound to required CI vs. being validated -- **Bound to required CI pass evidence (14):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `ui-readback-separate-session`, `desktop-readback-separate-session`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported`, `compose-recorded-bundles` +- **Bound to required CI pass evidence (15):** `web-supported`, `deterministic-zero-model-replay`, `effect-verification-silent-writes`, `ui-readback-separate-session`, `desktop-readback-separate-session`, `server-response-evidence`, `identity-gate-halt-armed`, `identifier-crop-compile-emission`, `halt-teach-promote`, `typed-business-decisions-supported`, `reviewed-judgment-cases-supported`, `qualified-remote-decision-v2-supported`, `mockmed-benchmark-ci-reproducible`, `substrate-runtime-validation-supported`, `compose-recorded-bundles` - **Being validated — opt-in / infra-gated or field (8):** `effect-verifier-kit`, `windows-desktop-validating`, `macos-native-validating`, `linux-native-validating`, `rdp-validating`, `desktop-recording-validating`, `citrix-pixel-validating`, `openemr-field-benchmark` - **Roadmap / research (1):** `win32-window-replay-roadmap` @@ -143,6 +143,21 @@ - The WinForms fixture runs as a UI Automation stand-in that mirrors patient_notes.ps1 and its pn_db.py persistence, and the kiosk as pixels drawn at the Tk layout; neither is the live Windows VM or the Docker RDP fixture in this evidence. On the pixel path OpenAdapt cannot prove where keyboard focus is; it checks the search box shows the query before it submits. +### `server-response-evidence` — supported — bound to required CI pass evidence + +> In a browser, the server's answer to a save request is kept on the step and counted on the receipt at check level same_screen, with no body, URL or value. It comes from the same transaction as the save, so on its own it never confirms an effect or makes a run VERIFIED. + +- Surfaces: docs +- Strongest evidence strength: **supported** (tier is `supported`) + +| Backing evidence | Kind | Gating / CI stage | Strength | Proves | +|---|---|---|---|---| +| `tests/test_server_responses.py` | test | ci (required PR gate (test)) | supported | Only answers to write requests are kept, as method, status and digests with no URL, query or body text; a real Chromium save produces one; a 201 answer with an unconfirmed read-back still ends RECONCILIATION_REQUIRED and not billable; the receipt counts answers without changing its evidence tier; results without answers serialize as before. | + +**Caveats (honest limits):** + +- Only the browser backend sees server answers. An answer that arrives after the step's checks finish is not kept. + ### `identity-gate-halt-armed` — supported — bound to required CI pass evidence > On ambiguity the runtime halts instead of guessing, and an identity gate refuses a wrong-entity (wrong-patient) click — on ARMED steps only. diff --git a/packages/flow/docs/verification.json b/packages/flow/docs/verification.json index 35763c9f79..a7a9e5a32f 100644 --- a/packages/flow/docs/verification.json +++ b/packages/flow/docs/verification.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-10-18T13:20:41-04:00 (git HEAD commit date)", + "generated_at": "2026-10-18T15:02:08-04:00 (git HEAD commit date)", "green_check_run": false, "green_check_job": null, "green_check_scope": [], @@ -578,6 +578,34 @@ ], "errors": [] }, + { + "id": "server-response-evidence", + "claim": "In a browser, the server's answer to a save request is kept on the step and counted on the receipt at check level same_screen, with no body, URL or value. It comes from the same transaction as the save, so on its own it never confirms an effect or makes a run VERIFIED.", + "tier": "supported", + "reproducibility": null, + "surfaces": [ + "docs" + ], + "strongest_evidence": "supported", + "caveats": [ + "Only the browser backend sees server answers. An answer that arrives after the step's checks finish is not kept." + ], + "evidence": [ + { + "path": "tests/test_server_responses.py", + "kind": "test", + "exists": true, + "strength": "supported", + "gating": "ci (required PR gate (test))", + "node": null, + "node_found": null, + "ci_job": "test", + "junit_status": null, + "proves": "Only answers to write requests are kept, as method, status and digests with no URL, query or body text; a real Chromium save produces one; a 201 answer with an unconfirmed read-back still ends RECONCILIATION_REQUIRED and not billable; the receipt counts answers without changing its evidence tier; results without answers serialize as before." + } + ], + "errors": [] + }, { "id": "identity-gate-halt-armed", "claim": "On ambiguity the runtime halts instead of guessing, and an identity gate refuses a wrong-entity (wrong-patient) click \u2014 on ARMED steps only.", diff --git a/packages/flow/openadapt_flow/backends/_server_responses.py b/packages/flow/openadapt_flow/backends/_server_responses.py new file mode 100644 index 0000000000..d1c4f9e111 --- /dev/null +++ b/packages/flow/openadapt_flow/backends/_server_responses.py @@ -0,0 +1,132 @@ +"""The server's answers to write requests, as the browser sees them. + +When a person or a program presses Save in a web application, the browser +usually sends a write request (POST, PUT, PATCH, DELETE) and the server +answers it. That answer is worth keeping: a 500 behind a green "Saved" +banner is a clear sign something went wrong. It is not proof that the record +is right, because it comes from the same transaction as the save. The +replayer keeps it on the step as evidence at check level ``same_screen``, +and only a read-back decides whether the save is confirmed. + +:class:`ServerResponseLog` listens to one Playwright page. It keeps a bounded +log of answers to write requests and returns them as +:class:`~openadapt_flow.ir.ServerResponseEvidence`: method, status, digests +of the endpoint and the body, the body's size and media type. It never keeps +a body, a URL, a header, or a query string. +""" + +from __future__ import annotations + +import hashlib +import threading +from collections import deque +from typing import Any, Optional +from urllib.parse import urlsplit + +from openadapt_flow.ir import ServerResponseEvidence + +WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) + +#: Answers kept in the log; older ones drop off. +_MAX_LOG = 64 +#: Bodies above this size are not read to compute a digest. +_MAX_BODY_BYTES = 5 * 1024 * 1024 + + +def _endpoint_digest(url: str) -> str: + parts = urlsplit(url) + endpoint = f"{parts.scheme}://{parts.netloc}{parts.path}".lower() + return hashlib.sha256(endpoint.encode("utf-8")).hexdigest() + + +class ServerResponseLog: + """A bounded log of answers to write requests on one page. + + The Playwright event handler only stores the response object; the body is + read later, outside the handler, because a sync handler must not block on + the page. + """ + + def __init__(self, page: Any) -> None: + self._lock = threading.Lock() + self._log: deque[tuple[int, Any]] = deque(maxlen=_MAX_LOG) + self._seq = 0 + self.armed = False + try: + page.on("response", self._on_response) + self.armed = True + except Exception: # noqa: BLE001 - evidence only + self.armed = False + + def _on_response(self, response: Any) -> None: + try: + method = str(response.request.method).upper() + except Exception: # noqa: BLE001 + return + if method not in WRITE_METHODS: + return + with self._lock: + self._seq += 1 + self._log.append((self._seq, response)) + + def mark(self) -> int: + """The log position now; answers after it belong to what comes next.""" + + with self._lock: + return self._seq + + def since(self, mark: int) -> list[ServerResponseEvidence]: + """Answers logged after ``mark``, as evidence (oldest first).""" + + with self._lock: + entries = [(seq, r) for seq, r in self._log if seq > mark] + out: list[ServerResponseEvidence] = [] + for _seq, response in entries: + evidence = _evidence(response) + if evidence is not None: + out.append(evidence) + return out + + +def _evidence(response: Any) -> Optional[ServerResponseEvidence]: + try: + method = str(response.request.method).upper() + status = int(response.status) + url = str(response.url) + except Exception: # noqa: BLE001 - a response we can't describe is skipped + return None + if method not in WRITE_METHODS or not 100 <= status <= 599: + return None + content_type: Optional[str] = None + try: + headers = response.headers or {} + raw = headers.get("content-type") + if raw: + content_type = str(raw).split(";", 1)[0].strip().lower()[:100] or None + except Exception: # noqa: BLE001 + content_type = None + body_sha256: Optional[str] = None + body_bytes: Optional[int] = None + try: + length = int((response.headers or {}).get("content-length") or 0) + except (TypeError, ValueError): + length = 0 + if length <= _MAX_BODY_BYTES: + try: + body = response.body() + except Exception: # noqa: BLE001 - the page moved on; no body + body = None + if isinstance(body, (bytes, bytearray)) and len(body) <= _MAX_BODY_BYTES: + body_sha256 = hashlib.sha256(bytes(body)).hexdigest() + body_bytes = len(body) + return ServerResponseEvidence( + method=method, # type: ignore[arg-type] + status=status, + endpoint_sha256=_endpoint_digest(url), + body_sha256=body_sha256, + body_bytes=body_bytes, + content_type=content_type, + ) + + +__all__ = ["ServerResponseLog", "WRITE_METHODS"] diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index 7741310397..1d997960ce 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -21,6 +21,7 @@ from urllib.parse import urlsplit from openadapt_flow.backends._page_quiescence import PageQuiescence +from openadapt_flow.backends._server_responses import ServerResponseLog if TYPE_CHECKING: # pragma: no cover from playwright.sync_api import Page @@ -1090,6 +1091,17 @@ def __init__( # openadapt_flow.backends._page_quiescence). Armed at construction: # network requests issued before this point are not tracked. self._quiescence = PageQuiescence(page) + # The server's answers to write requests, kept as save evidence (see + # openadapt_flow.backends._server_responses). Never a check. + self._server_responses = ServerResponseLog(page) + + def server_response_mark(self) -> int: + """Where the log of server answers to write requests stands now.""" + return self._server_responses.mark() + + def server_responses_since(self, mark: int) -> list[Any]: + """Server answers to write requests made after ``mark``.""" + return self._server_responses.since(mark) @property def viewport(self) -> tuple[int, int]: diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 7ae47ffe36..7425348283 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3521,6 +3521,34 @@ class EffectFieldCheck(BaseModel): verification_tier: Optional[int] = None +class ServerResponseEvidence(BaseModel): + """The application server's answer to a save request, as the browser saw it. + + Kept on the step as evidence at check level ``same_screen``: it comes from + the same transaction as the save, so on its own it never confirms an + effect and never makes a run VERIFIED. It holds no body, URL, or record + value: the method, the status, and digests. + """ + + model_config = ConfigDict(extra="forbid") + + method: Literal["POST", "PUT", "PATCH", "DELETE"] + status: int = Field(ge=100, le=599) + #: SHA-256 of the request's origin and path (no query string). + endpoint_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + #: SHA-256 of the response body, when the browser still had it. + body_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + body_bytes: Optional[int] = Field(default=None, ge=0) + #: The response's media type (``application/json``), without parameters. + content_type: Optional[str] = Field(default=None, max_length=100) + #: Where this evidence ranks (:mod:`openadapt_flow.check_level`). + check_level: Literal["same_screen"] = "same_screen" + + @property + def ok(self) -> bool: + return 200 <= self.status < 300 + + class StepResult(BaseModel): step_id: str intent: str @@ -3572,6 +3600,10 @@ class StepResult(BaseModel): safety_halt: bool = False effect_results: list[str] = Field(default_factory=list) effect_evidence: list[EffectVerificationEvidence] = Field(default_factory=list) + #: The server's answers to this step's write requests, when the browser + #: saw them. Evidence at check level ``same_screen``; never a check on its + #: own. Empty (and omitted when serialized) on every other backend. + server_responses: list[ServerResponseEvidence] = Field(default_factory=list) #: Field-by-field read-back comparisons; omitted from the report when empty. effect_field_checks: list[EffectFieldCheck] = Field(default_factory=list) #: Machine-readable terminal category. ``report.halt`` remains useful @@ -3654,6 +3686,8 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: data.pop("surface", None) if isinstance(data, dict) and not self.effect_field_checks: data.pop("effect_field_checks", None) + if isinstance(data, dict) and not self.server_responses: + data.pop("server_responses", None) return data diff --git a/packages/flow/openadapt_flow/receipt.py b/packages/flow/openadapt_flow/receipt.py index 4420a5c8c2..cd2d038778 100644 --- a/packages/flow/openadapt_flow/receipt.py +++ b/packages/flow/openadapt_flow/receipt.py @@ -130,6 +130,27 @@ _HOUR_UTC_RE = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:00:00Z$") +class ReceiptServerResponses(BaseModel): + """Counts of the server's answers to the run's save requests. + + Supporting evidence at check level ``same_screen``: the answers come from + the same transaction as the saves, so they never stand in for a read-back. + """ + + model_config = ConfigDict(extra="forbid") + + observed: ReceiptCount + #: Answers with a 2xx status. + ok: ReceiptCount + check_level: Literal["same_screen"] = "same_screen" + + @model_validator(mode="after") + def _ok_within_observed(self) -> "ReceiptServerResponses": + if self.ok > self.observed: + raise ValueError("server_responses.ok exceeds observed") + return self + + class ReceiptError(ValueError): """A receipt could not be built or written.""" @@ -166,6 +187,9 @@ class RunReceipt(BaseModel): #: :attr:`effect_tier_reached`. Optional so receipts written before it #: existed still validate (``exclude_none`` keeps their digest). effect_assurance: Optional[ReceiptAssurance] = None + #: The server's answers to the save requests, when the browser saw them. + #: Omitted when there were none, so older receipts keep their bytes. + server_responses: Optional[ReceiptServerResponses] = None authorization_required: ReceiptCount authorization_confirmed: ReceiptCount identity_required: ReceiptCount @@ -271,6 +295,8 @@ def _drop_unset_assurance(self, handler: Any) -> dict[str, Any]: data: dict[str, Any] = handler(self) if self.effect_assurance is None: data.pop("effect_assurance", None) + if self.server_responses is None: + data.pop("server_responses", None) return data def canonical_json(self, *, include_digest: bool = True) -> bytes: @@ -856,6 +882,17 @@ def _build_receipt( bundle_digest=bundle_digest, generated_at=_hour_utc(getattr(report, "started_at", None)), ) + answers = [ + response + for result in report.results + for response in getattr(result, "server_responses", None) or [] + ] + if answers: + payload["server_responses"] = { + "observed": len(answers), + "ok": sum(1 for answer in answers if answer.ok), + "check_level": "same_screen", + } except ValueError as exc: raise ReceiptError(f"run cannot produce a VERIFIED receipt: {exc}") from exc digest = hashlib.sha256( @@ -982,6 +1019,18 @@ def _receipt_rows(receipt: RunReceipt) -> list[tuple[str, str]]: if receipt.effect_assurance else [] ), + *( + [ + ( + "server answers", + f"{receipt.server_responses.ok} of " + f"{receipt.server_responses.observed} ok (same transaction; " + "not a check)", + ) + ] + if receipt.server_responses + else [] + ), ( "identity-armed", f"{receipt.identity_armed} of {receipt.identity_applicable}", diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 7d9e12e6c1..53b5f7366e 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -603,6 +603,10 @@ def better_identity_read(first: Any, retry: Any) -> Any: return first +#: Server answers kept per step (a save makes one or a few write requests). +_MAX_SERVER_RESPONSES = 16 + + class Replayer: """Replays a Workflow against a Backend using injected vision. @@ -6750,6 +6754,9 @@ def _run_step_on_active_surface( # System-of-record pre-state, snapshotted just before the action when # the step declares effects (see the block guarding self._act below). effect_pre_state: Optional[EffectState] = None + # Where the backend's log of server answers stood before the save, so + # the answers to this step's write requests can be kept as evidence. + server_mark: Optional[int] = None # The step's effect contracts with every ValueExpr bound to THIS run's # params (P0-3); resolved BEFORE the pre-state snapshot so verification # targets the record this run wrote, not the demonstration's. @@ -7200,6 +7207,8 @@ def _run_step_on_active_surface( else "safety_halt" ) break + if server_mark is None and effect_pre_state is not None: + server_mark = self._server_response_mark() try: self._active_delivery_resolution = resolution self._active_delivery_region = matched_region @@ -7554,6 +7563,11 @@ def _run_step_on_active_surface( # save, a phantom optimistic-UI success, a duplicate write, or a # lost update -- the EffectVerifier reads the REAL record and HALTs # on any non-CONFIRMED verdict (docs/design/EFFECT_VERIFIER.md). + if server_mark is not None: + # The server's own answer to the save: kept as evidence at a + # lower check level. It comes from the same transaction, so it + # never confirms an effect; the verifier below still decides. + self._collect_server_responses(result, server_mark) if (error is None or delivery_uncertain) and effect_pre_state is not None: effect_error = self._verify_effects( step, @@ -8714,6 +8728,28 @@ def _auto_readback_verifier(self) -> Any: self._readback_verifier = verifier return verifier + def _server_response_mark(self) -> Optional[int]: + """Where the backend's server-answer log stands, or None when the + backend doesn't see server answers (every non-browser backend).""" + + mark = getattr(self.backend, "server_response_mark", None) + if not callable(mark): + return None + try: + return int(mark()) + except Exception: # noqa: BLE001 - evidence only, never a failure + return None + + def _collect_server_responses(self, result: StepResult, mark: int) -> None: + since = getattr(self.backend, "server_responses_since", None) + if not callable(since): + return + try: + responses = list(since(mark)) + except Exception: # noqa: BLE001 - evidence only, never a failure + return + result.server_responses = responses[:_MAX_SERVER_RESPONSES] + def _verify_effects( self, step: Step, diff --git a/packages/flow/tests/test_run_receipt.py b/packages/flow/tests/test_run_receipt.py index e539bfc74c..cb2e7ec8da 100644 --- a/packages/flow/tests/test_run_receipt.py +++ b/packages/flow/tests/test_run_receipt.py @@ -80,6 +80,9 @@ "evidence_classes", "effect_tier_reached", "effect_assurance", + # Counts of the server's answers to save requests (same transaction; never + # a check). Counts and a fixed label only. + "server_responses", "authorization_required", "authorization_confirmed", "identity_required", diff --git a/packages/flow/tests/test_server_responses.py b/packages/flow/tests/test_server_responses.py new file mode 100644 index 0000000000..0fd48fad21 --- /dev/null +++ b/packages/flow/tests/test_server_responses.py @@ -0,0 +1,293 @@ +"""The server's own answer to a save, kept as evidence and labelled honestly. + +When the browser sees the server answer a save request, the answer is kept on +the step and counted on the receipt at check level ``same_screen``. It comes +from the same transaction as the save, so on its own it never confirms an +effect and never makes a run VERIFIED. No body, URL or value is kept. +""" + +from __future__ import annotations + +import hashlib +import http.server +import threading +from pathlib import Path +from typing import Any, Iterator + +import pytest + +from openadapt_flow.backends._server_responses import ServerResponseLog +from openadapt_flow.ir import ( + ActionKind, + Postcondition, + PostconditionKind, + ServerResponseEvidence, + Step, + Workflow, +) +from openadapt_flow.receipt import build_receipt +from openadapt_flow.runtime.effects import ( + Effect, + EffectKind, + EffectState, + EffectVerdict, + Verdict, +) +from openadapt_flow.runtime.replayer import Replayer +from tests.test_replayer import FakeBackend, FakeVision, Match + +BODY = b'{"id": "enc-7", "status": "saved"}' + + +# -- the log, on fakes ------------------------------------------------------------- + + +class _Request: + def __init__(self, method: str) -> None: + self.method = method + + +class _Response: + def __init__(self, method: str, status: int, url: str, body: bytes) -> None: + self.request = _Request(method) + self.status = status + self.url = url + self.headers = {"content-type": "application/json; charset=utf-8"} + self._body = body + + def body(self) -> bytes: + return self._body + + +class _Page: + def __init__(self) -> None: + self.handlers: dict[str, Any] = {} + + def on(self, event: str, handler: Any) -> None: + self.handlers[event] = handler + + +def test_only_answers_to_write_requests_are_kept_without_values() -> None: + page = _Page() + log = ServerResponseLog(page) + emit = page.handlers["response"] + emit(_Response("GET", 200, "https://emr.test/patients/MRN-1", b"x")) + mark = log.mark() + emit(_Response("POST", 201, "https://emr.test/api/encounters?patient=MRN-1", BODY)) + emit(_Response("GET", 200, "https://emr.test/api/encounters", b"[]")) + [answer] = log.since(mark) + assert answer.method == "POST" and answer.status == 201 and answer.ok + assert answer.body_sha256 == hashlib.sha256(BODY).hexdigest() + assert answer.content_type == "application/json" + assert answer.check_level == "same_screen" + dumped = answer.model_dump_json() + for leak in ("MRN-1", "emr.test", "enc-7", "patient="): + assert leak not in dumped + # The query string is not part of the endpoint digest. + other = _Response("POST", 201, "https://emr.test/api/encounters", BODY) + emit(other) + assert log.since(mark)[1].endpoint_sha256 == answer.endpoint_sha256 + + +def test_an_answer_whose_body_is_gone_keeps_its_status() -> None: + page = _Page() + log = ServerResponseLog(page) + + class Gone(_Response): + def body(self) -> bytes: + raise RuntimeError("navigated") + + page.handlers["response"](Gone("PUT", 500, "https://emr.test/api/x", b"")) + [answer] = log.since(0) + assert answer.status == 500 and not answer.ok and answer.body_sha256 is None + + +# -- the replayer -------------------------------------------------------------------- + + +class AnsweringBackend(FakeBackend): + """A fake backend whose save gets a server answer, as a browser would see.""" + + def __init__(self, status: int = 201) -> None: + super().__init__() + self.status = status + self.answers: list[ServerResponseEvidence] = [] + + def server_response_mark(self) -> int: + return len(self.answers) + + def server_responses_since(self, mark: int) -> list[ServerResponseEvidence]: + return self.answers[mark:] + + def press(self, key: str) -> None: + super().press(key) + self.answers.append( + ServerResponseEvidence( + method="POST", + status=self.status, + endpoint_sha256="e" * 64, + body_sha256=hashlib.sha256(BODY).hexdigest(), + body_bytes=len(BODY), + content_type="application/json", + ) + ) + + +class _Verifier: + """A verifier that can't read the record: the save is unconfirmed.""" + + substrate = "rest" + verification_tier = 1 + + def capture_pre_state(self, context: Any = None) -> EffectState: + return EffectState(substrate="rest", reachable=True, records=[]) + + def verify(self, effect: Effect, before: EffectState, context: Any = None): + return EffectVerdict( + verdict=Verdict.INDETERMINATE, + kind=effect.kind, + substrate="rest", + reason="record store unreachable", + unavailable=True, + ) + + +def _workflow() -> Workflow: + return Workflow( + name="save", + steps=[ + Step( + id="save", + intent="save encounter", + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + expect=[ + Postcondition( + kind=PostconditionKind.TEXT_PRESENT, text="Saved", timeout_s=0.2 + ) + ], + effects=[ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"patient_id": "p1"}, + expected_count=1, + timeout_s=0.0, + ) + ], + ) + ], + ) + + +def test_a_201_from_the_server_never_makes_the_run_verified(tmp_path: Path) -> None: + vision = FakeVision() + vision.text_results = { + "Saved": Match(point=(50, 10), region=(30, 5, 40, 10), confidence=0.9) + } + bundle = tmp_path / "bundle" + (bundle / "templates").mkdir(parents=True) + backend = AnsweringBackend(201) + report = Replayer( + backend, vision=vision, effect_verifier=_Verifier(), poll_interval_s=0.01 + ).run(_workflow(), bundle_dir=bundle, run_dir=tmp_path / "run") + [result] = report.results + assert [a.status for a in result.server_responses] == [201] + assert result.server_responses[0].check_level == "same_screen" + # The server said "created"; the record read couldn't confirm it. + assert report.execution_outcome != "VERIFIED" + assert report.transaction_outcome == "RECONCILIATION_REQUIRED" + assert report.transaction_billable is False + saved = (tmp_path / "run" / "report.json").read_text() + assert '"server_responses"' in saved + + +def test_steps_without_answers_serialize_as_before() -> None: + from openadapt_flow.ir import StepResult + + assert ( + "server_responses" + not in StepResult(step_id="a", intent="b", ok=True).model_dump_json() + ) + + +def test_the_receipt_counts_answers_as_supporting_evidence() -> None: + from tests.test_run_receipt import _report + + report = _report() + plain = build_receipt(report) + assert "server_responses" not in plain.model_dump() + report.results[-1].server_responses = [ + ServerResponseEvidence(method="POST", status=201, endpoint_sha256="e" * 64), + ServerResponseEvidence(method="POST", status=500, endpoint_sha256="e" * 64), + ] + receipt = build_receipt(report) + assert receipt.server_responses is not None + assert (receipt.server_responses.observed, receipt.server_responses.ok) == (2, 1) + assert receipt.server_responses.check_level == "same_screen" + # Evidence strength still comes from the read-back alone. + assert receipt.effect_tier_reached == plain.effect_tier_reached + + +# -- a real browser ------------------------------------------------------------------- + + +@pytest.fixture() +def save_server() -> Iterator[str]: + page = ( + b"" + ) + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args: object) -> None: + return None + + def do_GET(self) -> None: # noqa: N802 + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.end_headers() + self.wfile.write(page) + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length") or 0) + self.rfile.read(length) + self.send_response(201) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(BODY))) + self.end_headers() + self.wfile.write(BODY) + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}/" + finally: + server.shutdown() + server.server_close() + + +def test_the_browser_backend_keeps_the_answer_to_a_real_save(save_server: str) -> None: + sync_api = pytest.importorskip("playwright.sync_api") + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + with sync_api.sync_playwright() as pw: + try: + browser = pw.chromium.launch(headless=True) + except Exception as exc: # pragma: no cover - env specific + pytest.skip(f"no chromium: {exc}") + try: + page = browser.new_page() + backend = PlaywrightBackend(page) + page.goto(save_server) + mark = backend.server_response_mark() + page.click("#save") + page.wait_for_function("document.body.innerText.includes('Saved')") + [answer] = backend.server_responses_since(mark) + finally: + browser.close() + assert answer.method == "POST" and answer.status == 201 + assert answer.body_sha256 == hashlib.sha256(BODY).hexdigest() + assert "MRN-9" not in answer.model_dump_json() From 354cbc3dcd52cef5adc4278e85d702f76c7ac654 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 15:31:20 -0400 Subject: [PATCH 2/3] flow: server answer evidence review fixes - Keep only method, status and media type: no endpoint or body digests (a digest of a short path or body can be guessed). - Never read a response body, so a streaming answer can't hold up a run. - Count only the page's own fetch, XHR and form writes to its origin, sent after the save began; never beacons, other origins, requests already in flight, or service-worker answers. - "Not an error" is a status below 400: a form save often redirects. - Keep up to 64 answers per step. Co-Authored-By: Claude Opus 5.5 --- packages/flow/claims.yaml | 28 +++-- .../flow/docs/APPROVAL_AND_VERIFICATION.md | 4 +- packages/flow/docs/EFFECT_KIT.md | 15 ++- packages/flow/docs/VERIFICATION.md | 8 +- packages/flow/docs/verification.json | 8 +- .../backends/_server_responses.py | 117 ++++++++++-------- .../backends/playwright_backend.py | 3 +- packages/flow/openadapt_flow/ir.py | 13 +- packages/flow/openadapt_flow/receipt.py | 3 +- .../flow/openadapt_flow/runtime/replayer.py | 2 +- packages/flow/tests/test_server_responses.py | 71 ++++++----- 11 files changed, 146 insertions(+), 126 deletions(-) diff --git a/packages/flow/claims.yaml b/packages/flow/claims.yaml index 976b7508f5..afbbb462c9 100644 --- a/packages/flow/claims.yaml +++ b/packages/flow/claims.yaml @@ -419,25 +419,29 @@ claims: - id: server-response-evidence claim: >- - In a browser, the server's answer to a save request is kept on the step - and counted on the receipt at check level same_screen, with no body, - URL or value. It comes from the same transaction as the save, so on its - own it never confirms an effect or makes a run VERIFIED. + In a browser, the server's answer to a save's own write request is + kept on the step and counted on the receipt at check level + same_screen, as method, status and media type only. It comes from the + same transaction as the save, so on its own it never confirms an effect + or makes a run VERIFIED. surfaces: [docs] tier: supported evidence: - path: tests/test_server_responses.py proves: >- - Only answers to write requests are kept, as method, status and - digests with no URL, query or body text; a real Chromium save - produces one; a 201 answer with an unconfirmed read-back still ends - RECONCILIATION_REQUIRED and not billable; the receipt counts answers - without changing its evidence tier; results without answers - serialize as before. + Only same-origin fetch, XHR and form writes sent after the mark are + kept, never beacons, other origins, requests already in flight or + service-worker answers, and never a URL, path or body; a real + Chromium save produces one; a 201 answer with an unconfirmed + read-back still ends RECONCILIATION_REQUIRED and not billable; a + redirect counts as not an error; the receipt counts answers without + changing its evidence tier; results without answers serialize as + before. caveats: - >- - Only the browser backend sees server answers. An answer that arrives - after the step's checks finish is not kept. + Only the browser backend sees server answers. A save that writes to + another origin (a separate API host) leaves none. An answer that + arrives after the screen postconditions is not kept. # --------------------------------------------------- identity gate / halt - id: identity-gate-halt-armed diff --git a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md index 09e355769f..641a6485a6 100644 --- a/packages/flow/docs/APPROVAL_AND_VERIFICATION.md +++ b/packages/flow/docs/APPROVAL_AND_VERIFICATION.md @@ -197,8 +197,8 @@ older scales keep their published numbers and map onto it: | 0 `unchecked` | No read, unreachable or stale store, no tier | (no confirmed tier) | (no Seal) | `unchecked` | No | The server's own answer to a save, when the browser sees it, is kept too -(`StepResult.server_responses`: method, status, digests, never a body or a -URL; counted on the receipt as `server_responses`). It ranks `same_screen`: +(`StepResult.server_responses`: method, status and media type, never a URL, +a body or a digest of one; counted on the receipt as `server_responses`). It ranks `same_screen`: it comes from the same transaction as the save, so on its own it never confirms a save, never makes a run `VERIFIED`, and never makes a save clean. A 500 behind a "Saved" banner is still worth seeing. diff --git a/packages/flow/docs/EFFECT_KIT.md b/packages/flow/docs/EFFECT_KIT.md index 562abc2372..fc41c93092 100644 --- a/packages/flow/docs/EFFECT_KIT.md +++ b/packages/flow/docs/EFFECT_KIT.md @@ -703,10 +703,17 @@ Citrix session. In a browser, a save usually sends a write request (POST, PUT, PATCH, DELETE) and the server answers it. Flow keeps that answer on the save step (`StepResult.server_responses`) and counts it on the receipt -(`server_responses`: observed, 2xx, `check_level: same_screen`). It keeps the -method, the status, a digest of the endpoint (origin and path, no query -string), a digest and size of the body, and the media type. It never keeps a -body, a URL, a header or a value. +(`server_responses`: observed, not errors, `check_level: same_screen`). It +keeps the method, the status and the media type. It never keeps a URL, a path, +a header, a body, or a digest of any of them (a digest of a short path or body +can be guessed), and it never reads a response body, so a streaming answer +can't hold up the run. + +Only the page's own write requests count: `fetch`, XHR and form submissions +to the page's origin, sent after the save began. A beacon, a request to +another origin (analytics, a widget), a request already in flight, and an +answer from a service worker are left out. "Not an error" means a status +below 400, because a form save often answers with a redirect. The answer comes from the same transaction as the save. A server can say `201 Created` and store the wrong value, or store it under another record, or diff --git a/packages/flow/docs/VERIFICATION.md b/packages/flow/docs/VERIFICATION.md index 62ff656f8f..62b77f8559 100644 --- a/packages/flow/docs/VERIFICATION.md +++ b/packages/flow/docs/VERIFICATION.md @@ -2,7 +2,7 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims.yaml`. Do not edit by hand — edit the registry and regenerate. -- Generated at: **2026-10-18T15:02:08-04:00 (git HEAD commit date)** +- Generated at: **2026-10-18T15:29:50-04:00 (git HEAD commit date)** - JUnit pass check: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) - Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). - Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. @@ -145,18 +145,18 @@ ### `server-response-evidence` — supported — bound to required CI pass evidence -> In a browser, the server's answer to a save request is kept on the step and counted on the receipt at check level same_screen, with no body, URL or value. It comes from the same transaction as the save, so on its own it never confirms an effect or makes a run VERIFIED. +> In a browser, the server's answer to a save's own write request is kept on the step and counted on the receipt at check level same_screen, as method, status and media type only. It comes from the same transaction as the save, so on its own it never confirms an effect or makes a run VERIFIED. - Surfaces: docs - Strongest evidence strength: **supported** (tier is `supported`) | Backing evidence | Kind | Gating / CI stage | Strength | Proves | |---|---|---|---|---| -| `tests/test_server_responses.py` | test | ci (required PR gate (test)) | supported | Only answers to write requests are kept, as method, status and digests with no URL, query or body text; a real Chromium save produces one; a 201 answer with an unconfirmed read-back still ends RECONCILIATION_REQUIRED and not billable; the receipt counts answers without changing its evidence tier; results without answers serialize as before. | +| `tests/test_server_responses.py` | test | ci (required PR gate (test)) | supported | Only same-origin fetch, XHR and form writes sent after the mark are kept, never beacons, other origins, requests already in flight or service-worker answers, and never a URL, path or body; a real Chromium save produces one; a 201 answer with an unconfirmed read-back still ends RECONCILIATION_REQUIRED and not billable; a redirect counts as not an error; the receipt counts answers without changing its evidence tier; results without answers serialize as before. | **Caveats (honest limits):** -- Only the browser backend sees server answers. An answer that arrives after the step's checks finish is not kept. +- Only the browser backend sees server answers. A save that writes to another origin (a separate API host) leaves none. An answer that arrives after the screen postconditions is not kept. ### `identity-gate-halt-armed` — supported — bound to required CI pass evidence diff --git a/packages/flow/docs/verification.json b/packages/flow/docs/verification.json index 3f5d9d9edd..bdd6047912 100644 --- a/packages/flow/docs/verification.json +++ b/packages/flow/docs/verification.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-10-18T15:02:08-04:00 (git HEAD commit date)", + "generated_at": "2026-10-18T15:29:50-04:00 (git HEAD commit date)", "green_check_run": false, "green_check_job": null, "green_check_scope": [], @@ -580,7 +580,7 @@ }, { "id": "server-response-evidence", - "claim": "In a browser, the server's answer to a save request is kept on the step and counted on the receipt at check level same_screen, with no body, URL or value. It comes from the same transaction as the save, so on its own it never confirms an effect or makes a run VERIFIED.", + "claim": "In a browser, the server's answer to a save's own write request is kept on the step and counted on the receipt at check level same_screen, as method, status and media type only. It comes from the same transaction as the save, so on its own it never confirms an effect or makes a run VERIFIED.", "tier": "supported", "reproducibility": null, "surfaces": [ @@ -588,7 +588,7 @@ ], "strongest_evidence": "supported", "caveats": [ - "Only the browser backend sees server answers. An answer that arrives after the step's checks finish is not kept." + "Only the browser backend sees server answers. A save that writes to another origin (a separate API host) leaves none. An answer that arrives after the screen postconditions is not kept." ], "evidence": [ { @@ -601,7 +601,7 @@ "node_found": null, "ci_job": "test", "junit_status": null, - "proves": "Only answers to write requests are kept, as method, status and digests with no URL, query or body text; a real Chromium save produces one; a 201 answer with an unconfirmed read-back still ends RECONCILIATION_REQUIRED and not billable; the receipt counts answers without changing its evidence tier; results without answers serialize as before." + "proves": "Only same-origin fetch, XHR and form writes sent after the mark are kept, never beacons, other origins, requests already in flight or service-worker answers, and never a URL, path or body; a real Chromium save produces one; a 201 answer with an unconfirmed read-back still ends RECONCILIATION_REQUIRED and not billable; a redirect counts as not an error; the receipt counts answers without changing its evidence tier; results without answers serialize as before." } ], "errors": [] diff --git a/packages/flow/openadapt_flow/backends/_server_responses.py b/packages/flow/openadapt_flow/backends/_server_responses.py index d1c4f9e111..83b3555c20 100644 --- a/packages/flow/openadapt_flow/backends/_server_responses.py +++ b/packages/flow/openadapt_flow/backends/_server_responses.py @@ -1,4 +1,4 @@ -"""The server's answers to write requests, as the browser sees them. +"""The server's answers to a save's write requests, as the browser sees them. When a person or a program presses Save in a web application, the browser usually sends a write request (POST, PUT, PATCH, DELETE) and the server @@ -8,16 +8,24 @@ replayer keeps it on the step as evidence at check level ``same_screen``, and only a read-back decides whether the save is confirmed. -:class:`ServerResponseLog` listens to one Playwright page. It keeps a bounded -log of answers to write requests and returns them as -:class:`~openadapt_flow.ir.ServerResponseEvidence`: method, status, digests -of the endpoint and the body, the body's size and media type. It never keeps -a body, a URL, a header, or a query string. +:class:`ServerResponseLog` listens to one Playwright page. It keeps answers +to write requests that: + +- the page's own code sent (``fetch``, XHR, or a form submission), not a + beacon, an image or a script; +- went to the page's own origin; +- were sent after the save began (requests already in flight are ignored); +- came from the network, not from a service worker (an offline-first app's + worker can answer a write that never reached the server). + +It keeps the method, the status and the media type, and whether the request +went to the page's origin. It never keeps a URL, a path, a header, a body or +a digest of any of them: a digest of a short path or body can be guessed. It +never reads a response body, so a streaming answer can't hold up the run. """ from __future__ import annotations -import hashlib import threading from collections import deque from typing import Any, Optional @@ -26,70 +34,84 @@ from openadapt_flow.ir import ServerResponseEvidence WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) - +#: Requests the page's own code makes: fetch, XHR, and form submissions. +_PAGE_REQUESTS = frozenset({"fetch", "xhr", "document"}) #: Answers kept in the log; older ones drop off. _MAX_LOG = 64 -#: Bodies above this size are not read to compute a digest. -_MAX_BODY_BYTES = 5 * 1024 * 1024 -def _endpoint_digest(url: str) -> str: +def _origin(url: str) -> str: parts = urlsplit(url) - endpoint = f"{parts.scheme}://{parts.netloc}{parts.path}".lower() - return hashlib.sha256(endpoint.encode("utf-8")).hexdigest() + return f"{parts.scheme}://{parts.netloc}".lower() class ServerResponseLog: - """A bounded log of answers to write requests on one page. - - The Playwright event handler only stores the response object; the body is - read later, outside the handler, because a sync handler must not block on - the page. - """ + """A bounded log of answers to the page's own write requests.""" def __init__(self, page: Any) -> None: + self._page = page self._lock = threading.Lock() - self._log: deque[tuple[int, Any]] = deque(maxlen=_MAX_LOG) + #: Sequence number of each tracked write request, by request object. + self._issued: dict[int, int] = {} self._seq = 0 + self._log: deque[tuple[int, ServerResponseEvidence]] = deque(maxlen=_MAX_LOG) self.armed = False try: + page.on("request", self._on_request) page.on("response", self._on_response) self.armed = True except Exception: # noqa: BLE001 - evidence only self.armed = False - def _on_response(self, response: Any) -> None: + def _on_request(self, request: Any) -> None: try: - method = str(response.request.method).upper() + method = str(request.method).upper() + kind = str(request.resource_type) except Exception: # noqa: BLE001 return - if method not in WRITE_METHODS: + if method not in WRITE_METHODS or kind not in _PAGE_REQUESTS: return with self._lock: self._seq += 1 - self._log.append((self._seq, response)) + self._issued[id(request)] = self._seq + if len(self._issued) > 4 * _MAX_LOG: + # Requests that never got an answer don't pile up. + oldest = sorted(self._issued.items(), key=lambda kv: kv[1]) + for key, _ in oldest[: len(oldest) - 2 * _MAX_LOG]: + self._issued.pop(key, None) + + def _on_response(self, response: Any) -> None: + try: + request = response.request + except Exception: # noqa: BLE001 + return + with self._lock: + seq = self._issued.pop(id(request), None) + if seq is None: + return + evidence = _evidence(self._page, response) + if evidence is None: + return + with self._lock: + self._log.append((seq, evidence)) def mark(self) -> int: - """The log position now; answers after it belong to what comes next.""" + """Requests sent after this mark belong to what comes next.""" with self._lock: return self._seq def since(self, mark: int) -> list[ServerResponseEvidence]: - """Answers logged after ``mark``, as evidence (oldest first).""" + """Answers to write requests sent after ``mark`` (oldest first).""" with self._lock: - entries = [(seq, r) for seq, r in self._log if seq > mark] - out: list[ServerResponseEvidence] = [] - for _seq, response in entries: - evidence = _evidence(response) - if evidence is not None: - out.append(evidence) - return out + return [evidence for seq, evidence in self._log if seq > mark] -def _evidence(response: Any) -> Optional[ServerResponseEvidence]: +def _evidence(page: Any, response: Any) -> Optional[ServerResponseEvidence]: try: + if getattr(response, "from_service_worker", False): + return None method = str(response.request.method).upper() status = int(response.status) url = str(response.url) @@ -97,34 +119,23 @@ def _evidence(response: Any) -> Optional[ServerResponseEvidence]: return None if method not in WRITE_METHODS or not 100 <= status <= 599: return None + try: + page_origin = _origin(str(page.url)) + except Exception: # noqa: BLE001 + return None + if _origin(url) != page_origin: + # Another origin: analytics, a third-party widget. Not the save. + return None content_type: Optional[str] = None try: - headers = response.headers or {} - raw = headers.get("content-type") + raw = (response.headers or {}).get("content-type") if raw: content_type = str(raw).split(";", 1)[0].strip().lower()[:100] or None except Exception: # noqa: BLE001 content_type = None - body_sha256: Optional[str] = None - body_bytes: Optional[int] = None - try: - length = int((response.headers or {}).get("content-length") or 0) - except (TypeError, ValueError): - length = 0 - if length <= _MAX_BODY_BYTES: - try: - body = response.body() - except Exception: # noqa: BLE001 - the page moved on; no body - body = None - if isinstance(body, (bytes, bytearray)) and len(body) <= _MAX_BODY_BYTES: - body_sha256 = hashlib.sha256(bytes(body)).hexdigest() - body_bytes = len(body) return ServerResponseEvidence( method=method, # type: ignore[arg-type] status=status, - endpoint_sha256=_endpoint_digest(url), - body_sha256=body_sha256, - body_bytes=body_bytes, content_type=content_type, ) diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index 1d997960ce..7a59245149 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -1100,7 +1100,8 @@ def server_response_mark(self) -> int: return self._server_responses.mark() def server_responses_since(self, mark: int) -> list[Any]: - """Server answers to write requests made after ``mark``.""" + """Server answers to the page's own write requests sent after + ``mark``; never blocks on a response body.""" return self._server_responses.since(mark) @property diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 7425348283..aeb3bbb51a 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3526,19 +3526,15 @@ class ServerResponseEvidence(BaseModel): Kept on the step as evidence at check level ``same_screen``: it comes from the same transaction as the save, so on its own it never confirms an - effect and never makes a run VERIFIED. It holds no body, URL, or record - value: the method, the status, and digests. + effect and never makes a run VERIFIED. It holds the method, the status and + the media type only: no URL, path, body, or digest of one (a digest of a + short path or body can be guessed). """ model_config = ConfigDict(extra="forbid") method: Literal["POST", "PUT", "PATCH", "DELETE"] status: int = Field(ge=100, le=599) - #: SHA-256 of the request's origin and path (no query string). - endpoint_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") - #: SHA-256 of the response body, when the browser still had it. - body_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") - body_bytes: Optional[int] = Field(default=None, ge=0) #: The response's media type (``application/json``), without parameters. content_type: Optional[str] = Field(default=None, max_length=100) #: Where this evidence ranks (:mod:`openadapt_flow.check_level`). @@ -3546,7 +3542,8 @@ class ServerResponseEvidence(BaseModel): @property def ok(self) -> bool: - return 200 <= self.status < 300 + """Not an error: 2xx, or a 3xx redirect after a form post.""" + return self.status < 400 class StepResult(BaseModel): diff --git a/packages/flow/openadapt_flow/receipt.py b/packages/flow/openadapt_flow/receipt.py index cd2d038778..229205bd3f 100644 --- a/packages/flow/openadapt_flow/receipt.py +++ b/packages/flow/openadapt_flow/receipt.py @@ -140,7 +140,8 @@ class ReceiptServerResponses(BaseModel): model_config = ConfigDict(extra="forbid") observed: ReceiptCount - #: Answers with a 2xx status. + #: Answers that were not errors (status below 400; a form save often + #: answers with a redirect). ok: ReceiptCount check_level: Literal["same_screen"] = "same_screen" diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 53b5f7366e..039313502c 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -604,7 +604,7 @@ def better_identity_read(first: Any, retry: Any) -> Any: #: Server answers kept per step (a save makes one or a few write requests). -_MAX_SERVER_RESPONSES = 16 +_MAX_SERVER_RESPONSES = 64 class Replayer: diff --git a/packages/flow/tests/test_server_responses.py b/packages/flow/tests/test_server_responses.py index 0fd48fad21..b45ac89fe9 100644 --- a/packages/flow/tests/test_server_responses.py +++ b/packages/flow/tests/test_server_responses.py @@ -8,7 +8,6 @@ from __future__ import annotations -import hashlib import http.server import threading from pathlib import Path @@ -43,23 +42,28 @@ class _Request: - def __init__(self, method: str) -> None: + def __init__(self, method: str, kind: str = "fetch") -> None: self.method = method + self.resource_type = kind class _Response: - def __init__(self, method: str, status: int, url: str, body: bytes) -> None: - self.request = _Request(method) + def __init__( + self, request: _Request, status: int, url: str, *, worker: bool = False + ) -> None: + self.request = request self.status = status self.url = url self.headers = {"content-type": "application/json; charset=utf-8"} - self._body = body + self.from_service_worker = worker - def body(self) -> bytes: - return self._body + def body(self) -> bytes: # pragma: no cover - must never be called + raise AssertionError("the log must not read response bodies") class _Page: + url = "https://emr.test/patients/MRN-1/encounters" + def __init__(self) -> None: self.handlers: dict[str, Any] = {} @@ -67,39 +71,39 @@ def on(self, event: str, handler: Any) -> None: self.handlers[event] = handler -def test_only_answers_to_write_requests_are_kept_without_values() -> None: +def _send(page: _Page, request: _Request, status: int, url: str, **kw: Any) -> None: + page.handlers["request"](request) + page.handlers["response"](_Response(request, status, url, **kw)) + + +def test_only_the_saves_own_writes_are_kept_without_values() -> None: page = _Page() log = ServerResponseLog(page) - emit = page.handlers["response"] - emit(_Response("GET", 200, "https://emr.test/patients/MRN-1", b"x")) + in_flight = _Request("POST") + page.handlers["request"](in_flight) # an autosave sent before the save mark = log.mark() - emit(_Response("POST", 201, "https://emr.test/api/encounters?patient=MRN-1", BODY)) - emit(_Response("GET", 200, "https://emr.test/api/encounters", b"[]")) + page.handlers["response"](_Response(in_flight, 200, "https://emr.test/autosave")) + _send(page, _Request("POST"), 201, "https://emr.test/api/encounters?p=MRN-1") + _send(page, _Request("GET"), 200, "https://emr.test/api/encounters") + _send(page, _Request("POST", "ping"), 204, "https://emr.test/beacon") + _send(page, _Request("POST"), 200, "https://analytics.example/collect") + _send(page, _Request("POST"), 200, "https://emr.test/queue", worker=True) [answer] = log.since(mark) assert answer.method == "POST" and answer.status == 201 and answer.ok - assert answer.body_sha256 == hashlib.sha256(BODY).hexdigest() assert answer.content_type == "application/json" assert answer.check_level == "same_screen" dumped = answer.model_dump_json() - for leak in ("MRN-1", "emr.test", "enc-7", "patient="): + for leak in ("MRN-1", "emr.test", "encounters", "p="): assert leak not in dumped - # The query string is not part of the endpoint digest. - other = _Response("POST", 201, "https://emr.test/api/encounters", BODY) - emit(other) - assert log.since(mark)[1].endpoint_sha256 == answer.endpoint_sha256 -def test_an_answer_whose_body_is_gone_keeps_its_status() -> None: +def test_a_form_redirect_is_not_an_error_but_a_500_is() -> None: page = _Page() log = ServerResponseLog(page) - - class Gone(_Response): - def body(self) -> bytes: - raise RuntimeError("navigated") - - page.handlers["response"](Gone("PUT", 500, "https://emr.test/api/x", b"")) - [answer] = log.since(0) - assert answer.status == 500 and not answer.ok and answer.body_sha256 is None + _send(page, _Request("POST", "document"), 302, "https://emr.test/save") + _send(page, _Request("PUT"), 500, "https://emr.test/api/x") + redirect, failure = log.since(0) + assert redirect.ok and not failure.ok # -- the replayer -------------------------------------------------------------------- @@ -123,12 +127,7 @@ def press(self, key: str) -> None: super().press(key) self.answers.append( ServerResponseEvidence( - method="POST", - status=self.status, - endpoint_sha256="e" * 64, - body_sha256=hashlib.sha256(BODY).hexdigest(), - body_bytes=len(BODY), - content_type="application/json", + method="POST", status=self.status, content_type="application/json" ) ) @@ -218,8 +217,8 @@ def test_the_receipt_counts_answers_as_supporting_evidence() -> None: plain = build_receipt(report) assert "server_responses" not in plain.model_dump() report.results[-1].server_responses = [ - ServerResponseEvidence(method="POST", status=201, endpoint_sha256="e" * 64), - ServerResponseEvidence(method="POST", status=500, endpoint_sha256="e" * 64), + ServerResponseEvidence(method="POST", status=201), + ServerResponseEvidence(method="POST", status=500), ] receipt = build_receipt(report) assert receipt.server_responses is not None @@ -289,5 +288,5 @@ def test_the_browser_backend_keeps_the_answer_to_a_real_save(save_server: str) - finally: browser.close() assert answer.method == "POST" and answer.status == 201 - assert answer.body_sha256 == hashlib.sha256(BODY).hexdigest() + assert answer.content_type == "application/json" assert "MRN-9" not in answer.model_dump_json() From 21105d2e5b83c67dcce4c9965258d2de992ac125 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 17:03:19 -0400 Subject: [PATCH 3/3] Merge flow/desktop-readback; regenerate claims report and artifact inventory Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/VERIFICATION.md | 2 +- packages/flow/docs/verification.json | 2 +- packages/flow/public-artifacts.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/flow/docs/VERIFICATION.md b/packages/flow/docs/VERIFICATION.md index 62b77f8559..7b8c3dc3b0 100644 --- a/packages/flow/docs/VERIFICATION.md +++ b/packages/flow/docs/VERIFICATION.md @@ -2,7 +2,7 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims.yaml`. Do not edit by hand — edit the registry and regenerate. -- Generated at: **2026-10-18T15:29:50-04:00 (git HEAD commit date)** +- Generated at: **2026-10-18T17:03:19-04:00 (git HEAD commit date)** - JUnit pass check: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) - Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). - Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. diff --git a/packages/flow/docs/verification.json b/packages/flow/docs/verification.json index bdd6047912..d24383f196 100644 --- a/packages/flow/docs/verification.json +++ b/packages/flow/docs/verification.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-10-18T15:29:50-04:00 (git HEAD commit date)", + "generated_at": "2026-10-18T17:03:19-04:00 (git HEAD commit date)", "green_check_run": false, "green_check_job": null, "green_check_scope": [], diff --git a/packages/flow/public-artifacts.json b/packages/flow/public-artifacts.json index fdd24b6d50..052acf685c 100644 --- a/packages/flow/public-artifacts.json +++ b/packages/flow/public-artifacts.json @@ -621,7 +621,7 @@ }, { "path": "claims.yaml", - "sha256": "ec1bd4acfa3d487e78809d6a5ecb742a60c6d8e858ab6fa7d50c75ea76083e7e" + "sha256": "416980c4e39abe5d0d4f3c8a9056365bb7b846867839def1b5d5af0307c4698a" }, { "path": "deploy/on-prem/docker-compose.yml", @@ -1857,7 +1857,7 @@ }, { "path": "docs/verification.json", - "sha256": "a0512978638a854e31a1b6f6f2538f717345168c6e89d42369637499223c6c9f" + "sha256": "ba542789703d75627379444437689cbee209055348a8c1a7e0e1d21abda4728b" }, { "path": "examples/claude-code-demo/demo.gif",