diff --git a/packages/flow/benchmark/hover_identity/run_hover_sweep.py b/packages/flow/benchmark/hover_identity/run_hover_sweep.py new file mode 100644 index 000000000..5bff60300 --- /dev/null +++ b/packages/flow/benchmark/hover_identity/run_hover_sweep.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Hover-before-click evidence on a pixel-only kiosk: before and after. + +The appointment kiosk of ``benchmark/pixel_scale`` with its Save button turned +into a toolbar icon. A second icon, ``Clear form``, draws the same glyph; the +two differ only by their tooltip. The kiosk draws a button under the pointer +in its active colour, as Tk does, and shows the icon's tooltip. + +The kiosk is recorded once (100% scaling, ClearType on) with the production +Recorder, compiled, and replayed: + +- ``healthy``: same layout. A correct run saves one row for Ada Lovelace. +- ``swapped``: the two icons trade places, as a toolbar reorder between + versions would. At the recorded Save position there is now an identical + Clear icon. A correct run does not click it. + +Each at 100% ClearType on (same render), 100% ClearType off (another +rasterizer) and 125% (display scaling). Identity evidence per click step is +read from the compiled bundle. No model is called. + +The same script runs against an older checkout for the before numbers: where +the Recorder cannot capture hover frames, it records without them. + +Usage:: + + python benchmark/hover_identity/run_hover_sweep.py --output results_after.json +""" + +from __future__ import annotations + +import argparse +import inspect +import json +import platform +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +PIXEL_SCALE = HERE.parent / "pixel_scale" +for _path in (str(HERE.parents[1]), str(PIXEL_SCALE)): + if _path not in sys.path: + sys.path.insert(0, _path) + +from kiosk_render import ( # noqa: E402 + KioskBackend, + RenderProfile, + ScaledKiosk, + rdp_appointments_icons_spec, +) + +SCHEMA = "openadapt.hover-identity-sweep.v1" +RECORD_PROFILE = RenderProfile(scale=1.0, cleartype=True) +PROFILES = ( + RenderProfile(scale=1.0, cleartype=True), + RenderProfile(scale=1.0, cleartype=False), + RenderProfile(scale=1.25, cleartype=True), +) +DEMO = { + "appointment_slot": "2026-08-12 09:30", + "visit_type": "Cardiology follow-up", + "request_id": "REQ-DEMO-001", +} +REPLAY = { + "appointment_slot": "2026-08-14 10:45", + "visit_type": "Intake consultation", + "request_id": "REQ-LIVE-2048", +} +FIELD_FOR_PARAM = { + "appointment_slot": "slot", + "visit_type": "visit_type", + "request_id": "request_id", +} + + +def _recorder_can_hover() -> bool: + from openadapt_flow.recorder import Recorder + + return "capture_hover" in inspect.signature(Recorder.__init__).parameters + + +def _record(work: Path) -> tuple[Any, Path]: + from openadapt_flow.compiler import compile_recording + from openadapt_flow.recorder import Recorder + + kiosk = ScaledKiosk( + rdp_appointments_icons_spec(), RECORD_PROFILE, hover_feedback=True + ) + backend = KioskBackend(kiosk) + rec_dir, bundle_dir = work / "recording", work / "bundle" + options = {"capture_hover": True} if _recorder_can_hover() else {} + rec = Recorder( + backend, + rec_dir, + settle_interval_s=0.0, + settle_stable_frames=2, + settle_timeout_s=2.0, + **options, + ) + rec.click(*kiosk.logical_center("row0")) + for param, value in DEMO.items(): + rec.click(*kiosk.logical_center(FIELD_FOR_PARAM[param])) + rec.type_text(value, param=param) + rec.click(*kiosk.logical_center("save")) + rec.finish() + workflow = compile_recording(rec_dir, bundle_dir, name="hover-identity") + return workflow, bundle_dir + + +def coverage(workflow: Any) -> list[dict[str, Any]]: + """Identity evidence per click step of the compiled bundle.""" + out = [] + for step in workflow.steps: + if str(getattr(step.action, "value", step.action)) not in ( + "click", + "double_click", + "right_click", + ): + continue + a = step.anchor + record_identity = bool( + a is not None + and ( + a.context_text + or a.structured_identity + or a.identity_template + or a.identifier_crop + ) + ) + hover = getattr(step, "hover", None) + out.append( + { + "step_id": step.id, + "intent": step.intent, + "record_identity": record_identity, + "hover_response": bool(hover is not None and hover.response), + "has_evidence": record_identity + or bool(hover is not None and hover.response), + } + ) + return out + + +def _replay( + bundle_dir: Path, profile: RenderProfile, *, swapped: bool, run_dir: Path +) -> dict[str, Any]: + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime.replayer import Replayer + + workflow = Workflow.load(bundle_dir) + kiosk = ScaledKiosk( + rdp_appointments_icons_spec(swapped=swapped), profile, hover_feedback=True + ) + backend = KioskBackend(kiosk) + crash: Optional[str] = None + report = None + try: + report = Replayer(backend, poll_interval_s=0.0, layout_drift=True).run( + workflow, params=REPLAY, bundle_dir=bundle_dir, run_dir=run_dir + ) + except Exception as exc: # noqa: BLE001 - a crash is an outcome + crash = f"{type(exc).__name__}: {exc}"[:300] + clear_rect = kiosk._rect("clear") + clear_clicked = any( + clear_rect[0] <= x < clear_rect[0] + clear_rect[2] + and clear_rect[1] <= y < clear_rect[1] + clear_rect[3] + for x, y in backend.clicks + ) + rows = kiosk.state.saved + expected = { + "patient_name": "Ada Lovelace", + "patient_mrn": kiosk.spec.patients[0][1], + **{FIELD_FOR_PARAM[k]: v for k, v in REPLAY.items()}, + } + success = bool(report is not None and report.success) + if crash is not None: + outcome = "crash" + elif clear_clicked: + outcome = "wrong_control_clicked" + elif rows and rows != [expected]: + outcome = "wrong_write" + elif success and rows == [expected]: + outcome = "pass" + elif success: + outcome = "false_completion" + else: + outcome = "halted" + steps = [] + stop = None + for result in report.results if report is not None else []: + ident = result.identity + hover = getattr(ident, "hover", None) if ident is not None else None + steps.append( + { + "step_id": result.step_id, + "ok": result.ok, + "rung": result.resolution.rung if result.resolution else None, + "identity": ident.status if ident is not None else None, + "identity_mode": ident.mode if ident is not None else None, + "hover": [hover.verdict, hover.reason] if hover is not None else None, + } + ) + if not result.ok and stop is None: + stop = {"step_id": result.step_id, "error": (result.error or "")[:300]} + return { + "layout": "swapped" if swapped else "recorded", + "profile": profile.label, + "outcome": outcome, + "rows_written": len(rows), + "clear_clicked": clear_clicked, + "model_calls": int(report.model_calls) if report is not None else 0, + "stopped_at": stop, + "steps": steps, + "crash": crash, + } + + +def _git(*args: str) -> Optional[str]: + try: + import openadapt_flow + + cwd = Path(openadapt_flow.__file__).resolve().parent + return subprocess.run( + ["git", *args], cwd=cwd, capture_output=True, text=True, check=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def run(work: Path) -> dict[str, Any]: + workflow, bundle_dir = _record(work) + points = [] + for profile in PROFILES: + for swapped in (False, True): + tag = ( + f"{round(profile.scale * 100)}_{'ct' if profile.cleartype else 'noct'}" + ) + tag += "_swapped" if swapped else "" + point = _replay(bundle_dir, profile, swapped=swapped, run_dir=work / tag) + print( + f"{point['layout']:9s} {point['profile']:24s} {point['outcome']}", + flush=True, + ) + points.append(point) + cov = coverage(workflow) + return { + "schema_version": SCHEMA, + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "commit": _git("rev-parse", "HEAD"), + "platform": f"{platform.system()} {platform.machine()} py{platform.python_version()}", + "recorder_captured_hover": _recorder_can_hover(), + "coverage": { + "click_steps": len(cov), + "with_record_identity": sum(c["record_identity"] for c in cov), + "with_hover_response": sum(c["hover_response"] for c in cov), + "with_any_evidence": sum(c["has_evidence"] for c in cov), + "steps": cov, + }, + "summary": { + "healthy_points": sum(p["layout"] == "recorded" for p in points), + "healthy_passed": sum( + p["layout"] == "recorded" and p["outcome"] == "pass" for p in points + ), + "swapped_points": sum(p["layout"] == "swapped" for p in points), + "swapped_wrong_control_clicked": sum( + p["layout"] == "swapped" and p["clear_clicked"] for p in points + ), + "swapped_stopped_before_click": sum( + p["layout"] == "swapped" + and not p["clear_clicked"] + and p["outcome"] == "halted" + for p in points + ), + "wrong_writes": sum(p["outcome"] == "wrong_write" for p in points), + "model_calls": sum(p["model_calls"] for p in points), + }, + "points": points, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--work-dir", type=Path) + args = parser.parse_args() + work = args.work_dir or Path(tempfile.mkdtemp(prefix="oaflow-hover-")) + work.mkdir(parents=True, exist_ok=True) + results = run(work) + args.output.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + **results["summary"], + **{k: v for k, v in results["coverage"].items() if k != "steps"}, + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/pixel_scale/kiosk_render.py b/packages/flow/benchmark/pixel_scale/kiosk_render.py index 6813e2be0..187840fcd 100644 --- a/packages/flow/benchmark/pixel_scale/kiosk_render.py +++ b/packages/flow/benchmark/pixel_scale/kiosk_render.py @@ -208,7 +208,7 @@ def linespace(pt: int, bold: bool, scale: float = 1.0) -> int: class Widget: """A Tk widget placed at logical (100%) coordinates.""" - kind: str # label | button | entry + kind: str # label | button | entry | icon name: str x: int y: int @@ -223,6 +223,8 @@ class Widget: anchor_w: bool = False relief: str = "flat" bd: int = 0 + #: Tooltip shown while the pointer rests on the widget (hover feedback on). + tooltip: Optional[str] = None def size(self) -> tuple[int, int]: """Logical size, from Tk's geometry rules at 100%.""" @@ -239,6 +241,8 @@ def size(self) -> tuple[int, int]: return (w, h) if self.kind == "entry": return ((self.chars or 20) * avg + 2 * (1 + 1) + 2, line + 2 * (1 + 1) + 2) + if self.kind == "icon": + return (self.width or 48, self.height or 44) raise ValueError(self.kind) @@ -258,6 +262,8 @@ class KioskSpec: fields: list[str] save_button: str rows: list[str] + #: Icon buttons and what they do: ``save`` or ``clear`` (empty the form). + icon_actions: dict[str, str] = field(default_factory=dict) def rdp_appointments_spec( @@ -499,6 +505,44 @@ def rdp_appointments_unambiguous_spec() -> KioskSpec: ) +def rdp_appointments_icons_spec(swapped: bool = False) -> KioskSpec: + """The appointment kiosk with its Save button turned into a toolbar icon. + + Two icons draw the same glyph and differ only by their tooltip: + ``Save appointment`` and ``Clear form``. ``swapped`` exchanges their + places, as a toolbar reorder between versions would. Record IDs are free + of OCR-confusable glyphs, so identity is not what stops a run here. + """ + spec = rdp_appointments_spec( + (("Ada Lovelace", "MRN A3467"), ("Grace Hopper", "MRN B7346")), + name="rdp_appointments_icons_swapped" if swapped else "rdp_appointments_icons", + ) + btn_bg = "#2d6fd8" + save_x, clear_x = (750, 550) if swapped else (550, 750) + widgets = [w for w in spec.widgets if w.name != "save"] + for name, x, tip in ( + ("save", save_x, "Save appointment"), + ("clear", clear_x, "Clear form"), + ): + widgets.append( + Widget( + "icon", + name, + x, + 570, + bg=btn_bg, + width=56, + height=48, + relief="raised", + bd=2, + tooltip=tip, + ) + ) + spec.widgets = widgets + spec.icon_actions = {"save": "save", "clear": "clear"} + return spec + + SPECS: dict[str, Callable[[], KioskSpec]] = { "rdp_appointments": rdp_appointments_spec, "rdp_appointments_unambiguous_ids": rdp_appointments_unambiguous_spec, @@ -519,11 +563,18 @@ class KioskState: class ScaledKiosk: """Kiosk state machine plus a renderer for one :class:`RenderProfile`.""" - def __init__(self, spec: KioskSpec, profile: RenderProfile) -> None: + def __init__( + self, spec: KioskSpec, profile: RenderProfile, *, hover_feedback: bool = False + ) -> None: self.spec = spec self.profile = profile self.state = KioskState(values={name: "" for name in spec.fields}) self._by_name = {w.name: w for w in spec.widgets} + # Tk draws a button under the pointer in its active colour and the + # application may show a tooltip. Off by default: the display-scale + # sweep was measured without it. + self.hover_feedback = hover_feedback + self.pointer: Optional[tuple[int, int]] = None # -- geometry -------------------------------------------------------- def logical_rect(self, name: str) -> tuple[int, int, int, int]: @@ -550,7 +601,7 @@ def logical_center(self, name: str) -> tuple[int, int]: # -- interaction ----------------------------------------------------- def _hit(self, x: int, y: int) -> Optional[str]: for w in self.spec.widgets: - if w.kind not in ("button", "entry"): + if w.kind not in ("button", "entry", "icon"): continue rx, ry, rw, rh = self._rect(w.name) if rx <= x < rx + rw and ry <= y < ry + rh: @@ -565,6 +616,10 @@ def click(self, x: int, y: int) -> None: self.state.active = self.spec.rows.index(name) elif name in self.spec.fields: self.state.focus = name + elif self.spec.icon_actions.get(name) == "clear": + self.state.values = {k: "" for k in self.state.values} + self.state.status = "Form cleared" + self.state.status_fg = "#b42318" elif name == self.spec.save_button: self.save() @@ -612,6 +667,8 @@ def render(self) -> Image.Image: self.state.status_fg if w.name == self.spec.status_label else w.fg ) bg = _rgb(w.bg) if w.bg else _rgb(self.spec.bg) + if w.kind in ("button", "icon") and self._under_pointer(w.name): + bg = _shade(bg, 1.18) # Tk activebackground if w.kind != "label": canvas[ry : ry + rh, rx : rx + rw] = bg relief = w.relief @@ -622,6 +679,9 @@ def render(self) -> Image.Image: ): relief, bd = "sunken", 4 self._bevel(canvas, (rx, ry, rw, rh), bg, relief, max(1, self._px(bd))) + if w.kind == "icon": + self._glyph(canvas, (rx, ry, rw, rh)) + continue pad = self._px(3) line = linespace(w.pt, w.bold, fs) if w.kind == "label": @@ -646,8 +706,53 @@ def render(self) -> Image.Image: cx = origin[0] + text_width(text, w.pt, w.bold, fs) + 1 cw = max(1, self._px(2)) canvas[origin[1] : origin[1] + line, cx : cx + cw] = fg + self._tooltip(canvas) return Image.fromarray(canvas, "RGB") + def _under_pointer(self, name: str) -> bool: + if not self.hover_feedback or self.pointer is None: + return False + return self._hit(*self.pointer) == name + + def _glyph(self, canvas: np.ndarray, rect: tuple[int, int, int, int]) -> None: + """The same white disc-and-bar glyph on every icon.""" + x, y, w, h = rect + cx, cy = x + w // 2, y + h // 2 + r = max(3, min(w, h) // 4) + yy, xx = np.ogrid[: canvas.shape[0], : canvas.shape[1]] + disc = (xx - cx) ** 2 + (yy - cy) ** 2 <= r * r + canvas[disc] = (255, 255, 255) + bar = max(1, r // 3) + canvas[cy - bar : cy + bar, cx - 2 * r : cx + 2 * r] = (255, 255, 255) + + def _tooltip(self, canvas: np.ndarray) -> None: + if not self.hover_feedback or self.pointer is None: + return + name = self._hit(*self.pointer) + widget = self._by_name.get(name) if name else None + if widget is None or not widget.tooltip: + return + fs = self.profile.scale * self.profile.font_scale + px, py = self.pointer + tx, ty = px + self._px(12), py + self._px(20) + tw = text_width(widget.tooltip, 11, False, fs) + self._px(8) + th = linespace(11, False, fs) + self._px(6) + vh, vw = canvas.shape[:2] + tx, ty = min(tx, vw - tw - 1), min(ty, vh - th - 1) + canvas[ty : ty + th, tx : tx + tw] = (255, 255, 225) + canvas[ty, tx : tx + tw] = canvas[ty + th - 1, tx : tx + tw] = (0, 0, 0) + canvas[ty : ty + th, tx] = canvas[ty : ty + th, tx + tw - 1] = (0, 0, 0) + _draw_text( + canvas, + (tx + self._px(4), ty + self._px(3)), + widget.tooltip, + font_px=font_px(11, fs), + bold=False, + fg=(0, 0, 0), + cleartype=self.profile.cleartype, + clip=(tx, ty, tw, th), + ) + @staticmethod def _bevel( canvas: np.ndarray, @@ -702,8 +807,14 @@ def screenshot(self) -> bytes: def click(self, x: int, y: int, *, double: bool = False) -> None: self.clicks.append((int(x), int(y))) + self.kiosk.pointer = (int(x), int(y)) self.kiosk.click(int(x), int(y)) + def hover(self, x: int, y: int) -> None: + """Rest the pointer on (x, y); with hover feedback on, the kiosk draws + the control's active colour and tooltip.""" + self.kiosk.pointer = (int(x), int(y)) + def type_text(self, text: str) -> None: self.kiosk.type_text(text) if self._after_type is not None: diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index 7027ad2d1..fc03f9e48 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -188,6 +188,30 @@ halts with a named reason when the confirmation is missing before and after numbers are in [`benchmark/known_ids/KNOWN_IDS.md`](../benchmark/known_ids/KNOWN_IDS.md). +Before a step stops on a look-alike identifier, the replayer takes extra looks +that change nothing +([`openadapt_flow/runtime/identity_looks.py`](../openadapt_flow/runtime/identity_looks.py)): +the same read on two fresh screenshots, the 2x re-read, the known-ID list when +one is configured, and, where the surface allows it, a select-and-copy of the +identifier's text box (`runtime.identity_copy_look`). OCR looks can't settle +the record on their own, because a look-alike twin can read the same on every +frame and at every size. Only two looks can: the known-ID list, and a copied +identifier that equals a run parameter in the identity band. A look that names +another record stops the run. Each look is recorded on the step's identity +check. + +The copy look has limits. It needs a surface where selecting the identifier +and copying it changes nothing, and a clipboard that reaches the machine Flow +runs on (RDP or Citrix clipboard redirection). It drags across only the one +text box that holds the identifier, and the run stops if the screen changes +anywhere else during the look. A canvas with no selectable text has no copy +look. A recording's own identifier is OCR text and can be a misread (the RDP +kiosk recorded `A1001` as `Alo01`), so in a workflow that always opens the +recorded record, the copy can stop a run but never confirm one. It confirms +only a record chosen by a run parameter. Governed and qualification runs never +take it. The check just before the click takes no input; it reuses the copy +only when the identifier's pixels are unchanged since the look. + Permissive `replay` may proceed on an unreadable reversible target and reports that condition. Governed `run` requires an affirmative live verdict for every identity-required admitted step; mismatch, unreadable, or abstain halts before @@ -216,6 +240,39 @@ irreversible, and qualification requires its record check. An icon-only click that only navigates after data entry is therefore over-classified until an operator reclassifies it. +On a pixel surface an icon-only click carries no text that names its control. +When the recorder can move the pointer without clicking +(`Recorder(capture_hover=True)` on a backend with `hover`), it rests the +pointer on each click target first and keeps that frame. For a click with no +other identity evidence, the compiler stores what changed near the target, such +as the control's highlight or tooltip, as the step's hover evidence +([`openadapt_flow/runtime/hover_evidence.py`](../openadapt_flow/runtime/hover_evidence.py)). +At replay, on the pixel rungs, the runtime rests the pointer on the resolved +target, waits the same 0.8 s, and compares. A different response, or a +response where the recording saw none, stops the run before the click. A +missing response, or one that can't be compared (another render with no +readable text), is recorded on the step and changes nothing. A tooltip that +opens later than the wait looks like a different response and stops the run. +Controls that don't answer the pointer, such as most text fields, get no +evidence this way. The browser, remote-display (Citrix and RDP client window) +and FreeRDP backends can hover. The Aardwolf RDP transport and the native +Windows, macOS and Linux backends can't move the pointer without a button +yet, so they take no hover look. + +A pixel step is checked on one frame and clicked a moment later. Consequential +identity-gated clicks were already bound to the exact checked frame. Every +other pixel click now gets a check-to-click guard (`Replayer._pixel_recheck`): +right before the click, the runtime takes a fresh frame and compares the +target region, the identity region and the hover response with the frame the +step was checked on. If one changed (a dialog opened, the list re-sorted), it +doesn't click. It re-resolves the target and re-runs the identity check on +the fresh frame, with no input, and clicks only when the next frame matches +that one. After two re-observations that still see a change, or when the +target or its identity can't be confirmed again, it refuses the click. Each +change is recorded on the step (`StepResult.check_to_click`). A change smaller +than 4 pixels, or under 24 levels per channel, doesn't count. Structured +targets are bound by element tokens instead and aren't rechecked. + A consequential click must be tied to a record before Flow delivers it. Inside a table row, the row is that record. An icon-only Save that floats beside a form (openIMIS's claim Save has no text, accessible name, or tooltip) has no diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index 75e2dd6d8..0db58a5d6 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -639,6 +639,9 @@ def start_record(self) -> None: settle_stable_frames=self._settle_stable_frames, settle_timeout_s=self._settle_timeout_s, system_of_record_reader=self._system_of_record_reader, + # Remote sessions are screen-only: keep each click target's hover + # response as identity evidence (runtime.hover_evidence). + capture_hover=self._kind in ("rdp", "citrix"), ) self._actors = {} diff --git a/packages/flow/openadapt_flow/backend.py b/packages/flow/openadapt_flow/backend.py index 5d60a75c0..4e591623b 100644 --- a/packages/flow/openadapt_flow/backend.py +++ b/packages/flow/openadapt_flow/backend.py @@ -647,6 +647,19 @@ def prepare_pointer_actuation(self, x: int, y: int) -> None: ... +@runtime_checkable +class PointerHoverBackend(Protocol): + """Optional: rest the pointer on a point without pressing a button. + + Used to read a control's hover response (highlight, tooltip, status-bar + text) before an icon-only click on a pixel surface + (:mod:`openadapt_flow.runtime.hover_evidence`), and by the recorder to + capture that response. Emits no button edge. + """ + + def hover(self, x: int, y: int) -> None: ... + + @runtime_checkable class RichPointerActionBackend(Protocol): """Pointer actions beyond the primary/double click contract.""" diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index a9bad239c..d30d4598f 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -3335,6 +3335,12 @@ def right_click(self, x: int, y: int) -> None: cause_type=type(exc).__name__, ) from exc + def hover(self, x: int, y: int) -> None: + """Rest the mouse on pixel coordinates without pressing a button.""" + + self._assert_qualification_environment_current() + self.page.mouse.move(x, y) + def drag(self, x: int, y: int, end_x: int, end_y: int) -> None: """Drag between two independently resolved points.""" diff --git a/packages/flow/openadapt_flow/backends/rdp_backend.py b/packages/flow/openadapt_flow/backends/rdp_backend.py index 52f5ffc84..5133d4567 100644 --- a/packages/flow/openadapt_flow/backends/rdp_backend.py +++ b/packages/flow/openadapt_flow/backends/rdp_backend.py @@ -728,6 +728,20 @@ def right_click_guarded( delivered_at=datetime.now(timezone.utc).isoformat(), ) + def hover(self, x: int, y: int) -> None: + """Rest the pointer on (x, y) without a button edge. + + Needs a transport with ``move(x, y)``; raises NotImplementedError + otherwise, and the caller treats the hover look as unavailable. + """ + move = getattr(self._transport, "move", None) + if not callable(move): + raise NotImplementedError("this RDP transport cannot move the pointer") + with self._input_lock: + self._ensure_input_ready(point=(int(x), int(y)), operation="rdp_hover") + self._assert_frame_fresh() + move(int(x), int(y)) + def drag(self, x: int, y: int, end_x: int, end_y: int) -> None: """Drag between two points resolved from the same fresh framebuffer.""" diff --git a/packages/flow/openadapt_flow/backends/remote_display.py b/packages/flow/openadapt_flow/backends/remote_display.py index abae34e27..94b6ff986 100644 --- a/packages/flow/openadapt_flow/backends/remote_display.py +++ b/packages/flow/openadapt_flow/backends/remote_display.py @@ -982,6 +982,11 @@ def prepare_pointer_actuation(self, x: int, y: int) -> None: self._wait_for_pointer_settle() self._prepared_pointer_point = point + def hover(self, x: int, y: int) -> None: + """Rest the pointer on (x, y) with the same checks as a pre-positioned + click (:meth:`prepare_pointer_actuation`); emits no button edge.""" + self.prepare_pointer_actuation(x, y) + def click(self, x: int, y: int, *, double: bool = False) -> None: """Click (or double-click) at captured-pixel coordinates (x, y).""" with self._input_lock: diff --git a/packages/flow/openadapt_flow/backends/xfreerdp.py b/packages/flow/openadapt_flow/backends/xfreerdp.py index 1ec3ba293..0d94dfe30 100644 --- a/packages/flow/openadapt_flow/backends/xfreerdp.py +++ b/packages/flow/openadapt_flow/backends/xfreerdp.py @@ -446,6 +446,11 @@ def pointer(self, x: int, y: int, button: str, down: bool) -> None: btn, ) + def move(self, x: int, y: int) -> None: + """Move the pointer with no button edge (a hover).""" + self._last_pointer = (int(x), int(y)) + self._xdotool("mousemove", str(int(x)), str(int(y))) + def key(self, keysym_or_char: str, down: bool) -> None: name = _KEYSYMS.get(keysym_or_char.lower()) if len(keysym_or_char) > 1 else None if name is not None: diff --git a/packages/flow/openadapt_flow/bundle_validation.py b/packages/flow/openadapt_flow/bundle_validation.py index 3387d3645..4df208263 100644 --- a/packages/flow/openadapt_flow/bundle_validation.py +++ b/packages/flow/openadapt_flow/bundle_validation.py @@ -184,6 +184,15 @@ class _SealedCanonicalOmission: "field; a populated band remains sealed content" ), ), + _SealedCanonicalOmission( + owner=Step, + field_name="hover", + omitted_values=(None,), + reason=( + "absent hover evidence predates the v2 field; a step CARRYING " + "a recorded hover response is new content and changes the digest" + ), + ), _SealedCanonicalOmission( owner=Step, field_name="submit_binding", diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index 5a953566a..0fa8193af 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -43,6 +43,7 @@ DialogResponse, ExecutionMode, ExecutionTargetKind, + HoverEvidence, Landmark, ParamKind, ParamSpec, @@ -1426,6 +1427,56 @@ def _marked_identifier_region(value: object, *, source: str) -> Optional[Region] return (int(value[0]), int(value[1]), int(value[2]), int(value[3])) +HOVER_CROP_DIR = "templates/hover" + + +def _hover_evidence( + bundle: Path, + step_id: str, + before_png: bytes, + hover_path: Path, + click: Point, + *, + dwell_ms: object = None, +) -> Optional[HoverEvidence]: + """The recorded hover response for a click with no record identity. + + The recorder saved ``frames/{i}_hover.png`` after resting the pointer on + the target (``Recorder(capture_hover=True)``). What changed near the + target between the before frame and that frame is the response + (:func:`openadapt_flow.runtime.hover_evidence.measure_response`); its + pixels land under ``templates/hover/`` so they are sealed with the other + image crops. None when there is no hover frame, nothing near the target + changed, or the frames cannot be compared. + """ + from openadapt_flow.runtime import hover_evidence as hover_mod + + if not hover_path.is_file(): + return None + hover_png = hover_path.read_bytes() + try: + response = hover_mod.measure_response(before_png, hover_png, click) + except ValueError: + logger.warning("hover frame for %s differs in size; no hover evidence", step_id) + return None + dwell = ( + int(dwell_ms) + if isinstance(dwell_ms, int) and 0 <= dwell_ms <= 5000 + else hover_mod.HOVER_DWELL_MS + ) + if response is None: + return None # nothing answered the pointer: no evidence to keep + crop_rel = f"{HOVER_CROP_DIR}/{step_id}.png" + (bundle / HOVER_CROP_DIR).mkdir(parents=True, exist_ok=True) + (bundle / crop_rel).write_bytes(response.crop) + return HoverEvidence( + response=True, + region=hover_mod.relative(response.region, click), + crop=crop_rel, + dwell_ms=dwell, + ) + + def _emit_identifier_crop( bundle: Path, step_id: str, @@ -2357,6 +2408,18 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ), exclude_region=crop_region, ) + hover = ( + _hover_evidence( + bundle, + step_id, + before_png, + recording / "frames" / f"{i:04d}_hover.png", + click, + dwell_ms=event.get("hover_dwell_ms"), + ) + if not identity_armed and kind != "drag" + else None + ) if kind == "drag": destination = drag_end_anchor.ocr_text if drag_end_anchor else None intent = "drag " + ( @@ -2395,6 +2458,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: identity_armed=identity_armed, identity_unarmed_reason=unarmed_reason, identifier_crop_missing_reason=idcrop_missing, + hover=hover, ), before_png, after_png, diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 3ab74fe59..17e273e73 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -712,6 +712,24 @@ class GroundingModelConfig(BaseModel): api_key_env: str = "" +class IdentityCopyLookSection(BaseModel): + """Select-and-copy look for a look-alike identifier on a pixel surface. + + Turn on only where selecting the recorded identifier cell and copying it + changes nothing (a read-only text cell) and the session's clipboard reaches + the machine Flow runs on (RDP/Citrix clipboard redirection). See + :mod:`openadapt_flow.runtime.identity_looks`. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + #: ``drag`` across the cell at mid height, or ``double_click`` its centre. + gesture: Literal["drag", "double_click"] = "drag" + #: Keys that copy the selection in the driven session. + copy_keys: list[str] = Field(default_factory=lambda: ["Control+c"], min_length=1) + + class RuntimeSection(BaseModel): """Runtime posture: durability and model-egress opt-in.""" @@ -761,6 +779,13 @@ class RuntimeSection(BaseModel): known_ids_confirmation_path: Optional[str] = None #: Oldest export (by its export time) replay will use. known_ids_max_age_hours: float = Field(default=24.0, gt=0) + #: Before stopping on a look-alike identifier, select the identifier cell, + #: copy it and read the clipboard. Settles the record only when the copy + #: equals a run parameter in the identity band; halts when the screen + #: changes outside the cell. Off by default. Never used by governed runs. + identity_copy_look: IdentityCopyLookSection = Field( + default_factory=IdentityCopyLookSection + ) #: Font/layout drift check before each recorded click step #: (:mod:`openadapt_flow.runtime.layout_drift`). None (default) turns it on #: for remote-desktop replays (``rdp``/``citrix``) and off elsewhere; @@ -1562,8 +1587,21 @@ def build_replayer( confirmation_path=runtime_config.known_ids_confirmation_path, max_age_s=runtime_config.known_ids_max_age_hours * 3600.0, ) + identity_copy = None + read_clipboard = None + copy_cfg = runtime_config.identity_copy_look if runtime_config is not None else None + if copy_cfg is not None and copy_cfg.enabled: + from openadapt_flow.records.read import SystemClipboard + from openadapt_flow.runtime.identity_looks import IdentityCopyConfig + + identity_copy = IdentityCopyConfig( + gesture=copy_cfg.gesture, copy_keys=tuple(copy_cfg.copy_keys) + ) + read_clipboard = SystemClipboard() return Replayer( backend, + identity_copy=identity_copy, + read_clipboard=read_clipboard, layout_drift=( runtime_config.layout_drift_check if runtime_config is not None else None ), diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 1dcce099a..dca06b1fd 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -1501,6 +1501,37 @@ def _one_identity(self) -> "DialogResponse": return self +class HoverEvidence(BaseModel): + """What the recording saw when the pointer rested on a click target. + + See :mod:`openadapt_flow.runtime.hover_evidence`. Captured for click steps + whose target carries no record identity (an icon-only control) on a + pixel surface. Replay moves the pointer onto the resolved target, waits + ``dwell_ms`` and compares its response with this one before clicking. + """ + + model_config = ConfigDict(extra="forbid") + + response: bool = Field( + description="Whether anything near the target changed under the pointer" + ) + region: Optional[Region] = Field( + default=None, + description=( + "The changed box, as an offset from the recorded click point" + " (dx, dy, w, h). Set iff ``response``." + ), + ) + crop: Optional[str] = Field( + default=None, + description=( + "Bundle-relative PNG of the response pixels (templates/hover/)," + " sealed with the other image crops. Set iff ``response``." + ), + ) + dwell_ms: int = Field(default=800, ge=0, le=5000) + + class Step(BaseModel): id: str intent: str = Field(description="Human-readable purpose of the step") @@ -1810,6 +1841,15 @@ def _validate_rich_action_contract(self) -> "Step": " band tier (docs/LIMITS.md)." ), ) + hover: Optional[HoverEvidence] = Field( + default=None, + description=( + "Pointer-hover response captured at record time for a click whose" + " target carries no record identity (runtime.hover_evidence)." + " None when the recorder could not hover or the step has other" + " identity evidence, and on bundles compiled before this field." + ), + ) submit_binding: Optional[SubmitBinding] = Field( default=None, description=( @@ -3293,6 +3333,39 @@ class KnownIdEvidence(BaseModel): export_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") +class IdentityLook(BaseModel): + """One extra look taken before stopping on a look-alike identifier. + + See :mod:`openadapt_flow.runtime.identity_looks`. Holds no identifier + values: only what kind of look it was, what it concluded, and the + SHA-256 of the frame it read when it read one. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["frame", "magnified", "known_ids", "clipboard"] + #: ``agrees``: consistent with the first read, settles nothing alone. + #: ``resolves``: names exactly one record, the expected one. + #: ``contradicts``: names another record; the step stops. + #: ``unavailable``: the look could not be taken or read nothing usable. + outcome: Literal["agrees", "resolves", "contradicts", "unavailable"] + reason: Optional[str] = Field(default=None, max_length=80) + frame_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + + +class HoverLook(BaseModel): + """The live hover response compared with the recorded one. + + Holds no pixels: the verdict, why, and the SHA-256 of the hover frame. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + verdict: Literal["verified", "mismatch", "unexpected", "absent", "unverifiable"] + reason: str = Field(max_length=80) + frame_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + + class IdentityCheck(BaseModel): """Outcome of the pre-click target-identity check (runtime.identity). @@ -3336,6 +3409,7 @@ class IdentityCheck(BaseModel): "pixel", "vlm", "signal_quorum", + "hover", ] = "context" coverage: float = 0.0 expected: str = "" @@ -3377,6 +3451,27 @@ class IdentityCheck(BaseModel): "confirmed, or it is older than the configured age. The step halts." ), ) + hover: Optional[HoverLook] = Field( + default=None, + description=( + "Set for mode ``hover``: the live pointer-hover response compared " + "with the recorded one (runtime.hover_evidence)." + ), + ) + looks: list[IdentityLook] = Field( + default_factory=list, + description=( + "Extra looks taken because the first read abstained only on a " + "look-alike identifier (runtime.identity_looks), in order." + ), + ) + look_halt: Optional[Literal["identity_look_changed_screen"]] = Field( + default=None, + description=( + "Set when an extra look changed the screen outside the " + "identifier it read. The step halts." + ), + ) class HealEvent(BaseModel): @@ -3810,6 +3905,20 @@ class EffectFieldCheck(BaseModel): verification_tier: Optional[int] = None +class CheckToClickRecheck(BaseModel): + """The screen changed between a pixel step's check and its click. + + Records which checked regions changed on the fresh frame taken just + before the click, and what the runtime did: re-observed the target and + its identity on that frame, or refused the click. No pixels. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + changed: list[Literal["target", "identity", "hover", "frame_size"]] + outcome: Literal["reobserved", "aborted"] + + class StepResult(BaseModel): step_id: str intent: str @@ -3938,6 +4047,10 @@ class StepResult(BaseModel): identity_reobserved: bool = False identity_first_read: Optional[IdentityCheck] = None identity_first_frame_png: Optional[str] = None + #: Pixel clicks only: each time the target, identity or hover region + #: changed between the step's check and its click (see + #: ``Replayer._pixel_recheck``). Empty, and omitted, when nothing changed. + check_to_click: list[CheckToClickRecheck] = Field(default_factory=list) heal: Optional[HealEvent] = None error: Optional[str] = None before_png: Optional[str] = None # run-dir-relative paths @@ -3966,6 +4079,8 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: data.pop("timings", None) if isinstance(data, dict) and not self.effect_field_checks: data.pop("effect_field_checks", None) + if isinstance(data, dict) and not self.check_to_click: + data.pop("check_to_click", None) return data diff --git a/packages/flow/openadapt_flow/recorder.py b/packages/flow/openadapt_flow/recorder.py index ef686ecf0..8bac4349d 100644 --- a/packages/flow/openadapt_flow/recorder.py +++ b/packages/flow/openadapt_flow/recorder.py @@ -74,6 +74,11 @@ #: Bound on waiting for a newly opened page's first navigation. _NEW_PAGE_LOAD_TIMEOUT_S = 10.0 + +# Mirrors openadapt_flow.runtime.hover_evidence.HOVER_DWELL_MS. Not imported: +# the recorder must not pull in the runtime's vision dependencies. +HOVER_DWELL_MS = 800 + # How long a recorded action that raises a browser dialog waits for it. _DIALOG_WAIT_S = 5.0 @@ -140,7 +145,13 @@ def __init__( surface_id: str = PRIMARY_SURFACE_ID, surface_kind: Optional[str] = None, page_backend_factory: Optional[Callable[[Any], Backend]] = None, + capture_hover: bool = False, ) -> None: + # Pixel surfaces: before each click, rest the pointer on the target + # and keep that frame (frames/{i}_hover.png) so the compiler can store + # the hover response as identity evidence (runtime.hover_evidence). + # Needs a backend with ``hover(x, y)``; ignored otherwise. + self._capture_hover = capture_hover # ``_backend`` is always the ACTIVE surface's backend; on a # single-surface recording it never changes. self._backend = backend @@ -665,6 +676,9 @@ def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: ) if page_backend is not None: page_backend.expect_page_close() + hover_png = self._hover_frame(event) + if hover_png is not None: + event = {**event, "hover_dwell_ms": HOVER_DWELL_MS} try: act() finally: @@ -692,8 +706,29 @@ def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: # click that closes it. if self._closed_its_page(event): event = {**event, "closes_surface": True} + if hover_png is not None: + (self._frames_dir / f"{self._i:04d}_hover.png").write_bytes(hover_png) self._commit(event, before, after, structural_before) + def _hover_frame(self, event: dict[str, Any]) -> Optional[bytes]: + """Rest the pointer on a click target and capture the settled frame. + + Only with ``capture_hover`` and a backend that can hover, and only for + click events. The pointer stays there for the click that follows. + """ + if not self._capture_hover or event.get("kind") not in ( + "click", + "double_click", + "right_click", + ): + return None + hover = getattr(self._backend, "hover", None) + if not callable(hover): + return None + hover(int(event["x"]), int(event["y"])) + time.sleep(HOVER_DWELL_MS / 1000.0) + return self._wait_settled() + def record_observed( self, event: dict[str, Any], diff --git a/packages/flow/openadapt_flow/runtime/hover_evidence.py b/packages/flow/openadapt_flow/runtime/hover_evidence.py new file mode 100644 index 000000000..d4e5afa69 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/hover_evidence.py @@ -0,0 +1,290 @@ +"""Hover-before-click evidence for steps with no record identity. + +An icon-only click (a toolbar icon, a floppy-disk Save) on a pixel surface +carries no text that says which control it is, so the identity check has +nothing to compare. Most applications answer a pointer resting on a control: +a highlight, a raised border, a tooltip or status-bar text near the pointer. +The recording can capture that answer, and replay can ask the same question +before it clicks. + +At record time the recorder moves the pointer onto the target, waits +:data:`HOVER_DWELL_MS`, and saves the frame. The compiler keeps the pixels +that changed near the target (:func:`measure_response`) as the step's +:class:`~openadapt_flow.ir.HoverEvidence`. At replay, on a pixel surface, the +runtime moves the pointer onto the resolved target, waits the same time, and +compares (:func:`compare`): + +- ``verified``: the live response has the recorded shape and position and + the same content (pixels on the same render; OCR text otherwise); +- ``mismatch``: a response with other content, or in another place or + shape; the run stops before the click; +- ``unexpected``: the recording saw no response and the live screen + answered; the run stops before the click; +- ``absent``: the recording saw a response and the live screen did not. + No evidence; the step goes on as it did before hover evidence existed; +- ``unverifiable``: shape and position match but neither pixels nor text + can be compared (another render with no readable text). No evidence. + +Hover evidence only adds stops: ``mismatch`` and ``unexpected`` stop the run; +the other verdicts are recorded on the step and change nothing. + +Moving the pointer clicks nothing. Hover-opened menus are the one known +hazard: a response far from the target falls outside the measured window and +is ignored, but a menu that opens under the pointer is a response like any +other and is compared like one. +""" + +from __future__ import annotations + +import io +from dataclasses import dataclass +from typing import Any, Callable, Literal, Optional + +from openadapt_flow.ir import Point, Region +from openadapt_flow.runtime import identity as identity_mod + +HOVER_DWELL_MS = 800 +"""How long the pointer rests on the target before the hover frame.""" + +# The measured window around the pointer. Tooltips usually open below and to +# the right of the pointer; highlights cover the control under it. +WINDOW_LEFT = 160 +WINDOW_RIGHT = 320 +WINDOW_UP = 80 +WINDOW_DOWN = 160 + +CHANGE_LEVEL = 24 +"""Per-channel difference (0-255) for a pixel to count as changed.""" + +MIN_CHANGED_PX = 12 +"""Fewer changed pixels than this is noise, not a response.""" + +GEOMETRY_TOL_PX = 10 +GEOMETRY_TOL_FRAC = 0.25 +CROP_MATCH_MAX_MEAN = 0.04 +"""Mean grey difference (0-1) over the whole aligned response, at most.""" + +BLOCK_PX = 8 +BLOCK_MAX_MEAN = 0.12 +"""Mean grey difference in any :data:`BLOCK_PX` square, at most.""" + +ALIGN_MAX_SHIFT = 3 + +Verdict = Literal["verified", "mismatch", "unexpected", "absent", "unverifiable"] + + +@dataclass(frozen=True) +class Response: + """What changed near the pointer when it came to rest on the target.""" + + region: Region + crop: bytes + changed_px: int + + +def hover_window(point: Point, viewport: tuple[int, int]) -> Region: + x, y = int(point[0]), int(point[1]) + w, h = int(viewport[0]), int(viewport[1]) + x0 = max(0, x - WINDOW_LEFT) + y0 = max(0, y - WINDOW_UP) + x1 = min(w, x + WINDOW_RIGHT) + y1 = min(h, y + WINDOW_DOWN) + return (x0, y0, max(0, x1 - x0), max(0, y1 - y0)) + + +def _rgb(png: bytes) -> Any: + import numpy as np + from PIL import Image + + with Image.open(io.BytesIO(png)) as im: + return np.asarray(im.convert("RGB"), dtype=np.int16) + + +def measure_response( + pre_png: bytes, hover_png: bytes, point: Point +) -> Optional[Response]: + """The changed box near ``point`` between two frames, or None. + + Raises ValueError when the frames differ in size (the screen changed + shape, which is not a hover response). + """ + import numpy as np + + a, b = _rgb(pre_png), _rgb(hover_png) + if a.shape != b.shape: + raise ValueError("hover frame differs in size from the frame before it") + x, y, w, h = hover_window(point, (a.shape[1], a.shape[0])) + if w <= 0 or h <= 0: + return None + diff = np.abs(a[y : y + h, x : x + w] - b[y : y + h, x : x + w]).max(axis=2) + mask = diff > CHANGE_LEVEL + count = int(mask.sum()) + if count < MIN_CHANGED_PX: + return None + ys, xs = np.nonzero(mask) + region = ( + x + int(xs.min()), + y + int(ys.min()), + int(xs.max() - xs.min() + 1), + int(ys.max() - ys.min() + 1), + ) + crop = identity_mod.crop_region(hover_png, region) + if crop is None: + return None + return Response(region=region, crop=crop, changed_px=count) + + +def relative(region: Region, point: Point) -> Region: + """``region`` as an offset from ``point`` (dx, dy, w, h).""" + return ( + int(region[0] - point[0]), + int(region[1] - point[1]), + int(region[2]), + int(region[3]), + ) + + +def _close(a: int, b: int, size: int) -> bool: + return abs(a - b) <= max(GEOMETRY_TOL_PX, GEOMETRY_TOL_FRAC * size) + + +def geometry_matches(recorded: Region, live: Region) -> bool: + """Same place and shape relative to the target, within tolerance.""" + rx, ry, rw, rh = recorded + lx, ly, lw, lh = live + return ( + _close(rx + rw // 2, lx + lw // 2, rw) + and _close(ry + rh // 2, ly + lh // 2, rh) + and _close(rw, lw, rw) + and _close(rh, lh, rh) + ) + + +def crops_look_same(recorded_png: bytes, live_png: bytes) -> bool: + """Whether two response crops show the same thing. + + Grey images. The live crop is aligned onto the recorded one by the best + shift of up to :data:`ALIGN_MAX_SHIFT` pixels (a response drawn a pixel + or two away), or resized when the sizes differ by more. Then every + :data:`BLOCK_PX` square must agree: a changed word in a tooltip is a + local difference that an average over the whole crop would hide. + """ + import numpy as np + from PIL import Image + + with ( + Image.open(io.BytesIO(recorded_png)) as r, + Image.open(io.BytesIO(live_png)) as lv, + ): + a = np.asarray(r.convert("L"), dtype=np.float32) / 255.0 + lg = lv.convert("L") + if ( + abs(lg.size[0] - a.shape[1]) > ALIGN_MAX_SHIFT + or abs(lg.size[1] - a.shape[0]) > ALIGN_MAX_SHIFT + ): + lg = lg.resize((a.shape[1], a.shape[0])) + b = np.asarray(lg, dtype=np.float32) / 255.0 + pad = ALIGN_MAX_SHIFT + h, w = a.shape + canvas = np.full((h + 2 * pad, w + 2 * pad), np.nan, dtype=np.float32) + bh, bw = min(b.shape[0], h + pad), min(b.shape[1], w + pad) + best: Optional[Any] = None + best_mean = float("inf") + for dy in range(-pad, pad + 1): + for dx in range(-pad, pad + 1): + canvas[:] = np.nan + y0, x0 = pad + dy, pad + dx + canvas[y0 : y0 + bh, x0 : x0 + bw] = b[:bh, :bw] + window = canvas[pad : pad + h, pad : pad + w] + diff = np.abs(window - a) + covered = ~np.isnan(diff) + if covered.mean() < 0.9: + continue + mean = float(np.nanmean(diff)) + if mean < best_mean: + best_mean, best = mean, np.where(covered, diff, 0.0) + if best is None or best_mean > CROP_MATCH_MAX_MEAN: + return False + for y in range(0, h, BLOCK_PX): + for x in range(0, w, BLOCK_PX): + if float(best[y : y + BLOCK_PX, x : x + BLOCK_PX].mean()) > BLOCK_MAX_MEAN: + return False + return True + + +def _text(png: bytes, ocr: Callable[..., Any]) -> str: + try: + lines = ocr(png) or [] + except Exception: # noqa: BLE001 - unreadable is no text + return "" + words = " ".join(str(getattr(line, "text", "")) for line in lines) + return identity_mod.ocr_canonical(identity_mod.squash(words)) + + +def compare( + *, + recorded_response: bool, + recorded_region: Optional[Region], + recorded_crop: Optional[bytes], + live: Optional[Response], + point: Point, + same_render: bool, + ocr: Optional[Callable[..., Any]] = None, +) -> tuple[Verdict, str]: + """Compare the live hover response with the recorded one. + + Args: + recorded_response: Whether the recording saw any response. + recorded_region: The recorded response box, relative to the click. + recorded_crop: The recorded response pixels. + live: The live response at the resolved point, or None. + point: The resolved point. + same_render: The live frame is drawn at the recorded scale and by the + same rasterizer, so pixels can be compared. + ocr: OCR for the other-render comparison. + + Returns: + ``(verdict, reason)``. + """ + if not recorded_response: + if live is None: + # Agrees, but no response is not evidence of which control it is. + return "unverifiable", "no_response_as_recorded" + return "unexpected", "response_where_none_was_recorded" + if live is None: + return "absent", "no_response" + if recorded_region is None or recorded_crop is None: + return "unverifiable", "recording_kept_no_response" + if not geometry_matches(recorded_region, relative(live.region, point)): + return "mismatch", "response_moved_or_resized" + if same_render and not identity_mod.text_render_differs(recorded_crop, live.crop): + if crops_look_same(recorded_crop, live.crop): + return "verified", "same_pixels" + if ocr is None: + return "mismatch", "different_pixels" + # Another rasterizer can redraw the same tooltip with other pixels + # (ClearType switched); its text still decides. + if ocr is not None: + rec_text, live_text = _text(recorded_crop, ocr), _text(live.crop, ocr) + if rec_text and live_text: + if rec_text == live_text: + return "verified", "same_text" + return "mismatch", "different_text" + if rec_text or live_text: + return "mismatch", "text_on_one_side_only" + if same_render: + return "mismatch", "different_pixels" + return "unverifiable", "no_comparable_content" + + +__all__ = [ + "HOVER_DWELL_MS", + "Response", + "Verdict", + "compare", + "crops_look_same", + "geometry_matches", + "hover_window", + "measure_response", + "relative", +] diff --git a/packages/flow/openadapt_flow/runtime/identity_looks.py b/packages/flow/openadapt_flow/runtime/identity_looks.py new file mode 100644 index 000000000..5e28d6ed2 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/identity_looks.py @@ -0,0 +1,555 @@ +"""Extra looks before stopping on a look-alike identifier. + +On a pixel-only surface (RDP, Citrix, canvas) the OCR identity tier abstains +when the record's identifier carries a look-alike glyph (O/0, l/1/I): OCR can +read ``A1001`` and a different record's ``Al001`` as the same string. Before +the run stops on such an abstain, the replayer takes extra looks that change +nothing on the screen: + +- ``frame``: the same OCR read on :data:`FRAME_LOOKS` fresh screenshots. Each + must abstain on the same look-alike identifier classes as the first read. + A frame that names another record ends the check with a mismatch. +- ``magnified``: the replayer's existing 2x re-read of the first frame. It + must not name another record, and when it reads the identifier it must read + it in the same look-alike classes. +- ``known_ids``: the complete known-ID list (:mod:`.known_ids`) when the + deployment configured one. +- ``clipboard``: where the surface allows it, a drag across the recorded + identifier cell, a copy, and a read of the clipboard (:func:`copy_identifier`). + The clipboard holds the application's own characters, so ``0`` and ``O`` + differ. + +OCR looks never settle the question on their own: a look-alike twin can read +the same on every frame and at every magnification. Only two looks can +resolve: the known-ID list, and a clipboard read that equals an EXACT +expected identifier. An exact expected identifier comes only from a run +parameter embedded in the band (the operator's typed or supplied value). A +recorded OCR band is not exact (a recording can read ``A1001`` as ``Alo01``), +so in recorded-record mode the clipboard can contradict but never accept. + +:func:`decide` accepts only when every frame agrees, no look contradicts, and +at least one look resolves to the expected record. Any contradiction is +final. Otherwise the original abstain stands, with the looks recorded on it. +""" + +from __future__ import annotations + +import hashlib +import io +import secrets +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Literal, Mapping, Optional, Sequence + +from openadapt_flow.ir import IdentityCheck, IdentityLook, Region +from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime.known_ids import ( + ABSTAIN_REASON_GLYPH, + KnownIdList, + id_core, + lookalike_class, + resolve_lookalike_abstain, +) + +FRAME_LOOKS = 2 +"""Fresh screenshots read after the first one.""" + +SELECTION_PAD_PX = 6 +"""How far outside the identifier cell a selection highlight may paint.""" + +COPY_GESTURES = ("drag", "double_click") + + +def identifier_tokens(text: str) -> list[str]: + """Identifier-shaped tokens of a read, in :func:`known_ids.id_core` form.""" + out = [] + for raw in (text or "").split(): + tok = identity_mod.squash(raw) + if tok and identity_mod._is_identifier_shaped(tok): + out.append(id_core(tok)) + return out + + +def lookalike_classes(text: str) -> frozenset[str]: + """Look-alike classes of the identifier-shaped tokens in ``text``.""" + return frozenset(lookalike_class(t) for t in identifier_tokens(text)) + + +def vulnerable_classes(text: str) -> frozenset[str]: + """Classes of the identifiers OCR could have collapsed (O/0, l/1/I).""" + return frozenset( + lookalike_class(t) + for t in identifier_tokens(text) + if identity_mod._is_glyph_vulnerable_identifier(t) + ) + + +def is_glyph_abstain(check: Optional[IdentityCheck]) -> bool: + return ( + check is not None + and check.status == "abstain" + and check.abstain_reason == ABSTAIN_REASON_GLYPH + ) + + +def exact_expected_ids( + run_values: Mapping[str, Any], embedded: Iterable[str] +) -> tuple[str, ...]: + """Exact identifiers the operator gave for parameters in the band. + + Only parameters embedded in the recorded band count: a run value that is + not part of the identity band says nothing about which record this is. + """ + out: list[str] = [] + for name in embedded: + value = run_values.get(name) + if isinstance(value, str): + out.extend(identifier_tokens(value)) + return tuple(dict.fromkeys(out)) + + +@dataclass(frozen=True) +class IdentityCopyConfig: + """Turns on the select-and-copy look for a deployment. + + Only for surfaces where selecting the identifier cell and copying it + changes nothing: a read-only text cell, with the remote session's + clipboard redirected to the machine Flow runs on. Any screen change + outside the cell during the look halts the run. + """ + + gesture: str = "drag" + copy_keys: tuple[str, ...] = ("Control+c",) + + def __post_init__(self) -> None: + if self.gesture not in COPY_GESTURES: + raise ValueError( + f"identity copy gesture must be one of {COPY_GESTURES}, " + f"not {self.gesture!r}" + ) + if not self.copy_keys: + raise ValueError("identity copy needs at least one copy key") + + +@dataclass(frozen=True) +class LookBinding: + """A clipboard read taken at step entry, reusable at actuation. + + The actuation-time identity check takes no input. It reuses this read, or + the verdict it settled, only when the identifier cell on its frame shows + the same pixels as one of the crops seen at step entry. + """ + + step_id: str + classes: frozenset[str] + read: "ClipboardRead" + crops: tuple[bytes, ...] + #: The step-entry verdict when the copy settled the record. + verdict: Optional[IdentityCheck] = None + + def same_pixels(self, step_id: str, crop: Optional[bytes]) -> bool: + return step_id == self.step_id and any(crops_match(crop, c) for c in self.crops) + + def matches( + self, step_id: str, classes: frozenset[str], crop: Optional[bytes] + ) -> bool: + return classes == self.classes and self.same_pixels(step_id, crop) + + +@dataclass(frozen=True) +class ClipboardRead: + """What one select-and-copy look returned. + + Attributes: + text: The copied text, or None when nothing was copied. + changed_outside: The screen changed outside the identifier cell + while the look ran. The step halts. + pre_crop: The identifier cell's pixels just before the gesture. + post_crop: The identifier cell's pixels just after the copy. + frame_sha256: SHA-256 of the frame taken just after the copy. + reason: Why nothing was copied, when nothing was. + """ + + text: Optional[str] + changed_outside: bool = False + pre_crop: Optional[bytes] = None + post_crop: Optional[bytes] = None + frame_sha256: Optional[str] = None + reason: Optional[str] = None + + +def _look( + kind: Literal["frame", "magnified", "known_ids", "clipboard"], + outcome: Literal["agrees", "resolves", "contradicts", "unavailable"], + reason: Optional[str] = None, + frame_sha256: Optional[str] = None, +) -> IdentityLook: + return IdentityLook( + kind=kind, outcome=outcome, reason=reason, frame_sha256=frame_sha256 + ) + + +def _frame_look( + first_classes: frozenset[str], + read: IdentityCheck, + frame_sha256: Optional[str], + contradicts: Callable[[IdentityCheck], bool], +) -> IdentityLook: + if read.status == "mismatch": + reason = "names_another_record" if contradicts(read) else "read_incomplete" + return _look("frame", "contradicts", reason, frame_sha256) + if not is_glyph_abstain(read): + return _look("frame", "unavailable", f"read_{read.status}", frame_sha256) + if lookalike_classes(read.observed) != first_classes: + return _look("frame", "unavailable", "identifier_read_differs", frame_sha256) + return _look("frame", "agrees", None, frame_sha256) + + +def _magnified_look( + first_classes: frozenset[str], + read: Optional[IdentityCheck], + contradicts: Callable[[IdentityCheck], bool], +) -> IdentityLook: + if read is None: + return _look("magnified", "unavailable", "not_taken") + if read.status == "mismatch" and contradicts(read): + return _look("magnified", "contradicts", "names_another_record") + if read.status in ("unreadable", "mismatch"): + # Nothing readable, or only part of the band: says nothing either way. + return _look("magnified", "unavailable", f"read_{read.status}") + classes = lookalike_classes(read.observed) + if not classes: + return _look("magnified", "unavailable", "no_identifier_read") + if classes != first_classes: + return _look("magnified", "unavailable", "identifier_read_differs") + return _look("magnified", "agrees") + + +def _listed(known_ids: KnownIdList, core: str) -> bool: + return core in known_ids.members_of_class(lookalike_class(core)) + + +def _clipboard_look( + first_tokens: Sequence[str], + copied: Optional[ClipboardRead], + exact: Sequence[str], + known_ids: Optional[KnownIdList], +) -> IdentityLook: + """Judge a select-and-copy read against the first OCR read. + + The copy and the read must name the same identifiers up to look-alike + glyphs: token by token, or, when one side splits an identifier the other + keeps whole (``103 045`` against ``103045``), as the joined string. + """ + if copied is None: + return _look("clipboard", "unavailable", "not_configured") + sha = copied.frame_sha256 + if copied.text is None: + return _look("clipboard", "unavailable", copied.reason or "nothing_copied", sha) + tokens = identifier_tokens(copied.text) + if not tokens: + return _look("clipboard", "unavailable", "no_identifier_copied", sha) + vulnerable_read = any( + identity_mod._is_glyph_vulnerable_identifier(t) for t in first_tokens + ) + by_class: dict[str, set[str]] = {} + for tok in tokens: + by_class.setdefault(lookalike_class(tok), set()).add(tok) + read_classes = {lookalike_class(t) for t in first_tokens} + exact_set = set(exact) + if set(by_class) == read_classes: + if any(len(v) != 1 for v in by_class.values()): + return _look("clipboard", "unavailable", "copy_ambiguous", sha) + copied_ids = {k: next(iter(v)) for k, v in by_class.items()} + vulnerable = { + lookalike_class(t) + for t in first_tokens + if identity_mod._is_glyph_vulnerable_identifier(t) + } + elif lookalike_class("".join(tokens)) == lookalike_class("".join(first_tokens)): + joined = "".join(tokens) + copied_ids = {lookalike_class(joined): joined} + vulnerable = set(copied_ids) if vulnerable_read else set() + exact_set |= {"".join(exact)} if exact else set() + else: + # The copy did not cover the identifier OCR read (or covered more). + return _look("clipboard", "unavailable", "copy_differs_from_read", sha) + if known_ids is not None and known_ids.complete: + all_listed = all(_listed(known_ids, c) for c in copied_ids.values()) + if not all_listed and not _listed(known_ids, "".join(tokens)): + return _look("clipboard", "contradicts", "copied_id_not_listed", sha) + exact_by_class: dict[str, set[str]] = {} + for e in exact_set: + exact_by_class.setdefault(lookalike_class(e), set()).add(e) + for cls, core in copied_ids.items(): + expected = exact_by_class.get(cls) + if expected and core not in expected: + return _look("clipboard", "contradicts", "copied_id_differs", sha) + if vulnerable and all( + cls in exact_by_class and copied_ids[cls] in exact_by_class[cls] + for cls in vulnerable + ): + return _look("clipboard", "resolves", "copied_id_equals_run_value", sha) + return _look("clipboard", "agrees", "no_exact_expected_id", sha) + + +def decide( + first: IdentityCheck, + *, + frames: Sequence[tuple[IdentityCheck, Optional[str]]], + magnified: Optional[IdentityCheck], + contradicts: Callable[[IdentityCheck], bool], + known_ids: Optional[KnownIdList] = None, + known_ids_lift: Optional[IdentityCheck] = None, + clipboard: Optional[ClipboardRead] = None, + exact_expected: Sequence[str] = (), +) -> IdentityCheck: + """Combine the looks into one verdict. + + Args: + first: The first read: a glyph-only abstain. + frames: ``(read, frame_sha256)`` per fresh screenshot. + magnified: The 2x re-read of the first frame, if taken. + contradicts: Whether a ``mismatch`` read names another record + (``replayer.identity_read_contradicts``). + known_ids: The complete known-ID list, for the clipboard cross-check. + known_ids_lift: The list's verdict on the frame reads + (:func:`known_ids.resolve_lookalike_abstain`), or None. + clipboard: The select-and-copy read, or None when not taken. + exact_expected: Exact identifiers from run parameters in the band. + + Returns: + ``verified`` (with every look recorded) when every frame agrees, no + look contradicts and one resolves; the contradicting read (or the + first, as ``mismatch``) when any look contradicts; otherwise + ``first`` with the looks recorded. + """ + if not is_glyph_abstain(first): + return first + first_classes = lookalike_classes(first.observed) + looks: list[IdentityLook] = [] + for read, sha in frames: + look = _frame_look(first_classes, read, sha, contradicts) + looks.append(look) + if look.outcome == "contradicts": + # A fresh frame that names another record is final. + return read.model_copy(update={"looks": looks}) + mag = _magnified_look(first_classes, magnified, contradicts) + looks.append(mag) + if mag.outcome == "contradicts": + assert magnified is not None + return magnified.model_copy(update={"looks": looks}) + if known_ids is not None or known_ids_lift is not None: + looks.append( + _look("known_ids", "resolves", "single_listed_id") + if known_ids_lift is not None + else _look("known_ids", "unavailable", "list_does_not_settle_it") + ) + if clipboard is not None: + clip = _clipboard_look( + identifier_tokens(first.observed), clipboard, exact_expected, known_ids + ) + looks.append(clip) + if clip.outcome == "contradicts": + return first.model_copy( + update={ + "status": "mismatch", + "abstain_reason": None, + "looks": looks, + } + ) + frames_agree = bool(frames) and all( + look.outcome == "agrees" for look in looks if look.kind == "frame" + ) + magnified_ok = mag.outcome in ("agrees", "unavailable") + resolved = any(look.outcome == "resolves" for look in looks) + if frames_agree and magnified_ok and resolved: + base = known_ids_lift if known_ids_lift is not None else first + return base.model_copy( + update={"status": "verified", "abstain_reason": None, "looks": looks} + ) + return first.model_copy(update={"looks": looks}) + + +# ----------------------------------------------------------- copy look + + +def copy_region(lines: Iterable[Any], within: Region) -> Optional[Region]: + """The text box to select: the one line inside ``within`` that holds a + look-alike identifier. + + Selecting only that line keeps the gesture off every other control in the + row. None when no line, or more than one, holds such an identifier: the + copy look is then not taken. + """ + hits = [] + for line in lines: + region = getattr(line, "region", None) + if region is None or not identity_mod.regions_intersect(region, within): + continue + if vulnerable_classes(str(getattr(line, "text", ""))): + hits.append(region) + if len(hits) != 1: + return None + x, y, w, h = (int(v) for v in hits[0]) + return (x - 4, y - 2, w + 8, h + 4) + + +def crop_png(png: bytes, region: Region) -> Optional[bytes]: + return identity_mod.crop_region(png, region) + + +def _diff_outside(pre: bytes, post: bytes, allowed: Region) -> Optional[bool]: + """Whether ``post`` differs from ``pre`` outside ``allowed``. + + None when the frames cannot be compared (a different size counts as a + change). + """ + try: + import numpy as np + from PIL import Image + + a = np.asarray(Image.open(io.BytesIO(pre)).convert("RGB"), dtype=np.int16) + b = np.asarray(Image.open(io.BytesIO(post)).convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - an unreadable frame cannot be judged + return None + if a.shape != b.shape: + return True + changed = np.abs(a - b).max(axis=2) > 0 + x, y, w, h = allowed + x0 = max(0, x - SELECTION_PAD_PX) + y0 = max(0, y - SELECTION_PAD_PX) + x1 = min(changed.shape[1], x + w + SELECTION_PAD_PX) + y1 = min(changed.shape[0], y + h + SELECTION_PAD_PX) + changed[y0:y1, x0:x1] = False + return bool(changed.any()) + + +def copy_identifier( + backend: Any, + clipboard: Any, + region: Region, + *, + gesture: str = "drag", + copy_keys: Sequence[str] = ("Control+c",), + settle: Optional[Callable[[], bytes]] = None, + crop_region: Optional[Region] = None, +) -> ClipboardRead: + """Select the identifier cell, copy it, and read the clipboard. + + The clipboard's previous text is restored afterwards. The clipboard is + first set to a random marker; a clipboard still holding the marker counts + as nothing copied. The frames just before the gesture and just after the + copy must be identical outside the identifier cell (a selection highlight + may paint inside it); otherwise ``changed_outside`` is set and the caller + halts. + + Args: + backend: The run's backend. ``drag`` needs a ``drag(x, y, ex, ey)`` + method; ``double_click`` uses ``click(x, y, double=True)``. + clipboard: A :class:`~openadapt_flow.records.read.ClipboardReader`. + region: The identifier cell on the live frame. + gesture: ``drag`` (across the cell, left to right, at mid height) or + ``double_click`` (at the cell's centre). + copy_keys: Keys that copy the selection on the driven session. + settle: Returns the next settled frame; defaults to a screenshot. + crop_region: Where to cut ``pre_crop`` / ``post_crop``; defaults to + ``region``. + """ + x, y, w, h = (int(v) for v in region) + if w < 4 or h < 4: + return ClipboardRead(text=None, reason="cell_too_small") + if gesture == "drag" and not callable(getattr(backend, "drag", None)): + return ClipboardRead(text=None, reason="backend_cannot_drag") + if gesture not in COPY_GESTURES: + return ClipboardRead(text=None, reason="unknown_gesture") + capture = settle or backend.screenshot + try: + previous = clipboard.read_text() + except Exception: # noqa: BLE001 - no clipboard channel, no look + return ClipboardRead(text=None, reason="clipboard_unreadable") + marker = f"openadapt-identity-look-{secrets.token_hex(8)}" + copied: Optional[str] = None + pre = post = None + try: + clipboard.write_text(marker) + pre = capture() + mid_y = y + h // 2 + if gesture == "drag": + # From just before the first glyph to just past the last one: a + # drag that ends inside the last glyph leaves it unselected. + backend.drag(x + 1, mid_y, x + w - 1, mid_y) + else: + backend.click(x + w // 2, mid_y, double=True) + for key in copy_keys: + backend.press(key) + post = capture() + copied = clipboard.read_text() + except Exception as exc: # noqa: BLE001 - a failed look is no look + return ClipboardRead( + text=None, + changed_outside=bool( + pre is not None + and post is not None + and _diff_outside(pre, post, region) is not False + ), + reason=f"look_failed:{type(exc).__name__}"[:80], + ) + finally: + if previous is not None: + try: + clipboard.write_text(previous) + except Exception: # noqa: BLE001 - restoring is best effort + pass + outside = _diff_outside(pre, post, region) if pre and post else None + read = ClipboardRead( + text=None if copied in (None, marker) else copied, + changed_outside=outside is not False, + pre_crop=crop_png(pre, crop_region or region) if pre else None, + post_crop=crop_png(post, crop_region or region) if post else None, + frame_sha256=hashlib.sha256(post).hexdigest() if post else None, + reason="nothing_copied" if copied in (None, marker) else None, + ) + return read + + +def crops_match(a: Optional[bytes], b: Optional[bytes]) -> bool: + """Whether two identifier-cell crops show the same pixels. + + Tolerates encoder noise (every channel within 8 levels, mean under 1), + never a changed glyph: a different character moves whole strokes. + """ + if a is None or b is None: + return False + if a == b: + return True + try: + import numpy as np + from PIL import Image + + x = np.asarray(Image.open(io.BytesIO(a)).convert("RGB"), dtype=np.int16) + y = np.asarray(Image.open(io.BytesIO(b)).convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - unreadable crops never match + return False + if x.shape != y.shape: + return False + d = np.abs(x - y) + return bool(d.max() <= 8 and d.mean() < 1.0) + + +__all__ = [ + "COPY_GESTURES", + "ClipboardRead", + "FRAME_LOOKS", + "IdentityCopyConfig", + "LookBinding", + "copy_identifier", + "copy_region", + "crops_match", + "decide", + "exact_expected_ids", + "identifier_tokens", + "is_glyph_abstain", + "lookalike_classes", + "resolve_lookalike_abstain", + "vulnerable_classes", +] diff --git a/packages/flow/openadapt_flow/runtime/known_ids.py b/packages/flow/openadapt_flow/runtime/known_ids.py index 0fa17ec9f..7d528d3ef 100644 --- a/packages/flow/openadapt_flow/runtime/known_ids.py +++ b/packages/flow/openadapt_flow/runtime/known_ids.py @@ -137,6 +137,10 @@ def members(self, token: str) -> tuple[str, ...]: """Listed identifiers OCR could read as ``token``.""" return self._classes.get(lookalike_class(token), ()) + def members_of_class(self, key: str) -> tuple[str, ...]: + """Listed identifiers in one look-alike class (a :func:`lookalike_class` key).""" + return self._classes.get(key, ()) + @property def size(self) -> int: return sum(len(v) for v in self._classes.values()) diff --git a/packages/flow/openadapt_flow/runtime/render_scale.py b/packages/flow/openadapt_flow/runtime/render_scale.py index cceededc0..a3f6818cb 100644 --- a/packages/flow/openadapt_flow/runtime/render_scale.py +++ b/packages/flow/openadapt_flow/runtime/render_scale.py @@ -707,6 +707,7 @@ def _passthrough(name: str) -> property: "act_guarded_coordinate": _point_method, "arm_guarded_keyboard": _point_method, "prepare_pointer_actuation": _point_method, + "hover": _point_method, "drag": _drag_method, "drag_guarded": _drag_method, "acquire_actuation_frame": _frame_method, diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 6dcfdf5fe..5310a2a01 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -35,6 +35,7 @@ from __future__ import annotations +import dataclasses import functools import hashlib import hmac @@ -59,6 +60,7 @@ Optional, Sequence, TypeVar, + Union, cast, ) from urllib.parse import urlsplit @@ -94,11 +96,14 @@ Anchor, ApiBinding, BusinessDecisionEvidence, + CheckToClickRecheck, EffectVerificationEvidence, ExecutionTargetKind, FreshActuationEvent, HaltObservation, + HoverLook, IdentityCheck, + IdentityLook, IdentitySignalEvidence, Interstitial, InterstitialActionResult, @@ -151,6 +156,7 @@ from openadapt_flow.runtime import heal as heal_mod from openadapt_flow.runtime import healing as healing_mod from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime import identity_looks as looks_mod from openadapt_flow.runtime.authorization import ( GovernedRunAuthorization, RuntimeParamScalar, @@ -185,6 +191,9 @@ from openadapt_flow.runtime.effects.adapter import ( verifier_identity as effect_verifier_identity, ) +from openadapt_flow.runtime.known_ids import ( + lookalike_class as known_ids_lookalike_class, +) from openadapt_flow.runtime.program_predicates import ( evaluate_program_predicate, exact_png_size, @@ -211,6 +220,13 @@ _DeliveryResultT = TypeVar("_DeliveryResultT") _MAX_FRESH_ACTUATION_REACQUISITIONS = 2 +# Check-to-click guard on the pixel path (``Replayer._pixel_recheck``): how +# often a changed target or identity region is re-observed before the click +# is refused, and how much a pixel must change to count (remote encoders +# shift colours by a few levels; a dialog or a new row changes far more). +_PIXEL_RECHECK_REOBSERVES = 2 +_PIXEL_RECHECK_LEVEL = 24 +_PIXEL_RECHECK_MIN_PIXELS = 4 # How long a step that raises a browser dialog waits for it to open. _DIALOG_WAIT_S = 5.0 # Private module capability. The public ``run`` API cannot mint a durable @@ -675,9 +691,19 @@ class Replayer: structured-text, pixel-compare, OCR, and halt with no model. known_ids: Optional complete ``runtime.known_ids.KnownIdList``. When set, an OCR identity check that abstained only because the band - rests on a look-alike identifier is lifted to verified if two OCR - reads agree and exactly one listed identifier is in that - look-alike class. None (default) keeps the abstain. + rests on a look-alike identifier is lifted to verified if the + reads of three separate screenshots agree and exactly one listed + identifier is in that look-alike class. None (default) keeps the + abstain. + identity_copy: Optional + :class:`~openadapt_flow.runtime.identity_looks.IdentityCopyConfig`. + With a ``read_clipboard`` channel, a look-alike abstain on a step + with a recorded identifier cell takes one more look: select the + cell, copy it, read the clipboard. The copy settles the record only + when it equals a run parameter embedded in the band; it stops the + run when it names another record or the screen changed outside the + cell. Never taken in governed or qualification runs. None + (default) takes no copy look. effect_verifier: Optional system-of-record ``EffectVerifier`` (``runtime.effects``) bound to the deployment's system of record (a REST/FHIR API, a document store). When set, any step that @@ -764,6 +790,8 @@ class Replayer: _render_scale_attempted: bool = False _render_scale_inner: Optional[Backend] = None _render_scale_evidence: Optional[dict[str, Any]] = None + identity_copy: Optional[looks_mod.IdentityCopyConfig] = None + _identity_input_looks: bool = False def __init__( self, @@ -808,6 +836,7 @@ def __init__( attended_draft_by: Optional[Any] = None, draft_key_dir: Optional[Path] = None, read_clipboard: Optional[Any] = None, + identity_copy: Optional[looks_mod.IdentityCopyConfig] = None, deployment_surface: Optional[str] = None, layout_drift: Optional[bool] = None, layout_drift_tolerances: Optional[Any] = None, @@ -1073,6 +1102,13 @@ def __init__( # Optional complete known-ID list (runtime.known_ids): lifts a # look-alike identifier abstain only when no listed twin exists. self.known_ids = known_ids + # Optional select-and-copy look for look-alike identifiers + # (runtime.identity_looks). It is input, so only the step-entry + # identity check takes it; the actuation-time check reuses its read + # through a pixel-matched binding. + self.identity_copy = identity_copy + self._identity_input_looks = False + self._identity_look_bindings: dict[str, looks_mod.LookBinding] = {} # Optional VLM drift-oracle (state-verifier), OFF by default. When set, # it may CONFIRM a render-drift-sensitive postcondition that # deterministically false-failed (the same heal-under-drift the @@ -7446,7 +7482,7 @@ def _run_step_on_active_surface( ) last_frame = before_png pre_identity_halt = (result.safety_halt, result.failure_category) - error = self._identity_gate_error( + error = self._entry_identity_gate_error( step, resolution, before_png, @@ -7517,7 +7553,7 @@ def _run_step_on_active_surface( result.resolution = resolution # The first read's halt flags belong to that read. result.safety_halt, result.failure_category = pre_identity_halt - error = self._identity_gate_error( + error = self._entry_identity_gate_error( step, resolution, before_png, @@ -9685,6 +9721,27 @@ def _identity_band_still_changing( return False return False + def _entry_identity_gate_error( + self, + step: Step, + *args: Any, + **kwargs: Any, + ) -> Optional[str]: + """The identity gate at step entry, where an extra look may use input. + + Only this check may select and copy the identifier cell + (:mod:`openadapt_flow.runtime.identity_looks`). The actuation-time + check that follows takes no input and reuses that read only through a + pixel-matched binding. + """ + self.__dict__.setdefault("_identity_look_bindings", {}).pop(step.id, None) + self.__dict__.setdefault("_hover_frames", {}).pop(step.id, None) + self._identity_input_looks = True + try: + return self._identity_gate_error(step, *args, **kwargs) + finally: + self._identity_input_looks = False + def _identity_gate_error( self, step: Step, @@ -9704,7 +9761,12 @@ def _identity_gate_error( if qualification is not None else None ) - if ( + hover_check = ( + self._hover_check(step, resolution, frame_png, workflow, bundle_dir) + if qualified_policy is None + else None + ) + if hover_check is None and ( step.action not in ( ActionKind.CLICK, @@ -9728,9 +9790,15 @@ def _identity_gate_error( ) ): return None + # A hover check needs an anchor too (see _hover_check). + assert step.anchor is not None - check = self._verify_identity( - step, resolution, frame_png, params, workflow, bundle_dir + check = ( + hover_check + if hover_check is not None + else self._verify_identity( + step, resolution, frame_png, params, workflow, bundle_dir + ) ) if ( check.status in {"verified", "mismatch"} @@ -9788,6 +9856,13 @@ def _identity_gate_error( f"{check.quorum_required or 0} independent signals verified " "on the fresh frame — refusing to act; run aborted" ) + elif check.look_halt is not None: + error = ( + f"Step '{step.id}' ({step.intent}) took an extra look at a " + "look-alike identifier and the screen changed outside the " + f"identifier ({check.look_halt}) — refusing to act" + ) + result.safety_halt = True elif check.known_id_halt is not None: names = { "known_ids_unconfirmed": "has no admission confirmation", @@ -9801,6 +9876,18 @@ def _identity_gate_error( "confirm a current export; refusing to act" ) result.safety_halt = True + elif check.mode == "hover" and check.status == "mismatch": + reason = check.hover.reason if check.hover is not None else "differs" + error = ( + f"Step '{step.id}' ({step.intent}): with the pointer on the " + "target, the screen answered differently from the recording " + f"({reason}) — refusing to click; run aborted" + ) + elif check.mode == "hover": + # Hover evidence only adds a stop. A step that had no identity + # check before keeps going when the response is absent or cannot + # be compared; the verdict stays on the step result. + error = None elif check.status == "mismatch": error = ( f"Identity check failed for step '{step.id}' " @@ -12184,6 +12271,20 @@ def _act_without_dialog( "— refusing raw coordinate delivery; run aborted" ) else: + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + resolution = rechecked + x, y = resolution.point refusal = self._delivery_authorization_refusal( workflow, params, step, result ) @@ -12302,6 +12403,19 @@ def _act_without_dialog( "identity verification to delivery; run aborted" ) elif isinstance(self.backend, RichPointerActionBackend): + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + x, y = rechecked.point refusal = self._delivery_authorization_refusal( workflow, params, step, result ) @@ -14687,6 +14801,26 @@ def _verify_identity_ocr( rw, rh, ) + if identifier_region is not None and not self._identity_input_looks: + # The check just before the click takes no input. When the step-entry + # check settled the record with a copy and the identifier shows the + # same pixels now, the same record is on screen: reuse that verdict. + binding = self.__dict__.get("_identity_look_bindings", {}).get(step.id) + if ( + binding is not None + and binding.verdict is not None + and binding.same_pixels( + step.id, looks_mod.crop_png(before_png, identifier_region) + ) + ): + reused = IdentityLook( + kind="clipboard", + outcome="resolves", + reason="same_pixels_as_step_entry_copy", + ) + return binding.verdict.model_copy( + update={"looks": [*binding.verdict.looks, reused]} + ) band = identity_mod.band_region( resolution.point, anchor.region[3], self.backend.viewport ) @@ -14786,10 +14920,23 @@ def native_attempt( ) check = native_attempt(before_png, first_row_lines) - if check.status == "verified" or identity_read_contradicts(check): - # A read that names a different record is final: a later read of - # the same screen must never overrule it into "verified". + if check.status == "verified": return check + if identity_read_contradicts(check): + # A read that names a different record is final: a later read of + # the same screen must never overrule it into "verified". The one + # exception is a run parameter OCR read with look-alike glyphs + # (C0X1234 read as COX1234): extra looks may confirm the exact + # value; otherwise this mismatch stands. + return self._lookalike_param_looks( + step, + check, + native_attempt, + params, + workflow, + identifier_region, + before_png, + ) upscaled = identity_mod.upscale_crop(before_png, live_region) if upscaled is None: return check @@ -14808,16 +14955,20 @@ def native_attempt( ), scoped_identifier=identifier_region is not None, ) + pending_looks: list[Any] = [] if ( - self.known_ids is not None - and retry.status not in ("verified", "mismatch") - and check.status == "abstain" - and check.abstain_reason == "glyph_confusable_identifier" + retry.status not in ("verified", "mismatch") + and looks_mod.is_glyph_abstain(check) + and ( + self.known_ids is not None + or self._copy_look_possible(identifier_region) + ) ): - # The band rests on a look-alike identifier. A confirmed, - # complete known-ID list with exactly one member in that class - # rules out a twin (runtime.known_ids), provided a second read - # from a FRESH screenshot agrees with the first. + # The band rests on a look-alike identifier. Take extra looks that + # change nothing (runtime.identity_looks): fresh frames, the 2x + # re-read above, the known-ID list, and, where configured, a + # select-and-copy of the identifier cell. Only the list or an + # exact copy can settle it; any contradiction is final. matches: Optional[Callable[[str], bool]] = None if anchor.identity_template is not None and anchor.identity_template.tokens: from openadapt_flow.runtime import identity_template as itmpl @@ -14825,9 +14976,30 @@ def native_attempt( matches = functools.partial( itmpl.recorded_identifier_matches, anchor.identity_template ) - lifted = self._known_id_lift(check, native_attempt, params, matches) - if lifted is not None: - return lifted + looked = self._look_again( + step, + check, + retry, + native_attempt, + params, + workflow, + identifier_region, + before_png, + matches, + ) + if ( + looked.status != "abstain" + or looked.known_id_halt is not None + or looked.look_halt is not None + ): + return looked + pending_looks = list(looked.looks) + + def with_looks(final: IdentityCheck) -> IdentityCheck: + if not pending_looks or final.status == "verified" or final.looks: + return final + return final.model_copy(update={"looks": pending_looks}) + check = better_identity_read(check, retry) if ( check.status == "verified" @@ -14836,7 +15008,7 @@ def native_attempt( ): # A read of a different record is final: the row re-read below # must never overrule it into "verified". - return check + return with_looks(check) # Last, re-read the point's own text row alone (see # identity.row_region): the same verdict rules, on a crop that holds # no part of the neighbouring rows. @@ -14849,7 +15021,7 @@ def native_attempt( ) ) if row is None or row_png is None: - return check + return with_looks(check) f = identity_mod.ROW_REREAD_FACTOR row_check = attempt( row_png, @@ -14862,22 +15034,431 @@ def native_attempt( exclude[3] * f, ), ) - return better_identity_read(check, row_check) + return with_looks(better_identity_read(check, row_check)) + + def _hover_check( + self, + step: Step, + resolution: Resolution, + frame_png: bytes, + workflow: Workflow, + bundle_dir: Optional[Path], + ) -> Optional[IdentityCheck]: + """Compare the target's hover response with the recording. + + Only at the step-entry check (moving the pointer is input), only for a + click whose recording saw a hover response and that has no other + identity evidence, + only on the pixel rungs, and only when the backend can hover + (:class:`~openadapt_flow.backend.PointerHoverBackend`). None when any + of that does not hold: the step then has no identity check, as + before. See :mod:`openadapt_flow.runtime.hover_evidence`. + """ + from openadapt_flow.runtime import hover_evidence as hover_mod + + evidence = step.hover + anchor = step.anchor + if ( + evidence is None + or not evidence.response + or anchor is None + or not self._identity_input_looks + or step.action + not in (ActionKind.CLICK, ActionKind.DOUBLE_CLICK, ActionKind.RIGHT_CLICK) + or resolution.rung == "structural" + or anchor.context_text + or anchor.structured_identity + or anchor.identity_template + or anchor.identifier_crop + ): + return None + hover = getattr(self.backend, "hover", None) + if not callable(hover): + return None + point = (int(resolution.point[0]), int(resolution.point[1])) + + def check(verdict: str, reason: str, png: Optional[bytes]) -> IdentityCheck: + status = { + "verified": "verified", + "mismatch": "mismatch", + "unexpected": "mismatch", + "absent": "abstain", + "unverifiable": "unreadable", + }[verdict] + return IdentityCheck( + status=status, # type: ignore[arg-type] + mode="hover", + expected="recorded hover response", + observed=reason, + hover=HoverLook( + verdict=verdict, # type: ignore[arg-type] + reason=reason, + frame_sha256=sha256_bytes(png) if png is not None else None, + ), + ) + + try: + hover(*point) + except NotImplementedError: + return None + except Exception as exc: # noqa: BLE001 - a failed hover confirms nothing + return check( + "unverifiable", f"hover_failed:{type(exc).__name__}"[:80], None + ) + time.sleep(evidence.dwell_ms / 1000.0) + hover_png = self.vision.wait_settled(self.backend) + try: + live = hover_mod.measure_response(frame_png, hover_png, point) + except ValueError: + return check("mismatch", "screen_changed_size", hover_png) + recorded_crop = ( + self._asset_bytes(bundle_dir, evidence.crop, workflow=workflow) + if evidence.crop and bundle_dir is not None + else None + ) + # The click that follows is checked against this frame, not the one + # before the pointer moved (see _pixel_recheck). + self.__dict__.setdefault("_hover_frames", {})[step.id] = hover_png + verdict, reason = hover_mod.compare( + recorded_response=evidence.response, + recorded_region=evidence.region, + recorded_crop=recorded_crop, + live=live, + point=point, + same_render=self._render_scale_inner is None, + ocr=self.vision.ocr, + ) + return check(verdict, reason, hover_png) + + def _band_check( + self, + anchor: Anchor, + observed: str, + run_values: Mapping[str, str], + workflow: Workflow, + ) -> IdentityCheck: + """The OCR-tier verdict for ``observed`` with the given run values.""" + tmpl = anchor.identity_template + if tmpl is not None and tmpl.tokens: + from openadapt_flow.runtime import identity_template as itmpl + + return itmpl.verify_template_identity( + tmpl, observed, params=dict(run_values), param_examples=workflow.params + ) + return identity_mod.verify_target_identity( + anchor.context_text or "", + observed, + params=dict(run_values), + param_examples=workflow.params, + ) + + def _lookalike_param_read( + self, + check: IdentityCheck, + anchor: Anchor, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + ) -> Optional[IdentityCheck]: + """A run-parameter mismatch that is only a look-alike misread, as the + look-alike abstain it would be, or None. + + The parameter's identifier is read with look-alike glyphs (the run + value ``C0X1234`` read as ``COX1234``). The band is checked again with + the value as OCR read it; when every other identity budget then passes + and only the look-alike identifier remains, OCR cannot say which of the + two it is. Used only to decide whether extra looks may run; the caller + keeps the mismatch unless a look settles the record. + """ + if check.status != "mismatch" or check.mode != "param" or not check.param: + return None + run_values = runtime_params_for_gui(params) + run = run_values.get(check.param) + if not isinstance(run, str): + return None + run_tokens = looks_mod.identifier_tokens(run) + if not any(identity_mod._is_glyph_vulnerable_identifier(t) for t in run_tokens): + return None + seen = looks_mod.identifier_tokens(check.observed) + as_read: list[str] = [] + for token in run_tokens: + same = { + o + for o in seen + if known_ids_lookalike_class(o) == known_ids_lookalike_class(token) + } + if len(same) != 1: + return None + as_read.append(same.pop()) + if as_read == run_tokens: + return None + again = self._band_check( + anchor, + check.observed, + {**run_values, check.param: " ".join(as_read)}, + workflow, + ) + if not looks_mod.is_glyph_abstain(again): + return None + return again - def _known_id_lift( + def _lookalike_param_looks( self, + step: Step, + check: IdentityCheck, + native_attempt: Callable[[bytes], IdentityCheck], + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + identifier_region: Optional[Region], + frame_png: bytes, + ) -> IdentityCheck: + """Extra looks for a look-alike run-parameter mismatch. + + Returns the looks' ``verified`` verdict, a named halt, or ``check`` + unchanged (with the looks recorded). Never turns a mismatch into an + abstain. + """ + anchor = step.anchor + assert anchor is not None + first = self._lookalike_param_read(check, anchor, params, workflow) + if first is None or not ( + self.known_ids is not None or self._copy_look_possible(identifier_region) + ): + return check + + def read(png: bytes) -> IdentityCheck: + got = native_attempt(png) + return self._lookalike_param_read(got, anchor, params, workflow) or got + + matches: Optional[Callable[[str], bool]] = None + if anchor.identity_template is not None and anchor.identity_template.tokens: + from openadapt_flow.runtime import identity_template as itmpl + + matches = functools.partial( + itmpl.recorded_identifier_matches, anchor.identity_template + ) + looked = self._look_again( + step, + first, + None, + read, + params, + workflow, + identifier_region, + frame_png, + matches, + ) + if ( + looked.status == "verified" + or looked.known_id_halt is not None + or looked.look_halt is not None + ): + return looked + return check.model_copy(update={"looks": list(looked.looks)}) + + def _check_regions( + self, step: Step, resolution: Resolution, matched_region: Optional[Region] + ) -> dict[str, Region]: + """The regions a pixel step was checked on, at ``resolution``. + + ``target``: where the target matched (or a box of the recorded + target's size at the point). ``identity``: the identifier region, or + the identity band's row, when the step has identity evidence. + ``hover``: the recorded hover response, when there is one. + """ + anchor = step.anchor + px, py = int(resolution.point[0]), int(resolution.point[1]) + regions: dict[str, Region] = {} + if matched_region is not None: + regions["target"] = matched_region + elif anchor is not None: + w, h = int(anchor.region[2]), int(anchor.region[3]) + regions["target"] = (px - w // 2, py - h // 2, w, h) + else: + regions["target"] = (px - 8, py - 8, 16, 16) + if anchor is not None: + if anchor.identifier_region is not None: + rx, ry, rw, rh = anchor.identifier_region + regions["identity"] = ( + rx + px - anchor.click_point[0], + ry + py - anchor.click_point[1], + rw, + rh, + ) + elif anchor.context_text or anchor.identity_template: + h = int(anchor.region[3]) + regions["identity"] = ( + 0, + py - h // 2 - 4, + int(self.backend.viewport[0]), + h + 8, + ) + if step.hover is not None and step.hover.region is not None: + dx, dy, w, h = step.hover.region + regions["hover"] = (px + dx, py + dy, w, h) + return regions + + @staticmethod + def _changed_regions( + before_png: bytes, fresh_png: bytes, regions: Mapping[str, Region] + ) -> list[str]: + """Names of ``regions`` whose pixels differ between two frames.""" + import numpy as np + from PIL import Image + + try: + with ( + Image.open(io.BytesIO(before_png)) as a_img, + Image.open(io.BytesIO(fresh_png)) as b_img, + ): + a = np.asarray(a_img.convert("RGB"), dtype=np.int16) + b = np.asarray(b_img.convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - an unreadable frame is a change + return ["frame_size"] + if a.shape != b.shape: + return ["frame_size"] + height, width = a.shape[:2] + changed: list[str] = [] + for name, (x, y, w, h) in regions.items(): + x0, y0 = max(0, int(x)), max(0, int(y)) + x1, y1 = min(width, int(x + w)), min(height, int(y + h)) + if x1 <= x0 or y1 <= y0: + continue + diff = np.abs(a[y0:y1, x0:x1] - b[y0:y1, x0:x1]).max(axis=2) + if int((diff > _PIXEL_RECHECK_LEVEL).sum()) >= _PIXEL_RECHECK_MIN_PIXELS: + changed.append(name) + return changed + + def _pixel_recheck( + self, + step: Step, + resolution: Resolution, + checked_png: bytes, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + bundle_dir: Path, + result: StepResult, + run_dir: Path, + ) -> Union[Resolution, str]: + """Guard the gap between a pixel step's check and its click. + + Right before the click, take a fresh frame and compare the target, + identity and hover regions with the frame the step was checked on (the + hover frame when the step took a hover look). When one changed (a + dialog opened over the target, the list re-sorted), do not click: + re-resolve the target and re-run the identity check on the fresh + frame, with no input, and compare again on the next frame. After + :data:`_PIXEL_RECHECK_REOBSERVES` re-observations that still see + changes, or when the target or its identity cannot be confirmed again, + refuse the click. Structured (DOM / accessibility) targets are bound + by element tokens instead and are not rechecked here. + + Returns the resolution to click, or the refusal. + """ + if resolution.rung == "structural": + return resolution + baseline = self.__dict__.get("_hover_frames", {}).get(step.id) or checked_png + current = resolution + region = self._active_delivery_region + for attempt in range(_PIXEL_RECHECK_REOBSERVES + 1): + fresh = self.backend.screenshot() + changed = self._changed_regions( + baseline, fresh, self._check_regions(step, current, region) + ) + if not changed: + if current is not resolution: + result.resolution = current + return current + what = ", ".join(changed) + if attempt == _PIXEL_RECHECK_REOBSERVES: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + break + fresh_resolution, fresh_region, error = self._resolve_step( + step, fresh, bundle_dir, workflow + ) + if error is None and fresh_resolution is not None: + error = self._identity_gate_error( + step, + fresh_resolution, + fresh, + params, + workflow, + bundle_dir, + result, + evidence_run_dir=run_dir, + ) + if error is not None or fresh_resolution is None: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}): the screen changed at the " + f"{what} region between the check and the click, and the " + "target could not be confirmed again on a fresh frame " + f"({error or 'target not found'}) — click not delivered" + ) + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="reobserved") # type: ignore[arg-type] + ) + current, region, baseline = fresh_resolution, fresh_region, fresh + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}): the screen kept changing at the " + f"{what} region between the check and the click — click not " + "delivered; run aborted" + ) + + def _copy_look_allowed(self) -> bool: + """The copy look is input: never in governed or qualification runs.""" + return ( + self.identity_copy is not None + and self._read_clipboard is not None + and self.governed_authorization is None + and self.managed_dispatch_binding is None + and self.production_qualification_guard is None + and self.qualification_campaign_guard is None + and self.qualification_fault_driver is None + and getattr(self, "approval_gate", None) is None + ) + + def _copy_look_possible(self, identifier_region: Optional[Region]) -> bool: + return identifier_region is not None and self._copy_look_allowed() + + def _embedded_identity_params( + self, anchor: Anchor, workflow: Workflow + ) -> list[str]: + """Workflow parameters whose demonstrated value is in the identity band.""" + tmpl = anchor.identity_template + if tmpl is not None and tmpl.tokens: + return list(tmpl.param_token_indices) + if anchor.context_text: + return identity_mod.embedded_params(anchor.context_text, workflow.params) + return [] + + def _look_again( + self, + step: Step, first: IdentityCheck, + magnified: Optional[IdentityCheck], read: Callable[[bytes], IdentityCheck], params: Optional[Mapping[str, RuntimeParamScalar]], + workflow: Workflow, + identifier_region: Optional[Region], + frame_png: bytes, recorded_id_matches: Optional[Callable[[str], bool]] = None, - ) -> Optional[IdentityCheck]: - """Known-ID cross-check for a look-alike abstain. - - Returns a ``verified`` check, the abstain marked with a named - ``known_id_halt`` when the list may not be used now, the fresh read - when it is a ``mismatch`` (final), or None (keep the original - verdict). The second read comes from a new screenshot, - so a transient render state on one capture cannot satisfy both. + ) -> IdentityCheck: + """Extra looks before stopping on a look-alike identifier. + + Returns the verdict of :func:`identity_looks.decide`: ``verified`` + when every fresh frame agrees and the known-ID list or an exact copy + settles the record, the contradicting read when a look names another + record, or ``first`` carrying the looks. A known-ID list that may not + be used now, or a copy look that changed the screen, returns ``first`` + marked with a named halt. """ from openadapt_flow.runtime import known_ids as known_ids_mod @@ -14886,34 +15467,136 @@ def _known_id_lift( if isinstance(self.known_ids, known_ids_mod.KnownIdAdmission) else None ) + known: Any = self.known_ids if admission is not None: halt = admission.check() if halt is not None: return first.model_copy(update={"known_id_halt": halt.reason}) - known: Any = admission.known_ids - else: - known = self.known_ids - second = read(self.backend.screenshot()) - if second.status == "mismatch": - # The fresh read names a different record; that is final. - return second - lifted = known_ids_mod.resolve_lookalike_abstain( - [first, second], - known, - run_values=runtime_params_for_gui(params or {}), - recorded_id_matches=recorded_id_matches, - ) - if lifted is None or lifted.known_id_evidence is None: - return None - update: dict[str, Any] = {"frames": 2} - if admission is not None: - update["confirmed_at"] = admission.confirmation.confirmed_at - update["export_sha256"] = admission.confirmation.sha256 - return lifted.model_copy( - update={ - "known_id_evidence": lifted.known_id_evidence.model_copy(update=update) - } + known = admission.known_ids + run_values = runtime_params_for_gui(params or {}) + + frames: list[tuple[IdentityCheck, Optional[str]]] = [] + for _ in range(looks_mod.FRAME_LOOKS): + time.sleep(self.poll_interval_s) + png = self.backend.screenshot() + frames.append((read(png), sha256_bytes(png))) + if frames[-1][0].status == "mismatch": + break # final; no further looks + + frame_reads = [first, *(r for r, _ in frames)] + known_lift: Optional[IdentityCheck] = None + if known is not None and all( + looks_mod.is_glyph_abstain(r) for r in frame_reads + ): + lifted = known_ids_mod.resolve_lookalike_abstain( + frame_reads, + known, + run_values=run_values, + recorded_id_matches=recorded_id_matches, + ) + if lifted is not None and lifted.known_id_evidence is not None: + update: dict[str, Any] = {"frames": len(frame_reads)} + if admission is not None: + update["confirmed_at"] = admission.confirmation.confirmed_at + update["export_sha256"] = admission.confirmation.sha256 + known_lift = lifted.model_copy( + update={ + "known_id_evidence": lifted.known_id_evidence.model_copy( + update=update + ) + } + ) + + copied: Optional[looks_mod.ClipboardRead] = None + classes = looks_mod.lookalike_classes(first.observed) + bindings = self.__dict__.setdefault("_identity_look_bindings", {}) + if ( + identifier_region is not None + and self._copy_look_allowed() + and all(r.status != "mismatch" for r, _ in frames) + ): + assert self.identity_copy is not None + if self._identity_input_looks: + try: + cell_lines = self.vision.ocr(frame_png, region=identifier_region) + except Exception: # noqa: BLE001 - no box, no copy look + cell_lines = [] + cell = looks_mod.copy_region(cell_lines, identifier_region) + copied = ( + looks_mod.copy_identifier( + self.backend, + self._read_clipboard, + cell, + gesture=self.identity_copy.gesture, + copy_keys=self.identity_copy.copy_keys, + crop_region=identifier_region, + ) + if cell is not None + else looks_mod.ClipboardRead( + text=None, reason="no_single_identifier_box" + ) + ) + if copied.changed_outside: + look = IdentityLook( + kind="clipboard", + outcome="unavailable", + reason="screen_changed_outside_cell", + frame_sha256=copied.frame_sha256, + ) + return first.model_copy( + update={ + "look_halt": "identity_look_changed_screen", + "looks": [look], + } + ) + crops = tuple( + c + for c in ( + looks_mod.crop_png(frame_png, identifier_region), + copied.pre_crop, + copied.post_crop, + ) + if c is not None + ) + bindings[step.id] = looks_mod.LookBinding( + step_id=step.id, classes=classes, read=copied, crops=crops + ) + else: + binding = bindings.get(step.id) + live_crop = looks_mod.crop_png(frame_png, identifier_region) + if binding is not None and binding.matches(step.id, classes, live_crop): + copied = binding.read + + known_list = None + if known is not None: + known_list = getattr(known, "known_ids", known) + decided = looks_mod.decide( + first, + frames=frames, + magnified=magnified, + contradicts=identity_read_contradicts, + known_ids=known_list, + known_ids_lift=known_lift, + clipboard=copied, + exact_expected=looks_mod.exact_expected_ids( + run_values, + self._embedded_identity_params(step.anchor, workflow) + if step.anchor is not None + else [], + ), ) + binding = bindings.get(step.id) + if ( + self._identity_input_looks + and binding is not None + and decided.status == "verified" + and any( + look.kind == "clipboard" and look.outcome == "resolves" + for look in decided.looks + ) + ): + bindings[step.id] = dataclasses.replace(binding, verdict=decided) + return decided # -- typed-input verification ------------------------------------------------- diff --git a/packages/flow/openadapt_flow/validation/identity_looks_dense.py b/packages/flow/openadapt_flow/validation/identity_looks_dense.py new file mode 100644 index 000000000..259d02e9b --- /dev/null +++ b/packages/flow/openadapt_flow/validation/identity_looks_dense.py @@ -0,0 +1,508 @@ +"""Extra identity looks on the dense record list: before and after. + +Drives the REAL ``Replayer._verify_identity`` (identifier-cell OCR, the 2x +re-read, the known-ID list and the extra looks of +:mod:`openadapt_flow.runtime.identity_looks`) against the dense record list of +:mod:`.dense_surface`, rendered live in a browser. The pixel path is the one +the compiler builds for a pixel recording: the identifier region is the tight +box of the row's identity text (``context_region_from_lines``) and the +recorded identity is the OCR text inside it. + +Each armed click is scored twice, at the step-entry check and, when that +check verified, again at the actuation-time check, which takes no input: + +- **correct row**: the resolver landed on the recorded record. A stop is a + false stop (an over-halt). +- **sibling row**: the resolver landed on the adjacent different record. A + verify is a wrong-record accept. Must stay 0. + +Variants: no list and no copy (the shipped default), the known-ID list, the +copy look, and both. Modes: recorded record (the band names a fixed record) +and parameter (the MRN is a run parameter, as in a worklist). The copy look +drags across the identifier's text box with the real mouse; the copied text +is the page's own selection, which is what a redirected clipboard receives. + +The same script runs on ``origin/main`` for the before numbers: variants the +code under test does not support are skipped. + +Run: + python -m openadapt_flow.validation.identity_looks_dense --out DIR +""" + +from __future__ import annotations + +import argparse +import dataclasses +import hashlib +import inspect +import json +import subprocess +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any, Optional + +SCHEMA = "openadapt.identity-looks-dense.v1" +VARIANTS = ("shipped", "list", "copy", "list_copy") +MODES = ("recorded", "param") + + +def _git_commit() -> Optional[str]: + """Commit of the openadapt_flow under test (the before run imports + another checkout's package with this script).""" + try: + import openadapt_flow + + here = Path(openadapt_flow.__file__).resolve().parent + out = subprocess.run( + ["git", "-C", str(here), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + return out.stdout.strip() or None + except Exception: # noqa: BLE001 - provenance only + return None + + +class CachedVision: + """``openadapt_flow.vision`` with OCR memoized by (frame bytes, region). + + The fixture renders deterministically, so identical frames get identical + reads; caching changes no verdict, only the run time. + """ + + def __init__(self) -> None: + import openadapt_flow.vision as vision + + self._vision = vision + self._cache: dict[tuple[str, Any], Any] = {} + self.calls = 0 + + def __getattr__(self, name: str) -> Any: + return getattr(self._vision, name) + + def ocr(self, png: bytes, region: Any = None, **kw: Any) -> Any: + key = (hashlib.sha256(png).hexdigest(), tuple(region) if region else None) + if key not in self._cache: + self.calls += 1 + self._cache[key] = self._vision.ocr(png, region=region, **kw) + return self._cache[key] + + +@dataclass +class PageBackend: + """A live browser page seen only as pixels, with mouse drag and copy. + + Coordinates are screen pixels (CSS pixels times the device scale). The + clipboard is the page's selection at the copy key, which is what a + redirected remote clipboard would receive. + """ + + page: Any + dsf: int + viewport: tuple[int, int] + clean_png: bytes = b"" + dirty: bool = False + copied: Optional[str] = None + drags: int = 0 + # (clean frame sha, drag) -> (selection text, post frame) + _memo: dict[tuple[str, tuple[int, int, int, int]], tuple[str, bytes]] = field( + default_factory=dict + ) + _pending: Optional[tuple[str, tuple[int, int, int, int]]] = None + _post: Optional[bytes] = None + + def screenshot(self) -> bytes: + if not self.dirty: + return self.clean_png + if self._post is not None: + return self._post + return self.page.screenshot(full_page=True) + + def drag(self, x: int, y: int, end_x: int, end_y: int) -> None: + key = (hashlib.sha256(self.clean_png).hexdigest()[:16], (x, y, end_x, end_y)) + self.drags += 1 + self.dirty = True + if key in self._memo: + self._pending = key + self._post = self._memo[key][1] + return + d = self.dsf + self.page.mouse.move(x / d, y / d) + self.page.mouse.down() + self.page.mouse.move(end_x / d, end_y / d, steps=6) + self.page.mouse.up() + self._pending = key + self._post = None + + def press(self, key: str) -> None: + if key.lower() not in ("control+c", "meta+c", "controlormeta+c"): + raise AssertionError(f"unexpected key {key!r}") + if self._pending is None: + return + if self._pending in self._memo: + self.copied = self._memo[self._pending][0] + return + text = self.page.evaluate("() => String(window.getSelection())") + post = self.page.screenshot(full_page=True) + self._memo[self._pending] = (text, post) + self._post = post + self.copied = text + + def reset(self) -> None: + """Harness only: clear the selection between trials.""" + self.page.evaluate("() => window.getSelection().removeAllRanges()") + self.page.mouse.move(1, 1) + self.dirty = False + self._pending = None + self._post = None + + +class PageClipboard: + def __init__(self, backend: PageBackend) -> None: + self.backend = backend + + def read_text(self) -> Optional[str]: + return self.backend.copied + + def write_text(self, text: str) -> None: + self.backend.copied = text + + +def _open_page(pw: Any, table: Any, cond: Any, top_offset_px: int) -> tuple[Any, Any]: + from openadapt_flow.validation.dense_surface import render_table_html + + html = render_table_html( + table, + font_family=cond.font_family, + font_px=cond.font_px, + row_pad_px=cond.row_pad_px, + top_offset_px=top_offset_px, + ) + browser = pw.chromium.launch(headless=True) + page = browser.new_page( + viewport={"width": 1120, "height": 1600}, + device_scale_factor=cond.device_scale_factor, + ) + page.set_content(html, wait_until="networkidle") + full_h = page.evaluate("document.body.scrollHeight") + page.set_viewport_size({"width": 1120, "height": int(full_h) + 20}) + return browser, page + + +def _points(page: Any, n: int, dsf: int) -> dict[int, tuple[tuple[int, int], ...]]: + out: dict[int, tuple[tuple[int, int], ...]] = {} + for i in range(n): + pts = [] + for sel in (f'[data-name="{i}"]', f'[data-open="{i}"]'): + bb = page.eval_on_selector( + sel, + "el => { const r = el.getBoundingClientRect();" + " return [r.x, r.y, r.width, r.height]; }", + ) + pts.append((int((bb[0] + bb[2] / 2) * dsf), int((bb[1] + bb[3] / 2) * dsf))) + out[i] = tuple(pts) + return out + + +def _supports_copy() -> bool: + from openadapt_flow.runtime.replayer import Replayer + + return "identity_copy" in inspect.signature(Replayer.__init__).parameters + + +def _replayer(backend: Any, vision: Any, variant: str, known_ids: Any) -> Any: + from openadapt_flow.runtime.replayer import Replayer + + kwargs: dict[str, Any] = {"vision": vision, "poll_interval_s": 0.0} + if variant in ("list", "list_copy"): + kwargs["known_ids"] = known_ids + if variant in ("copy", "list_copy"): + from openadapt_flow.runtime.identity_looks import IdentityCopyConfig + + kwargs["identity_copy"] = IdentityCopyConfig() + kwargs["read_clipboard"] = PageClipboard(backend) + return Replayer(backend, **kwargs) + + +def _check_both_gates( + replayer: Any, backend: PageBackend, step: Any, res: Any, wf: Any, params: dict +) -> dict[str, Any]: + entry_png = backend.screenshot() + if hasattr(replayer, "_entry_identity_gate_error"): + replayer.__dict__.setdefault("_identity_look_bindings", {}).pop(step.id, None) + replayer._identity_input_looks = True + try: + entry = replayer._verify_identity( + step, res, entry_png, params, wf, bundle_dir=None + ) + finally: + if hasattr(replayer, "_identity_input_looks"): + replayer._identity_input_looks = False + out: dict[str, Any] = { + "entry": entry.status, + "entry_reason": entry.abstain_reason, + "looks": [ + [look.kind, look.outcome, look.reason] + for look in getattr(entry, "looks", []) + ], + "look_halt": getattr(entry, "look_halt", None), + } + final = entry.status + if entry.status == "verified" and backend.dirty: + act_png = backend.screenshot() # the selection highlight is still drawn + act = replayer._verify_identity(step, res, act_png, params, wf, bundle_dir=None) + out["actuation"] = act.status + out["actuation_looks"] = [[x.kind, x.outcome, x.reason] for x in act.looks] + final = act.status + elif entry.status == "verified": + out["actuation"] = "verified" # no input was used: same frame, same verdict + out["final"] = final + backend.reset() + return out + + +def run( + seeds: list[int], + *, + n_rows: int = 40, + progress: bool = False, +) -> dict[str, Any]: + import cv2 + import numpy as np + from playwright.sync_api import sync_playwright + + from openadapt_flow._browser_setup import ensure_chromium_installed + from openadapt_flow.compiler.compile import ( + MIN_OCR_CONFIDENCE, + _discriminative_crop_region, + ) + from openadapt_flow.ir import ActionKind, Anchor, Resolution, Step, Workflow + from openadapt_flow.runtime.identity import ( + band_region, + context_region_from_lines, + identifier_text_from_lines, + ) + from openadapt_flow.runtime.known_ids import KnownIdList + from openadapt_flow.validation.dense_surface import ( + CLICK_CONFIGS, + RECORD_CONDITION, + REPLAY_CONDITIONS, + build_dense_table, + render_frame, + ) + + ensure_chromium_installed() + vision = CachedVision() + variants = [v for v in VARIANTS if _supports_copy() or "copy" not in v] + today = date.today() + trials: list[dict[str, Any]] = [] + # Record frames first: render_frame runs its own Playwright session. Each + # replay condition is recorded at its own device scale: the identifier + # box is a pixel region, and a display-scale change is handled by a + # separate mechanism (runtime.render_scale), not by this check. + dsfs = sorted({c.device_scale_factor for c in REPLAY_CONDITIONS}) + record_frames = { + (seed, d): render_frame( + build_dense_table(seed, n_rows=n_rows), + dataclasses.replace(RECORD_CONDITION, device_scale_factor=d), + top_offset_px=0, + ) + for seed in seeds + for d in dsfs + } + with sync_playwright() as pw: + for seed in seeds: + table = build_dense_table(seed, n_rows=n_rows) + known = KnownIdList( + ids=tuple(r.mrn for r in table.rows), + source=f"synthetic seed {seed}", + complete=True, + ) + idx = {id(r): i for i, r in enumerate(table.rows)} + for k, cond in enumerate(REPLAY_CONDITIONS): + rec = record_frames[(seed, cond.device_scale_factor)] + rec_lines = vision.ocr(rec.png) + rec_bgr = cv2.imdecode( + np.frombuffer(rec.png, np.uint8), cv2.IMREAD_COLOR + ) + browser, page = _open_page(pw, table, cond, 4 + k * 3) + try: + dsf = cond.device_scale_factor + png = page.screenshot(full_page=True) + import io + + from PIL import Image + + with Image.open(io.BytesIO(png)) as im: + viewport = im.size + backend = PageBackend(page=page, dsf=dsf, viewport=viewport) + backend.clean_png = png + live_pts = _points(page, len(table.rows), dsf) + for pair in table.pairs: + ti, si = idx[id(pair.target)], idx[id(pair.sibling)] + for config in CLICK_CONFIGS: + pi = 0 if config == "click_name" else 1 + name_pt, open_pt = rec.points[ti][0], rec.points[ti][1] + rec_click = name_pt if pi == 0 else open_pt + crop = _discriminative_crop_region(rec_bgr, rec_click) + box = context_region_from_lines( + rec_lines, + exclude_region=crop, + band=band_region(rec_click, crop[3], rec.viewport), + point=rec_click, + min_confidence=MIN_OCR_CONFIDENCE, + reference_date=today, + ) + context = ( + identifier_text_from_lines( + rec_lines, + region=box, + min_confidence=MIN_OCR_CONFIDENCE, + reference_date=today, + ) + if box + else None + ) + base = { + "seed": seed, + "collision_class": pair.collision_class, + "click_config": config, + "replay_condition": cond.name, + "armed": bool(context), + } + if not context: + trials.append(base) + continue + anchor = Anchor( + template="t.png", + region=crop, + click_point=rec_click, + context_text=context, + identifier_region=box, + ) + step = Step( + id=f"s{ti}_{config}", + intent="open record", + action=ActionKind.CLICK, + anchor=anchor, + risk="irreversible", + ) + for mode in MODES: + params = ( + {"mrn": pair.target.mrn} if mode == "param" else {} + ) + wf = Workflow( + name="dense", params=dict(params), steps=[step] + ) + for variant in variants: + row = dict(base, mode=mode, variant=variant) + for side, ri in (("correct", ti), ("sibling", si)): + res = Resolution( + rung="ocr", + point=live_pts[ri][pi], + confidence=0.9, + elapsed_ms=1.0, + ) + rp = _replayer(backend, vision, variant, known) + row[side] = _check_both_gates( + rp, backend, step, res, wf, params + ) + trials.append(row) + if progress: + print( + f"seed {seed} {cond.name} {pair.collision_class} done" + ) + finally: + browser.close() + return { + "schema": SCHEMA, + "meta": { + "commit": _git_commit(), + "seeds": seeds, + "n_rows": n_rows, + "replay_conditions": [c.name for c in REPLAY_CONDITIONS], + "record_condition": f"{RECORD_CONDITION.name} at each replay's device scale", + "click_configs": list(CLICK_CONFIGS), + "variants": variants, + "modes": list(MODES), + "ocr_calls": vision.calls, + }, + "trials": trials, + } + + +def score(result: dict[str, Any]) -> dict[str, Any]: + """Counts per (mode, variant), overall and by collision class.""" + out: dict[str, Any] = {} + for t in result["trials"]: + if not t.get("armed") or "variant" not in t: + continue + key = f"{t['mode']}/{t['variant']}" + for scope in ("all", t["collision_class"]): + b = out.setdefault(key, {}).setdefault( + scope, + { + "correct_row_clicks": 0, + "false_stops": 0, + "entry_verified_actuation_stopped": 0, + "sibling_row_clicks": 0, + "wrong_record_accepts": 0, + "copy_looks": 0, + }, + ) + b["correct_row_clicks"] += 1 + b["sibling_row_clicks"] += 1 + c, s = t["correct"], t["sibling"] + b["false_stops"] += c["final"] != "verified" + b["entry_verified_actuation_stopped"] += ( + c["entry"] == "verified" and c["final"] != "verified" + ) + b["wrong_record_accepts"] += ( + s["entry"] == "verified" or s.get("final") == "verified" + ) + b["copy_looks"] += sum( + 1 for x in (c, s) for look in x["looks"] if look[0] == "clipboard" + ) + return out + + +def unarmed(result: dict[str, Any]) -> int: + return sum(1 for t in result["trials"] if not t.get("armed")) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True) + parser.add_argument("--seeds", type=int, nargs="+", default=[1, 2, 3]) + parser.add_argument("--n-rows", type=int, default=40) + parser.add_argument("--progress", action="store_true") + args = parser.parse_args(argv) + result = run(args.seeds, n_rows=args.n_rows, progress=args.progress) + scored = score(result) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + { + "meta": result["meta"], + "unarmed": unarmed(result), + "scored": scored, + "trials": result["trials"], + }, + indent=1, + ) + + "\n" + ) + for key, groups in scored.items(): + a = groups["all"] + print( + f"{key}: false stops {a['false_stops']}/{a['correct_row_clicks']}, " + f"wrong-record accepts {a['wrong_record_accepts']}/{a['sibling_row_clicks']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/tests/test_bundle_schema_v2.py b/packages/flow/tests/test_bundle_schema_v2.py index f2b29a689..f6190b43b 100644 --- a/packages/flow/tests/test_bundle_schema_v2.py +++ b/packages/flow/tests/test_bundle_schema_v2.py @@ -189,6 +189,8 @@ def _pre_frame_path_content(wf: Workflow, shape: str) -> dict: assert step.pop("drag_end_anchor") is None assert step.pop("selection_commit_key") is None assert step.pop("selection_region") is None + # Reviewed v2 omission: hover evidence postdates these seals. + assert step.pop("hover") is None # Reviewed v2 omission: submit-binding evidence postdates these seals. assert step.pop("submit_binding") is None assert step.pop("submit_binding_unbound_reason") is None @@ -254,6 +256,7 @@ def _write_synthetic_pre_field_bundle( assert step.pop("field_label") is None assert step.pop("selection_commit_key") is None assert step.pop("selection_region") is None + assert step.pop("hover") is None assert step.pop("submit_binding") is None assert step.pop("submit_binding_unbound_reason") is None assert step.pop("surface") is None @@ -603,6 +606,36 @@ def _binding(**changes): ) +def test_recorded_hover_evidence_is_sealed_content(tmp_path): + """Only an absent hover field is omitted; a recorded response is sealed.""" + + from openadapt_flow.ir import ActionKind, Anchor, HoverEvidence, Step + + b = _write_bundle_dir(tmp_path) + + def _bundle(hover=None): + step = Step( + id="save", + intent="click recorded visual target", + action=ActionKind.CLICK, + anchor=Anchor( + template="templates/btn.png", region=(0, 0, 10, 10), click_point=(5, 5) + ), + hover=hover, + ) + return Workflow(name="sealed-hover", steps=[step]) + + bare = _bundle() + file_hashes = bv.compute_file_hashes(bare, b) + bare_digest = bv.compute_content_digest(bare, file_hashes) + assert "hover" not in _step_payloads(bv._workflow_content(bare), "linear")[0] + ev = HoverEvidence(response=True, region=(-5, -5, 60, 40), crop="templates/h.png") + hovered = bv.compute_content_digest(_bundle(ev), file_hashes) + assert hovered != bare_digest + moved = ev.model_copy(update={"region": (-5, -5, 60, 41)}) + assert bv.compute_content_digest(_bundle(moved), file_hashes) != hovered + + def test_sealed_empty_defaults_do_not_implicitly_cross_schema_versions(): wf = _effect_workflow([Effect(kind=EffectKind.RECORD_WRITTEN, match={"a": "b"})]) rendered = wf.model_dump(mode="json", exclude={"manifest"}) diff --git a/packages/flow/tests/test_check_to_click.py b/packages/flow/tests/test_check_to_click.py new file mode 100644 index 000000000..ee8c60725 --- /dev/null +++ b/packages/flow/tests/test_check_to_click.py @@ -0,0 +1,228 @@ +"""Check-to-click guard on the pixel path (``Replayer._pixel_recheck``). + +A pixel step is resolved and its identity checked on one frame, and clicked a +moment later. If the screen changes in between (a dialog opens over the +target, the list re-sorts), the click must not land on whatever is there now. +Right before the click the runtime compares the target and identity regions on +a fresh frame with the checked frame. On a change it re-resolves and re-checks +on the fresh frame, and clicks only when that holds; otherwise it refuses. +""" + +from __future__ import annotations + +import io +from pathlib import Path + +from PIL import Image, ImageDraw + +import openadapt_flow.vision as vision +from openadapt_flow.ir import Anchor, Resolution, Step, StepResult, Workflow +from openadapt_flow.runtime.replayer import Replayer + +SIZE = (640, 400) +BUTTON = (100, 120, 140, 44) # x, y, w, h of the recorded target + + +def _frame( + *, + button_y: int = BUTTON[1], + dialog: bool = False, + clock: str = "10:00", + noise: int = 0, +) -> bytes: + img = Image.new("RGB", SIZE, (244, 246, 248)) + d = ImageDraw.Draw(img) + x, _, w, h = BUTTON + d.rectangle((x, button_y, x + w, button_y + h), fill=(45, 111, 216)) + d.text((x + 22, button_y + 15), "Open chart", fill="white") + d.rectangle((x, button_y + 70, x + w, button_y + 70 + h), fill=(180, 60, 60)) + d.text((x + 22, button_y + 85), "Delete", fill="white") + d.text((560, 380), clock, fill="black") # a clock far from the target + if dialog: + d.rectangle((60, 90, 420, 260), fill="white", outline="black") + d.text((80, 110), "Session expires soon. Continue?", fill="black") + d.rectangle((150, 125, 260, 165), fill=(220, 220, 220), outline="black") + d.text((175, 138), "Continue", fill="black") + if noise: + px = img.load() + for i in range(0, SIZE[0], 7): + r, g, b = px[i, 5] + px[i, 5] = (r - noise, g, b) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +class _Backend: + """Serves a scripted frame sequence; records clicks.""" + + viewport = SIZE + + def __init__(self, frames: list[bytes]): + self.frames = list(frames) + self.shots = 0 + self.clicks: list[tuple[int, int]] = [] + + def screenshot(self) -> bytes: + frame = self.frames[min(self.shots, len(self.frames) - 1)] + self.shots += 1 + return frame + + def click(self, x: int, y: int, *, double: bool = False) -> None: + self.clicks.append((x, y)) + + def type_text(self, text: str) -> None: # pragma: no cover - unused + pass + + def press(self, key: str) -> None: # pragma: no cover - unused + pass + + def scroll(self, dx: int, dy: int) -> None: # pragma: no cover - unused + pass + + +def _bundle(tmp_path: Path) -> tuple[Path, Step, Workflow]: + checked = Image.open(io.BytesIO(_frame())) + x, y, w, h = BUTTON + (tmp_path / "templates").mkdir() + checked.crop((x, y, x + w + 1, y + h + 1)).save(tmp_path / "templates" / "t.png") + step = Step( + id="step_000", + intent="click 'Open chart'", + action="click", + anchor=Anchor( + template="templates/t.png", + region=(x, y, w + 1, h + 1), + click_point=(x + w // 2, y + h // 2), + ), + ) + return tmp_path, step, Workflow(name="c2c", params={}, steps=[step]) + + +def _click(tmp_path: Path, frames: list[bytes], rung: str = "template"): + bundle, step, wf = _bundle(tmp_path) + backend = _Backend(frames) + replayer = Replayer(backend, vision=vision) + x, y, w, h = BUTTON + res = Resolution( + rung=rung, point=(x + w // 2, y + h // 2), confidence=0.99, elapsed_ms=1.0 + ) + replayer._active_delivery_region = (x, y, w + 1, h + 1) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + error = replayer._act( + step, + res, + {}, + workflow=wf, + step_index=0, + bundle_dir=bundle, + before_png=_frame(), + result=result, + run_dir=tmp_path / "run", + ) + return error, result, backend + + +def test_unchanged_screen_clicks_once(tmp_path): + error, result, backend = _click(tmp_path, [_frame()]) + assert error is None + assert backend.clicks == [(170, 142)] + assert result.check_to_click == [] + assert "check_to_click" not in result.model_dump() + + +def test_change_away_from_the_target_does_not_stop_the_click(tmp_path): + error, result, backend = _click(tmp_path, [_frame(clock="10:01")]) + assert error is None and len(backend.clicks) == 1 + assert result.check_to_click == [] + + +def test_encoder_noise_is_not_a_change(tmp_path): + error, _, backend = _click(tmp_path, [_frame(noise=6)]) + assert error is None and len(backend.clicks) == 1 + + +def test_dialog_between_check_and_click_aborts_the_click(tmp_path): + error, result, backend = _click(tmp_path, [_frame(dialog=True)]) + assert backend.clicks == [] # nothing was delivered under the dialog + assert error is not None and "click not delivered" in error + assert result.safety_halt is True + assert [e.outcome for e in result.check_to_click] == ["aborted"] + assert "target" in result.check_to_click[0].changed + + +def test_moved_target_is_reobserved_and_clicked_where_it_is_now(tmp_path): + moved = _frame(button_y=BUTTON[1] + 30) + error, result, backend = _click(tmp_path, [moved, moved]) + assert error is None + assert backend.clicks == [(170, 172)] + assert [e.outcome for e in result.check_to_click] == ["reobserved"] + + +def test_a_screen_that_keeps_changing_is_refused(tmp_path): + frames = [_frame(button_y=BUTTON[1] + d) for d in (10, 20, 30, 40)] + error, result, backend = _click(tmp_path, frames) + assert backend.clicks == [] + assert error is not None and "kept changing" in error + assert [e.outcome for e in result.check_to_click] == [ + "reobserved", + "reobserved", + "aborted", + ] + + +def test_structural_targets_are_not_rechecked(tmp_path): + backend = _Backend([_frame(dialog=True)]) + replayer = Replayer(backend, vision=vision) + bundle, step, wf = _bundle(tmp_path) + res = Resolution(rung="structural", point=(170, 142), confidence=1.0, elapsed_ms=1) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + got = replayer._pixel_recheck( + step, res, _frame(), {}, wf, bundle, result, tmp_path / "run" + ) + assert got is res and backend.shots == 0 + + +def test_identity_region_change_is_detected(): + replayer = Replayer(_Backend([_frame()]), vision=vision) + step = Step( + id="s", + intent="open", + action="click", + anchor=Anchor( + template="t.png", + region=(100, 120, 141, 45), + click_point=(170, 142), + context_text="Open chart row identity text", + ), + ) + res = Resolution(rung="template", point=(170, 142), confidence=1.0, elapsed_ms=1) + regions = replayer._check_regions(step, res, None) + assert set(regions) == {"target", "identity"} + assert regions["identity"][2] == SIZE[0] # the whole row + changed = Replayer._changed_regions( + _frame(), _frame(button_y=BUTTON[1] + 30), regions + ) + assert "target" in changed and "identity" in changed + + +def test_a_hover_look_is_the_baseline_for_the_click(tmp_path): + # The hover look moved the pointer and the target now shows its hover + # state; the click is checked against that frame, not the earlier one. + hovered = _frame(clock="10:00", noise=0) + img = Image.open(io.BytesIO(hovered)).convert("RGB") + ImageDraw.Draw(img).rectangle((100, 120, 240, 164), outline="yellow", width=3) + out = io.BytesIO() + img.save(out, format="PNG") + hovered = out.getvalue() + bundle, step, wf = _bundle(tmp_path) + backend = _Backend([hovered]) + replayer = Replayer(backend, vision=vision) + replayer.__dict__["_hover_frames"] = {step.id: hovered} + res = Resolution(rung="template", point=(170, 142), confidence=1.0, elapsed_ms=1) + replayer._active_delivery_region = (100, 120, 141, 45) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + got = replayer._pixel_recheck( + step, res, _frame(), {}, wf, bundle, result, tmp_path / "run" + ) + assert got is res and result.check_to_click == [] diff --git a/packages/flow/tests/test_hover_evidence.py b/packages/flow/tests/test_hover_evidence.py new file mode 100644 index 000000000..5a0cfe8d8 --- /dev/null +++ b/packages/flow/tests/test_hover_evidence.py @@ -0,0 +1,350 @@ +"""Hover-before-click evidence (runtime.hover_evidence) and its wiring. + +Synthetic frames: an icon button that lightens under the pointer and opens a +tooltip. Two icons look the same and differ only by their tooltip. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from typing import Optional + +from PIL import Image, ImageDraw + +from openadapt_flow.ir import ( + Anchor, + HoverEvidence, + Resolution, + Step, + StepResult, + Workflow, +) +from openadapt_flow.runtime import hover_evidence as H +from openadapt_flow.runtime.replayer import Replayer + +ICON = (100, 100, 40, 40) +POINT = (120, 120) + + +def _frame( + pointer: Optional[tuple[int, int]] = None, tooltip: str = "Save note" +) -> bytes: + img = Image.new("RGB", (480, 320), (244, 246, 248)) + d = ImageDraw.Draw(img) + x, y, w, h = ICON + over = pointer is not None and x <= pointer[0] < x + w and y <= pointer[1] < y + h + d.rectangle( + (x, y, x + w - 1, y + h - 1), fill=(90, 140, 230) if over else (45, 111, 216) + ) + d.ellipse((x + 10, y + 10, x + 30, y + 30), fill="white") + if over and tooltip: + tx, ty = pointer[0] + 12, pointer[1] + 20 + d.rectangle( + (tx, ty, tx + 9 * len(tooltip), ty + 18), + fill=(255, 255, 225), + outline="black", + ) + d.text((tx + 4, ty + 3), tooltip, fill="black") + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def test_measure_response_finds_highlight_and_tooltip(): + r = H.measure_response(_frame(), _frame(POINT), POINT) + assert r is not None + x, y, w, h = r.region + assert x <= ICON[0] and y <= ICON[1] + assert x + w > POINT[0] + 12 and y + h > POINT[1] + 20 # tooltip included + assert H.measure_response(_frame(), _frame(), POINT) is None + + +def _recorded(tooltip: str = "Save note"): + r = H.measure_response(_frame(), _frame(POINT, tooltip), POINT) + assert r is not None + return H.relative(r.region, POINT), r.crop + + +def _compare(live_tooltip: Optional[str], *, same_render=True, recorded_response=True): + region, crop = _recorded() + live = ( + H.measure_response(_frame(), _frame(POINT, live_tooltip), POINT) + if live_tooltip is not None + else None + ) + return H.compare( + recorded_response=recorded_response, + recorded_region=region if recorded_response else None, + recorded_crop=crop if recorded_response else None, + live=live, + point=POINT, + same_render=same_render, + ) + + +def test_same_response_verifies(): + assert _compare("Save note") == ("verified", "same_pixels") + + +def test_other_tooltip_on_a_look_alike_icon_is_a_mismatch(): + verdict, _ = _compare("Delete note") + assert verdict == "mismatch" + + +def test_no_live_response_is_absent_not_a_stop(): + assert _compare(None)[0] == "absent" + + +def test_response_where_none_was_recorded_is_unexpected(): + assert _compare("Save note", recorded_response=False)[0] == "unexpected" + assert _compare(None, recorded_response=False)[0] == "unverifiable" + + +def test_moved_response_is_a_mismatch(): + region, crop = _recorded() + far = H.Response(region=(300, 250, 40, 20), crop=crop, changed_px=100) + verdict, reason = H.compare( + recorded_response=True, + recorded_region=region, + recorded_crop=crop, + live=far, + point=POINT, + same_render=True, + ) + assert verdict == "mismatch" and reason == "response_moved_or_resized" + + +def test_other_render_without_text_is_unverifiable(): + verdict, _ = _compare("Save note", same_render=False) + assert verdict == "unverifiable" + + +def test_same_text_redrawn_by_another_rasterizer_verifies(): + class Line: + def __init__(self, text): + self.text = text + + region, crop = _recorded() + live = H.measure_response(_frame(), _frame(POINT, "Save note"), POINT) + assert live is not None + blurred = H.Response( + region=live.region, + crop=_invert(live.crop), + changed_px=live.changed_px, + ) + verdict, reason = H.compare( + recorded_response=True, + recorded_region=region, + recorded_crop=crop, + live=blurred, + point=POINT, + same_render=True, + ocr=lambda png: [Line("Save note")], + ) + assert (verdict, reason) == ("verified", "same_text") + + +def _invert(png: bytes) -> bytes: + from PIL import ImageOps + + img = ImageOps.invert(Image.open(io.BytesIO(png)).convert("RGB")) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def test_other_render_compares_ocr_text(): + class Line: + def __init__(self, text): + self.text = text + + region, crop = _recorded() + live = H.measure_response(_frame(), _frame(POINT, "Delete note"), POINT) + texts = iter(["Save note", "Delete note"]) + + def ocr(png): + return [Line(next(texts))] + + verdict, reason = H.compare( + recorded_response=True, + recorded_region=region, + recorded_crop=crop, + live=live, + point=POINT, + same_render=False, + ocr=ocr, + ) + assert (verdict, reason) == ("mismatch", "different_text") + + +# ------------------------------------------------------------- replayer + + +class _HoverBackend: + viewport = (480, 320) + + def __init__(self, tooltip: str = "Save note", can_hover: bool = True): + self.tooltip = tooltip + self.pointer: Optional[tuple[int, int]] = None + self.hovers: list[tuple[int, int]] = [] + if not can_hover: + self.hover = None # type: ignore[assignment] + + def screenshot(self) -> bytes: + return _frame(self.pointer, self.tooltip) + + def hover(self, x, y): # noqa: D401 - backend seam + self.hovers.append((x, y)) + self.pointer = (x, y) + + +class _Vision: + @staticmethod + def wait_settled(backend): + return backend.screenshot() + + @staticmethod + def ocr(png, region=None): + return [] + + +def _bundle(tmp_path: Path) -> tuple[Path, Step, Workflow]: + region, crop = _recorded() + (tmp_path / "templates" / "hover").mkdir(parents=True) + (tmp_path / "templates" / "hover" / "step_000.png").write_bytes(crop) + step = Step( + id="step_000", + intent="click recorded visual target", + action="click", + anchor=Anchor(template="t.png", region=ICON, click_point=POINT), + risk="irreversible", + hover=HoverEvidence( + response=True, + region=region, + crop="templates/hover/step_000.png", + dwell_ms=0, + ), + ) + return tmp_path, step, Workflow(name="hover", params={}, steps=[step]) + + +def _gate(tmp_path, backend, *, entry=True, rung="template"): + bundle, step, wf = _bundle(tmp_path) + replayer = Replayer(backend, vision=_Vision()) + res = Resolution(rung=rung, point=POINT, confidence=0.9, elapsed_ms=1.0) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + call = ( + replayer._entry_identity_gate_error if entry else replayer._identity_gate_error + ) + error = call(step, res, _frame(), {}, wf, bundle, result) + return error, result + + +def test_same_hover_response_verifies_the_icon(tmp_path): + error, result = _gate(tmp_path, _HoverBackend("Save note")) + assert error is None + assert result.identity is not None and result.identity.mode == "hover" + assert result.identity.status == "verified" + assert result.identity.hover is not None + assert result.identity.hover.verdict == "verified" + + +def test_look_alike_icon_with_other_tooltip_stops_before_the_click(tmp_path): + backend = _HoverBackend("Delete note") + error, result = _gate(tmp_path, backend) + assert error is not None and "refusing to click" in error + assert result.identity is not None and result.identity.status == "mismatch" + assert backend.hovers == [POINT] + + +def test_missing_tooltip_changes_the_response_and_stops(tmp_path): + # A highlight with no tooltip (the tooltip opened late) is another shape. + error, result = _gate(tmp_path, _HoverBackend("")) + assert result.identity is not None and result.identity.status == "mismatch" + assert result.identity.hover.reason == "response_moved_or_resized" + assert error is not None + + +def test_absent_response_is_recorded_and_does_not_stop(tmp_path): + class Frozen(_HoverBackend): + def screenshot(self): + return _frame() + + error, result = _gate(tmp_path, Frozen()) + assert error is None + assert result.identity is not None and result.identity.status == "abstain" + assert result.identity.hover.verdict == "absent" + + +def test_actuation_check_takes_no_hover(tmp_path): + backend = _HoverBackend("Delete note") + error, result = _gate(tmp_path, backend, entry=False) + assert error is None and result.identity is None + assert backend.hovers == [] + + +def test_structural_resolution_takes_no_hover(tmp_path): + backend = _HoverBackend("Delete note") + error, result = _gate(tmp_path, backend, rung="structural") + assert error is None and backend.hovers == [] + + +def test_backend_without_hover_keeps_old_behaviour(tmp_path): + backend = _HoverBackend("Delete note", can_hover=False) + error, result = _gate(tmp_path, backend) + assert error is None and result.identity is None + + +# -------------------------------------------------------------- compiler + + +def test_compiler_keeps_the_hover_response(tmp_path): + from openadapt_flow.compiler.compile import _hover_evidence + + hover_path = tmp_path / "0000_hover.png" + hover_path.write_bytes(_frame(POINT)) + bundle = tmp_path / "bundle" + bundle.mkdir() + ev = _hover_evidence(bundle, "step_000", _frame(), hover_path, POINT, dwell_ms=800) + assert ev is not None and ev.response and ev.crop == "templates/hover/step_000.png" + assert (bundle / ev.crop).is_file() + hover_path.write_bytes(_frame()) + assert _hover_evidence(bundle, "step_001", _frame(), hover_path, POINT) is None + assert ( + _hover_evidence(bundle, "s", _frame(), tmp_path / "missing.png", POINT) is None + ) + + +def test_recorder_saves_a_hover_frame_before_the_click(tmp_path): + from openadapt_flow.recorder import Recorder + + class Backend(_HoverBackend): + def click(self, x, y, *, double=False): + self.pointer = (x, y) + + backend = Backend() + rec = Recorder(backend, tmp_path / "rec", capture_hover=True, settle_timeout_s=0.2) + rec.click(*POINT) + events = [ + json.loads(line) + for line in (tmp_path / "rec" / "events.jsonl").read_text().splitlines() + ] + assert events[0]["hover_dwell_ms"] == H.HOVER_DWELL_MS + assert (tmp_path / "rec" / "frames" / "0000_hover.png").is_file() + assert backend.hovers == [POINT] + + +def test_recorder_without_capture_hover_does_not_hover(tmp_path): + from openadapt_flow.recorder import Recorder + + class Backend(_HoverBackend): + def click(self, x, y, *, double=False): + self.pointer = (x, y) + + backend = Backend() + rec = Recorder(backend, tmp_path / "rec", settle_timeout_s=0.2) + rec.click(*POINT) + assert backend.hovers == [] + assert not (tmp_path / "rec" / "frames" / "0000_hover.png").exists() diff --git a/packages/flow/tests/test_identity_looks.py b/packages/flow/tests/test_identity_looks.py new file mode 100644 index 000000000..03e47211e --- /dev/null +++ b/packages/flow/tests/test_identity_looks.py @@ -0,0 +1,363 @@ +"""Extra looks before stopping on a look-alike identifier (runtime.identity_looks). + +Pure tests of the decision rule and the copy look. The replayer path, with a +real render and real OCR, is in ``test_identity_looks_replayer.py``. +""" + +from __future__ import annotations + +import io + +import pytest +from PIL import Image, ImageDraw + +from openadapt_flow.ir import IdentityCheck +from openadapt_flow.runtime import identity_looks as L +from openadapt_flow.runtime.known_ids import KnownIdList +from openadapt_flow.runtime.replayer import identity_read_contradicts + +BAND = "AC50061 Smith, John 01/15/1980" + + +def _abstain(observed: str = BAND) -> IdentityCheck: + return IdentityCheck( + status="abstain", + abstain_reason="glyph_confusable_identifier", + coverage=1.0, + expected=BAND, + observed=observed, + ) + + +def _frames(*reads: IdentityCheck): + return [(r, "a" * 64) for r in reads] + + +def _decide(**kw): + kw.setdefault("frames", _frames(_abstain(), _abstain())) + kw.setdefault("magnified", _abstain()) + return L.decide(_abstain(), contradicts=identity_read_contradicts, **kw) + + +def _copy(text: str) -> L.ClipboardRead: + return L.ClipboardRead(text=text, frame_sha256="b" * 64) + + +def test_ocr_looks_alone_never_accept(): + check = _decide() + assert check.status == "abstain" + assert [look.kind for look in check.looks] == ["frame", "frame", "magnified"] + assert all(look.outcome == "agrees" for look in check.looks) + + +def test_copy_equal_to_run_value_settles_the_record(): + check = _decide( + clipboard=_copy("AC50061\tSmith, John\t01/15/1980"), exact_expected=("ac50061",) + ) + assert check.status == "verified" + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "resolves" + + +def test_copy_of_look_alike_twin_is_a_mismatch(): + check = _decide( + clipboard=_copy("AC5OO61 Smith, John 01/15/1980"), exact_expected=("ac50061",) + ) + assert check.status == "mismatch" + assert check.looks[-1].outcome == "contradicts" + + +def test_copy_without_an_exact_expected_id_cannot_accept(): + # Recorded-record mode: the recorded band is OCR text, not exact. + check = _decide(clipboard=_copy("AC50061 Smith, John 01/15/1980")) + assert check.status == "abstain" + assert check.looks[-1].outcome == "agrees" + + +def test_copy_of_an_unlisted_id_contradicts_the_list(): + ids = KnownIdList(ids=("AC50061", "ZZ1000"), source="t", complete=True) + check = _decide( + clipboard=_copy("AC5OO61"), + known_ids=ids, + known_ids_lift=_abstain().model_copy(update={"status": "verified"}), + ) + assert check.status == "mismatch" + + +def test_known_id_list_resolves_when_frames_agree(): + ids = KnownIdList(ids=("AC50061", "ZZ1000"), source="t", complete=True) + lift = _abstain().model_copy(update={"status": "verified"}) + check = _decide(known_ids=ids, known_ids_lift=lift) + assert check.status == "verified" + assert check.looks[-1].kind == "known_ids" + + +def test_a_frame_naming_another_record_is_final(): + other = IdentityCheck( + status="mismatch", expected=BAND, observed="ZZ9999 Doe, Jane 02/02/1990" + ) + check = _decide( + frames=_frames(other, _abstain()), + clipboard=_copy("AC50061"), + exact_expected=("ac50061",), + ) + assert check.status == "mismatch" + assert check.looks[0].outcome == "contradicts" + + +def test_frames_that_read_another_identifier_block_acceptance(): + drifted = _abstain("AC5OO6l Smith, John 01/15/1980") # same class, still ok + moved = _abstain("AC50071 Smith, John 01/15/1980") # different class + assert ( + _decide( + frames=_frames(drifted, _abstain()), + clipboard=_copy("AC50061"), + exact_expected=("ac50061",), + ).status + == "verified" + ) + check = _decide( + frames=_frames(moved, _abstain()), + clipboard=_copy("AC50061"), + exact_expected=("ac50061",), + ) + assert check.status == "abstain" + assert check.looks[0].outcome == "unavailable" + + +def test_no_fresh_frames_means_no_acceptance(): + check = _decide(frames=[], clipboard=_copy("AC50061"), exact_expected=("ac50061",)) + assert check.status == "abstain" + + +def test_magnified_read_naming_another_record_is_final(): + other = IdentityCheck( + status="mismatch", expected=BAND, observed="ZZ9999 Doe, Jane 02/02/1990" + ) + check = _decide( + magnified=other, clipboard=_copy("AC50061"), exact_expected=("ac50061",) + ) + assert check.status == "mismatch" + + +def test_copy_that_covers_a_different_cell_settles_nothing(): + check = _decide(clipboard=_copy("ZZ1000 Filler, Pat"), exact_expected=("ac50061",)) + assert check.status == "abstain" + assert check.looks[-1].reason == "copy_differs_from_read" + + +def test_only_glyph_abstains_take_looks(): + short = IdentityCheck(status="abstain", abstain_reason="identity_evidence_short") + assert ( + L.decide( + short, frames=[], magnified=None, contradicts=identity_read_contradicts + ) + is short + ) + + +def test_exact_expected_ids_come_only_from_band_params(): + run = {"mrn": "AC50061", "note": "AC99999"} + assert L.exact_expected_ids(run, ["mrn"]) == ("ac50061",) + assert L.exact_expected_ids(run, []) == () + + +def test_copy_config_rejects_unknown_gesture(): + with pytest.raises(ValueError): + L.IdentityCopyConfig(gesture="triple_click") + + +# ------------------------------------------------------------ copy look + + +def _png(highlight=None, extra=None) -> bytes: + img = Image.new("RGB", (200, 100), "white") + draw = ImageDraw.Draw(img) + draw.text((20, 40), "AC50061", fill="black") + if highlight: + draw.rectangle(highlight, fill=(51, 144, 255)) + if extra: + draw.rectangle(extra, fill="red") + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +class _Clipboard: + def __init__(self, text="user's own text"): + self.text = text + self.writes = [] + + def read_text(self): + return self.text + + def write_text(self, text): + self.writes.append(text) + self.text = text + + +class _Backend: + def __init__(self, clipboard, copied="AC50061", after=None): + self.clipboard = clipboard + self.copied = copied + self.after = after + self.frame = _png() + self.drags = [] + self.keys = [] + + def screenshot(self): + return self.frame + + def drag(self, x, y, ex, ey): + self.drags.append((x, y, ex, ey)) + self.frame = self.after if self.after is not None else self.frame + + def click(self, x, y, *, double=False): + raise AssertionError("drag gesture must not click") + + def press(self, key): + self.keys.append(key) + if self.copied is not None: + self.clipboard.text = self.copied + + +REGION = (15, 35, 80, 20) + + +def test_copy_reads_the_cell_and_restores_the_clipboard(): + clip = _Clipboard() + backend = _Backend(clip, after=_png(highlight=(16, 36, 90, 52))) + read = L.copy_identifier(backend, clip, REGION) + assert read.text == "AC50061" + assert not read.changed_outside + assert backend.drags and backend.drags[0][1] == 45 # mid height + assert backend.keys == ["Control+c"] + assert clip.text == "user's own text" # restored + + +def test_marker_left_in_clipboard_counts_as_nothing_copied(): + clip = _Clipboard() + read = L.copy_identifier(_Backend(clip, copied=None), clip, REGION) + assert read.text is None and read.reason == "nothing_copied" + + +def test_screen_change_outside_the_cell_is_flagged(): + clip = _Clipboard() + backend = _Backend(clip, after=_png(extra=(150, 80, 190, 95))) + read = L.copy_identifier(backend, clip, REGION) + assert read.changed_outside + + +def test_backend_without_drag_takes_no_look(): + class NoDrag: + def screenshot(self): + return _png() + + read = L.copy_identifier(NoDrag(), _Clipboard(), REGION) + assert read.text is None and read.reason == "backend_cannot_drag" + + +def test_crops_match_tolerates_noise_not_a_changed_glyph(): + a = L.crop_png(_png(), REGION) + img = Image.open(io.BytesIO(a)).convert("RGB") + px = img.load() + px[0, 0] = tuple(max(0, c - 3) for c in px[0, 0]) + noisy = io.BytesIO() + img.save(noisy, format="PNG") + assert L.crops_match(a, noisy.getvalue()) + other = Image.new("RGB", (200, 100), "white") + ImageDraw.Draw(other).text((20, 40), "AC5OO61", fill="black") + buf = io.BytesIO() + other.save(buf, format="PNG") + assert not L.crops_match(a, L.crop_png(buf.getvalue(), REGION)) + + +def test_copy_region_is_the_single_line_holding_a_lookalike_id(): + from types import SimpleNamespace as NS + + within = (0, 0, 600, 40) + mrn = NS(text="AC50061", region=(10, 5, 90, 20)) + name = NS(text="Smith, John", region=(200, 5, 120, 20)) + dob = NS(text="01/15/1980", region=(400, 5, 110, 20)) + assert L.copy_region([mrn, name, dob], within) == (6, 3, 98, 24) + other = NS(text="ZZ10O1", region=(520, 5, 60, 20)) + assert L.copy_region([mrn, other], within) is None # two candidates + assert L.copy_region([name, dob], within) is None # none + outside = NS(text="AC50061", region=(10, 100, 90, 20)) + assert L.copy_region([outside], within) is None + + +def test_a_split_identifier_is_matched_as_one_string(): + first = _abstain("103 O45 Smith, John 01/15/1980") + kw = dict( + frames=_frames(first, first), + magnified=first, + contradicts=identity_read_contradicts, + exact_expected=("103", "045"), + ) + assert L.decide(first, clipboard=_copy("103045"), **kw).status == "verified" + assert L.decide(first, clipboard=_copy("1O3045"), **kw).status == "mismatch" + ids = KnownIdList(ids=("103 045", "ZZ1000"), source="t", complete=True) + listed = L.decide(first, clipboard=_copy("103 045"), known_ids=ids, **kw) + assert listed.status == "verified" + + +def test_lookalike_param_mismatch_takes_looks_but_never_softens(monkeypatch): + from openadapt_flow.ir import Anchor, Step, Workflow + from openadapt_flow.runtime import identity as I + from openadapt_flow.runtime.replayer import Replayer + + band = "AC50061 Smith, John 01/15/1980" + anchor = Anchor( + template="t.png", + region=(0, 0, 10, 10), + click_point=(5, 5), + context_text=band, + identifier_region=(0, 0, 400, 20), + ) + step = Step(id="s", intent="open", action="click", anchor=anchor) + wf = Workflow(name="p", params={"mrn": "AC50061"}, steps=[step]) + params = {"mrn": "AC50061"} + + def read(observed): + return I.verify_target_identity( + band, observed, params=params, param_examples=wf.params + ) + + misread = read("AC5OO61 Smith, John 01/15/1980") + assert misread.status == "mismatch" and misread.mode == "param" + + class _Clip: + def read_text(self): + return None + + def write_text(self, text): + pass + + r = Replayer( + object(), # type: ignore[arg-type] + vision=object(), + read_clipboard=_Clip(), + identity_copy=L.IdentityCopyConfig(), + ) + normal = r._lookalike_param_read(misread, anchor, params, wf) + assert L.is_glyph_abstain(normal) + other = read("AC5OO61 Jones, Mary 02/02/1990") + assert r._lookalike_param_read(other, anchor, params, wf) is None + + def looked(status): + return lambda *a, **k: misread.model_copy( + update={"status": status, "abstain_reason": None} + ) + + native = lambda png: misread # noqa: E731 + monkeypatch.setattr(r, "_look_again", looked("verified")) + got = r._lookalike_param_looks( + step, misread, native, params, wf, (0, 0, 400, 20), b"" + ) + assert got.status == "verified" + monkeypatch.setattr(r, "_look_again", looked("abstain")) + got = r._lookalike_param_looks( + step, misread, native, params, wf, (0, 0, 400, 20), b"" + ) + assert got.status == "mismatch" # the original mismatch stands diff --git a/packages/flow/tests/test_identity_looks_replayer.py b/packages/flow/tests/test_identity_looks_replayer.py new file mode 100644 index 000000000..8f4f3f2d9 --- /dev/null +++ b/packages/flow/tests/test_identity_looks_replayer.py @@ -0,0 +1,225 @@ +"""Extra looks on the real replayer path, real render and real OCR. + +A record row whose identifier carries look-alike glyphs (``AC50061``). OCR +abstains on it. The select-and-copy look settles the record only when the +copied identifier equals the run's parameter; the look-alike twin +``AC5OO61`` stops the run. The actuation-time check reuses the step-entry +copy only when the identifier cell shows the same pixels. +""" + +from __future__ import annotations + +import io +from datetime import date + +import pytest + +pytest.importorskip("playwright") +pytest.importorskip("rapidocr_onnxruntime") + +from PIL import Image, ImageDraw # noqa: E402 + +import openadapt_flow.vision as vision # noqa: E402 +from openadapt_flow.ir import ( # noqa: E402 + Anchor, + IdentityCheck, + Resolution, + Step, + StepResult, + Workflow, +) +from openadapt_flow.runtime import identity as I # noqa: E402 +from openadapt_flow.runtime.identity_looks import IdentityCopyConfig # noqa: E402 +from openadapt_flow.runtime.replayer import Replayer # noqa: E402 +from tests.test_identity_homonym_8th import _ROW_K, _render # noqa: E402 + +_FRAMES: dict[str, object] = {} + + +def _frame(mrn: str): + if mrn not in _FRAMES: + _FRAMES[mrn] = _render(mrn) + return _FRAMES[mrn] + + +def _identifier_region(frame) -> tuple[int, int, int, int]: + """The MRN + name + DOB cells of the row under test.""" + _, _, _, row = frame.points[_ROW_K] + lines = sorted(vision.ocr(frame.png, region=row), key=lambda ln: ln.region[0]) + mrn = next(ln for ln in lines if "AC5" in ln.text.upper()) + dob = next(ln for ln in lines if "1980" in ln.text) + x0 = mrn.region[0] - 4 + x1 = dob.region[0] + dob.region[2] + 4 + return (x0, row[1], x1 - x0, row[3]) + + +class _Clipboard: + def __init__(self) -> None: + self.text = "operator's clipboard" + + def read_text(self): + return self.text + + def write_text(self, text): + self.text = text + + +def _highlight(png: bytes, region) -> bytes: + img = Image.open(io.BytesIO(png)).convert("RGB") + x, y, w, h = region + draw = ImageDraw.Draw(img) + draw.rectangle((x + 2, y + 2, x + w - 3, y + 4), fill=(51, 144, 255)) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +class _CopyBackend: + """Pixel-only session whose identifier cell can be selected and copied.""" + + def __init__(self, frame, mrn: str, clipboard: _Clipboard, *, spill=False): + self.viewport = frame.viewport + self.frame = frame.png + self.region = _identifier_region(frame) + self.copy_text = mrn # the drag spans the identifier's own text box + self.clipboard = clipboard + self.spill = spill + self.drags: list[tuple[int, int, int, int]] = [] + self.selected = False + + def screenshot(self) -> bytes: + return self.frame + + def drag(self, x, y, ex, ey): + self.drags.append((x, y, ex, ey)) + rx, ry, rw, rh = self.region + self.selected = ry <= y <= ry + rh + if self.selected: # a selection paints between the drag's ends + self.frame = _highlight(self.frame, (x, y - 8, ex - x, 16)) + if self.spill: # the gesture opened something elsewhere on screen + self.frame = _highlight(self.frame, (5, 5, 60, 20)) + + def press(self, key): + assert key == "Control+c" + if self.selected: + self.clipboard.text = self.copy_text + + +def _setup(live_mrn: str, *, param_mode: bool, spill=False, copy=True): + rec = _frame("AC50061") + live = _frame(live_mrn) + _, open_pt, _, _ = rec.points[_ROW_K] + _, live_pt, _, _ = live.points[_ROW_K] + region = _identifier_region(rec) + context = I.identifier_text_from_lines( + vision.ocr(rec.png), region=region, reference_date=date.today() + ) + assert context and "1980" in context + anchor = Anchor( + template="t.png", + region=(open_pt[0] - 30, open_pt[1] - 12, 60, 24), + click_point=(open_pt[0], open_pt[1]), + context_text=context, + identifier_region=region, + ) + step = Step( + id="open_record", + intent="open record", + action="click", + anchor=anchor, + risk="irreversible", + ) + params = {"mrn": "AC50061"} if param_mode else {} + wf = Workflow(name="looks", params=dict(params), steps=[step]) + res = Resolution(rung="ocr", point=live_pt, confidence=0.9, elapsed_ms=1.0) + clip = _Clipboard() + backend = _CopyBackend(live, live_mrn, clip, spill=spill) + replayer = Replayer( + backend, + vision=vision, + read_clipboard=clip if copy else None, + identity_copy=IdentityCopyConfig() if copy else None, + ) + return replayer, backend, step, res, wf, params, live + + +def _entry(replayer, step, res, wf, params, png) -> IdentityCheck: + replayer._identity_input_looks = True + try: + return replayer._verify_identity(step, res, png, params, wf, bundle_dir=None) + finally: + replayer._identity_input_looks = False + + +def test_without_copy_look_the_correct_row_still_abstains(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True, copy=False) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "abstain" + assert not b.drags + + +def test_copy_equal_to_the_run_value_verifies_the_correct_row(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "verified", check + kinds = [(look.kind, look.outcome) for look in check.looks] + assert ("clipboard", "resolves") in kinds + assert [k for k, _ in kinds].count("frame") == 2 + assert len(b.drags) == 1 + assert b.clipboard.text == "operator's clipboard" + + +def test_copy_of_the_look_alike_twin_stops_the_run(): + r, b, step, res, wf, params, live = _setup("AC5OO61", param_mode=True) + check = _entry(r, step, res, wf, params, live.png) + # It stops either way. Where OCR reads the twin as a look-alike of the + # run value, the copy is what names it; where OCR already reads another + # record (some OCR builds), the read is final and takes no looks. + assert check.status == "mismatch" + if check.looks: + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "contradicts" + + +def test_recorded_record_mode_copy_cannot_accept(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=False) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "abstain" + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "agrees" + + +def test_screen_change_outside_the_cell_halts(tmp_path): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True, spill=True) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + error = r._entry_identity_gate_error( + step, res, live.png, params, wf, tmp_path, result + ) + assert result.identity is not None + assert result.identity.look_halt == "identity_look_changed_screen" + assert error is not None and "identity_look_changed_screen" in error + assert result.safety_halt is True + + +def test_governed_runs_take_no_copy_look(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True) + r.governed_authorization = object() # type: ignore[assignment] + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "abstain" + assert not b.drags + + +def test_actuation_check_reuses_the_copy_only_on_matching_pixels(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True) + assert _entry(r, step, res, wf, params, live.png).status == "verified" + # Actuation-time check: no input, the selection highlight is still drawn. + again = r._verify_identity(step, res, b.frame, params, wf, bundle_dir=None) + assert len(b.drags) == 1 + assert again.status == "verified" + assert again.looks[-1].reason == "same_pixels_as_step_entry_copy" + # The twin now sits where the record was: the binding must not carry over. + twin = _frame("AC5OO61") + b.frame = twin.png + swapped = r._verify_identity(step, res, twin.png, params, wf, bundle_dir=None) + assert swapped.status != "verified" + assert len(b.drags) == 1 diff --git a/packages/flow/tests/test_known_ids_replayer.py b/packages/flow/tests/test_known_ids_replayer.py index c204fdeab..6564ef347 100644 --- a/packages/flow/tests/test_known_ids_replayer.py +++ b/packages/flow/tests/test_known_ids_replayer.py @@ -86,14 +86,21 @@ def test_correct_row_abstains_without_list(): assert _check("AC50061", "AC50061", None).status == "abstain" -def test_correct_row_verifies_with_complete_list_using_a_fresh_capture(): +def test_correct_row_verifies_with_complete_list_using_fresh_captures(): out: dict = {} check = _check("AC50061", "AC50061", _ids(), out=out) assert check.status == "verified" assert check.known_id_evidence is not None - assert check.known_id_evidence.reads == 2 - assert check.known_id_evidence.frames == 2 - assert out["captures"] == 1 # the second read took its own screenshot + assert check.known_id_evidence.reads == 3 + assert check.known_id_evidence.frames == 3 + assert out["captures"] == 2 # each later read took its own screenshot + assert [look.kind for look in check.looks] == [ + "frame", + "frame", + "magnified", + "known_ids", + ] + assert check.looks[-1].outcome == "resolves" def test_fresh_capture_showing_another_record_blocks_the_lift(): diff --git a/packages/flow/tests/test_state_dependency.py b/packages/flow/tests/test_state_dependency.py index 7f0127e37..547434718 100644 --- a/packages/flow/tests/test_state_dependency.py +++ b/packages/flow/tests/test_state_dependency.py @@ -79,13 +79,16 @@ def __init__(self, viewport=VIEWPORT): def screenshot(self): # Move a large black bar across the frame each poll so the PERCEPTUAL # hash actually differs frame to frame (a flat shade change would hash - # identically and count as settled). + # identically and count as settled). The bar runs along the bottom, + # clear of the click target (y 100-120): an animation over the target + # itself is a change between check and click, which the pixel + # check-to-click guard refuses (tests/test_check_to_click.py). self._tick += 1 img = Image.new("RGB", self._viewport, (240, 240, 240)) w, h = self._viewport x = (self._tick * 37) % max(1, w - 40) for px in range(x, min(x + 40, w)): - for py in range(h): + for py in range(max(0, h - 60), h): img.putpixel((px, py), (0, 0, 0)) buf = io.BytesIO() img.save(buf, format="PNG")