From 58fb10cdca9e5724de86d077e0622ec58f68c721 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 09:21:02 -0400 Subject: [PATCH 01/10] Take extra looks before stopping on a look-alike identifier On a pixel surface, an OCR identity read that abstains only because the record ID has a look-alike glyph (1/l, 0/O) now takes extra looks before the run stops: two fresh frames, the 2x re-read, the known-ID list, and, where configured, a select-and-copy of the identifier's text box. Only the list or a copy equal to a run parameter can settle the record; any look that names another record is final. Each look is recorded on the step's identity check. The copy look is input, so only the step-entry check takes it; the actuation check reuses it only on matching pixels. Co-Authored-By: Claude Opus 5.5 --- packages/flow/openadapt_flow/deployment.py | 38 ++ packages/flow/openadapt_flow/ir.py | 34 ++ .../openadapt_flow/runtime/identity_looks.py | 527 ++++++++++++++++++ .../flow/openadapt_flow/runtime/known_ids.py | 4 + .../flow/openadapt_flow/runtime/replayer.py | 299 ++++++++-- .../validation/identity_looks_dense.py | 492 ++++++++++++++++ packages/flow/tests/test_identity_looks.py | 287 ++++++++++ .../tests/test_identity_looks_replayer.py | 221 ++++++++ .../flow/tests/test_known_ids_replayer.py | 15 +- 9 files changed, 1862 insertions(+), 55 deletions(-) create mode 100644 packages/flow/openadapt_flow/runtime/identity_looks.py create mode 100644 packages/flow/openadapt_flow/validation/identity_looks_dense.py create mode 100644 packages/flow/tests/test_identity_looks.py create mode 100644 packages/flow/tests/test_identity_looks_replayer.py diff --git a/packages/flow/openadapt_flow/deployment.py b/packages/flow/openadapt_flow/deployment.py index 42c42a4844..d796aea390 100644 --- a/packages/flow/openadapt_flow/deployment.py +++ b/packages/flow/openadapt_flow/deployment.py @@ -678,6 +678,24 @@ class GroundingModelConfig(BaseModel): api_key_env: str = "" +class IdentityCopyLookSection(BaseModel): + """Select-and-copy look for a look-alike identifier on a pixel surface. + + Turn on only where selecting the recorded identifier cell and copying it + changes nothing (a read-only text cell) and the session's clipboard reaches + the machine Flow runs on (RDP/Citrix clipboard redirection). See + :mod:`openadapt_flow.runtime.identity_looks`. + """ + + model_config = ConfigDict(extra="forbid") + + enabled: bool = False + #: ``drag`` across the cell at mid height, or ``double_click`` its centre. + gesture: Literal["drag", "double_click"] = "drag" + #: Keys that copy the selection in the driven session. + copy_keys: list[str] = Field(default_factory=lambda: ["Control+c"], min_length=1) + + class RuntimeSection(BaseModel): """Runtime posture: durability and model-egress opt-in.""" @@ -727,6 +745,13 @@ class RuntimeSection(BaseModel): known_ids_confirmation_path: Optional[str] = None #: Oldest export (by its export time) replay will use. known_ids_max_age_hours: float = Field(default=24.0, gt=0) + #: Before stopping on a look-alike identifier, select the identifier cell, + #: copy it and read the clipboard. Settles the record only when the copy + #: equals a run parameter in the identity band; halts when the screen + #: changes outside the cell. Off by default. Never used by governed runs. + identity_copy_look: IdentityCopyLookSection = Field( + default_factory=IdentityCopyLookSection + ) #: Font/layout drift check before each recorded click step #: (:mod:`openadapt_flow.runtime.layout_drift`). None (default) turns it on #: for remote-desktop replays (``rdp``/``citrix``) and off elsewhere; @@ -1308,8 +1333,21 @@ def build_replayer( confirmation_path=runtime_config.known_ids_confirmation_path, max_age_s=runtime_config.known_ids_max_age_hours * 3600.0, ) + identity_copy = None + read_clipboard = None + copy_cfg = runtime_config.identity_copy_look if runtime_config is not None else None + if copy_cfg is not None and copy_cfg.enabled: + from openadapt_flow.records.read import SystemClipboard + from openadapt_flow.runtime.identity_looks import IdentityCopyConfig + + identity_copy = IdentityCopyConfig( + gesture=copy_cfg.gesture, copy_keys=tuple(copy_cfg.copy_keys) + ) + read_clipboard = SystemClipboard() return Replayer( backend, + identity_copy=identity_copy, + read_clipboard=read_clipboard, layout_drift=( runtime_config.layout_drift_check if runtime_config is not None else None ), diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 8b080ca654..16573be3c9 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3163,6 +3163,26 @@ class KnownIdEvidence(BaseModel): export_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") +class IdentityLook(BaseModel): + """One extra look taken before stopping on a look-alike identifier. + + See :mod:`openadapt_flow.runtime.identity_looks`. Holds no identifier + values: only what kind of look it was, what it concluded, and the + SHA-256 of the frame it read when it read one. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + kind: Literal["frame", "magnified", "known_ids", "clipboard"] + #: ``agrees``: consistent with the first read, settles nothing alone. + #: ``resolves``: names exactly one record, the expected one. + #: ``contradicts``: names another record; the step stops. + #: ``unavailable``: the look could not be taken or read nothing usable. + outcome: Literal["agrees", "resolves", "contradicts", "unavailable"] + reason: Optional[str] = Field(default=None, max_length=80) + frame_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + + class IdentityCheck(BaseModel): """Outcome of the pre-click target-identity check (runtime.identity). @@ -3247,6 +3267,20 @@ class IdentityCheck(BaseModel): "confirmed, or it is older than the configured age. The step halts." ), ) + looks: list[IdentityLook] = Field( + default_factory=list, + description=( + "Extra looks taken because the first read abstained only on a " + "look-alike identifier (runtime.identity_looks), in order." + ), + ) + look_halt: Optional[Literal["identity_look_changed_screen"]] = Field( + default=None, + description=( + "Set when an extra look changed the screen outside the " + "identifier it read. The step halts." + ), + ) class HealEvent(BaseModel): diff --git a/packages/flow/openadapt_flow/runtime/identity_looks.py b/packages/flow/openadapt_flow/runtime/identity_looks.py new file mode 100644 index 0000000000..2eb21206e8 --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/identity_looks.py @@ -0,0 +1,527 @@ +"""Extra looks before stopping on a look-alike identifier. + +On a pixel-only surface (RDP, Citrix, canvas) the OCR identity tier abstains +when the record's identifier carries a look-alike glyph (O/0, l/1/I): OCR can +read ``A1001`` and a different record's ``Al001`` as the same string. Before +the run stops on such an abstain, the replayer takes extra looks that change +nothing on the screen: + +- ``frame``: the same OCR read on :data:`FRAME_LOOKS` fresh screenshots. Each + must abstain on the same look-alike identifier classes as the first read. + A frame that names another record ends the check with a mismatch. +- ``magnified``: the replayer's existing 2x re-read of the first frame. It + must not name another record, and when it reads the identifier it must read + it in the same look-alike classes. +- ``known_ids``: the complete known-ID list (:mod:`.known_ids`) when the + deployment configured one. +- ``clipboard``: where the surface allows it, a drag across the recorded + identifier cell, a copy, and a read of the clipboard (:func:`copy_identifier`). + The clipboard holds the application's own characters, so ``0`` and ``O`` + differ. + +OCR looks never settle the question on their own: a look-alike twin can read +the same on every frame and at every magnification. Only two looks can +resolve: the known-ID list, and a clipboard read that equals an EXACT +expected identifier. An exact expected identifier comes only from a run +parameter embedded in the band (the operator's typed or supplied value). A +recorded OCR band is not exact (a recording can read ``A1001`` as ``Alo01``), +so in recorded-record mode the clipboard can contradict but never accept. + +:func:`decide` accepts only when every frame agrees, no look contradicts, and +at least one look resolves to the expected record. Any contradiction is +final. Otherwise the original abstain stands, with the looks recorded on it. +""" + +from __future__ import annotations + +import hashlib +import io +import secrets +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Literal, Mapping, Optional, Sequence + +from openadapt_flow.ir import IdentityCheck, IdentityLook, Region +from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime.known_ids import ( + ABSTAIN_REASON_GLYPH, + KnownIdList, + id_core, + lookalike_class, + resolve_lookalike_abstain, +) + +FRAME_LOOKS = 2 +"""Fresh screenshots read after the first one.""" + +SELECTION_PAD_PX = 6 +"""How far outside the identifier cell a selection highlight may paint.""" + +COPY_GESTURES = ("drag", "double_click") + + +def identifier_tokens(text: str) -> list[str]: + """Identifier-shaped tokens of a read, in :func:`known_ids.id_core` form.""" + out = [] + for raw in (text or "").split(): + tok = identity_mod.squash(raw) + if tok and identity_mod._is_identifier_shaped(tok): + out.append(id_core(tok)) + return out + + +def lookalike_classes(text: str) -> frozenset[str]: + """Look-alike classes of the identifier-shaped tokens in ``text``.""" + return frozenset(lookalike_class(t) for t in identifier_tokens(text)) + + +def vulnerable_classes(text: str) -> frozenset[str]: + """Classes of the identifiers OCR could have collapsed (O/0, l/1/I).""" + return frozenset( + lookalike_class(t) + for t in identifier_tokens(text) + if identity_mod._is_glyph_vulnerable_identifier(t) + ) + + +def is_glyph_abstain(check: Optional[IdentityCheck]) -> bool: + return ( + check is not None + and check.status == "abstain" + and check.abstain_reason == ABSTAIN_REASON_GLYPH + ) + + +def exact_expected_ids( + run_values: Mapping[str, Any], embedded: Iterable[str] +) -> tuple[str, ...]: + """Exact identifiers the operator gave for parameters in the band. + + Only parameters embedded in the recorded band count: a run value that is + not part of the identity band says nothing about which record this is. + """ + out: list[str] = [] + for name in embedded: + value = run_values.get(name) + if isinstance(value, str): + out.extend(identifier_tokens(value)) + return tuple(dict.fromkeys(out)) + + +@dataclass(frozen=True) +class IdentityCopyConfig: + """Turns on the select-and-copy look for a deployment. + + Only for surfaces where selecting the identifier cell and copying it + changes nothing: a read-only text cell, with the remote session's + clipboard redirected to the machine Flow runs on. Any screen change + outside the cell during the look halts the run. + """ + + gesture: str = "drag" + copy_keys: tuple[str, ...] = ("Control+c",) + + def __post_init__(self) -> None: + if self.gesture not in COPY_GESTURES: + raise ValueError( + f"identity copy gesture must be one of {COPY_GESTURES}, " + f"not {self.gesture!r}" + ) + if not self.copy_keys: + raise ValueError("identity copy needs at least one copy key") + + +@dataclass(frozen=True) +class LookBinding: + """A clipboard read taken at step entry, reusable at actuation. + + The actuation-time identity check takes no input. It reuses this read only + when the identifier cell on its frame shows the same pixels as one of the + crops seen at step entry. + """ + + step_id: str + classes: frozenset[str] + read: "ClipboardRead" + crops: tuple[bytes, ...] + + def matches( + self, step_id: str, classes: frozenset[str], crop: Optional[bytes] + ) -> bool: + return ( + step_id == self.step_id + and classes == self.classes + and any(crops_match(crop, c) for c in self.crops) + ) + + +@dataclass(frozen=True) +class ClipboardRead: + """What one select-and-copy look returned. + + Attributes: + text: The copied text, or None when nothing was copied. + changed_outside: The screen changed outside the identifier cell + while the look ran. The step halts. + pre_crop: The identifier cell's pixels just before the gesture. + post_crop: The identifier cell's pixels just after the copy. + frame_sha256: SHA-256 of the frame taken just after the copy. + reason: Why nothing was copied, when nothing was. + """ + + text: Optional[str] + changed_outside: bool = False + pre_crop: Optional[bytes] = None + post_crop: Optional[bytes] = None + frame_sha256: Optional[str] = None + reason: Optional[str] = None + + +def _look( + kind: Literal["frame", "magnified", "known_ids", "clipboard"], + outcome: Literal["agrees", "resolves", "contradicts", "unavailable"], + reason: Optional[str] = None, + frame_sha256: Optional[str] = None, +) -> IdentityLook: + return IdentityLook( + kind=kind, outcome=outcome, reason=reason, frame_sha256=frame_sha256 + ) + + +def _frame_look( + first_classes: frozenset[str], + read: IdentityCheck, + frame_sha256: Optional[str], + contradicts: Callable[[IdentityCheck], bool], +) -> IdentityLook: + if read.status == "mismatch": + reason = "names_another_record" if contradicts(read) else "read_incomplete" + return _look("frame", "contradicts", reason, frame_sha256) + if not is_glyph_abstain(read): + return _look("frame", "unavailable", f"read_{read.status}", frame_sha256) + if lookalike_classes(read.observed) != first_classes: + return _look("frame", "unavailable", "identifier_read_differs", frame_sha256) + return _look("frame", "agrees", None, frame_sha256) + + +def _magnified_look( + first_classes: frozenset[str], + read: Optional[IdentityCheck], + contradicts: Callable[[IdentityCheck], bool], +) -> IdentityLook: + if read is None: + return _look("magnified", "unavailable", "not_taken") + if read.status == "mismatch" and contradicts(read): + return _look("magnified", "contradicts", "names_another_record") + if read.status in ("unreadable", "mismatch"): + # Nothing readable, or only part of the band: says nothing either way. + return _look("magnified", "unavailable", f"read_{read.status}") + classes = lookalike_classes(read.observed) + if not classes: + return _look("magnified", "unavailable", "no_identifier_read") + if classes != first_classes: + return _look("magnified", "unavailable", "identifier_read_differs") + return _look("magnified", "agrees") + + +def _clipboard_look( + first_classes: frozenset[str], + vulnerable: frozenset[str], + copied: Optional[ClipboardRead], + exact: Sequence[str], + known_ids: Optional[KnownIdList], +) -> IdentityLook: + if copied is None: + return _look("clipboard", "unavailable", "not_configured") + sha = copied.frame_sha256 + if copied.text is None: + return _look("clipboard", "unavailable", copied.reason or "nothing_copied", sha) + tokens = identifier_tokens(copied.text) + by_class: dict[str, set[str]] = {} + for tok in tokens: + by_class.setdefault(lookalike_class(tok), set()).add(tok) + if set(by_class) != set(first_classes): + # The copy did not cover the identifier OCR read (or covered more). + return _look("clipboard", "unavailable", "copy_differs_from_read", sha) + if any(len(v) != 1 for v in by_class.values()): + return _look("clipboard", "unavailable", "copy_ambiguous", sha) + copied_ids = {k: next(iter(v)) for k, v in by_class.items()} + if known_ids is not None and known_ids.complete: + listed = {m for k in copied_ids for m in known_ids.members_of_class(k)} + if any(core not in listed for core in copied_ids.values()): + return _look("clipboard", "contradicts", "copied_id_not_listed", sha) + exact_by_class: dict[str, set[str]] = {} + for e in exact: + exact_by_class.setdefault(lookalike_class(e), set()).add(e) + for cls, core in copied_ids.items(): + expected = exact_by_class.get(cls) + if expected and core not in expected: + return _look("clipboard", "contradicts", "copied_id_differs", sha) + if vulnerable and all( + cls in exact_by_class and copied_ids[cls] in exact_by_class[cls] + for cls in vulnerable + ): + return _look("clipboard", "resolves", "copied_id_equals_run_value", sha) + return _look("clipboard", "agrees", "no_exact_expected_id", sha) + + +def decide( + first: IdentityCheck, + *, + frames: Sequence[tuple[IdentityCheck, Optional[str]]], + magnified: Optional[IdentityCheck], + contradicts: Callable[[IdentityCheck], bool], + known_ids: Optional[KnownIdList] = None, + known_ids_lift: Optional[IdentityCheck] = None, + clipboard: Optional[ClipboardRead] = None, + exact_expected: Sequence[str] = (), +) -> IdentityCheck: + """Combine the looks into one verdict. + + Args: + first: The first read: a glyph-only abstain. + frames: ``(read, frame_sha256)`` per fresh screenshot. + magnified: The 2x re-read of the first frame, if taken. + contradicts: Whether a ``mismatch`` read names another record + (``replayer.identity_read_contradicts``). + known_ids: The complete known-ID list, for the clipboard cross-check. + known_ids_lift: The list's verdict on the frame reads + (:func:`known_ids.resolve_lookalike_abstain`), or None. + clipboard: The select-and-copy read, or None when not taken. + exact_expected: Exact identifiers from run parameters in the band. + + Returns: + ``verified`` (with every look recorded) when every frame agrees, no + look contradicts and one resolves; the contradicting read (or the + first, as ``mismatch``) when any look contradicts; otherwise + ``first`` with the looks recorded. + """ + if not is_glyph_abstain(first): + return first + first_classes = lookalike_classes(first.observed) + vulnerable = vulnerable_classes(first.observed) + looks: list[IdentityLook] = [] + for read, sha in frames: + look = _frame_look(first_classes, read, sha, contradicts) + looks.append(look) + if look.outcome == "contradicts": + # A fresh frame that names another record is final. + return read.model_copy(update={"looks": looks}) + mag = _magnified_look(first_classes, magnified, contradicts) + looks.append(mag) + if mag.outcome == "contradicts": + assert magnified is not None + return magnified.model_copy(update={"looks": looks}) + if known_ids is not None or known_ids_lift is not None: + looks.append( + _look("known_ids", "resolves", "single_listed_id") + if known_ids_lift is not None + else _look("known_ids", "unavailable", "list_does_not_settle_it") + ) + if clipboard is not None: + clip = _clipboard_look( + first_classes, vulnerable, clipboard, exact_expected, known_ids + ) + looks.append(clip) + if clip.outcome == "contradicts": + return first.model_copy( + update={ + "status": "mismatch", + "abstain_reason": None, + "looks": looks, + } + ) + frames_agree = bool(frames) and all( + look.outcome == "agrees" for look in looks if look.kind == "frame" + ) + magnified_ok = mag.outcome in ("agrees", "unavailable") + resolved = any(look.outcome == "resolves" for look in looks) + if frames_agree and magnified_ok and resolved: + base = known_ids_lift if known_ids_lift is not None else first + return base.model_copy( + update={"status": "verified", "abstain_reason": None, "looks": looks} + ) + return first.model_copy(update={"looks": looks}) + + +# ----------------------------------------------------------- copy look + + +def copy_region(lines: Iterable[Any], within: Region) -> Optional[Region]: + """The text box to select: the one line inside ``within`` that holds a + look-alike identifier. + + Selecting only that line keeps the gesture off every other control in the + row. None when no line, or more than one, holds such an identifier: the + copy look is then not taken. + """ + hits = [] + for line in lines: + region = getattr(line, "region", None) + if region is None or not identity_mod.regions_intersect(region, within): + continue + if vulnerable_classes(str(getattr(line, "text", ""))): + hits.append(region) + if len(hits) != 1: + return None + x, y, w, h = (int(v) for v in hits[0]) + return (x - 2, y - 2, w + 4, h + 4) + + +def crop_png(png: bytes, region: Region) -> Optional[bytes]: + return identity_mod.crop_region(png, region) + + +def _diff_outside(pre: bytes, post: bytes, allowed: Region) -> Optional[bool]: + """Whether ``post`` differs from ``pre`` outside ``allowed``. + + None when the frames cannot be compared (a different size counts as a + change). + """ + try: + import numpy as np + from PIL import Image + + a = np.asarray(Image.open(io.BytesIO(pre)).convert("RGB"), dtype=np.int16) + b = np.asarray(Image.open(io.BytesIO(post)).convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - an unreadable frame cannot be judged + return None + if a.shape != b.shape: + return True + changed = np.abs(a - b).max(axis=2) > 0 + x, y, w, h = allowed + x0 = max(0, x - SELECTION_PAD_PX) + y0 = max(0, y - SELECTION_PAD_PX) + x1 = min(changed.shape[1], x + w + SELECTION_PAD_PX) + y1 = min(changed.shape[0], y + h + SELECTION_PAD_PX) + changed[y0:y1, x0:x1] = False + return bool(changed.any()) + + +def copy_identifier( + backend: Any, + clipboard: Any, + region: Region, + *, + gesture: str = "drag", + copy_keys: Sequence[str] = ("Control+c",), + settle: Optional[Callable[[], bytes]] = None, + crop_region: Optional[Region] = None, +) -> ClipboardRead: + """Select the identifier cell, copy it, and read the clipboard. + + The clipboard's previous text is restored afterwards. The clipboard is + first set to a random marker; a clipboard still holding the marker counts + as nothing copied. The frames just before the gesture and just after the + copy must be identical outside the identifier cell (a selection highlight + may paint inside it); otherwise ``changed_outside`` is set and the caller + halts. + + Args: + backend: The run's backend. ``drag`` needs a ``drag(x, y, ex, ey)`` + method; ``double_click`` uses ``click(x, y, double=True)``. + clipboard: A :class:`~openadapt_flow.records.read.ClipboardReader`. + region: The identifier cell on the live frame. + gesture: ``drag`` (across the cell, left to right, at mid height) or + ``double_click`` (at the cell's centre). + copy_keys: Keys that copy the selection on the driven session. + settle: Returns the next settled frame; defaults to a screenshot. + crop_region: Where to cut ``pre_crop`` / ``post_crop``; defaults to + ``region``. + """ + x, y, w, h = (int(v) for v in region) + if w < 4 or h < 4: + return ClipboardRead(text=None, reason="cell_too_small") + if gesture == "drag" and not callable(getattr(backend, "drag", None)): + return ClipboardRead(text=None, reason="backend_cannot_drag") + if gesture not in COPY_GESTURES: + return ClipboardRead(text=None, reason="unknown_gesture") + capture = settle or backend.screenshot + try: + previous = clipboard.read_text() + except Exception: # noqa: BLE001 - no clipboard channel, no look + return ClipboardRead(text=None, reason="clipboard_unreadable") + marker = f"openadapt-identity-look-{secrets.token_hex(8)}" + copied: Optional[str] = None + pre = post = None + try: + clipboard.write_text(marker) + pre = capture() + mid_y = y + h // 2 + inset = max(2, min(6, w // 10)) + if gesture == "drag": + backend.drag(x + inset, mid_y, x + w - inset, mid_y) + else: + backend.click(x + w // 2, mid_y, double=True) + for key in copy_keys: + backend.press(key) + post = capture() + copied = clipboard.read_text() + except Exception as exc: # noqa: BLE001 - a failed look is no look + return ClipboardRead( + text=None, + changed_outside=bool( + pre is not None + and post is not None + and _diff_outside(pre, post, region) is not False + ), + reason=f"look_failed:{type(exc).__name__}"[:80], + ) + finally: + if previous is not None: + try: + clipboard.write_text(previous) + except Exception: # noqa: BLE001 - restoring is best effort + pass + outside = _diff_outside(pre, post, region) if pre and post else None + read = ClipboardRead( + text=None if copied in (None, marker) else copied, + changed_outside=outside is not False, + pre_crop=crop_png(pre, crop_region or region) if pre else None, + post_crop=crop_png(post, crop_region or region) if post else None, + frame_sha256=hashlib.sha256(post).hexdigest() if post else None, + reason="nothing_copied" if copied in (None, marker) else None, + ) + return read + + +def crops_match(a: Optional[bytes], b: Optional[bytes]) -> bool: + """Whether two identifier-cell crops show the same pixels. + + Tolerates encoder noise (every channel within 8 levels, mean under 1), + never a changed glyph: a different character moves whole strokes. + """ + if a is None or b is None: + return False + if a == b: + return True + try: + import numpy as np + from PIL import Image + + x = np.asarray(Image.open(io.BytesIO(a)).convert("RGB"), dtype=np.int16) + y = np.asarray(Image.open(io.BytesIO(b)).convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - unreadable crops never match + return False + if x.shape != y.shape: + return False + d = np.abs(x - y) + return bool(d.max() <= 8 and d.mean() < 1.0) + + +__all__ = [ + "COPY_GESTURES", + "ClipboardRead", + "FRAME_LOOKS", + "IdentityCopyConfig", + "LookBinding", + "copy_identifier", + "copy_region", + "crops_match", + "decide", + "exact_expected_ids", + "identifier_tokens", + "is_glyph_abstain", + "lookalike_classes", + "resolve_lookalike_abstain", + "vulnerable_classes", +] diff --git a/packages/flow/openadapt_flow/runtime/known_ids.py b/packages/flow/openadapt_flow/runtime/known_ids.py index 0fa17ec9f8..7d528d3eff 100644 --- a/packages/flow/openadapt_flow/runtime/known_ids.py +++ b/packages/flow/openadapt_flow/runtime/known_ids.py @@ -137,6 +137,10 @@ def members(self, token: str) -> tuple[str, ...]: """Listed identifiers OCR could read as ``token``.""" return self._classes.get(lookalike_class(token), ()) + def members_of_class(self, key: str) -> tuple[str, ...]: + """Listed identifiers in one look-alike class (a :func:`lookalike_class` key).""" + return self._classes.get(key, ()) + @property def size(self) -> int: return sum(len(v) for v in self._classes.values()) diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 68c19a824b..5e3a125d64 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -97,6 +97,7 @@ FreshActuationEvent, HaltObservation, IdentityCheck, + IdentityLook, IdentitySignalEvidence, Interstitial, InterstitialActionResult, @@ -149,6 +150,7 @@ from openadapt_flow.runtime import heal as heal_mod from openadapt_flow.runtime import healing as healing_mod from openadapt_flow.runtime import identity as identity_mod +from openadapt_flow.runtime import identity_looks as looks_mod from openadapt_flow.runtime.authorization import ( GovernedRunAuthorization, RuntimeParamScalar, @@ -624,9 +626,19 @@ class Replayer: structured-text, pixel-compare, OCR, and halt with no model. known_ids: Optional complete ``runtime.known_ids.KnownIdList``. When set, an OCR identity check that abstained only because the band - rests on a look-alike identifier is lifted to verified if two OCR - reads agree and exactly one listed identifier is in that - look-alike class. None (default) keeps the abstain. + rests on a look-alike identifier is lifted to verified if the + reads of three separate screenshots agree and exactly one listed + identifier is in that look-alike class. None (default) keeps the + abstain. + identity_copy: Optional + :class:`~openadapt_flow.runtime.identity_looks.IdentityCopyConfig`. + With a ``read_clipboard`` channel, a look-alike abstain on a step + with a recorded identifier cell takes one more look: select the + cell, copy it, read the clipboard. The copy settles the record only + when it equals a run parameter embedded in the band; it stops the + run when it names another record or the screen changed outside the + cell. Never taken in governed or qualification runs. None + (default) takes no copy look. effect_verifier: Optional system-of-record ``EffectVerifier`` (``runtime.effects``) bound to the deployment's system of record (a REST/FHIR API, a document store). When set, any step that @@ -713,6 +725,8 @@ class Replayer: _render_scale_attempted: bool = False _render_scale_inner: Optional[Backend] = None _render_scale_evidence: Optional[dict[str, Any]] = None + identity_copy: Optional[looks_mod.IdentityCopyConfig] = None + _identity_input_looks: bool = False def __init__( self, @@ -755,6 +769,7 @@ def __init__( attended_draft_by: Optional[Any] = None, draft_key_dir: Optional[Path] = None, read_clipboard: Optional[Any] = None, + identity_copy: Optional[looks_mod.IdentityCopyConfig] = None, deployment_surface: Optional[str] = None, layout_drift: Optional[bool] = None, layout_drift_tolerances: Optional[Any] = None, @@ -995,6 +1010,13 @@ def __init__( # Optional complete known-ID list (runtime.known_ids): lifts a # look-alike identifier abstain only when no listed twin exists. self.known_ids = known_ids + # Optional select-and-copy look for look-alike identifiers + # (runtime.identity_looks). It is input, so only the step-entry + # identity check takes it; the actuation-time check reuses its read + # through a pixel-matched binding. + self.identity_copy = identity_copy + self._identity_input_looks = False + self._identity_look_bindings: dict[str, looks_mod.LookBinding] = {} # Optional VLM drift-oracle (state-verifier), OFF by default. When set, # it may CONFIRM a render-drift-sensitive postcondition that # deterministically false-failed (the same heal-under-drift the @@ -6947,7 +6969,7 @@ def _run_step_on_active_surface( ) last_frame = before_png pre_identity_halt = (result.safety_halt, result.failure_category) - error = self._identity_gate_error( + error = self._entry_identity_gate_error( step, resolution, before_png, @@ -7018,7 +7040,7 @@ def _run_step_on_active_surface( result.resolution = resolution # The first read's halt flags belong to that read. result.safety_halt, result.failure_category = pre_identity_halt - error = self._identity_gate_error( + error = self._entry_identity_gate_error( step, resolution, before_png, @@ -9100,6 +9122,26 @@ def _identity_band_still_changing( return False return False + def _entry_identity_gate_error( + self, + step: Step, + *args: Any, + **kwargs: Any, + ) -> Optional[str]: + """The identity gate at step entry, where an extra look may use input. + + Only this check may select and copy the identifier cell + (:mod:`openadapt_flow.runtime.identity_looks`). The actuation-time + check that follows takes no input and reuses that read only through a + pixel-matched binding. + """ + self.__dict__.setdefault("_identity_look_bindings", {}).pop(step.id, None) + self._identity_input_looks = True + try: + return self._identity_gate_error(step, *args, **kwargs) + finally: + self._identity_input_looks = False + def _identity_gate_error( self, step: Step, @@ -9203,6 +9245,13 @@ def _identity_gate_error( f"{check.quorum_required or 0} independent signals verified " "on the fresh frame — refusing to act; run aborted" ) + elif check.look_halt is not None: + error = ( + f"Step '{step.id}' ({step.intent}) took an extra look at a " + "look-alike identifier and the screen changed outside the " + f"identifier ({check.look_halt}) — refusing to act" + ) + result.safety_halt = True elif check.known_id_halt is not None: names = { "known_ids_unconfirmed": "has no admission confirmation", @@ -14110,16 +14159,20 @@ def native_attempt( ), scoped_identifier=identifier_region is not None, ) + pending_looks: list[Any] = [] if ( - self.known_ids is not None - and retry.status not in ("verified", "mismatch") - and check.status == "abstain" - and check.abstain_reason == "glyph_confusable_identifier" + retry.status not in ("verified", "mismatch") + and looks_mod.is_glyph_abstain(check) + and ( + self.known_ids is not None + or self._copy_look_possible(identifier_region) + ) ): - # The band rests on a look-alike identifier. A confirmed, - # complete known-ID list with exactly one member in that class - # rules out a twin (runtime.known_ids), provided a second read - # from a FRESH screenshot agrees with the first. + # The band rests on a look-alike identifier. Take extra looks that + # change nothing (runtime.identity_looks): fresh frames, the 2x + # re-read above, the known-ID list, and, where configured, a + # select-and-copy of the identifier cell. Only the list or an + # exact copy can settle it; any contradiction is final. matches: Optional[Callable[[str], bool]] = None if anchor.identity_template is not None and anchor.identity_template.tokens: from openadapt_flow.runtime import identity_template as itmpl @@ -14127,9 +14180,30 @@ def native_attempt( matches = functools.partial( itmpl.recorded_identifier_matches, anchor.identity_template ) - lifted = self._known_id_lift(check, native_attempt, params, matches) - if lifted is not None: - return lifted + looked = self._look_again( + step, + check, + retry, + native_attempt, + params, + workflow, + identifier_region, + before_png, + matches, + ) + if ( + looked.status != "abstain" + or looked.known_id_halt is not None + or looked.look_halt is not None + ): + return looked + pending_looks = list(looked.looks) + + def with_looks(final: IdentityCheck) -> IdentityCheck: + if not pending_looks or final.status == "verified" or final.looks: + return final + return final.model_copy(update={"looks": pending_looks}) + check = better_identity_read(check, retry) if ( check.status == "verified" @@ -14138,7 +14212,7 @@ def native_attempt( ): # A read of a different record is final: the row re-read below # must never overrule it into "verified". - return check + return with_looks(check) # Last, re-read the point's own text row alone (see # identity.row_region): the same verdict rules, on a crop that holds # no part of the neighbouring rows. @@ -14151,7 +14225,7 @@ def native_attempt( ) ) if row is None or row_png is None: - return check + return with_looks(check) f = identity_mod.ROW_REREAD_FACTOR row_check = attempt( row_png, @@ -14164,22 +14238,55 @@ def native_attempt( exclude[3] * f, ), ) - return better_identity_read(check, row_check) + return with_looks(better_identity_read(check, row_check)) - def _known_id_lift( + def _copy_look_allowed(self) -> bool: + """The copy look is input: never in governed or qualification runs.""" + return ( + self.identity_copy is not None + and self._read_clipboard is not None + and self.governed_authorization is None + and self.managed_dispatch_binding is None + and self.production_qualification_guard is None + and self.qualification_campaign_guard is None + and self.qualification_fault_driver is None + and getattr(self, "approval_gate", None) is None + ) + + def _copy_look_possible(self, identifier_region: Optional[Region]) -> bool: + return identifier_region is not None and self._copy_look_allowed() + + def _embedded_identity_params( + self, anchor: Anchor, workflow: Workflow + ) -> list[str]: + """Workflow parameters whose demonstrated value is in the identity band.""" + tmpl = anchor.identity_template + if tmpl is not None and tmpl.tokens: + return list(tmpl.param_token_indices) + if anchor.context_text: + return identity_mod.embedded_params(anchor.context_text, workflow.params) + return [] + + def _look_again( self, + step: Step, first: IdentityCheck, + magnified: IdentityCheck, read: Callable[[bytes], IdentityCheck], params: Optional[Mapping[str, RuntimeParamScalar]], + workflow: Workflow, + identifier_region: Optional[Region], + frame_png: bytes, recorded_id_matches: Optional[Callable[[str], bool]] = None, - ) -> Optional[IdentityCheck]: - """Known-ID cross-check for a look-alike abstain. - - Returns a ``verified`` check, the abstain marked with a named - ``known_id_halt`` when the list may not be used now, the fresh read - when it is a ``mismatch`` (final), or None (keep the original - verdict). The second read comes from a new screenshot, - so a transient render state on one capture cannot satisfy both. + ) -> IdentityCheck: + """Extra looks before stopping on a look-alike identifier. + + Returns the verdict of :func:`identity_looks.decide`: ``verified`` + when every fresh frame agrees and the known-ID list or an exact copy + settles the record, the contradicting read when a look names another + record, or ``first`` carrying the looks. A known-ID list that may not + be used now, or a copy look that changed the screen, returns ``first`` + marked with a named halt. """ from openadapt_flow.runtime import known_ids as known_ids_mod @@ -14188,33 +14295,123 @@ def _known_id_lift( if isinstance(self.known_ids, known_ids_mod.KnownIdAdmission) else None ) + known: Any = self.known_ids if admission is not None: halt = admission.check() if halt is not None: return first.model_copy(update={"known_id_halt": halt.reason}) - known: Any = admission.known_ids - else: - known = self.known_ids - second = read(self.backend.screenshot()) - if second.status == "mismatch": - # The fresh read names a different record; that is final. - return second - lifted = known_ids_mod.resolve_lookalike_abstain( - [first, second], - known, - run_values=runtime_params_for_gui(params or {}), - recorded_id_matches=recorded_id_matches, - ) - if lifted is None or lifted.known_id_evidence is None: - return None - update: dict[str, Any] = {"frames": 2} - if admission is not None: - update["confirmed_at"] = admission.confirmation.confirmed_at - update["export_sha256"] = admission.confirmation.sha256 - return lifted.model_copy( - update={ - "known_id_evidence": lifted.known_id_evidence.model_copy(update=update) - } + known = admission.known_ids + run_values = runtime_params_for_gui(params or {}) + + frames: list[tuple[IdentityCheck, Optional[str]]] = [] + for _ in range(looks_mod.FRAME_LOOKS): + time.sleep(self.poll_interval_s) + png = self.backend.screenshot() + frames.append((read(png), sha256_bytes(png))) + if frames[-1][0].status == "mismatch": + break # final; no further looks + + frame_reads = [first, *(r for r, _ in frames)] + known_lift: Optional[IdentityCheck] = None + if known is not None and all( + looks_mod.is_glyph_abstain(r) for r in frame_reads + ): + lifted = known_ids_mod.resolve_lookalike_abstain( + frame_reads, + known, + run_values=run_values, + recorded_id_matches=recorded_id_matches, + ) + if lifted is not None and lifted.known_id_evidence is not None: + update: dict[str, Any] = {"frames": len(frame_reads)} + if admission is not None: + update["confirmed_at"] = admission.confirmation.confirmed_at + update["export_sha256"] = admission.confirmation.sha256 + known_lift = lifted.model_copy( + update={ + "known_id_evidence": lifted.known_id_evidence.model_copy( + update=update + ) + } + ) + + copied: Optional[looks_mod.ClipboardRead] = None + classes = looks_mod.lookalike_classes(first.observed) + bindings = self.__dict__.setdefault("_identity_look_bindings", {}) + if ( + identifier_region is not None + and self._copy_look_allowed() + and all(r.status != "mismatch" for r, _ in frames) + ): + assert self.identity_copy is not None + if self._identity_input_looks: + try: + cell_lines = self.vision.ocr(frame_png, region=identifier_region) + except Exception: # noqa: BLE001 - no box, no copy look + cell_lines = [] + cell = looks_mod.copy_region(cell_lines, identifier_region) + copied = ( + looks_mod.copy_identifier( + self.backend, + self._read_clipboard, + cell, + gesture=self.identity_copy.gesture, + copy_keys=self.identity_copy.copy_keys, + crop_region=identifier_region, + ) + if cell is not None + else looks_mod.ClipboardRead( + text=None, reason="no_single_identifier_box" + ) + ) + if copied.changed_outside: + look = IdentityLook( + kind="clipboard", + outcome="unavailable", + reason="screen_changed_outside_cell", + frame_sha256=copied.frame_sha256, + ) + return first.model_copy( + update={ + "look_halt": "identity_look_changed_screen", + "looks": [look], + } + ) + crops = tuple( + c + for c in ( + looks_mod.crop_png(frame_png, identifier_region), + copied.pre_crop, + copied.post_crop, + ) + if c is not None + ) + bindings[step.id] = looks_mod.LookBinding( + step_id=step.id, classes=classes, read=copied, crops=crops + ) + else: + binding = bindings.get(step.id) + live_crop = looks_mod.crop_png(frame_png, identifier_region) + if binding is not None and binding.matches(step.id, classes, live_crop): + copied = binding.read + + known_list = None + if known is not None: + known_list = getattr(known, "known_ids", known) + return looks_mod.decide( + first, + frames=frames, + magnified=magnified, + contradicts=identity_read_contradicts, + known_ids=known_list, + known_ids_lift=known_lift, + clipboard=copied, + exact_expected=looks_mod.exact_expected_ids( + run_values, + self._embedded_identity_params(step.anchor, workflow) + if step.anchor is not None + else [], + ), ) # -- typed-input verification ------------------------------------------------- diff --git a/packages/flow/openadapt_flow/validation/identity_looks_dense.py b/packages/flow/openadapt_flow/validation/identity_looks_dense.py new file mode 100644 index 0000000000..e73bd9c5e3 --- /dev/null +++ b/packages/flow/openadapt_flow/validation/identity_looks_dense.py @@ -0,0 +1,492 @@ +"""Extra identity looks on the dense record list: before and after. + +Drives the REAL ``Replayer._verify_identity`` (identifier-cell OCR, the 2x +re-read, the known-ID list and the extra looks of +:mod:`openadapt_flow.runtime.identity_looks`) against the dense record list of +:mod:`.dense_surface`, rendered live in a browser. The pixel path is the one +the compiler builds for a pixel recording: the identifier region is the tight +box of the row's identity text (``context_region_from_lines``) and the +recorded identity is the OCR text inside it. + +Each armed click is scored twice, at the step-entry check and, when that +check verified, again at the actuation-time check, which takes no input: + +- **correct row**: the resolver landed on the recorded record. A stop is a + false stop (an over-halt). +- **sibling row**: the resolver landed on the adjacent different record. A + verify is a wrong-record accept. Must stay 0. + +Variants: no list and no copy (the shipped default), the known-ID list, the +copy look, and both. Modes: recorded record (the band names a fixed record) +and parameter (the MRN is a run parameter, as in a worklist). The copy look +drags across the identifier's text box with the real mouse; the copied text +is the page's own selection, which is what a redirected clipboard receives. + +The same script runs on ``origin/main`` for the before numbers: variants the +code under test does not support are skipped. + +Run: + python -m openadapt_flow.validation.identity_looks_dense --out DIR +""" + +from __future__ import annotations + +import argparse +import hashlib +import inspect +import json +import subprocess +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path +from typing import Any, Optional + +SCHEMA = "openadapt.identity-looks-dense.v1" +VARIANTS = ("shipped", "list", "copy", "list_copy") +MODES = ("recorded", "param") + + +def _git_commit() -> Optional[str]: + try: + here = Path(__file__).resolve().parent + out = subprocess.run( + ["git", "-C", str(here), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + ) + return out.stdout.strip() or None + except Exception: # noqa: BLE001 - provenance only + return None + + +class CachedVision: + """``openadapt_flow.vision`` with OCR memoized by (frame bytes, region). + + The fixture renders deterministically, so identical frames get identical + reads; caching changes no verdict, only the run time. + """ + + def __init__(self) -> None: + import openadapt_flow.vision as vision + + self._vision = vision + self._cache: dict[tuple[str, Any], Any] = {} + self.calls = 0 + + def __getattr__(self, name: str) -> Any: + return getattr(self._vision, name) + + def ocr(self, png: bytes, region: Any = None, **kw: Any) -> Any: + key = (hashlib.sha256(png).hexdigest(), tuple(region) if region else None) + if key not in self._cache: + self.calls += 1 + self._cache[key] = self._vision.ocr(png, region=region, **kw) + return self._cache[key] + + +@dataclass +class PageBackend: + """A live browser page seen only as pixels, with mouse drag and copy. + + Coordinates are screen pixels (CSS pixels times the device scale). The + clipboard is the page's selection at the copy key, which is what a + redirected remote clipboard would receive. + """ + + page: Any + dsf: int + viewport: tuple[int, int] + clean_png: bytes = b"" + dirty: bool = False + copied: Optional[str] = None + drags: int = 0 + # (clean frame sha, drag) -> (selection text, post frame) + _memo: dict[tuple[str, tuple[int, int, int, int]], tuple[str, bytes]] = field( + default_factory=dict + ) + _pending: Optional[tuple[str, tuple[int, int, int, int]]] = None + _post: Optional[bytes] = None + + def screenshot(self) -> bytes: + if not self.dirty: + return self.clean_png + if self._post is not None: + return self._post + return self.page.screenshot(full_page=True) + + def drag(self, x: int, y: int, end_x: int, end_y: int) -> None: + key = (hashlib.sha256(self.clean_png).hexdigest()[:16], (x, y, end_x, end_y)) + self.drags += 1 + self.dirty = True + if key in self._memo: + self._pending = key + self._post = self._memo[key][1] + return + d = self.dsf + self.page.mouse.move(x / d, y / d) + self.page.mouse.down() + self.page.mouse.move(end_x / d, end_y / d, steps=6) + self.page.mouse.up() + self._pending = key + self._post = None + + def press(self, key: str) -> None: + if key.lower() not in ("control+c", "meta+c", "controlormeta+c"): + raise AssertionError(f"unexpected key {key!r}") + if self._pending is None: + return + if self._pending in self._memo: + self.copied = self._memo[self._pending][0] + return + text = self.page.evaluate("() => String(window.getSelection())") + post = self.page.screenshot(full_page=True) + self._memo[self._pending] = (text, post) + self._post = post + self.copied = text + + def reset(self) -> None: + """Harness only: clear the selection between trials.""" + self.page.evaluate("() => window.getSelection().removeAllRanges()") + self.page.mouse.move(1, 1) + self.dirty = False + self._pending = None + self._post = None + + +class PageClipboard: + def __init__(self, backend: PageBackend) -> None: + self.backend = backend + + def read_text(self) -> Optional[str]: + return self.backend.copied + + def write_text(self, text: str) -> None: + self.backend.copied = text + + +def _open_page(pw: Any, table: Any, cond: Any, top_offset_px: int) -> tuple[Any, Any]: + from openadapt_flow.validation.dense_surface import render_table_html + + html = render_table_html( + table, + font_family=cond.font_family, + font_px=cond.font_px, + row_pad_px=cond.row_pad_px, + top_offset_px=top_offset_px, + ) + browser = pw.chromium.launch(headless=True) + page = browser.new_page( + viewport={"width": 1120, "height": 1600}, + device_scale_factor=cond.device_scale_factor, + ) + page.set_content(html, wait_until="networkidle") + full_h = page.evaluate("document.body.scrollHeight") + page.set_viewport_size({"width": 1120, "height": int(full_h) + 20}) + return browser, page + + +def _points(page: Any, n: int, dsf: int) -> dict[int, tuple[tuple[int, int], ...]]: + out: dict[int, tuple[tuple[int, int], ...]] = {} + for i in range(n): + pts = [] + for sel in (f'[data-name="{i}"]', f'[data-open="{i}"]'): + bb = page.eval_on_selector( + sel, + "el => { const r = el.getBoundingClientRect();" + " return [r.x, r.y, r.width, r.height]; }", + ) + pts.append((int((bb[0] + bb[2] / 2) * dsf), int((bb[1] + bb[3] / 2) * dsf))) + out[i] = tuple(pts) + return out + + +def _supports_copy() -> bool: + from openadapt_flow.runtime.replayer import Replayer + + return "identity_copy" in inspect.signature(Replayer.__init__).parameters + + +def _replayer(backend: Any, vision: Any, variant: str, known_ids: Any) -> Any: + from openadapt_flow.runtime.replayer import Replayer + + kwargs: dict[str, Any] = {"vision": vision, "poll_interval_s": 0.0} + if variant in ("list", "list_copy"): + kwargs["known_ids"] = known_ids + if variant in ("copy", "list_copy"): + from openadapt_flow.runtime.identity_looks import IdentityCopyConfig + + kwargs["identity_copy"] = IdentityCopyConfig() + kwargs["read_clipboard"] = PageClipboard(backend) + return Replayer(backend, **kwargs) + + +def _check_both_gates( + replayer: Any, backend: PageBackend, step: Any, res: Any, wf: Any, params: dict +) -> dict[str, Any]: + entry_png = backend.screenshot() + if hasattr(replayer, "_entry_identity_gate_error"): + replayer.__dict__.setdefault("_identity_look_bindings", {}).pop(step.id, None) + replayer._identity_input_looks = True + try: + entry = replayer._verify_identity( + step, res, entry_png, params, wf, bundle_dir=None + ) + finally: + if hasattr(replayer, "_identity_input_looks"): + replayer._identity_input_looks = False + out: dict[str, Any] = { + "entry": entry.status, + "entry_reason": entry.abstain_reason, + "looks": [ + [look.kind, look.outcome, look.reason] + for look in getattr(entry, "looks", []) + ], + "look_halt": getattr(entry, "look_halt", None), + } + final = entry.status + if entry.status == "verified" and backend.dirty: + act_png = backend.screenshot() # the selection highlight is still drawn + act = replayer._verify_identity(step, res, act_png, params, wf, bundle_dir=None) + out["actuation"] = act.status + out["actuation_looks"] = [[x.kind, x.outcome, x.reason] for x in act.looks] + final = act.status + elif entry.status == "verified": + out["actuation"] = "verified" # no input was used: same frame, same verdict + out["final"] = final + backend.reset() + return out + + +def run( + seeds: list[int], + *, + n_rows: int = 40, + progress: bool = False, +) -> dict[str, Any]: + import cv2 + import numpy as np + from playwright.sync_api import sync_playwright + + from openadapt_flow._browser_setup import ensure_chromium_installed + from openadapt_flow.compiler.compile import ( + MIN_OCR_CONFIDENCE, + _discriminative_crop_region, + ) + from openadapt_flow.ir import Anchor, Resolution, Step, Workflow + from openadapt_flow.runtime.identity import ( + band_region, + context_region_from_lines, + identifier_text_from_lines, + ) + from openadapt_flow.runtime.known_ids import KnownIdList + from openadapt_flow.validation.dense_surface import ( + CLICK_CONFIGS, + RECORD_CONDITION, + REPLAY_CONDITIONS, + build_dense_table, + render_frame, + ) + + ensure_chromium_installed() + vision = CachedVision() + variants = [v for v in VARIANTS if _supports_copy() or "copy" not in v] + today = date.today() + trials: list[dict[str, Any]] = [] + # Record frames first: render_frame runs its own Playwright session. + record_frames = { + seed: render_frame( + build_dense_table(seed, n_rows=n_rows), RECORD_CONDITION, top_offset_px=0 + ) + for seed in seeds + } + with sync_playwright() as pw: + for seed in seeds: + table = build_dense_table(seed, n_rows=n_rows) + known = KnownIdList( + ids=tuple(r.mrn for r in table.rows), + source=f"synthetic seed {seed}", + complete=True, + ) + rec = record_frames[seed] + rec_lines = vision.ocr(rec.png) + rec_bgr = cv2.imdecode(np.frombuffer(rec.png, np.uint8), cv2.IMREAD_COLOR) + idx = {id(r): i for i, r in enumerate(table.rows)} + for k, cond in enumerate(REPLAY_CONDITIONS): + browser, page = _open_page(pw, table, cond, 4 + k * 3) + try: + dsf = cond.device_scale_factor + png = page.screenshot(full_page=True) + import io + + from PIL import Image + + with Image.open(io.BytesIO(png)) as im: + viewport = im.size + backend = PageBackend(page=page, dsf=dsf, viewport=viewport) + backend.clean_png = png + live_pts = _points(page, len(table.rows), dsf) + for pair in table.pairs: + ti, si = idx[id(pair.target)], idx[id(pair.sibling)] + for config in CLICK_CONFIGS: + pi = 0 if config == "click_name" else 1 + rec_click = rec.points[ti][pi] + crop = _discriminative_crop_region(rec_bgr, rec_click) + box = context_region_from_lines( + rec_lines, + exclude_region=crop, + band=band_region(rec_click, crop[3], rec.viewport), + point=rec_click, + min_confidence=MIN_OCR_CONFIDENCE, + reference_date=today, + ) + context = ( + identifier_text_from_lines( + rec_lines, + region=box, + min_confidence=MIN_OCR_CONFIDENCE, + reference_date=today, + ) + if box + else None + ) + base = { + "seed": seed, + "collision_class": pair.collision_class, + "click_config": config, + "replay_condition": cond.name, + "armed": bool(context), + } + if not context: + trials.append(base) + continue + anchor = Anchor( + template="t.png", + region=crop, + click_point=rec_click, + context_text=context, + identifier_region=box, + ) + step = Step( + id=f"s{ti}_{config}", + intent="open record", + action="click", + anchor=anchor, + risk="irreversible", + ) + for mode in MODES: + params = ( + {"mrn": pair.target.mrn} if mode == "param" else {} + ) + wf = Workflow( + name="dense", params=dict(params), steps=[step] + ) + for variant in variants: + row = dict(base, mode=mode, variant=variant) + for side, ri in (("correct", ti), ("sibling", si)): + res = Resolution( + rung="ocr", + point=live_pts[ri][pi], + confidence=0.9, + elapsed_ms=1.0, + ) + rp = _replayer(backend, vision, variant, known) + row[side] = _check_both_gates( + rp, backend, step, res, wf, params + ) + trials.append(row) + if progress: + print( + f"seed {seed} {cond.name} {pair.collision_class} done" + ) + finally: + browser.close() + return { + "schema": SCHEMA, + "meta": { + "commit": _git_commit(), + "seeds": seeds, + "n_rows": n_rows, + "replay_conditions": [c.name for c in REPLAY_CONDITIONS], + "click_configs": list(CLICK_CONFIGS), + "variants": variants, + "modes": list(MODES), + "ocr_calls": vision.calls, + }, + "trials": trials, + } + + +def score(result: dict[str, Any]) -> dict[str, Any]: + """Counts per (mode, variant), overall and by collision class.""" + out: dict[str, Any] = {} + for t in result["trials"]: + if not t.get("armed") or "variant" not in t: + continue + key = f"{t['mode']}/{t['variant']}" + for scope in ("all", t["collision_class"]): + b = out.setdefault(key, {}).setdefault( + scope, + { + "correct_row_clicks": 0, + "false_stops": 0, + "entry_verified_actuation_stopped": 0, + "sibling_row_clicks": 0, + "wrong_record_accepts": 0, + "copy_looks": 0, + }, + ) + b["correct_row_clicks"] += 1 + b["sibling_row_clicks"] += 1 + c, s = t["correct"], t["sibling"] + b["false_stops"] += c["final"] != "verified" + b["entry_verified_actuation_stopped"] += ( + c["entry"] == "verified" and c["final"] != "verified" + ) + b["wrong_record_accepts"] += ( + s["entry"] == "verified" or s.get("final") == "verified" + ) + b["copy_looks"] += sum( + 1 for x in (c, s) for look in x["looks"] if look[0] == "clipboard" + ) + return out + + +def unarmed(result: dict[str, Any]) -> int: + return sum(1 for t in result["trials"] if not t.get("armed")) + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True) + parser.add_argument("--seeds", type=int, nargs="+", default=[1, 2, 3]) + parser.add_argument("--n-rows", type=int, default=40) + parser.add_argument("--progress", action="store_true") + args = parser.parse_args(argv) + result = run(args.seeds, n_rows=args.n_rows, progress=args.progress) + scored = score(result) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text( + json.dumps( + { + "meta": result["meta"], + "unarmed": unarmed(result), + "scored": scored, + "trials": result["trials"], + }, + indent=1, + ) + + "\n" + ) + for key, groups in scored.items(): + a = groups["all"] + print( + f"{key}: false stops {a['false_stops']}/{a['correct_row_clicks']}, " + f"wrong-record accepts {a['wrong_record_accepts']}/{a['sibling_row_clicks']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/tests/test_identity_looks.py b/packages/flow/tests/test_identity_looks.py new file mode 100644 index 0000000000..bb48353568 --- /dev/null +++ b/packages/flow/tests/test_identity_looks.py @@ -0,0 +1,287 @@ +"""Extra looks before stopping on a look-alike identifier (runtime.identity_looks). + +Pure tests of the decision rule and the copy look. The replayer path, with a +real render and real OCR, is in ``test_identity_looks_replayer.py``. +""" + +from __future__ import annotations + +import io + +import pytest +from PIL import Image, ImageDraw + +from openadapt_flow.ir import IdentityCheck +from openadapt_flow.runtime import identity_looks as L +from openadapt_flow.runtime.known_ids import KnownIdList +from openadapt_flow.runtime.replayer import identity_read_contradicts + +BAND = "AC50061 Smith, John 01/15/1980" + + +def _abstain(observed: str = BAND) -> IdentityCheck: + return IdentityCheck( + status="abstain", + abstain_reason="glyph_confusable_identifier", + coverage=1.0, + expected=BAND, + observed=observed, + ) + + +def _frames(*reads: IdentityCheck): + return [(r, "a" * 64) for r in reads] + + +def _decide(**kw): + kw.setdefault("frames", _frames(_abstain(), _abstain())) + kw.setdefault("magnified", _abstain()) + return L.decide(_abstain(), contradicts=identity_read_contradicts, **kw) + + +def _copy(text: str) -> L.ClipboardRead: + return L.ClipboardRead(text=text, frame_sha256="b" * 64) + + +def test_ocr_looks_alone_never_accept(): + check = _decide() + assert check.status == "abstain" + assert [look.kind for look in check.looks] == ["frame", "frame", "magnified"] + assert all(look.outcome == "agrees" for look in check.looks) + + +def test_copy_equal_to_run_value_settles_the_record(): + check = _decide( + clipboard=_copy("AC50061\tSmith, John\t01/15/1980"), exact_expected=("ac50061",) + ) + assert check.status == "verified" + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "resolves" + + +def test_copy_of_look_alike_twin_is_a_mismatch(): + check = _decide( + clipboard=_copy("AC5OO61 Smith, John 01/15/1980"), exact_expected=("ac50061",) + ) + assert check.status == "mismatch" + assert check.looks[-1].outcome == "contradicts" + + +def test_copy_without_an_exact_expected_id_cannot_accept(): + # Recorded-record mode: the recorded band is OCR text, not exact. + check = _decide(clipboard=_copy("AC50061 Smith, John 01/15/1980")) + assert check.status == "abstain" + assert check.looks[-1].outcome == "agrees" + + +def test_copy_of_an_unlisted_id_contradicts_the_list(): + ids = KnownIdList(ids=("AC50061", "ZZ1000"), source="t", complete=True) + check = _decide( + clipboard=_copy("AC5OO61"), + known_ids=ids, + known_ids_lift=_abstain().model_copy(update={"status": "verified"}), + ) + assert check.status == "mismatch" + + +def test_known_id_list_resolves_when_frames_agree(): + ids = KnownIdList(ids=("AC50061", "ZZ1000"), source="t", complete=True) + lift = _abstain().model_copy(update={"status": "verified"}) + check = _decide(known_ids=ids, known_ids_lift=lift) + assert check.status == "verified" + assert check.looks[-1].kind == "known_ids" + + +def test_a_frame_naming_another_record_is_final(): + other = IdentityCheck( + status="mismatch", expected=BAND, observed="ZZ9999 Doe, Jane 02/02/1990" + ) + check = _decide( + frames=_frames(other, _abstain()), + clipboard=_copy("AC50061"), + exact_expected=("ac50061",), + ) + assert check.status == "mismatch" + assert check.looks[0].outcome == "contradicts" + + +def test_frames_that_read_another_identifier_block_acceptance(): + drifted = _abstain("AC5OO6l Smith, John 01/15/1980") # same class, still ok + moved = _abstain("AC50071 Smith, John 01/15/1980") # different class + assert ( + _decide( + frames=_frames(drifted, _abstain()), + clipboard=_copy("AC50061"), + exact_expected=("ac50061",), + ).status + == "verified" + ) + check = _decide( + frames=_frames(moved, _abstain()), + clipboard=_copy("AC50061"), + exact_expected=("ac50061",), + ) + assert check.status == "abstain" + assert check.looks[0].outcome == "unavailable" + + +def test_no_fresh_frames_means_no_acceptance(): + check = _decide(frames=[], clipboard=_copy("AC50061"), exact_expected=("ac50061",)) + assert check.status == "abstain" + + +def test_magnified_read_naming_another_record_is_final(): + other = IdentityCheck( + status="mismatch", expected=BAND, observed="ZZ9999 Doe, Jane 02/02/1990" + ) + check = _decide( + magnified=other, clipboard=_copy("AC50061"), exact_expected=("ac50061",) + ) + assert check.status == "mismatch" + + +def test_copy_that_covers_a_different_cell_settles_nothing(): + check = _decide(clipboard=_copy("ZZ1000 Filler, Pat"), exact_expected=("ac50061",)) + assert check.status == "abstain" + assert check.looks[-1].reason == "copy_differs_from_read" + + +def test_only_glyph_abstains_take_looks(): + short = IdentityCheck(status="abstain", abstain_reason="identity_evidence_short") + assert ( + L.decide( + short, frames=[], magnified=None, contradicts=identity_read_contradicts + ) + is short + ) + + +def test_exact_expected_ids_come_only_from_band_params(): + run = {"mrn": "AC50061", "note": "AC99999"} + assert L.exact_expected_ids(run, ["mrn"]) == ("ac50061",) + assert L.exact_expected_ids(run, []) == () + + +def test_copy_config_rejects_unknown_gesture(): + with pytest.raises(ValueError): + L.IdentityCopyConfig(gesture="triple_click") + + +# ------------------------------------------------------------ copy look + + +def _png(highlight=None, extra=None) -> bytes: + img = Image.new("RGB", (200, 100), "white") + draw = ImageDraw.Draw(img) + draw.text((20, 40), "AC50061", fill="black") + if highlight: + draw.rectangle(highlight, fill=(51, 144, 255)) + if extra: + draw.rectangle(extra, fill="red") + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +class _Clipboard: + def __init__(self, text="user's own text"): + self.text = text + self.writes = [] + + def read_text(self): + return self.text + + def write_text(self, text): + self.writes.append(text) + self.text = text + + +class _Backend: + def __init__(self, clipboard, copied="AC50061", after=None): + self.clipboard = clipboard + self.copied = copied + self.after = after + self.frame = _png() + self.drags = [] + self.keys = [] + + def screenshot(self): + return self.frame + + def drag(self, x, y, ex, ey): + self.drags.append((x, y, ex, ey)) + self.frame = self.after if self.after is not None else self.frame + + def click(self, x, y, *, double=False): + raise AssertionError("drag gesture must not click") + + def press(self, key): + self.keys.append(key) + if self.copied is not None: + self.clipboard.text = self.copied + + +REGION = (15, 35, 80, 20) + + +def test_copy_reads_the_cell_and_restores_the_clipboard(): + clip = _Clipboard() + backend = _Backend(clip, after=_png(highlight=(16, 36, 90, 52))) + read = L.copy_identifier(backend, clip, REGION) + assert read.text == "AC50061" + assert not read.changed_outside + assert backend.drags and backend.drags[0][1] == 45 # mid height + assert backend.keys == ["Control+c"] + assert clip.text == "user's own text" # restored + + +def test_marker_left_in_clipboard_counts_as_nothing_copied(): + clip = _Clipboard() + read = L.copy_identifier(_Backend(clip, copied=None), clip, REGION) + assert read.text is None and read.reason == "nothing_copied" + + +def test_screen_change_outside_the_cell_is_flagged(): + clip = _Clipboard() + backend = _Backend(clip, after=_png(extra=(150, 80, 190, 95))) + read = L.copy_identifier(backend, clip, REGION) + assert read.changed_outside + + +def test_backend_without_drag_takes_no_look(): + class NoDrag: + def screenshot(self): + return _png() + + read = L.copy_identifier(NoDrag(), _Clipboard(), REGION) + assert read.text is None and read.reason == "backend_cannot_drag" + + +def test_crops_match_tolerates_noise_not_a_changed_glyph(): + a = L.crop_png(_png(), REGION) + img = Image.open(io.BytesIO(a)).convert("RGB") + px = img.load() + px[0, 0] = tuple(max(0, c - 3) for c in px[0, 0]) + noisy = io.BytesIO() + img.save(noisy, format="PNG") + assert L.crops_match(a, noisy.getvalue()) + other = Image.new("RGB", (200, 100), "white") + ImageDraw.Draw(other).text((20, 40), "AC5OO61", fill="black") + buf = io.BytesIO() + other.save(buf, format="PNG") + assert not L.crops_match(a, L.crop_png(buf.getvalue(), REGION)) + + +def test_copy_region_is_the_single_line_holding_a_lookalike_id(): + from types import SimpleNamespace as NS + + within = (0, 0, 600, 40) + mrn = NS(text="AC50061", region=(10, 5, 90, 20)) + name = NS(text="Smith, John", region=(200, 5, 120, 20)) + dob = NS(text="01/15/1980", region=(400, 5, 110, 20)) + assert L.copy_region([mrn, name, dob], within) == (8, 3, 94, 24) + other = NS(text="ZZ10O1", region=(520, 5, 60, 20)) + assert L.copy_region([mrn, other], within) is None # two candidates + assert L.copy_region([name, dob], within) is None # none + outside = NS(text="AC50061", region=(10, 100, 90, 20)) + assert L.copy_region([outside], within) is None diff --git a/packages/flow/tests/test_identity_looks_replayer.py b/packages/flow/tests/test_identity_looks_replayer.py new file mode 100644 index 0000000000..aadf7526ed --- /dev/null +++ b/packages/flow/tests/test_identity_looks_replayer.py @@ -0,0 +1,221 @@ +"""Extra looks on the real replayer path, real render and real OCR. + +A record row whose identifier carries look-alike glyphs (``AC50061``). OCR +abstains on it. The select-and-copy look settles the record only when the +copied identifier equals the run's parameter; the look-alike twin +``AC5OO61`` stops the run. The actuation-time check reuses the step-entry +copy only when the identifier cell shows the same pixels. +""" + +from __future__ import annotations + +import io +from datetime import date + +import pytest + +pytest.importorskip("playwright") +pytest.importorskip("rapidocr_onnxruntime") + +from PIL import Image, ImageDraw # noqa: E402 + +import openadapt_flow.vision as vision # noqa: E402 +from openadapt_flow.ir import ( # noqa: E402 + Anchor, + IdentityCheck, + Resolution, + Step, + StepResult, + Workflow, +) +from openadapt_flow.runtime import identity as I # noqa: E402 +from openadapt_flow.runtime.identity_looks import IdentityCopyConfig # noqa: E402 +from openadapt_flow.runtime.replayer import Replayer # noqa: E402 +from tests.test_identity_homonym_8th import _ROW_K, _render # noqa: E402 + +_FRAMES: dict[str, object] = {} + + +def _frame(mrn: str): + if mrn not in _FRAMES: + _FRAMES[mrn] = _render(mrn) + return _FRAMES[mrn] + + +def _identifier_region(frame) -> tuple[int, int, int, int]: + """The MRN + name + DOB cells of the row under test.""" + _, _, _, row = frame.points[_ROW_K] + lines = sorted(vision.ocr(frame.png, region=row), key=lambda ln: ln.region[0]) + mrn = next(ln for ln in lines if "AC5" in ln.text.upper()) + dob = next(ln for ln in lines if "1980" in ln.text) + x0 = mrn.region[0] - 4 + x1 = dob.region[0] + dob.region[2] + 4 + return (x0, row[1], x1 - x0, row[3]) + + +class _Clipboard: + def __init__(self) -> None: + self.text = "operator's clipboard" + + def read_text(self): + return self.text + + def write_text(self, text): + self.text = text + + +def _highlight(png: bytes, region) -> bytes: + img = Image.open(io.BytesIO(png)).convert("RGB") + x, y, w, h = region + draw = ImageDraw.Draw(img) + draw.rectangle((x + 2, y + 2, x + w - 3, y + 4), fill=(51, 144, 255)) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +class _CopyBackend: + """Pixel-only session whose identifier cell can be selected and copied.""" + + def __init__(self, frame, mrn: str, clipboard: _Clipboard, *, spill=False): + self.viewport = frame.viewport + self.frame = frame.png + self.region = _identifier_region(frame) + self.copy_text = mrn # the drag spans the identifier's own text box + self.clipboard = clipboard + self.spill = spill + self.drags: list[tuple[int, int, int, int]] = [] + self.selected = False + + def screenshot(self) -> bytes: + return self.frame + + def drag(self, x, y, ex, ey): + self.drags.append((x, y, ex, ey)) + rx, ry, rw, rh = self.region + self.selected = ry <= y <= ry + rh + if self.selected: # a selection paints between the drag's ends + self.frame = _highlight(self.frame, (x, y - 8, ex - x, 16)) + if self.spill: # the gesture opened something elsewhere on screen + self.frame = _highlight(self.frame, (5, 5, 60, 20)) + + def press(self, key): + assert key == "Control+c" + if self.selected: + self.clipboard.text = self.copy_text + + +def _setup(live_mrn: str, *, param_mode: bool, spill=False, copy=True): + rec = _frame("AC50061") + live = _frame(live_mrn) + _, open_pt, _, _ = rec.points[_ROW_K] + _, live_pt, _, _ = live.points[_ROW_K] + region = _identifier_region(rec) + context = I.identifier_text_from_lines( + vision.ocr(rec.png), region=region, reference_date=date.today() + ) + assert context and "1980" in context + anchor = Anchor( + template="t.png", + region=(open_pt[0] - 30, open_pt[1] - 12, 60, 24), + click_point=(open_pt[0], open_pt[1]), + context_text=context, + identifier_region=region, + ) + step = Step( + id="open_record", + intent="open record", + action="click", + anchor=anchor, + risk="irreversible", + ) + params = {"mrn": "AC50061"} if param_mode else {} + wf = Workflow(name="looks", params=dict(params), steps=[step]) + res = Resolution(rung="ocr", point=live_pt, confidence=0.9, elapsed_ms=1.0) + clip = _Clipboard() + backend = _CopyBackend(live, live_mrn, clip, spill=spill) + replayer = Replayer( + backend, + vision=vision, + read_clipboard=clip if copy else None, + identity_copy=IdentityCopyConfig() if copy else None, + ) + return replayer, backend, step, res, wf, params, live + + +def _entry(replayer, step, res, wf, params, png) -> IdentityCheck: + replayer._identity_input_looks = True + try: + return replayer._verify_identity(step, res, png, params, wf, bundle_dir=None) + finally: + replayer._identity_input_looks = False + + +def test_without_copy_look_the_correct_row_still_abstains(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True, copy=False) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "abstain" + assert not b.drags + + +def test_copy_equal_to_the_run_value_verifies_the_correct_row(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "verified", check + kinds = [(look.kind, look.outcome) for look in check.looks] + assert ("clipboard", "resolves") in kinds + assert [k for k, _ in kinds].count("frame") == 2 + assert len(b.drags) == 1 + assert b.clipboard.text == "operator's clipboard" + + +def test_copy_of_the_look_alike_twin_stops_the_run(): + r, b, step, res, wf, params, live = _setup("AC5OO61", param_mode=True) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "mismatch" + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "contradicts" + + +def test_recorded_record_mode_copy_cannot_accept(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=False) + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "abstain" + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "agrees" + + +def test_screen_change_outside_the_cell_halts(tmp_path): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True, spill=True) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + error = r._entry_identity_gate_error( + step, res, live.png, params, wf, tmp_path, result + ) + assert result.identity is not None + assert result.identity.look_halt == "identity_look_changed_screen" + assert error is not None and "identity_look_changed_screen" in error + assert result.safety_halt is True + + +def test_governed_runs_take_no_copy_look(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True) + r.governed_authorization = object() # type: ignore[assignment] + check = _entry(r, step, res, wf, params, live.png) + assert check.status == "abstain" + assert not b.drags + + +def test_actuation_check_reuses_the_copy_only_on_matching_pixels(): + r, b, step, res, wf, params, live = _setup("AC50061", param_mode=True) + assert _entry(r, step, res, wf, params, live.png).status == "verified" + # Actuation-time check: no input, the selection highlight is still drawn. + again = r._verify_identity(step, res, b.frame, params, wf, bundle_dir=None) + assert len(b.drags) == 1 + assert again.status == "verified" + assert ("clipboard", "resolves") in [(x.kind, x.outcome) for x in again.looks] + # The twin now sits where the record was: the binding must not carry over. + twin = _frame("AC5OO61") + b.frame = twin.png + swapped = r._verify_identity(step, res, twin.png, params, wf, bundle_dir=None) + assert swapped.status != "verified" + assert len(b.drags) == 1 diff --git a/packages/flow/tests/test_known_ids_replayer.py b/packages/flow/tests/test_known_ids_replayer.py index c204fdeab9..6564ef3473 100644 --- a/packages/flow/tests/test_known_ids_replayer.py +++ b/packages/flow/tests/test_known_ids_replayer.py @@ -86,14 +86,21 @@ def test_correct_row_abstains_without_list(): assert _check("AC50061", "AC50061", None).status == "abstain" -def test_correct_row_verifies_with_complete_list_using_a_fresh_capture(): +def test_correct_row_verifies_with_complete_list_using_fresh_captures(): out: dict = {} check = _check("AC50061", "AC50061", _ids(), out=out) assert check.status == "verified" assert check.known_id_evidence is not None - assert check.known_id_evidence.reads == 2 - assert check.known_id_evidence.frames == 2 - assert out["captures"] == 1 # the second read took its own screenshot + assert check.known_id_evidence.reads == 3 + assert check.known_id_evidence.frames == 3 + assert out["captures"] == 2 # each later read took its own screenshot + assert [look.kind for look in check.looks] == [ + "frame", + "frame", + "magnified", + "known_ids", + ] + assert check.looks[-1].outcome == "resolves" def test_fresh_capture_showing_another_record_blocks_the_lift(): From ebe48b92324b8c6f1ca72bc39d1a28018e45cf21 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 09:27:46 -0400 Subject: [PATCH 02/10] Record each dense-list condition at its replay's device scale Co-Authored-By: Claude Opus 5.5 --- .../validation/identity_looks_dense.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/packages/flow/openadapt_flow/validation/identity_looks_dense.py b/packages/flow/openadapt_flow/validation/identity_looks_dense.py index e73bd9c5e3..e901ddfb61 100644 --- a/packages/flow/openadapt_flow/validation/identity_looks_dense.py +++ b/packages/flow/openadapt_flow/validation/identity_looks_dense.py @@ -32,6 +32,7 @@ from __future__ import annotations import argparse +import dataclasses import hashlib import inspect import json @@ -293,12 +294,19 @@ def run( variants = [v for v in VARIANTS if _supports_copy() or "copy" not in v] today = date.today() trials: list[dict[str, Any]] = [] - # Record frames first: render_frame runs its own Playwright session. + # Record frames first: render_frame runs its own Playwright session. Each + # replay condition is recorded at its own device scale: the identifier + # box is a pixel region, and a display-scale change is handled by a + # separate mechanism (runtime.render_scale), not by this check. + dsfs = sorted({c.device_scale_factor for c in REPLAY_CONDITIONS}) record_frames = { - seed: render_frame( - build_dense_table(seed, n_rows=n_rows), RECORD_CONDITION, top_offset_px=0 + (seed, d): render_frame( + build_dense_table(seed, n_rows=n_rows), + dataclasses.replace(RECORD_CONDITION, device_scale_factor=d), + top_offset_px=0, ) for seed in seeds + for d in dsfs } with sync_playwright() as pw: for seed in seeds: @@ -308,11 +316,13 @@ def run( source=f"synthetic seed {seed}", complete=True, ) - rec = record_frames[seed] - rec_lines = vision.ocr(rec.png) - rec_bgr = cv2.imdecode(np.frombuffer(rec.png, np.uint8), cv2.IMREAD_COLOR) idx = {id(r): i for i, r in enumerate(table.rows)} for k, cond in enumerate(REPLAY_CONDITIONS): + rec = record_frames[(seed, cond.device_scale_factor)] + rec_lines = vision.ocr(rec.png) + rec_bgr = cv2.imdecode( + np.frombuffer(rec.png, np.uint8), cv2.IMREAD_COLOR + ) browser, page = _open_page(pw, table, cond, 4 + k * 3) try: dsf = cond.device_scale_factor @@ -408,6 +418,7 @@ def run( "seeds": seeds, "n_rows": n_rows, "replay_conditions": [c.name for c in REPLAY_CONDITIONS], + "record_condition": f"{RECORD_CONDITION.name} at each replay's device scale", "click_configs": list(CLICK_CONFIGS), "variants": variants, "modes": list(MODES), From 52f6dffe6aeaae20856c2b212480224b3dc483b6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 09:27:59 -0400 Subject: [PATCH 03/10] Report the commit of the package under test in the dense-list harness Co-Authored-By: Claude Opus 5.5 --- .../flow/openadapt_flow/validation/identity_looks_dense.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/flow/openadapt_flow/validation/identity_looks_dense.py b/packages/flow/openadapt_flow/validation/identity_looks_dense.py index e901ddfb61..59193e7722 100644 --- a/packages/flow/openadapt_flow/validation/identity_looks_dense.py +++ b/packages/flow/openadapt_flow/validation/identity_looks_dense.py @@ -48,8 +48,12 @@ def _git_commit() -> Optional[str]: + """Commit of the openadapt_flow under test (the before run imports + another checkout's package with this script).""" try: - here = Path(__file__).resolve().parent + import openadapt_flow + + here = Path(openadapt_flow.__file__).resolve().parent out = subprocess.run( ["git", "-C", str(here), "rev-parse", "HEAD"], capture_output=True, From a80564cbd2a2785c8567a30dacd5e263e35956b1 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 09:30:39 -0400 Subject: [PATCH 04/10] Describe the extra identity looks and their limits in LIMITS.md Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/LIMITS.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index e1a48764f2..df8274cf49 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -163,6 +163,30 @@ halts with a named reason when the confirmation is missing before and after numbers are in [`benchmark/known_ids/KNOWN_IDS.md`](../benchmark/known_ids/KNOWN_IDS.md). +Before a step stops on a look-alike identifier, the replayer takes extra looks +that change nothing +([`runtime/identity_looks.py`](../openadapt_flow/runtime/identity_looks.py)): +the same read on two fresh screenshots, the 2x re-read, the known-ID list when +one is configured, and, where the surface allows it, a select-and-copy of the +identifier's text box (`runtime.identity_copy_look`). OCR looks can't settle +the record on their own, because a look-alike twin can read the same on every +frame and at every size. Only two looks can: the known-ID list, and a copied +identifier that equals a run parameter in the identity band. A look that names +another record stops the run. Each look is recorded on the step's identity +check. + +The copy look has limits. It needs a surface where selecting the identifier +and copying it changes nothing, and a clipboard that reaches the machine Flow +runs on (RDP or Citrix clipboard redirection). It drags across only the one +text box that holds the identifier, and the run stops if the screen changes +anywhere else during the look. A canvas with no selectable text has no copy +look. A recording's own identifier is OCR text and can be a misread (the RDP +kiosk recorded `A1001` as `Alo01`), so in a workflow that always opens the +recorded record, the copy can stop a run but never confirm one. It confirms +only a record chosen by a run parameter. Governed and qualification runs never +take it. The check just before the click takes no input; it reuses the copy +only when the identifier's pixels are unchanged since the look. + Permissive `replay` may proceed on an unreadable reversible target and reports that condition. Governed `run` requires an affirmative live verdict for every identity-required admitted step; mismatch, unreadable, or abstain halts before From 663c10e876853bcf0d7ced648986dcadfca12f0f Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 09:37:22 -0400 Subject: [PATCH 05/10] Check a click target's hover response before an icon-only click On a pixel surface, an icon-only click has no text that names its control. The recorder can now rest the pointer on each click target and keep that frame; the compiler stores what changed near the target (the control's highlight or tooltip) as hover evidence for clicks with no other identity evidence. At replay, on the pixel rungs, the runtime rests the pointer on the resolved target and compares. A different response stops the run before the click; a missing or uncomparable one is recorded and changes nothing. Hover works on the browser, remote-display and FreeRDP backends. Agent-driven RDP and Citrix recordings capture it. Co-Authored-By: Claude Opus 5.5 --- .../hover_identity/run_hover_sweep.py | 306 +++++++++++++++ .../benchmark/pixel_scale/kiosk_render.py | 117 +++++- packages/flow/docs/LIMITS.md | 19 + packages/flow/openadapt_flow/authoring.py | 3 + packages/flow/openadapt_flow/backend.py | 13 + .../backends/playwright_backend.py | 6 + .../openadapt_flow/backends/rdp_backend.py | 14 + .../openadapt_flow/backends/remote_display.py | 5 + .../flow/openadapt_flow/backends/xfreerdp.py | 5 + .../flow/openadapt_flow/compiler/compile.py | 64 ++++ packages/flow/openadapt_flow/ir.py | 61 +++ packages/flow/openadapt_flow/recorder.py | 35 ++ .../openadapt_flow/runtime/hover_evidence.py | 290 +++++++++++++++ .../openadapt_flow/runtime/render_scale.py | 1 + .../flow/openadapt_flow/runtime/replayer.py | 119 +++++- packages/flow/tests/test_hover_evidence.py | 350 ++++++++++++++++++ 16 files changed, 1402 insertions(+), 6 deletions(-) create mode 100644 packages/flow/benchmark/hover_identity/run_hover_sweep.py create mode 100644 packages/flow/openadapt_flow/runtime/hover_evidence.py create mode 100644 packages/flow/tests/test_hover_evidence.py diff --git a/packages/flow/benchmark/hover_identity/run_hover_sweep.py b/packages/flow/benchmark/hover_identity/run_hover_sweep.py new file mode 100644 index 0000000000..5bff60300a --- /dev/null +++ b/packages/flow/benchmark/hover_identity/run_hover_sweep.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""Hover-before-click evidence on a pixel-only kiosk: before and after. + +The appointment kiosk of ``benchmark/pixel_scale`` with its Save button turned +into a toolbar icon. A second icon, ``Clear form``, draws the same glyph; the +two differ only by their tooltip. The kiosk draws a button under the pointer +in its active colour, as Tk does, and shows the icon's tooltip. + +The kiosk is recorded once (100% scaling, ClearType on) with the production +Recorder, compiled, and replayed: + +- ``healthy``: same layout. A correct run saves one row for Ada Lovelace. +- ``swapped``: the two icons trade places, as a toolbar reorder between + versions would. At the recorded Save position there is now an identical + Clear icon. A correct run does not click it. + +Each at 100% ClearType on (same render), 100% ClearType off (another +rasterizer) and 125% (display scaling). Identity evidence per click step is +read from the compiled bundle. No model is called. + +The same script runs against an older checkout for the before numbers: where +the Recorder cannot capture hover frames, it records without them. + +Usage:: + + python benchmark/hover_identity/run_hover_sweep.py --output results_after.json +""" + +from __future__ import annotations + +import argparse +import inspect +import json +import platform +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +PIXEL_SCALE = HERE.parent / "pixel_scale" +for _path in (str(HERE.parents[1]), str(PIXEL_SCALE)): + if _path not in sys.path: + sys.path.insert(0, _path) + +from kiosk_render import ( # noqa: E402 + KioskBackend, + RenderProfile, + ScaledKiosk, + rdp_appointments_icons_spec, +) + +SCHEMA = "openadapt.hover-identity-sweep.v1" +RECORD_PROFILE = RenderProfile(scale=1.0, cleartype=True) +PROFILES = ( + RenderProfile(scale=1.0, cleartype=True), + RenderProfile(scale=1.0, cleartype=False), + RenderProfile(scale=1.25, cleartype=True), +) +DEMO = { + "appointment_slot": "2026-08-12 09:30", + "visit_type": "Cardiology follow-up", + "request_id": "REQ-DEMO-001", +} +REPLAY = { + "appointment_slot": "2026-08-14 10:45", + "visit_type": "Intake consultation", + "request_id": "REQ-LIVE-2048", +} +FIELD_FOR_PARAM = { + "appointment_slot": "slot", + "visit_type": "visit_type", + "request_id": "request_id", +} + + +def _recorder_can_hover() -> bool: + from openadapt_flow.recorder import Recorder + + return "capture_hover" in inspect.signature(Recorder.__init__).parameters + + +def _record(work: Path) -> tuple[Any, Path]: + from openadapt_flow.compiler import compile_recording + from openadapt_flow.recorder import Recorder + + kiosk = ScaledKiosk( + rdp_appointments_icons_spec(), RECORD_PROFILE, hover_feedback=True + ) + backend = KioskBackend(kiosk) + rec_dir, bundle_dir = work / "recording", work / "bundle" + options = {"capture_hover": True} if _recorder_can_hover() else {} + rec = Recorder( + backend, + rec_dir, + settle_interval_s=0.0, + settle_stable_frames=2, + settle_timeout_s=2.0, + **options, + ) + rec.click(*kiosk.logical_center("row0")) + for param, value in DEMO.items(): + rec.click(*kiosk.logical_center(FIELD_FOR_PARAM[param])) + rec.type_text(value, param=param) + rec.click(*kiosk.logical_center("save")) + rec.finish() + workflow = compile_recording(rec_dir, bundle_dir, name="hover-identity") + return workflow, bundle_dir + + +def coverage(workflow: Any) -> list[dict[str, Any]]: + """Identity evidence per click step of the compiled bundle.""" + out = [] + for step in workflow.steps: + if str(getattr(step.action, "value", step.action)) not in ( + "click", + "double_click", + "right_click", + ): + continue + a = step.anchor + record_identity = bool( + a is not None + and ( + a.context_text + or a.structured_identity + or a.identity_template + or a.identifier_crop + ) + ) + hover = getattr(step, "hover", None) + out.append( + { + "step_id": step.id, + "intent": step.intent, + "record_identity": record_identity, + "hover_response": bool(hover is not None and hover.response), + "has_evidence": record_identity + or bool(hover is not None and hover.response), + } + ) + return out + + +def _replay( + bundle_dir: Path, profile: RenderProfile, *, swapped: bool, run_dir: Path +) -> dict[str, Any]: + from openadapt_flow.ir import Workflow + from openadapt_flow.runtime.replayer import Replayer + + workflow = Workflow.load(bundle_dir) + kiosk = ScaledKiosk( + rdp_appointments_icons_spec(swapped=swapped), profile, hover_feedback=True + ) + backend = KioskBackend(kiosk) + crash: Optional[str] = None + report = None + try: + report = Replayer(backend, poll_interval_s=0.0, layout_drift=True).run( + workflow, params=REPLAY, bundle_dir=bundle_dir, run_dir=run_dir + ) + except Exception as exc: # noqa: BLE001 - a crash is an outcome + crash = f"{type(exc).__name__}: {exc}"[:300] + clear_rect = kiosk._rect("clear") + clear_clicked = any( + clear_rect[0] <= x < clear_rect[0] + clear_rect[2] + and clear_rect[1] <= y < clear_rect[1] + clear_rect[3] + for x, y in backend.clicks + ) + rows = kiosk.state.saved + expected = { + "patient_name": "Ada Lovelace", + "patient_mrn": kiosk.spec.patients[0][1], + **{FIELD_FOR_PARAM[k]: v for k, v in REPLAY.items()}, + } + success = bool(report is not None and report.success) + if crash is not None: + outcome = "crash" + elif clear_clicked: + outcome = "wrong_control_clicked" + elif rows and rows != [expected]: + outcome = "wrong_write" + elif success and rows == [expected]: + outcome = "pass" + elif success: + outcome = "false_completion" + else: + outcome = "halted" + steps = [] + stop = None + for result in report.results if report is not None else []: + ident = result.identity + hover = getattr(ident, "hover", None) if ident is not None else None + steps.append( + { + "step_id": result.step_id, + "ok": result.ok, + "rung": result.resolution.rung if result.resolution else None, + "identity": ident.status if ident is not None else None, + "identity_mode": ident.mode if ident is not None else None, + "hover": [hover.verdict, hover.reason] if hover is not None else None, + } + ) + if not result.ok and stop is None: + stop = {"step_id": result.step_id, "error": (result.error or "")[:300]} + return { + "layout": "swapped" if swapped else "recorded", + "profile": profile.label, + "outcome": outcome, + "rows_written": len(rows), + "clear_clicked": clear_clicked, + "model_calls": int(report.model_calls) if report is not None else 0, + "stopped_at": stop, + "steps": steps, + "crash": crash, + } + + +def _git(*args: str) -> Optional[str]: + try: + import openadapt_flow + + cwd = Path(openadapt_flow.__file__).resolve().parent + return subprocess.run( + ["git", *args], cwd=cwd, capture_output=True, text=True, check=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def run(work: Path) -> dict[str, Any]: + workflow, bundle_dir = _record(work) + points = [] + for profile in PROFILES: + for swapped in (False, True): + tag = ( + f"{round(profile.scale * 100)}_{'ct' if profile.cleartype else 'noct'}" + ) + tag += "_swapped" if swapped else "" + point = _replay(bundle_dir, profile, swapped=swapped, run_dir=work / tag) + print( + f"{point['layout']:9s} {point['profile']:24s} {point['outcome']}", + flush=True, + ) + points.append(point) + cov = coverage(workflow) + return { + "schema_version": SCHEMA, + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "commit": _git("rev-parse", "HEAD"), + "platform": f"{platform.system()} {platform.machine()} py{platform.python_version()}", + "recorder_captured_hover": _recorder_can_hover(), + "coverage": { + "click_steps": len(cov), + "with_record_identity": sum(c["record_identity"] for c in cov), + "with_hover_response": sum(c["hover_response"] for c in cov), + "with_any_evidence": sum(c["has_evidence"] for c in cov), + "steps": cov, + }, + "summary": { + "healthy_points": sum(p["layout"] == "recorded" for p in points), + "healthy_passed": sum( + p["layout"] == "recorded" and p["outcome"] == "pass" for p in points + ), + "swapped_points": sum(p["layout"] == "swapped" for p in points), + "swapped_wrong_control_clicked": sum( + p["layout"] == "swapped" and p["clear_clicked"] for p in points + ), + "swapped_stopped_before_click": sum( + p["layout"] == "swapped" + and not p["clear_clicked"] + and p["outcome"] == "halted" + for p in points + ), + "wrong_writes": sum(p["outcome"] == "wrong_write" for p in points), + "model_calls": sum(p["model_calls"] for p in points), + }, + "points": points, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--work-dir", type=Path) + args = parser.parse_args() + work = args.work_dir or Path(tempfile.mkdtemp(prefix="oaflow-hover-")) + work.mkdir(parents=True, exist_ok=True) + results = run(work) + args.output.write_text(json.dumps(results, indent=2, sort_keys=True) + "\n") + print( + json.dumps( + { + **results["summary"], + **{k: v for k, v in results["coverage"].items() if k != "steps"}, + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/flow/benchmark/pixel_scale/kiosk_render.py b/packages/flow/benchmark/pixel_scale/kiosk_render.py index 6813e2be01..187840fcd4 100644 --- a/packages/flow/benchmark/pixel_scale/kiosk_render.py +++ b/packages/flow/benchmark/pixel_scale/kiosk_render.py @@ -208,7 +208,7 @@ def linespace(pt: int, bold: bool, scale: float = 1.0) -> int: class Widget: """A Tk widget placed at logical (100%) coordinates.""" - kind: str # label | button | entry + kind: str # label | button | entry | icon name: str x: int y: int @@ -223,6 +223,8 @@ class Widget: anchor_w: bool = False relief: str = "flat" bd: int = 0 + #: Tooltip shown while the pointer rests on the widget (hover feedback on). + tooltip: Optional[str] = None def size(self) -> tuple[int, int]: """Logical size, from Tk's geometry rules at 100%.""" @@ -239,6 +241,8 @@ def size(self) -> tuple[int, int]: return (w, h) if self.kind == "entry": return ((self.chars or 20) * avg + 2 * (1 + 1) + 2, line + 2 * (1 + 1) + 2) + if self.kind == "icon": + return (self.width or 48, self.height or 44) raise ValueError(self.kind) @@ -258,6 +262,8 @@ class KioskSpec: fields: list[str] save_button: str rows: list[str] + #: Icon buttons and what they do: ``save`` or ``clear`` (empty the form). + icon_actions: dict[str, str] = field(default_factory=dict) def rdp_appointments_spec( @@ -499,6 +505,44 @@ def rdp_appointments_unambiguous_spec() -> KioskSpec: ) +def rdp_appointments_icons_spec(swapped: bool = False) -> KioskSpec: + """The appointment kiosk with its Save button turned into a toolbar icon. + + Two icons draw the same glyph and differ only by their tooltip: + ``Save appointment`` and ``Clear form``. ``swapped`` exchanges their + places, as a toolbar reorder between versions would. Record IDs are free + of OCR-confusable glyphs, so identity is not what stops a run here. + """ + spec = rdp_appointments_spec( + (("Ada Lovelace", "MRN A3467"), ("Grace Hopper", "MRN B7346")), + name="rdp_appointments_icons_swapped" if swapped else "rdp_appointments_icons", + ) + btn_bg = "#2d6fd8" + save_x, clear_x = (750, 550) if swapped else (550, 750) + widgets = [w for w in spec.widgets if w.name != "save"] + for name, x, tip in ( + ("save", save_x, "Save appointment"), + ("clear", clear_x, "Clear form"), + ): + widgets.append( + Widget( + "icon", + name, + x, + 570, + bg=btn_bg, + width=56, + height=48, + relief="raised", + bd=2, + tooltip=tip, + ) + ) + spec.widgets = widgets + spec.icon_actions = {"save": "save", "clear": "clear"} + return spec + + SPECS: dict[str, Callable[[], KioskSpec]] = { "rdp_appointments": rdp_appointments_spec, "rdp_appointments_unambiguous_ids": rdp_appointments_unambiguous_spec, @@ -519,11 +563,18 @@ class KioskState: class ScaledKiosk: """Kiosk state machine plus a renderer for one :class:`RenderProfile`.""" - def __init__(self, spec: KioskSpec, profile: RenderProfile) -> None: + def __init__( + self, spec: KioskSpec, profile: RenderProfile, *, hover_feedback: bool = False + ) -> None: self.spec = spec self.profile = profile self.state = KioskState(values={name: "" for name in spec.fields}) self._by_name = {w.name: w for w in spec.widgets} + # Tk draws a button under the pointer in its active colour and the + # application may show a tooltip. Off by default: the display-scale + # sweep was measured without it. + self.hover_feedback = hover_feedback + self.pointer: Optional[tuple[int, int]] = None # -- geometry -------------------------------------------------------- def logical_rect(self, name: str) -> tuple[int, int, int, int]: @@ -550,7 +601,7 @@ def logical_center(self, name: str) -> tuple[int, int]: # -- interaction ----------------------------------------------------- def _hit(self, x: int, y: int) -> Optional[str]: for w in self.spec.widgets: - if w.kind not in ("button", "entry"): + if w.kind not in ("button", "entry", "icon"): continue rx, ry, rw, rh = self._rect(w.name) if rx <= x < rx + rw and ry <= y < ry + rh: @@ -565,6 +616,10 @@ def click(self, x: int, y: int) -> None: self.state.active = self.spec.rows.index(name) elif name in self.spec.fields: self.state.focus = name + elif self.spec.icon_actions.get(name) == "clear": + self.state.values = {k: "" for k in self.state.values} + self.state.status = "Form cleared" + self.state.status_fg = "#b42318" elif name == self.spec.save_button: self.save() @@ -612,6 +667,8 @@ def render(self) -> Image.Image: self.state.status_fg if w.name == self.spec.status_label else w.fg ) bg = _rgb(w.bg) if w.bg else _rgb(self.spec.bg) + if w.kind in ("button", "icon") and self._under_pointer(w.name): + bg = _shade(bg, 1.18) # Tk activebackground if w.kind != "label": canvas[ry : ry + rh, rx : rx + rw] = bg relief = w.relief @@ -622,6 +679,9 @@ def render(self) -> Image.Image: ): relief, bd = "sunken", 4 self._bevel(canvas, (rx, ry, rw, rh), bg, relief, max(1, self._px(bd))) + if w.kind == "icon": + self._glyph(canvas, (rx, ry, rw, rh)) + continue pad = self._px(3) line = linespace(w.pt, w.bold, fs) if w.kind == "label": @@ -646,8 +706,53 @@ def render(self) -> Image.Image: cx = origin[0] + text_width(text, w.pt, w.bold, fs) + 1 cw = max(1, self._px(2)) canvas[origin[1] : origin[1] + line, cx : cx + cw] = fg + self._tooltip(canvas) return Image.fromarray(canvas, "RGB") + def _under_pointer(self, name: str) -> bool: + if not self.hover_feedback or self.pointer is None: + return False + return self._hit(*self.pointer) == name + + def _glyph(self, canvas: np.ndarray, rect: tuple[int, int, int, int]) -> None: + """The same white disc-and-bar glyph on every icon.""" + x, y, w, h = rect + cx, cy = x + w // 2, y + h // 2 + r = max(3, min(w, h) // 4) + yy, xx = np.ogrid[: canvas.shape[0], : canvas.shape[1]] + disc = (xx - cx) ** 2 + (yy - cy) ** 2 <= r * r + canvas[disc] = (255, 255, 255) + bar = max(1, r // 3) + canvas[cy - bar : cy + bar, cx - 2 * r : cx + 2 * r] = (255, 255, 255) + + def _tooltip(self, canvas: np.ndarray) -> None: + if not self.hover_feedback or self.pointer is None: + return + name = self._hit(*self.pointer) + widget = self._by_name.get(name) if name else None + if widget is None or not widget.tooltip: + return + fs = self.profile.scale * self.profile.font_scale + px, py = self.pointer + tx, ty = px + self._px(12), py + self._px(20) + tw = text_width(widget.tooltip, 11, False, fs) + self._px(8) + th = linespace(11, False, fs) + self._px(6) + vh, vw = canvas.shape[:2] + tx, ty = min(tx, vw - tw - 1), min(ty, vh - th - 1) + canvas[ty : ty + th, tx : tx + tw] = (255, 255, 225) + canvas[ty, tx : tx + tw] = canvas[ty + th - 1, tx : tx + tw] = (0, 0, 0) + canvas[ty : ty + th, tx] = canvas[ty : ty + th, tx + tw - 1] = (0, 0, 0) + _draw_text( + canvas, + (tx + self._px(4), ty + self._px(3)), + widget.tooltip, + font_px=font_px(11, fs), + bold=False, + fg=(0, 0, 0), + cleartype=self.profile.cleartype, + clip=(tx, ty, tw, th), + ) + @staticmethod def _bevel( canvas: np.ndarray, @@ -702,8 +807,14 @@ def screenshot(self) -> bytes: def click(self, x: int, y: int, *, double: bool = False) -> None: self.clicks.append((int(x), int(y))) + self.kiosk.pointer = (int(x), int(y)) self.kiosk.click(int(x), int(y)) + def hover(self, x: int, y: int) -> None: + """Rest the pointer on (x, y); with hover feedback on, the kiosk draws + the control's active colour and tooltip.""" + self.kiosk.pointer = (int(x), int(y)) + def type_text(self, text: str) -> None: self.kiosk.type_text(text) if self._after_type is not None: diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index df8274cf49..0be440a291 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -215,6 +215,25 @@ irreversible, and qualification requires its record check. An icon-only click that only navigates after data entry is therefore over-classified until an operator reclassifies it. +On a pixel surface an icon-only click carries no text that names its control. +When the recorder can move the pointer without clicking +(`Recorder(capture_hover=True)` on a backend with `hover`), it rests the +pointer on each click target first and keeps that frame. For a click with no +other identity evidence, the compiler stores what changed near the target, such +as the control's highlight or tooltip, as the step's hover evidence +([`runtime/hover_evidence.py`](../openadapt_flow/runtime/hover_evidence.py)). +At replay, on the pixel rungs, the runtime rests the pointer on the resolved +target, waits the same 0.8 s, and compares. A different response, or a +response where the recording saw none, stops the run before the click. A +missing response, or one that can't be compared (another render with no +readable text), is recorded on the step and changes nothing. A tooltip that +opens later than the wait looks like a different response and stops the run. +Controls that don't answer the pointer, such as most text fields, get no +evidence this way. The browser, remote-display (Citrix and RDP client window) +and FreeRDP backends can hover. The Aardwolf RDP transport and the native +Windows, macOS and Linux backends can't move the pointer without a button +yet, so they take no hover look. + A consequential click must be tied to a record before Flow delivers it. Inside a table row, the row is that record. An icon-only Save that floats beside a form (openIMIS's claim Save has no text, accessible name, or tooltip) has no diff --git a/packages/flow/openadapt_flow/authoring.py b/packages/flow/openadapt_flow/authoring.py index 67a2926797..caf016ca05 100644 --- a/packages/flow/openadapt_flow/authoring.py +++ b/packages/flow/openadapt_flow/authoring.py @@ -632,6 +632,9 @@ def start_record(self) -> None: settle_stable_frames=self._settle_stable_frames, settle_timeout_s=self._settle_timeout_s, system_of_record_reader=self._system_of_record_reader, + # Remote sessions are screen-only: keep each click target's hover + # response as identity evidence (runtime.hover_evidence). + capture_hover=self._kind in ("rdp", "citrix"), ) self._actors = {} diff --git a/packages/flow/openadapt_flow/backend.py b/packages/flow/openadapt_flow/backend.py index 5d60a75c0a..4e591623bb 100644 --- a/packages/flow/openadapt_flow/backend.py +++ b/packages/flow/openadapt_flow/backend.py @@ -647,6 +647,19 @@ def prepare_pointer_actuation(self, x: int, y: int) -> None: ... +@runtime_checkable +class PointerHoverBackend(Protocol): + """Optional: rest the pointer on a point without pressing a button. + + Used to read a control's hover response (highlight, tooltip, status-bar + text) before an icon-only click on a pixel surface + (:mod:`openadapt_flow.runtime.hover_evidence`), and by the recorder to + capture that response. Emits no button edge. + """ + + def hover(self, x: int, y: int) -> None: ... + + @runtime_checkable class RichPointerActionBackend(Protocol): """Pointer actions beyond the primary/double click contract.""" diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index 7e6b82243b..9b5d37e333 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -3241,6 +3241,12 @@ def right_click(self, x: int, y: int) -> None: cause_type=type(exc).__name__, ) from exc + def hover(self, x: int, y: int) -> None: + """Rest the mouse on pixel coordinates without pressing a button.""" + + self._assert_qualification_environment_current() + self.page.mouse.move(x, y) + def drag(self, x: int, y: int, end_x: int, end_y: int) -> None: """Drag between two independently resolved points.""" diff --git a/packages/flow/openadapt_flow/backends/rdp_backend.py b/packages/flow/openadapt_flow/backends/rdp_backend.py index 64c9890fad..6e61902e4f 100644 --- a/packages/flow/openadapt_flow/backends/rdp_backend.py +++ b/packages/flow/openadapt_flow/backends/rdp_backend.py @@ -726,6 +726,20 @@ def right_click_guarded( delivered_at=datetime.now(timezone.utc).isoformat(), ) + def hover(self, x: int, y: int) -> None: + """Rest the pointer on (x, y) without a button edge. + + Needs a transport with ``move(x, y)``; raises NotImplementedError + otherwise, and the caller treats the hover look as unavailable. + """ + move = getattr(self._transport, "move", None) + if not callable(move): + raise NotImplementedError("this RDP transport cannot move the pointer") + with self._input_lock: + self._ensure_input_ready(point=(int(x), int(y)), operation="rdp_hover") + self._assert_frame_fresh() + move(int(x), int(y)) + def drag(self, x: int, y: int, end_x: int, end_y: int) -> None: """Drag between two points resolved from the same fresh framebuffer.""" diff --git a/packages/flow/openadapt_flow/backends/remote_display.py b/packages/flow/openadapt_flow/backends/remote_display.py index abdbf6f1dd..fa0bcf6eb2 100644 --- a/packages/flow/openadapt_flow/backends/remote_display.py +++ b/packages/flow/openadapt_flow/backends/remote_display.py @@ -980,6 +980,11 @@ def prepare_pointer_actuation(self, x: int, y: int) -> None: self._wait_for_pointer_settle() self._prepared_pointer_point = point + def hover(self, x: int, y: int) -> None: + """Rest the pointer on (x, y) with the same checks as a pre-positioned + click (:meth:`prepare_pointer_actuation`); emits no button edge.""" + self.prepare_pointer_actuation(x, y) + def click(self, x: int, y: int, *, double: bool = False) -> None: """Click (or double-click) at captured-pixel coordinates (x, y).""" with self._input_lock: diff --git a/packages/flow/openadapt_flow/backends/xfreerdp.py b/packages/flow/openadapt_flow/backends/xfreerdp.py index 1ec3ba2933..0d94dfe300 100644 --- a/packages/flow/openadapt_flow/backends/xfreerdp.py +++ b/packages/flow/openadapt_flow/backends/xfreerdp.py @@ -446,6 +446,11 @@ def pointer(self, x: int, y: int, button: str, down: bool) -> None: btn, ) + def move(self, x: int, y: int) -> None: + """Move the pointer with no button edge (a hover).""" + self._last_pointer = (int(x), int(y)) + self._xdotool("mousemove", str(int(x)), str(int(y))) + def key(self, keysym_or_char: str, down: bool) -> None: name = _KEYSYMS.get(keysym_or_char.lower()) if len(keysym_or_char) > 1 else None if name is not None: diff --git a/packages/flow/openadapt_flow/compiler/compile.py b/packages/flow/openadapt_flow/compiler/compile.py index e784e5eebf..bcf210c90f 100644 --- a/packages/flow/openadapt_flow/compiler/compile.py +++ b/packages/flow/openadapt_flow/compiler/compile.py @@ -43,6 +43,7 @@ DialogResponse, ExecutionMode, ExecutionTargetKind, + HoverEvidence, Landmark, ParamKind, ParamSpec, @@ -1426,6 +1427,56 @@ def _marked_identifier_region(value: object, *, source: str) -> Optional[Region] return (int(value[0]), int(value[1]), int(value[2]), int(value[3])) +HOVER_CROP_DIR = "templates/hover" + + +def _hover_evidence( + bundle: Path, + step_id: str, + before_png: bytes, + hover_path: Path, + click: Point, + *, + dwell_ms: object = None, +) -> Optional[HoverEvidence]: + """The recorded hover response for a click with no record identity. + + The recorder saved ``frames/{i}_hover.png`` after resting the pointer on + the target (``Recorder(capture_hover=True)``). What changed near the + target between the before frame and that frame is the response + (:func:`openadapt_flow.runtime.hover_evidence.measure_response`); its + pixels land under ``templates/hover/`` so they are sealed with the other + image crops. None when there is no hover frame, nothing near the target + changed, or the frames cannot be compared. + """ + from openadapt_flow.runtime import hover_evidence as hover_mod + + if not hover_path.is_file(): + return None + hover_png = hover_path.read_bytes() + try: + response = hover_mod.measure_response(before_png, hover_png, click) + except ValueError: + logger.warning("hover frame for %s differs in size; no hover evidence", step_id) + return None + dwell = ( + int(dwell_ms) + if isinstance(dwell_ms, int) and 0 <= dwell_ms <= 5000 + else hover_mod.HOVER_DWELL_MS + ) + if response is None: + return None # nothing answered the pointer: no evidence to keep + crop_rel = f"{HOVER_CROP_DIR}/{step_id}.png" + (bundle / HOVER_CROP_DIR).mkdir(parents=True, exist_ok=True) + (bundle / crop_rel).write_bytes(response.crop) + return HoverEvidence( + response=True, + region=hover_mod.relative(response.region, click), + crop=crop_rel, + dwell_ms=dwell, + ) + + def _emit_identifier_crop( bundle: Path, step_id: str, @@ -2357,6 +2408,18 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: ), exclude_region=crop_region, ) + hover = ( + _hover_evidence( + bundle, + step_id, + before_png, + recording / "frames" / f"{i:04d}_hover.png", + click, + dwell_ms=event.get("hover_dwell_ms"), + ) + if not identity_armed and kind != "drag" + else None + ) if kind == "drag": destination = drag_end_anchor.ocr_text if drag_end_anchor else None intent = "drag " + ( @@ -2395,6 +2458,7 @@ def cached_lines(i: int, suffix: str, png: bytes) -> list[OcrLine]: identity_armed=identity_armed, identity_unarmed_reason=unarmed_reason, identifier_crop_missing_reason=idcrop_missing, + hover=hover, ), before_png, after_png, diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 16573be3c9..87f6f181b6 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -1371,6 +1371,37 @@ def _one_identity(self) -> "DialogResponse": return self +class HoverEvidence(BaseModel): + """What the recording saw when the pointer rested on a click target. + + See :mod:`openadapt_flow.runtime.hover_evidence`. Captured for click steps + whose target carries no record identity (an icon-only control) on a + pixel surface. Replay moves the pointer onto the resolved target, waits + ``dwell_ms`` and compares its response with this one before clicking. + """ + + model_config = ConfigDict(extra="forbid") + + response: bool = Field( + description="Whether anything near the target changed under the pointer" + ) + region: Optional[Region] = Field( + default=None, + description=( + "The changed box, as an offset from the recorded click point" + " (dx, dy, w, h). Set iff ``response``." + ), + ) + crop: Optional[str] = Field( + default=None, + description=( + "Bundle-relative PNG of the response pixels (templates/hover/)," + " sealed with the other image crops. Set iff ``response``." + ), + ) + dwell_ms: int = Field(default=800, ge=0, le=5000) + + class Step(BaseModel): id: str intent: str = Field(description="Human-readable purpose of the step") @@ -1680,6 +1711,15 @@ def _validate_rich_action_contract(self) -> "Step": " band tier (docs/LIMITS.md)." ), ) + hover: Optional[HoverEvidence] = Field( + default=None, + description=( + "Pointer-hover response captured at record time for a click whose" + " target carries no record identity (runtime.hover_evidence)." + " None when the recorder could not hover or the step has other" + " identity evidence, and on bundles compiled before this field." + ), + ) submit_binding: Optional[SubmitBinding] = Field( default=None, description=( @@ -3183,6 +3223,19 @@ class IdentityLook(BaseModel): frame_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") +class HoverLook(BaseModel): + """The live hover response compared with the recorded one. + + Holds no pixels: the verdict, why, and the SHA-256 of the hover frame. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + verdict: Literal["verified", "mismatch", "unexpected", "absent", "unverifiable"] + reason: str = Field(max_length=80) + frame_sha256: Optional[str] = Field(default=None, pattern=r"^[0-9a-f]{64}$") + + class IdentityCheck(BaseModel): """Outcome of the pre-click target-identity check (runtime.identity). @@ -3226,6 +3279,7 @@ class IdentityCheck(BaseModel): "pixel", "vlm", "signal_quorum", + "hover", ] = "context" coverage: float = 0.0 expected: str = "" @@ -3267,6 +3321,13 @@ class IdentityCheck(BaseModel): "confirmed, or it is older than the configured age. The step halts." ), ) + hover: Optional[HoverLook] = Field( + default=None, + description=( + "Set for mode ``hover``: the live pointer-hover response compared " + "with the recorded one (runtime.hover_evidence)." + ), + ) looks: list[IdentityLook] = Field( default_factory=list, description=( diff --git a/packages/flow/openadapt_flow/recorder.py b/packages/flow/openadapt_flow/recorder.py index 432c0f6664..4e5b278ccc 100644 --- a/packages/flow/openadapt_flow/recorder.py +++ b/packages/flow/openadapt_flow/recorder.py @@ -74,6 +74,11 @@ #: Bound on waiting for a newly opened page's first navigation. _NEW_PAGE_LOAD_TIMEOUT_S = 10.0 + +# Mirrors openadapt_flow.runtime.hover_evidence.HOVER_DWELL_MS. Not imported: +# the recorder must not pull in the runtime's vision dependencies. +HOVER_DWELL_MS = 800 + # How long a recorded action that raises a browser dialog waits for it. _DIALOG_WAIT_S = 5.0 @@ -139,7 +144,13 @@ def __init__( surface_id: str = PRIMARY_SURFACE_ID, surface_kind: Optional[str] = None, page_backend_factory: Optional[Callable[[Any], Backend]] = None, + capture_hover: bool = False, ) -> None: + # Pixel surfaces: before each click, rest the pointer on the target + # and keep that frame (frames/{i}_hover.png) so the compiler can store + # the hover response as identity evidence (runtime.hover_evidence). + # Needs a backend with ``hover(x, y)``; ignored otherwise. + self._capture_hover = capture_hover # ``_backend`` is always the ACTIVE surface's backend; on a # single-surface recording it never changes. self._backend = backend @@ -651,6 +662,9 @@ def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: ) if page_backend is not None: page_backend.expect_page_close() + hover_png = self._hover_frame(event) + if hover_png is not None: + event = {**event, "hover_dwell_ms": HOVER_DWELL_MS} try: act() finally: @@ -678,8 +692,29 @@ def _record(self, event: dict[str, Any], act: Callable[[], None]) -> None: # click that closes it. if self._closed_its_page(event): event = {**event, "closes_surface": True} + if hover_png is not None: + (self._frames_dir / f"{self._i:04d}_hover.png").write_bytes(hover_png) self._commit(event, before, after, structural_before) + def _hover_frame(self, event: dict[str, Any]) -> Optional[bytes]: + """Rest the pointer on a click target and capture the settled frame. + + Only with ``capture_hover`` and a backend that can hover, and only for + click events. The pointer stays there for the click that follows. + """ + if not self._capture_hover or event.get("kind") not in ( + "click", + "double_click", + "right_click", + ): + return None + hover = getattr(self._backend, "hover", None) + if not callable(hover): + return None + hover(int(event["x"]), int(event["y"])) + time.sleep(HOVER_DWELL_MS / 1000.0) + return self._wait_settled() + def record_observed( self, event: dict[str, Any], diff --git a/packages/flow/openadapt_flow/runtime/hover_evidence.py b/packages/flow/openadapt_flow/runtime/hover_evidence.py new file mode 100644 index 0000000000..d4e5afa69d --- /dev/null +++ b/packages/flow/openadapt_flow/runtime/hover_evidence.py @@ -0,0 +1,290 @@ +"""Hover-before-click evidence for steps with no record identity. + +An icon-only click (a toolbar icon, a floppy-disk Save) on a pixel surface +carries no text that says which control it is, so the identity check has +nothing to compare. Most applications answer a pointer resting on a control: +a highlight, a raised border, a tooltip or status-bar text near the pointer. +The recording can capture that answer, and replay can ask the same question +before it clicks. + +At record time the recorder moves the pointer onto the target, waits +:data:`HOVER_DWELL_MS`, and saves the frame. The compiler keeps the pixels +that changed near the target (:func:`measure_response`) as the step's +:class:`~openadapt_flow.ir.HoverEvidence`. At replay, on a pixel surface, the +runtime moves the pointer onto the resolved target, waits the same time, and +compares (:func:`compare`): + +- ``verified``: the live response has the recorded shape and position and + the same content (pixels on the same render; OCR text otherwise); +- ``mismatch``: a response with other content, or in another place or + shape; the run stops before the click; +- ``unexpected``: the recording saw no response and the live screen + answered; the run stops before the click; +- ``absent``: the recording saw a response and the live screen did not. + No evidence; the step goes on as it did before hover evidence existed; +- ``unverifiable``: shape and position match but neither pixels nor text + can be compared (another render with no readable text). No evidence. + +Hover evidence only adds stops: ``mismatch`` and ``unexpected`` stop the run; +the other verdicts are recorded on the step and change nothing. + +Moving the pointer clicks nothing. Hover-opened menus are the one known +hazard: a response far from the target falls outside the measured window and +is ignored, but a menu that opens under the pointer is a response like any +other and is compared like one. +""" + +from __future__ import annotations + +import io +from dataclasses import dataclass +from typing import Any, Callable, Literal, Optional + +from openadapt_flow.ir import Point, Region +from openadapt_flow.runtime import identity as identity_mod + +HOVER_DWELL_MS = 800 +"""How long the pointer rests on the target before the hover frame.""" + +# The measured window around the pointer. Tooltips usually open below and to +# the right of the pointer; highlights cover the control under it. +WINDOW_LEFT = 160 +WINDOW_RIGHT = 320 +WINDOW_UP = 80 +WINDOW_DOWN = 160 + +CHANGE_LEVEL = 24 +"""Per-channel difference (0-255) for a pixel to count as changed.""" + +MIN_CHANGED_PX = 12 +"""Fewer changed pixels than this is noise, not a response.""" + +GEOMETRY_TOL_PX = 10 +GEOMETRY_TOL_FRAC = 0.25 +CROP_MATCH_MAX_MEAN = 0.04 +"""Mean grey difference (0-1) over the whole aligned response, at most.""" + +BLOCK_PX = 8 +BLOCK_MAX_MEAN = 0.12 +"""Mean grey difference in any :data:`BLOCK_PX` square, at most.""" + +ALIGN_MAX_SHIFT = 3 + +Verdict = Literal["verified", "mismatch", "unexpected", "absent", "unverifiable"] + + +@dataclass(frozen=True) +class Response: + """What changed near the pointer when it came to rest on the target.""" + + region: Region + crop: bytes + changed_px: int + + +def hover_window(point: Point, viewport: tuple[int, int]) -> Region: + x, y = int(point[0]), int(point[1]) + w, h = int(viewport[0]), int(viewport[1]) + x0 = max(0, x - WINDOW_LEFT) + y0 = max(0, y - WINDOW_UP) + x1 = min(w, x + WINDOW_RIGHT) + y1 = min(h, y + WINDOW_DOWN) + return (x0, y0, max(0, x1 - x0), max(0, y1 - y0)) + + +def _rgb(png: bytes) -> Any: + import numpy as np + from PIL import Image + + with Image.open(io.BytesIO(png)) as im: + return np.asarray(im.convert("RGB"), dtype=np.int16) + + +def measure_response( + pre_png: bytes, hover_png: bytes, point: Point +) -> Optional[Response]: + """The changed box near ``point`` between two frames, or None. + + Raises ValueError when the frames differ in size (the screen changed + shape, which is not a hover response). + """ + import numpy as np + + a, b = _rgb(pre_png), _rgb(hover_png) + if a.shape != b.shape: + raise ValueError("hover frame differs in size from the frame before it") + x, y, w, h = hover_window(point, (a.shape[1], a.shape[0])) + if w <= 0 or h <= 0: + return None + diff = np.abs(a[y : y + h, x : x + w] - b[y : y + h, x : x + w]).max(axis=2) + mask = diff > CHANGE_LEVEL + count = int(mask.sum()) + if count < MIN_CHANGED_PX: + return None + ys, xs = np.nonzero(mask) + region = ( + x + int(xs.min()), + y + int(ys.min()), + int(xs.max() - xs.min() + 1), + int(ys.max() - ys.min() + 1), + ) + crop = identity_mod.crop_region(hover_png, region) + if crop is None: + return None + return Response(region=region, crop=crop, changed_px=count) + + +def relative(region: Region, point: Point) -> Region: + """``region`` as an offset from ``point`` (dx, dy, w, h).""" + return ( + int(region[0] - point[0]), + int(region[1] - point[1]), + int(region[2]), + int(region[3]), + ) + + +def _close(a: int, b: int, size: int) -> bool: + return abs(a - b) <= max(GEOMETRY_TOL_PX, GEOMETRY_TOL_FRAC * size) + + +def geometry_matches(recorded: Region, live: Region) -> bool: + """Same place and shape relative to the target, within tolerance.""" + rx, ry, rw, rh = recorded + lx, ly, lw, lh = live + return ( + _close(rx + rw // 2, lx + lw // 2, rw) + and _close(ry + rh // 2, ly + lh // 2, rh) + and _close(rw, lw, rw) + and _close(rh, lh, rh) + ) + + +def crops_look_same(recorded_png: bytes, live_png: bytes) -> bool: + """Whether two response crops show the same thing. + + Grey images. The live crop is aligned onto the recorded one by the best + shift of up to :data:`ALIGN_MAX_SHIFT` pixels (a response drawn a pixel + or two away), or resized when the sizes differ by more. Then every + :data:`BLOCK_PX` square must agree: a changed word in a tooltip is a + local difference that an average over the whole crop would hide. + """ + import numpy as np + from PIL import Image + + with ( + Image.open(io.BytesIO(recorded_png)) as r, + Image.open(io.BytesIO(live_png)) as lv, + ): + a = np.asarray(r.convert("L"), dtype=np.float32) / 255.0 + lg = lv.convert("L") + if ( + abs(lg.size[0] - a.shape[1]) > ALIGN_MAX_SHIFT + or abs(lg.size[1] - a.shape[0]) > ALIGN_MAX_SHIFT + ): + lg = lg.resize((a.shape[1], a.shape[0])) + b = np.asarray(lg, dtype=np.float32) / 255.0 + pad = ALIGN_MAX_SHIFT + h, w = a.shape + canvas = np.full((h + 2 * pad, w + 2 * pad), np.nan, dtype=np.float32) + bh, bw = min(b.shape[0], h + pad), min(b.shape[1], w + pad) + best: Optional[Any] = None + best_mean = float("inf") + for dy in range(-pad, pad + 1): + for dx in range(-pad, pad + 1): + canvas[:] = np.nan + y0, x0 = pad + dy, pad + dx + canvas[y0 : y0 + bh, x0 : x0 + bw] = b[:bh, :bw] + window = canvas[pad : pad + h, pad : pad + w] + diff = np.abs(window - a) + covered = ~np.isnan(diff) + if covered.mean() < 0.9: + continue + mean = float(np.nanmean(diff)) + if mean < best_mean: + best_mean, best = mean, np.where(covered, diff, 0.0) + if best is None or best_mean > CROP_MATCH_MAX_MEAN: + return False + for y in range(0, h, BLOCK_PX): + for x in range(0, w, BLOCK_PX): + if float(best[y : y + BLOCK_PX, x : x + BLOCK_PX].mean()) > BLOCK_MAX_MEAN: + return False + return True + + +def _text(png: bytes, ocr: Callable[..., Any]) -> str: + try: + lines = ocr(png) or [] + except Exception: # noqa: BLE001 - unreadable is no text + return "" + words = " ".join(str(getattr(line, "text", "")) for line in lines) + return identity_mod.ocr_canonical(identity_mod.squash(words)) + + +def compare( + *, + recorded_response: bool, + recorded_region: Optional[Region], + recorded_crop: Optional[bytes], + live: Optional[Response], + point: Point, + same_render: bool, + ocr: Optional[Callable[..., Any]] = None, +) -> tuple[Verdict, str]: + """Compare the live hover response with the recorded one. + + Args: + recorded_response: Whether the recording saw any response. + recorded_region: The recorded response box, relative to the click. + recorded_crop: The recorded response pixels. + live: The live response at the resolved point, or None. + point: The resolved point. + same_render: The live frame is drawn at the recorded scale and by the + same rasterizer, so pixels can be compared. + ocr: OCR for the other-render comparison. + + Returns: + ``(verdict, reason)``. + """ + if not recorded_response: + if live is None: + # Agrees, but no response is not evidence of which control it is. + return "unverifiable", "no_response_as_recorded" + return "unexpected", "response_where_none_was_recorded" + if live is None: + return "absent", "no_response" + if recorded_region is None or recorded_crop is None: + return "unverifiable", "recording_kept_no_response" + if not geometry_matches(recorded_region, relative(live.region, point)): + return "mismatch", "response_moved_or_resized" + if same_render and not identity_mod.text_render_differs(recorded_crop, live.crop): + if crops_look_same(recorded_crop, live.crop): + return "verified", "same_pixels" + if ocr is None: + return "mismatch", "different_pixels" + # Another rasterizer can redraw the same tooltip with other pixels + # (ClearType switched); its text still decides. + if ocr is not None: + rec_text, live_text = _text(recorded_crop, ocr), _text(live.crop, ocr) + if rec_text and live_text: + if rec_text == live_text: + return "verified", "same_text" + return "mismatch", "different_text" + if rec_text or live_text: + return "mismatch", "text_on_one_side_only" + if same_render: + return "mismatch", "different_pixels" + return "unverifiable", "no_comparable_content" + + +__all__ = [ + "HOVER_DWELL_MS", + "Response", + "Verdict", + "compare", + "crops_look_same", + "geometry_matches", + "hover_window", + "measure_response", + "relative", +] diff --git a/packages/flow/openadapt_flow/runtime/render_scale.py b/packages/flow/openadapt_flow/runtime/render_scale.py index cceededc06..a3f6818cb3 100644 --- a/packages/flow/openadapt_flow/runtime/render_scale.py +++ b/packages/flow/openadapt_flow/runtime/render_scale.py @@ -707,6 +707,7 @@ def _passthrough(name: str) -> property: "act_guarded_coordinate": _point_method, "arm_guarded_keyboard": _point_method, "prepare_pointer_actuation": _point_method, + "hover": _point_method, "drag": _drag_method, "drag_guarded": _drag_method, "acquire_actuation_frame": _frame_method, diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 5e3a125d64..4aa78ba768 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -96,6 +96,7 @@ ExecutionTargetKind, FreshActuationEvent, HaltObservation, + HoverLook, IdentityCheck, IdentityLook, IdentitySignalEvidence, @@ -9161,7 +9162,12 @@ def _identity_gate_error( if qualification is not None else None ) - if ( + hover_check = ( + self._hover_check(step, resolution, frame_png, workflow, bundle_dir) + if qualified_policy is None + else None + ) + if hover_check is None and ( step.action not in ( ActionKind.CLICK, @@ -9186,8 +9192,12 @@ def _identity_gate_error( ): return None - check = self._verify_identity( - step, resolution, frame_png, params, workflow, bundle_dir + check = ( + hover_check + if hover_check is not None + else self._verify_identity( + step, resolution, frame_png, params, workflow, bundle_dir + ) ) if ( check.status in {"verified", "mismatch"} @@ -9265,6 +9275,18 @@ def _identity_gate_error( "confirm a current export; refusing to act" ) result.safety_halt = True + elif check.mode == "hover" and check.status == "mismatch": + reason = check.hover.reason if check.hover is not None else "differs" + error = ( + f"Step '{step.id}' ({step.intent}): with the pointer on the " + "target, the screen answered differently from the recording " + f"({reason}) — refusing to click; run aborted" + ) + elif check.mode == "hover": + # Hover evidence only adds a stop. A step that had no identity + # check before keeps going when the response is absent or cannot + # be compared; the verdict stays on the step result. + error = None elif check.status == "mismatch": error = ( f"Identity check failed for step '{step.id}' " @@ -14240,6 +14262,97 @@ def with_looks(final: IdentityCheck) -> IdentityCheck: ) return with_looks(better_identity_read(check, row_check)) + def _hover_check( + self, + step: Step, + resolution: Resolution, + frame_png: bytes, + workflow: Workflow, + bundle_dir: Optional[Path], + ) -> Optional[IdentityCheck]: + """Compare the target's hover response with the recording. + + Only at the step-entry check (moving the pointer is input), only for a + click whose recording saw a hover response and that has no other + identity evidence, + only on the pixel rungs, and only when the backend can hover + (:class:`~openadapt_flow.backend.PointerHoverBackend`). None when any + of that does not hold: the step then has no identity check, as + before. See :mod:`openadapt_flow.runtime.hover_evidence`. + """ + from openadapt_flow.runtime import hover_evidence as hover_mod + + evidence = step.hover + anchor = step.anchor + if ( + evidence is None + or not evidence.response + or anchor is None + or not self._identity_input_looks + or step.action + not in (ActionKind.CLICK, ActionKind.DOUBLE_CLICK, ActionKind.RIGHT_CLICK) + or resolution.rung == "structural" + or anchor.context_text + or anchor.structured_identity + or anchor.identity_template + or anchor.identifier_crop + ): + return None + hover = getattr(self.backend, "hover", None) + if not callable(hover): + return None + point = (int(resolution.point[0]), int(resolution.point[1])) + + def check(verdict: str, reason: str, png: Optional[bytes]) -> IdentityCheck: + status = { + "verified": "verified", + "mismatch": "mismatch", + "unexpected": "mismatch", + "absent": "abstain", + "unverifiable": "unreadable", + }[verdict] + return IdentityCheck( + status=status, # type: ignore[arg-type] + mode="hover", + expected="recorded hover response", + observed=reason, + hover=HoverLook( + verdict=verdict, # type: ignore[arg-type] + reason=reason, + frame_sha256=sha256_bytes(png) if png is not None else None, + ), + ) + + try: + hover(*point) + except NotImplementedError: + return None + except Exception as exc: # noqa: BLE001 - a failed hover confirms nothing + return check( + "unverifiable", f"hover_failed:{type(exc).__name__}"[:80], None + ) + time.sleep(evidence.dwell_ms / 1000.0) + hover_png = self.vision.wait_settled(self.backend) + try: + live = hover_mod.measure_response(frame_png, hover_png, point) + except ValueError: + return check("mismatch", "screen_changed_size", hover_png) + recorded_crop = ( + self._asset_bytes(bundle_dir, evidence.crop, workflow=workflow) + if evidence.crop and bundle_dir is not None + else None + ) + verdict, reason = hover_mod.compare( + recorded_response=evidence.response, + recorded_region=evidence.region, + recorded_crop=recorded_crop, + live=live, + point=point, + same_render=self._render_scale_inner is None, + ocr=self.vision.ocr, + ) + return check(verdict, reason, hover_png) + def _copy_look_allowed(self) -> bool: """The copy look is input: never in governed or qualification runs.""" return ( diff --git a/packages/flow/tests/test_hover_evidence.py b/packages/flow/tests/test_hover_evidence.py new file mode 100644 index 0000000000..5a0cfe8d8e --- /dev/null +++ b/packages/flow/tests/test_hover_evidence.py @@ -0,0 +1,350 @@ +"""Hover-before-click evidence (runtime.hover_evidence) and its wiring. + +Synthetic frames: an icon button that lightens under the pointer and opens a +tooltip. Two icons look the same and differ only by their tooltip. +""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from typing import Optional + +from PIL import Image, ImageDraw + +from openadapt_flow.ir import ( + Anchor, + HoverEvidence, + Resolution, + Step, + StepResult, + Workflow, +) +from openadapt_flow.runtime import hover_evidence as H +from openadapt_flow.runtime.replayer import Replayer + +ICON = (100, 100, 40, 40) +POINT = (120, 120) + + +def _frame( + pointer: Optional[tuple[int, int]] = None, tooltip: str = "Save note" +) -> bytes: + img = Image.new("RGB", (480, 320), (244, 246, 248)) + d = ImageDraw.Draw(img) + x, y, w, h = ICON + over = pointer is not None and x <= pointer[0] < x + w and y <= pointer[1] < y + h + d.rectangle( + (x, y, x + w - 1, y + h - 1), fill=(90, 140, 230) if over else (45, 111, 216) + ) + d.ellipse((x + 10, y + 10, x + 30, y + 30), fill="white") + if over and tooltip: + tx, ty = pointer[0] + 12, pointer[1] + 20 + d.rectangle( + (tx, ty, tx + 9 * len(tooltip), ty + 18), + fill=(255, 255, 225), + outline="black", + ) + d.text((tx + 4, ty + 3), tooltip, fill="black") + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def test_measure_response_finds_highlight_and_tooltip(): + r = H.measure_response(_frame(), _frame(POINT), POINT) + assert r is not None + x, y, w, h = r.region + assert x <= ICON[0] and y <= ICON[1] + assert x + w > POINT[0] + 12 and y + h > POINT[1] + 20 # tooltip included + assert H.measure_response(_frame(), _frame(), POINT) is None + + +def _recorded(tooltip: str = "Save note"): + r = H.measure_response(_frame(), _frame(POINT, tooltip), POINT) + assert r is not None + return H.relative(r.region, POINT), r.crop + + +def _compare(live_tooltip: Optional[str], *, same_render=True, recorded_response=True): + region, crop = _recorded() + live = ( + H.measure_response(_frame(), _frame(POINT, live_tooltip), POINT) + if live_tooltip is not None + else None + ) + return H.compare( + recorded_response=recorded_response, + recorded_region=region if recorded_response else None, + recorded_crop=crop if recorded_response else None, + live=live, + point=POINT, + same_render=same_render, + ) + + +def test_same_response_verifies(): + assert _compare("Save note") == ("verified", "same_pixels") + + +def test_other_tooltip_on_a_look_alike_icon_is_a_mismatch(): + verdict, _ = _compare("Delete note") + assert verdict == "mismatch" + + +def test_no_live_response_is_absent_not_a_stop(): + assert _compare(None)[0] == "absent" + + +def test_response_where_none_was_recorded_is_unexpected(): + assert _compare("Save note", recorded_response=False)[0] == "unexpected" + assert _compare(None, recorded_response=False)[0] == "unverifiable" + + +def test_moved_response_is_a_mismatch(): + region, crop = _recorded() + far = H.Response(region=(300, 250, 40, 20), crop=crop, changed_px=100) + verdict, reason = H.compare( + recorded_response=True, + recorded_region=region, + recorded_crop=crop, + live=far, + point=POINT, + same_render=True, + ) + assert verdict == "mismatch" and reason == "response_moved_or_resized" + + +def test_other_render_without_text_is_unverifiable(): + verdict, _ = _compare("Save note", same_render=False) + assert verdict == "unverifiable" + + +def test_same_text_redrawn_by_another_rasterizer_verifies(): + class Line: + def __init__(self, text): + self.text = text + + region, crop = _recorded() + live = H.measure_response(_frame(), _frame(POINT, "Save note"), POINT) + assert live is not None + blurred = H.Response( + region=live.region, + crop=_invert(live.crop), + changed_px=live.changed_px, + ) + verdict, reason = H.compare( + recorded_response=True, + recorded_region=region, + recorded_crop=crop, + live=blurred, + point=POINT, + same_render=True, + ocr=lambda png: [Line("Save note")], + ) + assert (verdict, reason) == ("verified", "same_text") + + +def _invert(png: bytes) -> bytes: + from PIL import ImageOps + + img = ImageOps.invert(Image.open(io.BytesIO(png)).convert("RGB")) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +def test_other_render_compares_ocr_text(): + class Line: + def __init__(self, text): + self.text = text + + region, crop = _recorded() + live = H.measure_response(_frame(), _frame(POINT, "Delete note"), POINT) + texts = iter(["Save note", "Delete note"]) + + def ocr(png): + return [Line(next(texts))] + + verdict, reason = H.compare( + recorded_response=True, + recorded_region=region, + recorded_crop=crop, + live=live, + point=POINT, + same_render=False, + ocr=ocr, + ) + assert (verdict, reason) == ("mismatch", "different_text") + + +# ------------------------------------------------------------- replayer + + +class _HoverBackend: + viewport = (480, 320) + + def __init__(self, tooltip: str = "Save note", can_hover: bool = True): + self.tooltip = tooltip + self.pointer: Optional[tuple[int, int]] = None + self.hovers: list[tuple[int, int]] = [] + if not can_hover: + self.hover = None # type: ignore[assignment] + + def screenshot(self) -> bytes: + return _frame(self.pointer, self.tooltip) + + def hover(self, x, y): # noqa: D401 - backend seam + self.hovers.append((x, y)) + self.pointer = (x, y) + + +class _Vision: + @staticmethod + def wait_settled(backend): + return backend.screenshot() + + @staticmethod + def ocr(png, region=None): + return [] + + +def _bundle(tmp_path: Path) -> tuple[Path, Step, Workflow]: + region, crop = _recorded() + (tmp_path / "templates" / "hover").mkdir(parents=True) + (tmp_path / "templates" / "hover" / "step_000.png").write_bytes(crop) + step = Step( + id="step_000", + intent="click recorded visual target", + action="click", + anchor=Anchor(template="t.png", region=ICON, click_point=POINT), + risk="irreversible", + hover=HoverEvidence( + response=True, + region=region, + crop="templates/hover/step_000.png", + dwell_ms=0, + ), + ) + return tmp_path, step, Workflow(name="hover", params={}, steps=[step]) + + +def _gate(tmp_path, backend, *, entry=True, rung="template"): + bundle, step, wf = _bundle(tmp_path) + replayer = Replayer(backend, vision=_Vision()) + res = Resolution(rung=rung, point=POINT, confidence=0.9, elapsed_ms=1.0) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + call = ( + replayer._entry_identity_gate_error if entry else replayer._identity_gate_error + ) + error = call(step, res, _frame(), {}, wf, bundle, result) + return error, result + + +def test_same_hover_response_verifies_the_icon(tmp_path): + error, result = _gate(tmp_path, _HoverBackend("Save note")) + assert error is None + assert result.identity is not None and result.identity.mode == "hover" + assert result.identity.status == "verified" + assert result.identity.hover is not None + assert result.identity.hover.verdict == "verified" + + +def test_look_alike_icon_with_other_tooltip_stops_before_the_click(tmp_path): + backend = _HoverBackend("Delete note") + error, result = _gate(tmp_path, backend) + assert error is not None and "refusing to click" in error + assert result.identity is not None and result.identity.status == "mismatch" + assert backend.hovers == [POINT] + + +def test_missing_tooltip_changes_the_response_and_stops(tmp_path): + # A highlight with no tooltip (the tooltip opened late) is another shape. + error, result = _gate(tmp_path, _HoverBackend("")) + assert result.identity is not None and result.identity.status == "mismatch" + assert result.identity.hover.reason == "response_moved_or_resized" + assert error is not None + + +def test_absent_response_is_recorded_and_does_not_stop(tmp_path): + class Frozen(_HoverBackend): + def screenshot(self): + return _frame() + + error, result = _gate(tmp_path, Frozen()) + assert error is None + assert result.identity is not None and result.identity.status == "abstain" + assert result.identity.hover.verdict == "absent" + + +def test_actuation_check_takes_no_hover(tmp_path): + backend = _HoverBackend("Delete note") + error, result = _gate(tmp_path, backend, entry=False) + assert error is None and result.identity is None + assert backend.hovers == [] + + +def test_structural_resolution_takes_no_hover(tmp_path): + backend = _HoverBackend("Delete note") + error, result = _gate(tmp_path, backend, rung="structural") + assert error is None and backend.hovers == [] + + +def test_backend_without_hover_keeps_old_behaviour(tmp_path): + backend = _HoverBackend("Delete note", can_hover=False) + error, result = _gate(tmp_path, backend) + assert error is None and result.identity is None + + +# -------------------------------------------------------------- compiler + + +def test_compiler_keeps_the_hover_response(tmp_path): + from openadapt_flow.compiler.compile import _hover_evidence + + hover_path = tmp_path / "0000_hover.png" + hover_path.write_bytes(_frame(POINT)) + bundle = tmp_path / "bundle" + bundle.mkdir() + ev = _hover_evidence(bundle, "step_000", _frame(), hover_path, POINT, dwell_ms=800) + assert ev is not None and ev.response and ev.crop == "templates/hover/step_000.png" + assert (bundle / ev.crop).is_file() + hover_path.write_bytes(_frame()) + assert _hover_evidence(bundle, "step_001", _frame(), hover_path, POINT) is None + assert ( + _hover_evidence(bundle, "s", _frame(), tmp_path / "missing.png", POINT) is None + ) + + +def test_recorder_saves_a_hover_frame_before_the_click(tmp_path): + from openadapt_flow.recorder import Recorder + + class Backend(_HoverBackend): + def click(self, x, y, *, double=False): + self.pointer = (x, y) + + backend = Backend() + rec = Recorder(backend, tmp_path / "rec", capture_hover=True, settle_timeout_s=0.2) + rec.click(*POINT) + events = [ + json.loads(line) + for line in (tmp_path / "rec" / "events.jsonl").read_text().splitlines() + ] + assert events[0]["hover_dwell_ms"] == H.HOVER_DWELL_MS + assert (tmp_path / "rec" / "frames" / "0000_hover.png").is_file() + assert backend.hovers == [POINT] + + +def test_recorder_without_capture_hover_does_not_hover(tmp_path): + from openadapt_flow.recorder import Recorder + + class Backend(_HoverBackend): + def click(self, x, y, *, double=False): + self.pointer = (x, y) + + backend = Backend() + rec = Recorder(backend, tmp_path / "rec", settle_timeout_s=0.2) + rec.click(*POINT) + assert backend.hovers == [] + assert not (tmp_path / "rec" / "frames" / "0000_hover.png").exists() From 7ea707dd679c40012a79e8c3621fd29201fa0b68 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 09:50:06 -0400 Subject: [PATCH 06/10] Keep pre-hover bundles' sealed digest unchanged An absent Step.hover is omitted from the content digest, like other additive v2 step fields; a recorded hover response is sealed content. Co-Authored-By: Claude Opus 5.5 --- .../flow/openadapt_flow/bundle_validation.py | 9 +++++ packages/flow/tests/test_bundle_schema_v2.py | 33 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/packages/flow/openadapt_flow/bundle_validation.py b/packages/flow/openadapt_flow/bundle_validation.py index 3387d3645e..4df2082638 100644 --- a/packages/flow/openadapt_flow/bundle_validation.py +++ b/packages/flow/openadapt_flow/bundle_validation.py @@ -184,6 +184,15 @@ class _SealedCanonicalOmission: "field; a populated band remains sealed content" ), ), + _SealedCanonicalOmission( + owner=Step, + field_name="hover", + omitted_values=(None,), + reason=( + "absent hover evidence predates the v2 field; a step CARRYING " + "a recorded hover response is new content and changes the digest" + ), + ), _SealedCanonicalOmission( owner=Step, field_name="submit_binding", diff --git a/packages/flow/tests/test_bundle_schema_v2.py b/packages/flow/tests/test_bundle_schema_v2.py index f2b29a6897..f6190b43bc 100644 --- a/packages/flow/tests/test_bundle_schema_v2.py +++ b/packages/flow/tests/test_bundle_schema_v2.py @@ -189,6 +189,8 @@ def _pre_frame_path_content(wf: Workflow, shape: str) -> dict: assert step.pop("drag_end_anchor") is None assert step.pop("selection_commit_key") is None assert step.pop("selection_region") is None + # Reviewed v2 omission: hover evidence postdates these seals. + assert step.pop("hover") is None # Reviewed v2 omission: submit-binding evidence postdates these seals. assert step.pop("submit_binding") is None assert step.pop("submit_binding_unbound_reason") is None @@ -254,6 +256,7 @@ def _write_synthetic_pre_field_bundle( assert step.pop("field_label") is None assert step.pop("selection_commit_key") is None assert step.pop("selection_region") is None + assert step.pop("hover") is None assert step.pop("submit_binding") is None assert step.pop("submit_binding_unbound_reason") is None assert step.pop("surface") is None @@ -603,6 +606,36 @@ def _binding(**changes): ) +def test_recorded_hover_evidence_is_sealed_content(tmp_path): + """Only an absent hover field is omitted; a recorded response is sealed.""" + + from openadapt_flow.ir import ActionKind, Anchor, HoverEvidence, Step + + b = _write_bundle_dir(tmp_path) + + def _bundle(hover=None): + step = Step( + id="save", + intent="click recorded visual target", + action=ActionKind.CLICK, + anchor=Anchor( + template="templates/btn.png", region=(0, 0, 10, 10), click_point=(5, 5) + ), + hover=hover, + ) + return Workflow(name="sealed-hover", steps=[step]) + + bare = _bundle() + file_hashes = bv.compute_file_hashes(bare, b) + bare_digest = bv.compute_content_digest(bare, file_hashes) + assert "hover" not in _step_payloads(bv._workflow_content(bare), "linear")[0] + ev = HoverEvidence(response=True, region=(-5, -5, 60, 40), crop="templates/h.png") + hovered = bv.compute_content_digest(_bundle(ev), file_hashes) + assert hovered != bare_digest + moved = ev.model_copy(update={"region": (-5, -5, 60, 41)}) + assert bv.compute_content_digest(_bundle(moved), file_hashes) != hovered + + def test_sealed_empty_defaults_do_not_implicitly_cross_schema_versions(): wf = _effect_workflow([Effect(kind=EffectKind.RECORD_WRITTEN, match={"a": "b"})]) rendered = wf.model_dump(mode="json", exclude={"manifest"}) From a708038895d806ef54aafa2d5fad43af4d60c48a Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 10:07:54 -0400 Subject: [PATCH 07/10] Fix four false stops in the extra identity looks - The copy drag now runs to the edges of the identifier's text box; it stopped inside the last glyph and copied one character short. - A split identifier (103 045 against 103045) is matched as one string, against the read and the known-ID list. - The check just before the click reuses the step-entry verdict when the identifier shows the same pixels, instead of re-reading the selected text. - A run-parameter mismatch that is only a look-alike misread (C0X1234 read as COX1234) gets the same looks. The mismatch stands unless the list or an exact copy of the run value settles the record. Co-Authored-By: Claude Opus 5.5 --- .../openadapt_flow/runtime/identity_looks.py | 72 ++++--- .../flow/openadapt_flow/runtime/replayer.py | 184 +++++++++++++++++- packages/flow/tests/test_identity_looks.py | 78 +++++++- .../tests/test_identity_looks_replayer.py | 2 +- 4 files changed, 307 insertions(+), 29 deletions(-) diff --git a/packages/flow/openadapt_flow/runtime/identity_looks.py b/packages/flow/openadapt_flow/runtime/identity_looks.py index 2eb21206e8..5e28d6ed28 100644 --- a/packages/flow/openadapt_flow/runtime/identity_looks.py +++ b/packages/flow/openadapt_flow/runtime/identity_looks.py @@ -134,24 +134,25 @@ def __post_init__(self) -> None: class LookBinding: """A clipboard read taken at step entry, reusable at actuation. - The actuation-time identity check takes no input. It reuses this read only - when the identifier cell on its frame shows the same pixels as one of the - crops seen at step entry. + The actuation-time identity check takes no input. It reuses this read, or + the verdict it settled, only when the identifier cell on its frame shows + the same pixels as one of the crops seen at step entry. """ step_id: str classes: frozenset[str] read: "ClipboardRead" crops: tuple[bytes, ...] + #: The step-entry verdict when the copy settled the record. + verdict: Optional[IdentityCheck] = None + + def same_pixels(self, step_id: str, crop: Optional[bytes]) -> bool: + return step_id == self.step_id and any(crops_match(crop, c) for c in self.crops) def matches( self, step_id: str, classes: frozenset[str], crop: Optional[bytes] ) -> bool: - return ( - step_id == self.step_id - and classes == self.classes - and any(crops_match(crop, c) for c in self.crops) - ) + return classes == self.classes and self.same_pixels(step_id, crop) @dataclass(frozen=True) @@ -223,34 +224,61 @@ def _magnified_look( return _look("magnified", "agrees") +def _listed(known_ids: KnownIdList, core: str) -> bool: + return core in known_ids.members_of_class(lookalike_class(core)) + + def _clipboard_look( - first_classes: frozenset[str], - vulnerable: frozenset[str], + first_tokens: Sequence[str], copied: Optional[ClipboardRead], exact: Sequence[str], known_ids: Optional[KnownIdList], ) -> IdentityLook: + """Judge a select-and-copy read against the first OCR read. + + The copy and the read must name the same identifiers up to look-alike + glyphs: token by token, or, when one side splits an identifier the other + keeps whole (``103 045`` against ``103045``), as the joined string. + """ if copied is None: return _look("clipboard", "unavailable", "not_configured") sha = copied.frame_sha256 if copied.text is None: return _look("clipboard", "unavailable", copied.reason or "nothing_copied", sha) tokens = identifier_tokens(copied.text) + if not tokens: + return _look("clipboard", "unavailable", "no_identifier_copied", sha) + vulnerable_read = any( + identity_mod._is_glyph_vulnerable_identifier(t) for t in first_tokens + ) by_class: dict[str, set[str]] = {} for tok in tokens: by_class.setdefault(lookalike_class(tok), set()).add(tok) - if set(by_class) != set(first_classes): + read_classes = {lookalike_class(t) for t in first_tokens} + exact_set = set(exact) + if set(by_class) == read_classes: + if any(len(v) != 1 for v in by_class.values()): + return _look("clipboard", "unavailable", "copy_ambiguous", sha) + copied_ids = {k: next(iter(v)) for k, v in by_class.items()} + vulnerable = { + lookalike_class(t) + for t in first_tokens + if identity_mod._is_glyph_vulnerable_identifier(t) + } + elif lookalike_class("".join(tokens)) == lookalike_class("".join(first_tokens)): + joined = "".join(tokens) + copied_ids = {lookalike_class(joined): joined} + vulnerable = set(copied_ids) if vulnerable_read else set() + exact_set |= {"".join(exact)} if exact else set() + else: # The copy did not cover the identifier OCR read (or covered more). return _look("clipboard", "unavailable", "copy_differs_from_read", sha) - if any(len(v) != 1 for v in by_class.values()): - return _look("clipboard", "unavailable", "copy_ambiguous", sha) - copied_ids = {k: next(iter(v)) for k, v in by_class.items()} if known_ids is not None and known_ids.complete: - listed = {m for k in copied_ids for m in known_ids.members_of_class(k)} - if any(core not in listed for core in copied_ids.values()): + all_listed = all(_listed(known_ids, c) for c in copied_ids.values()) + if not all_listed and not _listed(known_ids, "".join(tokens)): return _look("clipboard", "contradicts", "copied_id_not_listed", sha) exact_by_class: dict[str, set[str]] = {} - for e in exact: + for e in exact_set: exact_by_class.setdefault(lookalike_class(e), set()).add(e) for cls, core in copied_ids.items(): expected = exact_by_class.get(cls) @@ -298,7 +326,6 @@ def decide( if not is_glyph_abstain(first): return first first_classes = lookalike_classes(first.observed) - vulnerable = vulnerable_classes(first.observed) looks: list[IdentityLook] = [] for read, sha in frames: look = _frame_look(first_classes, read, sha, contradicts) @@ -319,7 +346,7 @@ def decide( ) if clipboard is not None: clip = _clipboard_look( - first_classes, vulnerable, clipboard, exact_expected, known_ids + identifier_tokens(first.observed), clipboard, exact_expected, known_ids ) looks.append(clip) if clip.outcome == "contradicts": @@ -364,7 +391,7 @@ def copy_region(lines: Iterable[Any], within: Region) -> Optional[Region]: if len(hits) != 1: return None x, y, w, h = (int(v) for v in hits[0]) - return (x - 2, y - 2, w + 4, h + 4) + return (x - 4, y - 2, w + 8, h + 4) def crop_png(png: bytes, region: Region) -> Optional[bytes]: @@ -447,9 +474,10 @@ def copy_identifier( clipboard.write_text(marker) pre = capture() mid_y = y + h // 2 - inset = max(2, min(6, w // 10)) if gesture == "drag": - backend.drag(x + inset, mid_y, x + w - inset, mid_y) + # From just before the first glyph to just past the last one: a + # drag that ends inside the last glyph leaves it unselected. + backend.drag(x + 1, mid_y, x + w - 1, mid_y) else: backend.click(x + w // 2, mid_y, double=True) for key in copy_keys: diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 5e3a125d64..a49614da29 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -35,6 +35,7 @@ from __future__ import annotations +import dataclasses import functools import hashlib import hmac @@ -185,6 +186,9 @@ from openadapt_flow.runtime.effects.adapter import ( verifier_identity as effect_verifier_identity, ) +from openadapt_flow.runtime.known_ids import ( + lookalike_class as known_ids_lookalike_class, +) from openadapt_flow.runtime.program_predicates import ( evaluate_program_predicate, exact_png_size, @@ -14038,6 +14042,26 @@ def _verify_identity_ocr( rw, rh, ) + if identifier_region is not None and not self._identity_input_looks: + # The check just before the click takes no input. When the step-entry + # check settled the record with a copy and the identifier shows the + # same pixels now, the same record is on screen: reuse that verdict. + binding = self.__dict__.get("_identity_look_bindings", {}).get(step.id) + if ( + binding is not None + and binding.verdict is not None + and binding.same_pixels( + step.id, looks_mod.crop_png(before_png, identifier_region) + ) + ): + reused = IdentityLook( + kind="clipboard", + outcome="resolves", + reason="same_pixels_as_step_entry_copy", + ) + return binding.verdict.model_copy( + update={"looks": [*binding.verdict.looks, reused]} + ) band = identity_mod.band_region( resolution.point, anchor.region[3], self.backend.viewport ) @@ -14137,10 +14161,23 @@ def native_attempt( ) check = native_attempt(before_png, first_row_lines) - if check.status == "verified" or identity_read_contradicts(check): - # A read that names a different record is final: a later read of - # the same screen must never overrule it into "verified". + if check.status == "verified": return check + if identity_read_contradicts(check): + # A read that names a different record is final: a later read of + # the same screen must never overrule it into "verified". The one + # exception is a run parameter OCR read with look-alike glyphs + # (C0X1234 read as COX1234): extra looks may confirm the exact + # value; otherwise this mismatch stands. + return self._lookalike_param_looks( + step, + check, + native_attempt, + params, + workflow, + identifier_region, + before_png, + ) upscaled = identity_mod.upscale_crop(before_png, live_region) if upscaled is None: return check @@ -14240,6 +14277,131 @@ def with_looks(final: IdentityCheck) -> IdentityCheck: ) return with_looks(better_identity_read(check, row_check)) + def _band_check( + self, + anchor: Anchor, + observed: str, + run_values: Mapping[str, str], + workflow: Workflow, + ) -> IdentityCheck: + """The OCR-tier verdict for ``observed`` with the given run values.""" + tmpl = anchor.identity_template + if tmpl is not None and tmpl.tokens: + from openadapt_flow.runtime import identity_template as itmpl + + return itmpl.verify_template_identity( + tmpl, observed, params=dict(run_values), param_examples=workflow.params + ) + return identity_mod.verify_target_identity( + anchor.context_text or "", + observed, + params=dict(run_values), + param_examples=workflow.params, + ) + + def _lookalike_param_read( + self, + check: IdentityCheck, + anchor: Anchor, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + ) -> Optional[IdentityCheck]: + """A run-parameter mismatch that is only a look-alike misread, as the + look-alike abstain it would be, or None. + + The parameter's identifier is read with look-alike glyphs (the run + value ``C0X1234`` read as ``COX1234``). The band is checked again with + the value as OCR read it; when every other identity budget then passes + and only the look-alike identifier remains, OCR cannot say which of the + two it is. Used only to decide whether extra looks may run; the caller + keeps the mismatch unless a look settles the record. + """ + if check.status != "mismatch" or check.mode != "param" or not check.param: + return None + run_values = runtime_params_for_gui(params) + run = run_values.get(check.param) + if not isinstance(run, str): + return None + run_tokens = looks_mod.identifier_tokens(run) + if not any(identity_mod._is_glyph_vulnerable_identifier(t) for t in run_tokens): + return None + seen = looks_mod.identifier_tokens(check.observed) + as_read: list[str] = [] + for token in run_tokens: + same = { + o + for o in seen + if known_ids_lookalike_class(o) == known_ids_lookalike_class(token) + } + if len(same) != 1: + return None + as_read.append(same.pop()) + if as_read == run_tokens: + return None + again = self._band_check( + anchor, + check.observed, + {**run_values, check.param: " ".join(as_read)}, + workflow, + ) + if not looks_mod.is_glyph_abstain(again): + return None + return again + + def _lookalike_param_looks( + self, + step: Step, + check: IdentityCheck, + native_attempt: Callable[[bytes], IdentityCheck], + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + identifier_region: Optional[Region], + frame_png: bytes, + ) -> IdentityCheck: + """Extra looks for a look-alike run-parameter mismatch. + + Returns the looks' ``verified`` verdict, a named halt, or ``check`` + unchanged (with the looks recorded). Never turns a mismatch into an + abstain. + """ + anchor = step.anchor + assert anchor is not None + first = self._lookalike_param_read(check, anchor, params, workflow) + if first is None or not ( + self.known_ids is not None or self._copy_look_possible(identifier_region) + ): + return check + + def read(png: bytes) -> IdentityCheck: + got = native_attempt(png) + return self._lookalike_param_read(got, anchor, params, workflow) or got + + matches: Optional[Callable[[str], bool]] = None + if anchor.identity_template is not None and anchor.identity_template.tokens: + from openadapt_flow.runtime import identity_template as itmpl + + matches = functools.partial( + itmpl.recorded_identifier_matches, anchor.identity_template + ) + looked = self._look_again( + step, + first, + None, + read, + params, + workflow, + identifier_region, + frame_png, + matches, + ) + if ( + looked.status == "verified" + or looked.known_id_halt is not None + or looked.look_halt is not None + ): + return looked + return check.model_copy(update={"looks": list(looked.looks)}) + def _copy_look_allowed(self) -> bool: """The copy look is input: never in governed or qualification runs.""" return ( @@ -14271,7 +14433,7 @@ def _look_again( self, step: Step, first: IdentityCheck, - magnified: IdentityCheck, + magnified: Optional[IdentityCheck], read: Callable[[bytes], IdentityCheck], params: Optional[Mapping[str, RuntimeParamScalar]], workflow: Workflow, @@ -14398,7 +14560,7 @@ def _look_again( known_list = None if known is not None: known_list = getattr(known, "known_ids", known) - return looks_mod.decide( + decided = looks_mod.decide( first, frames=frames, magnified=magnified, @@ -14413,6 +14575,18 @@ def _look_again( else [], ), ) + binding = bindings.get(step.id) + if ( + self._identity_input_looks + and binding is not None + and decided.status == "verified" + and any( + look.kind == "clipboard" and look.outcome == "resolves" + for look in decided.looks + ) + ): + bindings[step.id] = dataclasses.replace(binding, verdict=decided) + return decided # -- typed-input verification ------------------------------------------------- diff --git a/packages/flow/tests/test_identity_looks.py b/packages/flow/tests/test_identity_looks.py index bb48353568..03e47211ee 100644 --- a/packages/flow/tests/test_identity_looks.py +++ b/packages/flow/tests/test_identity_looks.py @@ -279,9 +279,85 @@ def test_copy_region_is_the_single_line_holding_a_lookalike_id(): mrn = NS(text="AC50061", region=(10, 5, 90, 20)) name = NS(text="Smith, John", region=(200, 5, 120, 20)) dob = NS(text="01/15/1980", region=(400, 5, 110, 20)) - assert L.copy_region([mrn, name, dob], within) == (8, 3, 94, 24) + assert L.copy_region([mrn, name, dob], within) == (6, 3, 98, 24) other = NS(text="ZZ10O1", region=(520, 5, 60, 20)) assert L.copy_region([mrn, other], within) is None # two candidates assert L.copy_region([name, dob], within) is None # none outside = NS(text="AC50061", region=(10, 100, 90, 20)) assert L.copy_region([outside], within) is None + + +def test_a_split_identifier_is_matched_as_one_string(): + first = _abstain("103 O45 Smith, John 01/15/1980") + kw = dict( + frames=_frames(first, first), + magnified=first, + contradicts=identity_read_contradicts, + exact_expected=("103", "045"), + ) + assert L.decide(first, clipboard=_copy("103045"), **kw).status == "verified" + assert L.decide(first, clipboard=_copy("1O3045"), **kw).status == "mismatch" + ids = KnownIdList(ids=("103 045", "ZZ1000"), source="t", complete=True) + listed = L.decide(first, clipboard=_copy("103 045"), known_ids=ids, **kw) + assert listed.status == "verified" + + +def test_lookalike_param_mismatch_takes_looks_but_never_softens(monkeypatch): + from openadapt_flow.ir import Anchor, Step, Workflow + from openadapt_flow.runtime import identity as I + from openadapt_flow.runtime.replayer import Replayer + + band = "AC50061 Smith, John 01/15/1980" + anchor = Anchor( + template="t.png", + region=(0, 0, 10, 10), + click_point=(5, 5), + context_text=band, + identifier_region=(0, 0, 400, 20), + ) + step = Step(id="s", intent="open", action="click", anchor=anchor) + wf = Workflow(name="p", params={"mrn": "AC50061"}, steps=[step]) + params = {"mrn": "AC50061"} + + def read(observed): + return I.verify_target_identity( + band, observed, params=params, param_examples=wf.params + ) + + misread = read("AC5OO61 Smith, John 01/15/1980") + assert misread.status == "mismatch" and misread.mode == "param" + + class _Clip: + def read_text(self): + return None + + def write_text(self, text): + pass + + r = Replayer( + object(), # type: ignore[arg-type] + vision=object(), + read_clipboard=_Clip(), + identity_copy=L.IdentityCopyConfig(), + ) + normal = r._lookalike_param_read(misread, anchor, params, wf) + assert L.is_glyph_abstain(normal) + other = read("AC5OO61 Jones, Mary 02/02/1990") + assert r._lookalike_param_read(other, anchor, params, wf) is None + + def looked(status): + return lambda *a, **k: misread.model_copy( + update={"status": status, "abstain_reason": None} + ) + + native = lambda png: misread # noqa: E731 + monkeypatch.setattr(r, "_look_again", looked("verified")) + got = r._lookalike_param_looks( + step, misread, native, params, wf, (0, 0, 400, 20), b"" + ) + assert got.status == "verified" + monkeypatch.setattr(r, "_look_again", looked("abstain")) + got = r._lookalike_param_looks( + step, misread, native, params, wf, (0, 0, 400, 20), b"" + ) + assert got.status == "mismatch" # the original mismatch stands diff --git a/packages/flow/tests/test_identity_looks_replayer.py b/packages/flow/tests/test_identity_looks_replayer.py index aadf7526ed..27d23ccd52 100644 --- a/packages/flow/tests/test_identity_looks_replayer.py +++ b/packages/flow/tests/test_identity_looks_replayer.py @@ -212,7 +212,7 @@ def test_actuation_check_reuses_the_copy_only_on_matching_pixels(): again = r._verify_identity(step, res, b.frame, params, wf, bundle_dir=None) assert len(b.drags) == 1 assert again.status == "verified" - assert ("clipboard", "resolves") in [(x.kind, x.outcome) for x in again.looks] + assert again.looks[-1].reason == "same_pixels_as_step_entry_copy" # The twin now sits where the record was: the binding must not carry over. twin = _frame("AC5OO61") b.frame = twin.png From 52a8c528ec79bdc9360e22c2e26d90ec3b03068c Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 10:38:18 -0400 Subject: [PATCH 08/10] Guard pixel clicks against a screen change between check and click Consequential identity-gated clicks were already bound to the exact checked frame. Every other pixel click now compares the target, identity and hover regions on a fresh frame taken right before the click with the frame the step was checked on (the hover frame after a hover look). On a change it does not click: it re-resolves the target and re-runs the identity check on the fresh frame with no input, and clicks only when the next frame matches. Two re-observations that still see a change, or a target or identity that cannot be confirmed again, refuse the click. Each change is recorded on the step. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/LIMITS.md | 14 ++ packages/flow/openadapt_flow/ir.py | 20 ++ .../flow/openadapt_flow/runtime/replayer.py | 197 +++++++++++++++ packages/flow/tests/test_check_to_click.py | 228 ++++++++++++++++++ 4 files changed, 459 insertions(+) create mode 100644 packages/flow/tests/test_check_to_click.py diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index 0be440a291..d526875029 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -234,6 +234,20 @@ and FreeRDP backends can hover. The Aardwolf RDP transport and the native Windows, macOS and Linux backends can't move the pointer without a button yet, so they take no hover look. +A pixel step is checked on one frame and clicked a moment later. Consequential +identity-gated clicks were already bound to the exact checked frame. Every +other pixel click now gets a check-to-click guard (`Replayer._pixel_recheck`): +right before the click, the runtime takes a fresh frame and compares the +target region, the identity region and the hover response with the frame the +step was checked on. If one changed (a dialog opened, the list re-sorted), it +doesn't click. It re-resolves the target and re-runs the identity check on +the fresh frame, with no input, and clicks only when the next frame matches +that one. After two re-observations that still see a change, or when the +target or its identity can't be confirmed again, it refuses the click. Each +change is recorded on the step (`StepResult.check_to_click`). A change smaller +than 4 pixels, or under 24 levels per channel, doesn't count. Structured +targets are bound by element tokens instead and aren't rechecked. + A consequential click must be tied to a record before Flow delivers it. Inside a table row, the row is that record. An icon-only Save that floats beside a form (openIMIS's claim Save has no text, accessible name, or tooltip) has no diff --git a/packages/flow/openadapt_flow/ir.py b/packages/flow/openadapt_flow/ir.py index 87f6f181b6..fc923f0c62 100644 --- a/packages/flow/openadapt_flow/ir.py +++ b/packages/flow/openadapt_flow/ir.py @@ -3766,6 +3766,20 @@ class EffectFieldCheck(BaseModel): verification_tier: Optional[int] = None +class CheckToClickRecheck(BaseModel): + """The screen changed between a pixel step's check and its click. + + Records which checked regions changed on the fresh frame taken just + before the click, and what the runtime did: re-observed the target and + its identity on that frame, or refused the click. No pixels. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + changed: list[Literal["target", "identity", "hover", "frame_size"]] + outcome: Literal["reobserved", "aborted"] + + class StepResult(BaseModel): step_id: str intent: str @@ -3891,6 +3905,10 @@ class StepResult(BaseModel): identity_reobserved: bool = False identity_first_read: Optional[IdentityCheck] = None identity_first_frame_png: Optional[str] = None + #: Pixel clicks only: each time the target, identity or hover region + #: changed between the step's check and its click (see + #: ``Replayer._pixel_recheck``). Empty, and omitted, when nothing changed. + check_to_click: list[CheckToClickRecheck] = Field(default_factory=list) heal: Optional[HealEvent] = None error: Optional[str] = None before_png: Optional[str] = None # run-dir-relative paths @@ -3911,6 +3929,8 @@ def _serialize_compatible(self, handler: Any) -> dict[str, Any]: data.pop("surface", None) if isinstance(data, dict) and not self.effect_field_checks: data.pop("effect_field_checks", None) + if isinstance(data, dict) and not self.check_to_click: + data.pop("check_to_click", None) return data diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 66871a1251..3b13c83d44 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -58,6 +58,7 @@ Mapping, Optional, TypeVar, + Union, cast, ) from urllib.parse import urlsplit @@ -93,6 +94,7 @@ Anchor, ApiBinding, BusinessDecisionEvidence, + CheckToClickRecheck, EffectVerificationEvidence, ExecutionTargetKind, FreshActuationEvent, @@ -215,6 +217,13 @@ _DeliveryResultT = TypeVar("_DeliveryResultT") _MAX_FRESH_ACTUATION_REACQUISITIONS = 2 +# Check-to-click guard on the pixel path (``Replayer._pixel_recheck``): how +# often a changed target or identity region is re-observed before the click +# is refused, and how much a pixel must change to count (remote encoders +# shift colours by a few levels; a dialog or a new row changes far more). +_PIXEL_RECHECK_REOBSERVES = 2 +_PIXEL_RECHECK_LEVEL = 24 +_PIXEL_RECHECK_MIN_PIXELS = 4 # How long a step that raises a browser dialog waits for it to open. _DIALOG_WAIT_S = 5.0 # Private module capability. The public ``run`` API cannot mint a durable @@ -9141,6 +9150,7 @@ def _entry_identity_gate_error( pixel-matched binding. """ self.__dict__.setdefault("_identity_look_bindings", {}).pop(step.id, None) + self.__dict__.setdefault("_hover_frames", {}).pop(step.id, None) self._identity_input_looks = True try: return self._identity_gate_error(step, *args, **kwargs) @@ -11566,6 +11576,20 @@ def _act_without_dialog( "— refusing raw coordinate delivery; run aborted" ) else: + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + resolution = rechecked + x, y = resolution.point refusal = self._delivery_authorization_refusal( workflow, params, step, result ) @@ -11684,6 +11708,19 @@ def _act_without_dialog( "identity verification to delivery; run aborted" ) elif isinstance(self.backend, RichPointerActionBackend): + rechecked = self._pixel_recheck( + step, + resolution, + before_png, + params, + workflow, + bundle_dir, + result, + run_dir, + ) + if isinstance(rechecked, str): + return rechecked + x, y = rechecked.point refusal = self._delivery_authorization_refusal( workflow, params, step, result ) @@ -14379,6 +14416,9 @@ def check(verdict: str, reason: str, png: Optional[bytes]) -> IdentityCheck: if evidence.crop and bundle_dir is not None else None ) + # The click that follows is checked against this frame, not the one + # before the pointer moved (see _pixel_recheck). + self.__dict__.setdefault("_hover_frames", {})[step.id] = hover_png verdict, reason = hover_mod.compare( recorded_response=evidence.response, recorded_region=evidence.region, @@ -14515,6 +14555,163 @@ def read(png: bytes) -> IdentityCheck: return looked return check.model_copy(update={"looks": list(looked.looks)}) + def _check_regions( + self, step: Step, resolution: Resolution, matched_region: Optional[Region] + ) -> dict[str, Region]: + """The regions a pixel step was checked on, at ``resolution``. + + ``target``: where the target matched (or a box of the recorded + target's size at the point). ``identity``: the identifier region, or + the identity band's row, when the step has identity evidence. + ``hover``: the recorded hover response, when there is one. + """ + anchor = step.anchor + px, py = int(resolution.point[0]), int(resolution.point[1]) + regions: dict[str, Region] = {} + if matched_region is not None: + regions["target"] = matched_region + elif anchor is not None: + w, h = int(anchor.region[2]), int(anchor.region[3]) + regions["target"] = (px - w // 2, py - h // 2, w, h) + else: + regions["target"] = (px - 8, py - 8, 16, 16) + if anchor is not None: + if anchor.identifier_region is not None: + rx, ry, rw, rh = anchor.identifier_region + regions["identity"] = ( + rx + px - anchor.click_point[0], + ry + py - anchor.click_point[1], + rw, + rh, + ) + elif anchor.context_text or anchor.identity_template: + h = int(anchor.region[3]) + regions["identity"] = ( + 0, + py - h // 2 - 4, + int(self.backend.viewport[0]), + h + 8, + ) + if step.hover is not None and step.hover.region is not None: + dx, dy, w, h = step.hover.region + regions["hover"] = (px + dx, py + dy, w, h) + return regions + + @staticmethod + def _changed_regions( + before_png: bytes, fresh_png: bytes, regions: Mapping[str, Region] + ) -> list[str]: + """Names of ``regions`` whose pixels differ between two frames.""" + import numpy as np + from PIL import Image + + try: + with ( + Image.open(io.BytesIO(before_png)) as a_img, + Image.open(io.BytesIO(fresh_png)) as b_img, + ): + a = np.asarray(a_img.convert("RGB"), dtype=np.int16) + b = np.asarray(b_img.convert("RGB"), dtype=np.int16) + except Exception: # noqa: BLE001 - an unreadable frame is a change + return ["frame_size"] + if a.shape != b.shape: + return ["frame_size"] + height, width = a.shape[:2] + changed: list[str] = [] + for name, (x, y, w, h) in regions.items(): + x0, y0 = max(0, int(x)), max(0, int(y)) + x1, y1 = min(width, int(x + w)), min(height, int(y + h)) + if x1 <= x0 or y1 <= y0: + continue + diff = np.abs(a[y0:y1, x0:x1] - b[y0:y1, x0:x1]).max(axis=2) + if int((diff > _PIXEL_RECHECK_LEVEL).sum()) >= _PIXEL_RECHECK_MIN_PIXELS: + changed.append(name) + return changed + + def _pixel_recheck( + self, + step: Step, + resolution: Resolution, + checked_png: bytes, + params: Mapping[str, RuntimeParamScalar], + workflow: Workflow, + bundle_dir: Path, + result: StepResult, + run_dir: Path, + ) -> Union[Resolution, str]: + """Guard the gap between a pixel step's check and its click. + + Right before the click, take a fresh frame and compare the target, + identity and hover regions with the frame the step was checked on (the + hover frame when the step took a hover look). When one changed (a + dialog opened over the target, the list re-sorted), do not click: + re-resolve the target and re-run the identity check on the fresh + frame, with no input, and compare again on the next frame. After + :data:`_PIXEL_RECHECK_REOBSERVES` re-observations that still see + changes, or when the target or its identity cannot be confirmed again, + refuse the click. Structured (DOM / accessibility) targets are bound + by element tokens instead and are not rechecked here. + + Returns the resolution to click, or the refusal. + """ + if resolution.rung == "structural": + return resolution + baseline = self.__dict__.get("_hover_frames", {}).get(step.id) or checked_png + current = resolution + region = self._active_delivery_region + for attempt in range(_PIXEL_RECHECK_REOBSERVES + 1): + fresh = self.backend.screenshot() + changed = self._changed_regions( + baseline, fresh, self._check_regions(step, current, region) + ) + if not changed: + if current is not resolution: + result.resolution = current + return current + what = ", ".join(changed) + if attempt == _PIXEL_RECHECK_REOBSERVES: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + break + fresh_resolution, fresh_region, error = self._resolve_step( + step, fresh, bundle_dir, workflow + ) + if error is None and fresh_resolution is not None: + error = self._identity_gate_error( + step, + fresh_resolution, + fresh, + params, + workflow, + bundle_dir, + result, + evidence_run_dir=run_dir, + ) + if error is not None or fresh_resolution is None: + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="aborted") # type: ignore[arg-type] + ) + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}): the screen changed at the " + f"{what} region between the check and the click, and the " + "target could not be confirmed again on a fresh frame " + f"({error or 'target not found'}) — click not delivered" + ) + result.check_to_click.append( + CheckToClickRecheck(changed=changed, outcome="reobserved") # type: ignore[arg-type] + ) + current, region, baseline = fresh_resolution, fresh_region, fresh + result.safety_halt = True + result.failure_category = "safety_halt" + return ( + f"Step '{step.id}' ({step.intent}): the screen kept changing at the " + f"{what} region between the check and the click — click not " + "delivered; run aborted" + ) + def _copy_look_allowed(self) -> bool: """The copy look is input: never in governed or qualification runs.""" return ( diff --git a/packages/flow/tests/test_check_to_click.py b/packages/flow/tests/test_check_to_click.py new file mode 100644 index 0000000000..ee8c607259 --- /dev/null +++ b/packages/flow/tests/test_check_to_click.py @@ -0,0 +1,228 @@ +"""Check-to-click guard on the pixel path (``Replayer._pixel_recheck``). + +A pixel step is resolved and its identity checked on one frame, and clicked a +moment later. If the screen changes in between (a dialog opens over the +target, the list re-sorts), the click must not land on whatever is there now. +Right before the click the runtime compares the target and identity regions on +a fresh frame with the checked frame. On a change it re-resolves and re-checks +on the fresh frame, and clicks only when that holds; otherwise it refuses. +""" + +from __future__ import annotations + +import io +from pathlib import Path + +from PIL import Image, ImageDraw + +import openadapt_flow.vision as vision +from openadapt_flow.ir import Anchor, Resolution, Step, StepResult, Workflow +from openadapt_flow.runtime.replayer import Replayer + +SIZE = (640, 400) +BUTTON = (100, 120, 140, 44) # x, y, w, h of the recorded target + + +def _frame( + *, + button_y: int = BUTTON[1], + dialog: bool = False, + clock: str = "10:00", + noise: int = 0, +) -> bytes: + img = Image.new("RGB", SIZE, (244, 246, 248)) + d = ImageDraw.Draw(img) + x, _, w, h = BUTTON + d.rectangle((x, button_y, x + w, button_y + h), fill=(45, 111, 216)) + d.text((x + 22, button_y + 15), "Open chart", fill="white") + d.rectangle((x, button_y + 70, x + w, button_y + 70 + h), fill=(180, 60, 60)) + d.text((x + 22, button_y + 85), "Delete", fill="white") + d.text((560, 380), clock, fill="black") # a clock far from the target + if dialog: + d.rectangle((60, 90, 420, 260), fill="white", outline="black") + d.text((80, 110), "Session expires soon. Continue?", fill="black") + d.rectangle((150, 125, 260, 165), fill=(220, 220, 220), outline="black") + d.text((175, 138), "Continue", fill="black") + if noise: + px = img.load() + for i in range(0, SIZE[0], 7): + r, g, b = px[i, 5] + px[i, 5] = (r - noise, g, b) + out = io.BytesIO() + img.save(out, format="PNG") + return out.getvalue() + + +class _Backend: + """Serves a scripted frame sequence; records clicks.""" + + viewport = SIZE + + def __init__(self, frames: list[bytes]): + self.frames = list(frames) + self.shots = 0 + self.clicks: list[tuple[int, int]] = [] + + def screenshot(self) -> bytes: + frame = self.frames[min(self.shots, len(self.frames) - 1)] + self.shots += 1 + return frame + + def click(self, x: int, y: int, *, double: bool = False) -> None: + self.clicks.append((x, y)) + + def type_text(self, text: str) -> None: # pragma: no cover - unused + pass + + def press(self, key: str) -> None: # pragma: no cover - unused + pass + + def scroll(self, dx: int, dy: int) -> None: # pragma: no cover - unused + pass + + +def _bundle(tmp_path: Path) -> tuple[Path, Step, Workflow]: + checked = Image.open(io.BytesIO(_frame())) + x, y, w, h = BUTTON + (tmp_path / "templates").mkdir() + checked.crop((x, y, x + w + 1, y + h + 1)).save(tmp_path / "templates" / "t.png") + step = Step( + id="step_000", + intent="click 'Open chart'", + action="click", + anchor=Anchor( + template="templates/t.png", + region=(x, y, w + 1, h + 1), + click_point=(x + w // 2, y + h // 2), + ), + ) + return tmp_path, step, Workflow(name="c2c", params={}, steps=[step]) + + +def _click(tmp_path: Path, frames: list[bytes], rung: str = "template"): + bundle, step, wf = _bundle(tmp_path) + backend = _Backend(frames) + replayer = Replayer(backend, vision=vision) + x, y, w, h = BUTTON + res = Resolution( + rung=rung, point=(x + w // 2, y + h // 2), confidence=0.99, elapsed_ms=1.0 + ) + replayer._active_delivery_region = (x, y, w + 1, h + 1) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + error = replayer._act( + step, + res, + {}, + workflow=wf, + step_index=0, + bundle_dir=bundle, + before_png=_frame(), + result=result, + run_dir=tmp_path / "run", + ) + return error, result, backend + + +def test_unchanged_screen_clicks_once(tmp_path): + error, result, backend = _click(tmp_path, [_frame()]) + assert error is None + assert backend.clicks == [(170, 142)] + assert result.check_to_click == [] + assert "check_to_click" not in result.model_dump() + + +def test_change_away_from_the_target_does_not_stop_the_click(tmp_path): + error, result, backend = _click(tmp_path, [_frame(clock="10:01")]) + assert error is None and len(backend.clicks) == 1 + assert result.check_to_click == [] + + +def test_encoder_noise_is_not_a_change(tmp_path): + error, _, backend = _click(tmp_path, [_frame(noise=6)]) + assert error is None and len(backend.clicks) == 1 + + +def test_dialog_between_check_and_click_aborts_the_click(tmp_path): + error, result, backend = _click(tmp_path, [_frame(dialog=True)]) + assert backend.clicks == [] # nothing was delivered under the dialog + assert error is not None and "click not delivered" in error + assert result.safety_halt is True + assert [e.outcome for e in result.check_to_click] == ["aborted"] + assert "target" in result.check_to_click[0].changed + + +def test_moved_target_is_reobserved_and_clicked_where_it_is_now(tmp_path): + moved = _frame(button_y=BUTTON[1] + 30) + error, result, backend = _click(tmp_path, [moved, moved]) + assert error is None + assert backend.clicks == [(170, 172)] + assert [e.outcome for e in result.check_to_click] == ["reobserved"] + + +def test_a_screen_that_keeps_changing_is_refused(tmp_path): + frames = [_frame(button_y=BUTTON[1] + d) for d in (10, 20, 30, 40)] + error, result, backend = _click(tmp_path, frames) + assert backend.clicks == [] + assert error is not None and "kept changing" in error + assert [e.outcome for e in result.check_to_click] == [ + "reobserved", + "reobserved", + "aborted", + ] + + +def test_structural_targets_are_not_rechecked(tmp_path): + backend = _Backend([_frame(dialog=True)]) + replayer = Replayer(backend, vision=vision) + bundle, step, wf = _bundle(tmp_path) + res = Resolution(rung="structural", point=(170, 142), confidence=1.0, elapsed_ms=1) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + got = replayer._pixel_recheck( + step, res, _frame(), {}, wf, bundle, result, tmp_path / "run" + ) + assert got is res and backend.shots == 0 + + +def test_identity_region_change_is_detected(): + replayer = Replayer(_Backend([_frame()]), vision=vision) + step = Step( + id="s", + intent="open", + action="click", + anchor=Anchor( + template="t.png", + region=(100, 120, 141, 45), + click_point=(170, 142), + context_text="Open chart row identity text", + ), + ) + res = Resolution(rung="template", point=(170, 142), confidence=1.0, elapsed_ms=1) + regions = replayer._check_regions(step, res, None) + assert set(regions) == {"target", "identity"} + assert regions["identity"][2] == SIZE[0] # the whole row + changed = Replayer._changed_regions( + _frame(), _frame(button_y=BUTTON[1] + 30), regions + ) + assert "target" in changed and "identity" in changed + + +def test_a_hover_look_is_the_baseline_for_the_click(tmp_path): + # The hover look moved the pointer and the target now shows its hover + # state; the click is checked against that frame, not the earlier one. + hovered = _frame(clock="10:00", noise=0) + img = Image.open(io.BytesIO(hovered)).convert("RGB") + ImageDraw.Draw(img).rectangle((100, 120, 240, 164), outline="yellow", width=3) + out = io.BytesIO() + img.save(out, format="PNG") + hovered = out.getvalue() + bundle, step, wf = _bundle(tmp_path) + backend = _Backend([hovered]) + replayer = Replayer(backend, vision=vision) + replayer.__dict__["_hover_frames"] = {step.id: hovered} + res = Resolution(rung="template", point=(170, 142), confidence=1.0, elapsed_ms=1) + replayer._active_delivery_region = (100, 120, 141, 45) + result = StepResult(step_id=step.id, intent=step.intent, ok=False) + got = replayer._pixel_recheck( + step, res, _frame(), {}, wf, bundle, result, tmp_path / "run" + ) + assert got is res and result.check_to_click == [] From b97f6715ac3daa2c22972b22a212b1bd62fcaa8b Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 15:02:02 -0400 Subject: [PATCH 09/10] Fix CI: harness types, a docs path, and an OCR-dependent test - identity_looks_dense: pass a typed click point and ActionKind (mypy). - LIMITS.md: name identity_looks.py by its package path, which the docs consistency check resolves. - The look-alike twin test asserts the stop; it accepts a read that is already final, as Linux OCR builds read it. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/LIMITS.md | 2 +- .../openadapt_flow/validation/identity_looks_dense.py | 7 ++++--- packages/flow/tests/test_identity_looks_replayer.py | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index c69bd8b658..01ca44f318 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -190,7 +190,7 @@ before and after numbers are in Before a step stops on a look-alike identifier, the replayer takes extra looks that change nothing -([`runtime/identity_looks.py`](../openadapt_flow/runtime/identity_looks.py)): +([`openadapt_flow/runtime/identity_looks.py`](../openadapt_flow/runtime/identity_looks.py)): the same read on two fresh screenshots, the 2x re-read, the known-ID list when one is configured, and, where the surface allows it, a select-and-copy of the identifier's text box (`runtime.identity_copy_look`). OCR looks can't settle diff --git a/packages/flow/openadapt_flow/validation/identity_looks_dense.py b/packages/flow/openadapt_flow/validation/identity_looks_dense.py index 59193e7722..259d02e9b9 100644 --- a/packages/flow/openadapt_flow/validation/identity_looks_dense.py +++ b/packages/flow/openadapt_flow/validation/identity_looks_dense.py @@ -278,7 +278,7 @@ def run( MIN_OCR_CONFIDENCE, _discriminative_crop_region, ) - from openadapt_flow.ir import Anchor, Resolution, Step, Workflow + from openadapt_flow.ir import ActionKind, Anchor, Resolution, Step, Workflow from openadapt_flow.runtime.identity import ( band_region, context_region_from_lines, @@ -344,7 +344,8 @@ def run( ti, si = idx[id(pair.target)], idx[id(pair.sibling)] for config in CLICK_CONFIGS: pi = 0 if config == "click_name" else 1 - rec_click = rec.points[ti][pi] + name_pt, open_pt = rec.points[ti][0], rec.points[ti][1] + rec_click = name_pt if pi == 0 else open_pt crop = _discriminative_crop_region(rec_bgr, rec_click) box = context_region_from_lines( rec_lines, @@ -384,7 +385,7 @@ def run( step = Step( id=f"s{ti}_{config}", intent="open record", - action="click", + action=ActionKind.CLICK, anchor=anchor, risk="irreversible", ) diff --git a/packages/flow/tests/test_identity_looks_replayer.py b/packages/flow/tests/test_identity_looks_replayer.py index 27d23ccd52..8f4f3f2d92 100644 --- a/packages/flow/tests/test_identity_looks_replayer.py +++ b/packages/flow/tests/test_identity_looks_replayer.py @@ -172,9 +172,13 @@ def test_copy_equal_to_the_run_value_verifies_the_correct_row(): def test_copy_of_the_look_alike_twin_stops_the_run(): r, b, step, res, wf, params, live = _setup("AC5OO61", param_mode=True) check = _entry(r, step, res, wf, params, live.png) + # It stops either way. Where OCR reads the twin as a look-alike of the + # run value, the copy is what names it; where OCR already reads another + # record (some OCR builds), the read is final and takes no looks. assert check.status == "mismatch" - assert check.looks[-1].kind == "clipboard" - assert check.looks[-1].outcome == "contradicts" + if check.looks: + assert check.looks[-1].kind == "clipboard" + assert check.looks[-1].outcome == "contradicts" def test_recorded_record_mode_copy_cannot_accept(): From 0b30ab132b1b70075600bbfe90d87eb8003ebaaa Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 19 Oct 2026 15:05:00 -0400 Subject: [PATCH 10/10] Fix CI for the hover and check-to-click change - replayer: assert the anchor before the identity check so mypy can narrow it when a hover check stands in for the OCR check. - LIMITS.md: name hover_evidence.py by its package path. - test_state_dependency: the never-settling fixture now animates a strip clear of the click target. An animation over the target itself is a change between check and click, which the new guard refuses; the test is about the readiness default, not about clicking under a moving bar. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/LIMITS.md | 2 +- packages/flow/openadapt_flow/runtime/replayer.py | 2 ++ packages/flow/tests/test_state_dependency.py | 7 +++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/flow/docs/LIMITS.md b/packages/flow/docs/LIMITS.md index 4a5b530bf4..fc03f9e482 100644 --- a/packages/flow/docs/LIMITS.md +++ b/packages/flow/docs/LIMITS.md @@ -246,7 +246,7 @@ When the recorder can move the pointer without clicking pointer on each click target first and keeps that frame. For a click with no other identity evidence, the compiler stores what changed near the target, such as the control's highlight or tooltip, as the step's hover evidence -([`runtime/hover_evidence.py`](../openadapt_flow/runtime/hover_evidence.py)). +([`openadapt_flow/runtime/hover_evidence.py`](../openadapt_flow/runtime/hover_evidence.py)). At replay, on the pixel rungs, the runtime rests the pointer on the resolved target, waits the same 0.8 s, and compares. A different response, or a response where the recording saw none, stops the run before the click. A diff --git a/packages/flow/openadapt_flow/runtime/replayer.py b/packages/flow/openadapt_flow/runtime/replayer.py index 1581015fd8..5310a2a010 100644 --- a/packages/flow/openadapt_flow/runtime/replayer.py +++ b/packages/flow/openadapt_flow/runtime/replayer.py @@ -9790,6 +9790,8 @@ def _identity_gate_error( ) ): return None + # A hover check needs an anchor too (see _hover_check). + assert step.anchor is not None check = ( hover_check diff --git a/packages/flow/tests/test_state_dependency.py b/packages/flow/tests/test_state_dependency.py index 7f0127e371..5474347186 100644 --- a/packages/flow/tests/test_state_dependency.py +++ b/packages/flow/tests/test_state_dependency.py @@ -79,13 +79,16 @@ def __init__(self, viewport=VIEWPORT): def screenshot(self): # Move a large black bar across the frame each poll so the PERCEPTUAL # hash actually differs frame to frame (a flat shade change would hash - # identically and count as settled). + # identically and count as settled). The bar runs along the bottom, + # clear of the click target (y 100-120): an animation over the target + # itself is a change between check and click, which the pixel + # check-to-click guard refuses (tests/test_check_to_click.py). self._tick += 1 img = Image.new("RGB", self._viewport, (240, 240, 240)) w, h = self._viewport x = (self._tick * 37) % max(1, w - 40) for px in range(x, min(x + 40, w)): - for py in range(h): + for py in range(max(0, h - 60), h): img.putpixel((px, py), (0, 0, 0)) buf = io.BytesIO() img.save(buf, format="PNG")