From d1080d40957b7da6a6171912d3635f2c4109d14d Mon Sep 17 00:00:00 2001 From: abrichr Date: Sat, 24 Oct 2026 23:47:28 -0400 Subject: [PATCH 1/2] fix: record native and widget selects, and give id-less save buttons a selector The interactive recorder stopped every recording at a native a textbox named after its name attribute. It now falls back to unique data-testid, data-test and name attributes, then a submit input's value, and names button inputs by their label, so id-less saves (Trac's input[name=submit], NetBox's button[name=_create]) carry the selector qualification needs. Found by the onboarding-effort benchmark (#1447). Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/BROWSER_RECORDING.md | 16 ++- .../backends/playwright_backend.py | 27 ++++ .../openadapt_flow/interactive_recorder.py | 59 +++++++- .../test_interactive_recorder_gestures.py | 136 ++++++++++++++++-- .../tests/test_structural_locator_idless.py | 121 ++++++++++++++++ 5 files changed, 339 insertions(+), 20 deletions(-) create mode 100644 packages/flow/tests/test_structural_locator_idless.py diff --git a/packages/flow/docs/BROWSER_RECORDING.md b/packages/flow/docs/BROWSER_RECORDING.md index f19b8a481..5e83e7aed 100644 --- a/packages/flow/docs/BROWSER_RECORDING.md +++ b/packages/flow/docs/BROWSER_RECORDING.md @@ -397,12 +397,20 @@ gesture compiles to one `DOUBLE_CLICK` step), right click, drag, text entry (including `contenteditable` and `role="textbox"`), special keys, keyboard shortcuts, and wheel scrolling. +A native `` selection. Its option commit happens in browser-native - dropdown UI that produces no recordable action events, so the choice would - be silently absent from the compiled workflow. The recording stops with an - explicit error at the moment of selection. +- A multiple-choice ``, + // ` + + + + + +""" + + +@pytest.fixture(scope="module") +def page_url(tmp_path_factory: pytest.TempPathFactory) -> str: + path = tmp_path_factory.mktemp("idless") / "newticket.html" + path.write_text(PAGE) + return path.as_uri() + + +@pytest.fixture(scope="module") +def browser() -> Iterator[Any]: + with sync_api.sync_playwright() as playwright: + launched = playwright.chromium.launch(headless=True) + try: + yield launched + finally: + launched.close() + + +def _center(page: Any, selector: str) -> tuple[int, int]: + box = page.locator(selector).first.bounding_box() + assert box is not None + return int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + + +def test_locator_names_idless_save_buttons(browser: Any, page_url: str) -> None: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(page_url) + backend = PlaywrightBackend(page) + + trac = backend.structural_locator_at(*_center(page, "input[name=submit]")) + assert trac is not None + assert trac.selector == 'input[name="submit"]' + assert (trac.role, trac.name) == ("button", "Create ticket") + + netbox = backend.structural_locator_at(*_center(page, "button[name=_create]")) + assert netbox is not None + assert netbox.selector == 'button[name="_create"]' + assert (netbox.role, netbox.name) == ("button", "Create") + + # Two identical unnamed buttons: no selector that could pick the + # wrong one, only role and name. + twin = backend.structural_locator_at(*_center(page, "button:text('Save')")) + assert twin is not None + assert twin.selector is None + assert (twin.role, twin.name) == ("button", "Save") + finally: + page.close() + + +def test_recorded_create_step_carries_its_selector( + browser: Any, page_url: str, tmp_path: Path +) -> None: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(page_url) + recorder = Recorder(PlaywrightBackend(page), tmp_path / "rec", app_url=page_url) + recorder.click(*_center(page, "input[name=field_summary]")) + recorder.type_text("Printer jams on tray 2", param="summary") + recorder.click(*_center(page, "input[name=submit]")) + recorder.finish() + assert page.evaluate("window.actions") == ["submit"] + finally: + page.close() + + workflow = compile_recording(tmp_path / "rec", tmp_path / "bundle", name="ticket") + save = workflow.steps[-1] + assert save.anchor is not None and save.anchor.structural is not None + assert save.anchor.structural.selector == 'input[name="submit"]' + assert save.anchor.structural.role == "button" + assert save.anchor.structural.name == "Create ticket" From 57946df981019d251028f8b4277ab1d780eb3baf Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 25 Oct 2026 01:11:27 -0400 Subject: [PATCH 2/2] fix: a selector names the clicked button only when it matches it alone Review of #1537: the uniqueness check counted matches in `document` but never confirmed the single match was the clicked element. A button in a shadow root with a same-named look-alike in the page got the look-alike's selector, and a replay would click the wrong Save. The check now runs in the clicked element's own document and requires the one match to be that element, for the id too. Inside a shadow root no selector is built: Playwright's CSS locator pierces open shadow roots at replay, so a selector unique in one root could match a look-alike elsewhere. Co-Authored-By: Claude Opus 5.5 --- .../backends/playwright_backend.py | 21 ++++-- .../tests/test_structural_locator_idless.py | 73 +++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/packages/flow/openadapt_flow/backends/playwright_backend.py b/packages/flow/openadapt_flow/backends/playwright_backend.py index f7a30d7e0..d6e1073f3 100644 --- a/packages/flow/openadapt_flow/backends/playwright_backend.py +++ b/packages/flow/openadapt_flow/backends/playwright_backend.py @@ -925,9 +925,22 @@ class _FramePoint: ) || el; const tag = actionable.tagName.toLowerCase(); let selector = null; + // A selector counts only when, in the clicked element's own document, + // it matches exactly one element and that element is the clicked one. A + // look-alike elsewhere must not make a selector "unique" for the wrong + // element. Inside a shadow root no selector is built: Playwright's CSS + // locator pierces open shadow roots at replay, so a selector unique in + // one root could match a look-alike in the page or another root. + const root = actionable.getRootNode(); + const unique = (candidate) => { + if (!(root instanceof Document)) return false; + try { + return root.querySelectorAll(candidate).length === 1 && + root.querySelector(candidate) === actionable; + } catch (_) { return false; } + }; const id = actionable.id; - if (id && document.querySelectorAll( - '#' + CSS.escape(id)).length === 1) { + if (id && unique('#' + CSS.escape(id))) { selector = '#' + CSS.escape(id); } // No unique id: the same unique attributes the interactive recorder uses, @@ -938,10 +951,6 @@ class _FramePoint: ? (actionable.getAttribute('type') || 'text').toLowerCase() : ''; const isButtonInput = ['submit', 'button', 'reset', 'image'] .indexOf(inputType) >= 0; - const unique = (candidate) => { - try { return document.querySelectorAll(candidate).length === 1; } - catch (_) { return false; } - }; for (const attr of ['data-testid', 'data-test', 'name']) { if (selector) break; const value = actionable.getAttribute(attr); diff --git a/packages/flow/tests/test_structural_locator_idless.py b/packages/flow/tests/test_structural_locator_idless.py index 12a69a52c..d3b6d4c79 100644 --- a/packages/flow/tests/test_structural_locator_idless.py +++ b/packages/flow/tests/test_structural_locator_idless.py @@ -119,3 +119,76 @@ def test_recorded_create_step_carries_its_selector( assert save.anchor.structural.selector == 'input[name="submit"]' assert save.anchor.structural.role == "button" assert save.anchor.structural.name == "Create ticket" + + +SHADOW_PAGE = """ + +

+ + +""" + + +def test_a_lookalike_in_another_root_never_names_the_clicked_button( + browser: Any, tmp_path: Path +) -> None: + """A Save inside a shadow root, and a look-alike Save in the page with + the same name. ``button[name="save"]`` matches exactly one element in the + document, the look-alike, so a count alone would call it unique for the + shadow button and a replay would click the wrong Save.""" + + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + + path = tmp_path / "shadow.html" + path.write_text(SHADOW_PAGE) + page = browser.new_page(viewport={"width": 1280, "height": 800}) + try: + page.goto(path.as_uri()) + backend = PlaywrightBackend(page) + shadow = page.locator("save-panel").locator("button") + box = shadow.bounding_box() + assert box is not None + x, y = int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + + # The page's own hit test stops at the shadow host. + located = backend.structural_locator_at(x, y) + assert located is None or located.selector != 'button[name="save"]' + + # A hit test that reaches the button inside the shadow root (as a + # piercing one would): still no selector, because the only match in + # the document is the look-alike. + page.evaluate( + """() => { + const inner = document.querySelector('save-panel') + .shadowRoot.querySelector('button'); + document.elementFromPoint = () => inner; + }""" + ) + located = backend.structural_locator_at(x, y) + assert located is not None + assert located.selector is None + assert (located.role, located.name) == ("button", "Save") + + # The light-DOM Save is the one element its selector matches, and it + # is the clicked one: it keeps the selector. + page.reload() + light = page.locator("p > button") + box = light.bounding_box() + assert box is not None + located = backend.structural_locator_at( + int(box["x"] + box["width"] / 2), int(box["y"] + box["height"] / 2) + ) + assert located is not None and located.selector == 'button[name="save"]' + finally: + page.close()