From 28aa2d6b711bf48ef9c99be191314d51cfe4e155 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 14:40:29 -0400 Subject: [PATCH 1/6] docs(flow): backend parity matrix, and a test that keeps --url from coming back docs/BACKEND_PARITY.md lists every command and Python entry point against web, Windows, macOS, Linux, RDP and Citrix: tested, works, refused (with the quoted reason), web-only demo, or n/a. Open gaps are listed with where they are fixed. tests/test_backend_parity.py walks the openadapt-flow and openadapt-agent parsers and fails when a command takes an app URL flag with no --backend or --config and no recorded reason in the doc's URL flags table. Co-Authored-By: Claude Opus 5.5 --- packages/flow/docs/BACKEND_PARITY.md | 151 +++++++++++++++++++ packages/flow/docs/SURFACES.md | 3 + packages/flow/tests/test_backend_parity.py | 159 +++++++++++++++++++++ 3 files changed, 313 insertions(+) create mode 100644 packages/flow/docs/BACKEND_PARITY.md create mode 100644 packages/flow/tests/test_backend_parity.py diff --git a/packages/flow/docs/BACKEND_PARITY.md b/packages/flow/docs/BACKEND_PARITY.md new file mode 100644 index 000000000..ebe27db34 --- /dev/null +++ b/packages/flow/docs/BACKEND_PARITY.md @@ -0,0 +1,151 @@ +# Backend parity + +Every OpenAdapt capability should work the same way on every app surface, or +say exactly why it can't. This page lists each command and Python entry point +against the six surfaces and says which of these is true: + +- **tested**: works, and a test drives it on that surface (file named). +- **works**: works through the shared backend factory; no test drives it on + that surface yet. +- **refused**: stops before any action, with the reason quoted. +- **web only (demo)**: runs the bundled MockMed demo app by design. +- **n/a**: does not drive an app (offline over files, a server, or a report). + +The six surfaces: + +| Surface | `backend.kind` | How Flow sees the app | +| --- | --- | --- | +| Web | `web` | Playwright page: DOM and pixels | +| Windows | `windows` | UI Automation through the in-guest agent (`agent_url`) | +| macOS | `macos` | Accessibility, one app window (`macos_app`) | +| Linux | `linux` | AT-SPI, one exact window (`linux_app`, `linux_window_title`) | +| RDP | `rdp` | pixels: network RDP (`rdp_host`) or a local client window (`rdp_window`) | +| Citrix | `citrix` | pixels: the local Citrix Workspace window | + +A command picks the surface from, in order: the bundle's recorded surface, the +`backend:` section of a deployment file (`--config deploy.yaml`), then the +`--backend` flags. `--url` alone means web. `backends/factory.py` +(`build_backend`) builds every surface and refuses a section that is missing +its target. + +## Commands that run an app + +| Command | Web | Windows | macOS | Linux | RDP | Citrix | +| --- | --- | --- | --- | --- | --- | --- | +| `record` | tested | tested (Capture; `--agent-url` refused) | tested (`--macos-app` scopes Capture) | works (whole desktop; `--linux-app` refused) | tested (local client window; `--rdp-host` refused) | tested | +| `replay` | tested | tested | works | works | works | tested | +| `run` | tested | tested | works | works | works | tested; refused without `rdp_readiness_text` | +| `resume` | tested | tested | works | works | works | tested; refused without `rdp_readiness_text` | +| `console --attend --allow-actions` | works | tested | works | works | works | works | +| `qualify run-case` | tested | works | works | works | works | works | +| `qualify campaign` | tested | refused unless `app.session` | refused unless `app.session` | refused unless `app.session` | tested (with `app.session`) | refused unless `app.session` | +| `connector run` | tested | works | works | works | tested | tested | +| `validate-hosted` | tested | tested | works | works | tested | works | +| `bench` | web only (demo) | web only (demo) | web only (demo) | web only (demo) | web only (demo) | web only (demo) | +| `demo`, `demo-record`, `tutorial` | web only (demo) | web only (demo) | web only (demo) | web only (demo) | web only (demo) | web only (demo) | +| `emit-mcp` (the generated server) | works | **web only (silent)** | **web only (silent)** | **web only (silent)** | **web only (silent)** | **web only (silent)** | +| `emit-skill` (the generated SKILL.md) | works | **tells the agent to pass `--url`** | same | same | same | same | +| `serve-execute` | works | **web only (silent)** | **web only (silent)** | **web only (silent)** | **web only (silent)** | **web only (silent)** | + +Evidence: + +- `record`: `tests/test_desktop_record.py`, `tests/test_backend_factory.py`, + `tests/test_citrix_workspace_backend.py`. Linux recording captures the whole + desktop; `--linux-app` is refused with "the current Capture component has no + Linux window-scoping primitive". +- `replay`, `run`: non-web runs go through `_replay_desktop` + (`__main__.py`). `tests/test_backend_factory.py` (windows), + `tests/test_citrix_workspace_backend.py`, `tests/test_run_gate.py` (Citrix + readiness refusal: "governed Citrix execution requires a stable + current-frame readiness marker before any action"). +- `resume`: `tests/test_cli_new_commands.py`. +- `console`: `tests/test_console.py` (windows, factory stubbed). +- `qualify campaign`: `qualify/session.py` has a built-in session for web + only. Other kinds refuse with "app.target ... needs app.session + (module:function returning (backend, close)); only web has a built-in + session". `tests/test_qualify_declarative.py`, + `tests/test_lookup_capabilities.py`. +- `connector run`: `tests/test_connector.py`. A non-web job that carries a URL + is refused: "native/remote byoc dispatch carries a browser target boundary". +- `validate-hosted`: `tests/test_runtime_validation.py`. A target URL for a + non-web run is refused: "Target URL is web-only and must be omitted". + +## Commands that compile or check files + +These read recordings, bundles, run directories or reports. They do not drive +an app, so the surface does not matter: **n/a** on all six. + +`compile`, `induce`, `for-each`, `compose`, `process` (their `run` hands each +child to `run`), `verify-run`, `scaffold-verifier`, `lint`, `explain`, +`visualize`, `seal`, `bind-record-checks`, `certify`, `qualify` (all verbs +except `campaign` and `run-case`), `propose-qualification`, `disambiguate`, +`teach`, `teach-branch`, `repair` (all verbs; they work on evidence frames and +the repair store), `admit`, `check`, `known-ids`, `triage`, `intake`, `pull`, +`push`, `report-break`, `report-run`, `sanitize`, `review-sanitized`, +`materialize-approved`, `approve-sanitized`, `connect`, `login`, +`local-seal`, `attach-seal`, `countersign`, `no-outbound`, `service`, +`approve`, `item-approval`, `approver-grant`, `jit` (admission verbs), +`fleet`, `business-decisions`, `serve-reward` (reads an oracle; it never +drives an app). + +## Python entry points + +| Entry point | Web | Windows | macOS | Linux | RDP | Citrix | +| --- | --- | --- | --- | --- | --- | --- | +| `jit.compile_agent(app=WebApp(...))` | tested (`tests/e2e/test_jit_cua_e2e.py`) | | | | | | +| `jit.compile_agent(app=SurfaceApp(...))` | | works | works | works | tested | tested | +| `AuthoringSession` (agent drives a recording) | tested | tested | tested | works | tested | tested | +| `runtime.effects.ui_readback` | tested | see below | see below | see below | see below | see below | +| `openadapt-agent serve --authoring` (pinning a window) | tested | tested | tested (frontmost window only) | **refused** | tested | tested | +| `openadapt-agent` run tools | tested | works (surface from `--config`) | works | works | works | works | + +- `SurfaceApp`: `tests/test_jit_surface_app.py` (every kind: validation, + origin binding, Citrix readiness refusal) and + `tests/test_jit_surface_pixels.py` (records MockMed through the real + `RemoteDisplayBackend` and `CitrixWorkspaceBackend`, pixels only). On RDP + and Citrix the canary is refused by the standard run gate: "Identity + coverage: ... UNARMED -- would act with no verified target identity". A + pixel click has no verified record identity until the bundle is qualified; + `run --profile standard` refuses the same bundle the same way. +- `AuthoringSession`: `packages/agent/tests/test_authoring_native.py` + (windows, rdp, citrix), `packages/agent/tests/test_authoring.py` (macOS). +- `ui_readback` opens its own browser at an `http(s)` record page. It works + with any replay surface when the app also has a web record page. A desktop + app without one needs a reopen read-back instead (see + [VERIFICATION.md](VERIFICATION.md)). +- `openadapt-agent serve --authoring` on Linux: there is no Linux pin flag, + so it stops with "stdio --authoring needs a locally pinned window". + +## URL flags + +`tests/test_backend_parity.py` fails when a command takes an app URL flag +(`--url`, `--target-url`, or any other `--...url` flag that names the app) +without `--backend` or `--config`, unless the command is listed here with the +reason. Service endpoints (`--tsa-url`, `--cloud-url`, `--execute-url`, +`--control-plane-url`, `--effects-base-url`, `--api-base-url`, `--agent-url`) +are not app URLs. + + +| Command | URL flag | Why it has no backend-neutral alternative | +| --- | --- | --- | +| `openadapt-flow validate-hosted` | `--target-url` | The surface comes from the signed run report. A non-web report refuses a target URL ("Target URL is web-only and must be omitted"). | +| `openadapt-agent authoring connect` | `--url` | `--url` opens a fresh browser. Without it the command pins the local app window (one macOS or Linux window). Windows, RDP and Citrix are coach-only on the mailbox: the person drives and Capture records (`COACH_ONLY_BACKENDS` in `agent/mailbox.py`). | +| `openadapt-agent seal keys pin` | `--url` | The seal key server's address, not an app. | + +## Open gaps + +| Gap | Where | Status | +| --- | --- | --- | +| `emit-mcp` server drives Playwright for every bundle | `emit/mcp_tool.py` | fixing | +| `emit-skill` tells the agent to pass `--url` for a desktop bundle | `emit/skill.py` | fixing | +| `serve-execute` runs every bundle in a browser | `execute/dispatch.py` | fixing | +| `bench` runs a desktop bundle against the web demo | `__main__.py` (`_cmd_bench`) | fixing | +| `console --attend` drives Citrix without the readiness marker | `runtime/durable/attended_service.py` | fixing | +| `resume --backend X` on a bundle recorded on Y is not refused | `__main__.py` (`_cmd_resume`) | fixing | +| Agent authoring can't pin a Linux window or a named macOS app | `agent/authoring.py`, `agent/cli.py` | fixing | +| JIT canary on RDP or Citrix needs qualified pixel identity | `jit/` | follow-up | +| `variants route` / `run` on a desktop surface: no fixture test | PR #1264 | after merge | +| `induce --probe-on` on a desktop test copy: no fixture test | PR #1276 | after merge | +| `run --self-repair` refuses non-web: "only browser runs can be repaired for now" | PR #1265 | after merge | +| `compile-agent` CLI takes only `--url` | PR #1277 | after merge | +| The GTK3 AT-SPI qualification job is in `packages/flow/.github/`, which GitHub does not run in the monorepo | `.github/workflows/` | follow-up | diff --git a/packages/flow/docs/SURFACES.md b/packages/flow/docs/SURFACES.md index e3ce3b6e6..8bc453939 100644 --- a/packages/flow/docs/SURFACES.md +++ b/packages/flow/docs/SURFACES.md @@ -4,6 +4,9 @@ OpenAdapt drives six execution surfaces through one governed runtime: `web`, `windows`, `macos`, `linux`, `rdp`, and `citrix`. The browser is one surface among six, not a privileged default. +[BACKEND_PARITY.md](BACKEND_PARITY.md) lists every command and Python entry +point against the six surfaces: tested, works, or refused with its reason. + ## Surface selection is explicit in production - Under `--profile standard` or `--profile regulated`, `record` and `run` diff --git a/packages/flow/tests/test_backend_parity.py b/packages/flow/tests/test_backend_parity.py new file mode 100644 index 000000000..d733a90f7 --- /dev/null +++ b/packages/flow/tests/test_backend_parity.py @@ -0,0 +1,159 @@ +"""No command may assume a browser where a deployment file can name the app. + +Every command that takes an app URL flag must also take ``--backend`` or +``--config``, or be listed in docs/BACKEND_PARITY.md (the "URL flags" table) +with the reason it can't. A new ``--...-url`` flag counts as an app URL unless +it is a known service endpoint below. See docs/BACKEND_PARITY.md. +""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path +from typing import Callable, Iterator + +import pytest + +DOC = Path(__file__).resolve().parents[1] / "docs" / "BACKEND_PARITY.md" + +#: URL flags that name a service, not the app being automated. +SERVICE_URL_FLAGS = frozenset( + { + "--agent-url", # the in-guest Windows agent: a backend target itself + "--windows-agent-url", + "--allow-url-override", + "--api-base-url", + "--cloud-url", + "--control-plane-url", + "--effects-base-url", + "--execute-url", + "--tsa-url", + } +) +#: Flags that let a command name any surface instead of a URL. +NEUTRAL_FLAGS = frozenset({"--backend", "--config"}) +_URL_FLAG = re.compile(r"^--(?:[a-z0-9]+-)*url$") + + +def _flow_parser() -> argparse.ArgumentParser: + from openadapt_flow.__main__ import build_parser + + return build_parser() + + +def _agent_parser() -> argparse.ArgumentParser: + from openadapt_flow.agent.cli import build_parser + + return build_parser() + + +PARSERS: dict[str, Callable[[], argparse.ArgumentParser]] = { + "openadapt-flow": _flow_parser, + "openadapt-agent": _agent_parser, +} + + +def _commands( + parser: argparse.ArgumentParser, prefix: str +) -> Iterator[tuple[str, set[str]]]: + options = { + option + for action in parser._actions + for option in action.option_strings + if option.startswith("--") + } + yield prefix, options + for action in parser._actions: + if isinstance(action, argparse._SubParsersAction): + for name, sub in action.choices.items(): + yield from _commands(sub, f"{prefix} {name}") + + +def url_flag_refusals(text: str) -> dict[str, set[str]]: + """``{command: {flag, ...}}`` from the doc's URL flags table.""" + + marker = "