From 0d790c13beda1400b1376857dab86511345732f0 Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 14:50:29 -0400 Subject: [PATCH 1/2] feat(agent): authoring pins the app a deployment file names openadapt-agent serve --authoring --config deploy.yaml now pins the app the file's backend: section names, through the same factory openadapt-flow run uses: a Windows app, one macOS or Linux window, or an RDP or Citrix client window. Linux had no pin at all, and macOS could pin only the frontmost window. Combining it with another pin flag, or a web section, is refused. Co-Authored-By: Claude Opus 5.5 --- packages/agent/README.md | 2 + packages/agent/tests/test_authoring_native.py | 77 +++++++++++++++++++ .../flow/openadapt_flow/agent/authoring.py | 51 ++++++++++++ packages/flow/openadapt_flow/agent/cli.py | 24 +++++- 4 files changed, 153 insertions(+), 1 deletion(-) diff --git a/packages/agent/README.md b/packages/agent/README.md index 29c07f103..ae6336b11 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -364,6 +364,8 @@ For a native Windows app, start Flow's Windows agent server in the signed-in des For an RDP or Citrix session, pass `--remote-window APP` (the client app, for example `"Microsoft Remote Desktop"`) or `--citrix`, and `--remote-window-title` when the app has more than one window. A remote session is screen-only: there's no DOM or UI Automation tree, so `observe` returns no nodes and your agent clicks x,y pixels in the client window and presses keys, the way a person would. To let your agent see that window, add `--authoring-screens scrubbed`, which turns on a `screen` tool that returns the frame with identifiers scrubbed and shares nothing if scrubbing fails. `--authoring-screens unscrubbed` skips scrubbing. Screens are off by default. +You can also name the app with a deployment file instead of these flags: `--authoring --config deploy.yaml`, where the file's `backend:` section names a Windows app (`agent_url`), one macOS window (`macos_app`, optionally `macos_window_title`), one Linux window (`linux_app` and `linux_window_title`), or an RDP or Citrix client window. It is the same section `openadapt-flow run --config` uses, so the recording and its runs drive the same app. This is how you record a Linux app, or a macOS app that isn't the frontmost window. + [Recording on Windows, RDP and Citrix](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RECORDING_ON_WINDOWS_RDP_CITRIX.md) explains the Windows agent server, why it's needed when the agent runs on another computer, how to start it, the token, and the RDP and Citrix pixel path. The recording marks every step as the agent's or the person's (`actor` in `events.jsonl`, `demonstrated_by` in `meta.json`), and a person still accepts the result. On Windows with none of these flags, the tools can only tell the person what to click. diff --git a/packages/agent/tests/test_authoring_native.py b/packages/agent/tests/test_authoring_native.py index e7b2d6b50..08edfe5a8 100644 --- a/packages/agent/tests/test_authoring_native.py +++ b/packages/agent/tests/test_authoring_native.py @@ -227,3 +227,80 @@ def test_desktop_pin_flags_need_authoring(): from openadapt_agent.cli import main assert main(["serve", "--citrix"]) == 2 + + +# -- the deployment file names the app (docs/BACKEND_PARITY.md) ----------------- + +SECTIONS = { + "windows": {"kind": "windows", "agent_url": "http://localhost:5001"}, + "macos": {"kind": "macos", "macos_app": "TextEdit"}, + "linux": {"kind": "linux", "linux_app": "gedit", "linux_window_title": "Notes"}, + "rdp": {"kind": "rdp", "rdp_window": "mstsc", "rdp_window_title": "Clinic"}, + "citrix": { + "kind": "citrix", + "rdp_window": "Citrix Viewer", + "rdp_readiness_text": "Patient Notes", + }, +} + + +@pytest.mark.parametrize("kind", sorted(SECTIONS)) +def test_a_deployment_section_pins_any_surface(factory, kind): + _backend, pinned, _close = pin_local_backend(deployment_backend=SECTIONS[kind]) + cfg, _built = factory[0] + assert pinned == kind and cfg.kind == kind + for key, value in SECTIONS[kind].items(): + assert getattr(cfg, key) == value + + +def test_a_deployment_section_is_one_target(factory): + with pytest.raises(AuthoringError, match="pin one target"): + pin_local_backend(deployment_backend=SECTIONS["linux"], url="https://example.invalid/") + with pytest.raises(AuthoringError, match="names a web app"): + pin_local_backend(deployment_backend={"kind": "web", "url": "https://x/"}) + assert factory == [] + + +@pytest.mark.parametrize("kind", ["linux", "citrix"]) +def test_agent_records_the_app_a_deployment_file_names(factory, tmp_path, kind): + session = open_authoring_session( + out_dir=tmp_path / "rec", + deployment_backend=SECTIONS[kind], + settle_interval_s=0.0, + settle_stable_frames=1, + settle_timeout_s=0.1, + ) + bridge = AuthoringBridge(session, out_dir=tmp_path) + assert bridge.dispatch("observe", {})["backend"] == kind + bridge.dispatch("start_record", {}) + bridge.dispatch("click", {"x": 40, "y": 50}) + bridge.dispatch("stop_record", {}) + meta = json.loads((session.recording_dir / "meta.json").read_text()) + assert meta["surface"] == kind and meta["demonstrated_by"] == "agent" + if kind == "citrix": + assert meta["backend_hints"] == { + "backend": "citrix", + "rdp_window": "Citrix Viewer", + "rdp_readiness_text": "Patient Notes", + } + + +def test_serve_pins_the_config_section_only_without_another_pin(tmp_path): + import argparse + + from openadapt_flow.agent.cli import _authoring_section + + linux = tmp_path / "linux.yaml" + linux.write_text("backend:\n kind: linux\n linux_app: gedit\n linux_window_title: Notes\n") + web = tmp_path / "web.yaml" + web.write_text("backend:\n kind: web\n url: https://example.invalid/\n") + + def ns(**kw): + base = dict(config=None, url=None, windows_agent_url=None, remote_window=None, citrix=False) + base.update(kw) + return argparse.Namespace(**base) + + assert _authoring_section(ns(config=str(linux))).kind == "linux" + assert _authoring_section(ns(config=str(web))) is None + assert _authoring_section(ns(config=str(linux), url="https://x/")) is None + assert _authoring_section(ns()) is None diff --git a/packages/flow/openadapt_flow/agent/authoring.py b/packages/flow/openadapt_flow/agent/authoring.py index 59b84c106..7d0cefedd 100644 --- a/packages/flow/openadapt_flow/agent/authoring.py +++ b/packages/flow/openadapt_flow/agent/authoring.py @@ -471,6 +471,7 @@ def pin_local_backend( remote_window: Optional[str] = None, remote_window_title: Optional[str] = None, citrix: bool = False, + deployment_backend: Any = None, ) -> tuple[Any, str, Any]: """Pin the local target. Titles never go to MCP. @@ -484,6 +485,10 @@ def pin_local_backend( operator already started (never started here). - ``remote_window`` / ``citrix`` → the RDP or Citrix client window, screen-only. + - ``deployment_backend`` (the ``backend:`` section of a deployment file) + → whatever app it names, built by the same factory ``openadapt-flow + run --config`` uses: a Windows app, one macOS or Linux window, or an + RDP or Citrix client window. - Windows with none of those → coach-only. - macOS / Linux → unique frontmost window via Flow backends when those constructors are importable (F1). Non-unique Linux titles are coach-only. @@ -494,6 +499,13 @@ def pin_local_backend( if backend is not None: kind = backend_kind or "web" return backend, kind, None + if deployment_backend is not None: + if url or windows_agent_url or remote_window or citrix: + raise AuthoringError( + "pin one target: the deployment file's backend section, or " + "--url / --windows-agent-url / --remote-window / --citrix" + ) + return _pin_deployment_backend(deployment_backend) pinned = _pin_desktop_driver( windows_agent_url=windows_agent_url, windows_agent_token=windows_agent_token, @@ -585,6 +597,33 @@ def _pin_desktop_driver( return backend, kind, getattr(backend, "close", None) +def _pin_deployment_backend(section: Any) -> tuple[Any, str, Any]: + """Build the app a deployment ``backend:`` section names (not ``web``).""" + + try: + from openadapt_flow.backends.factory import build_backend + from openadapt_flow.deployment import BackendConfig + except ImportError as exc: # pragma: no cover - Flow is a dependency + raise AuthoringError(f"openadapt-flow backends are unavailable: {exc}") from exc + config = ( + section + if isinstance(section, BackendConfig) + else BackendConfig.model_validate(dict(section)) + ) + kind = str(config.kind or "").strip().lower().replace("_", "-") + kind = "rdp" if kind == "remote-display" else kind + if kind == "web": + raise AuthoringError( + "the deployment file names a web app; pass --url URL --headed to " + "record it in a new Chromium window" + ) + try: + backend = build_backend(config) + except (ValueError, RuntimeError, OSError) as exc: + raise AuthoringError(f"could not pin the {kind} target: {exc}") from exc + return backend, kind, getattr(backend, "close", None) + + def _pin_web(url: str, *, headed: bool) -> tuple[Any, str, Any]: try: from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -715,6 +754,7 @@ def open_authoring_session( remote_window: Optional[str] = None, remote_window_title: Optional[str] = None, citrix: bool = False, + deployment_backend: Any = None, **kwargs: Any, ) -> object: """Construct Flow's public authoring session when that module exists. @@ -750,6 +790,7 @@ def open_authoring_session( remote_window=remote_window, remote_window_title=remote_window_title, citrix=citrix, + deployment_backend=deployment_backend, ), **kwargs, ) @@ -818,6 +859,16 @@ def _open_pinned_session( if pin.get("remote_window_title"): hints["rdp_window_title"] = pin["remote_window_title"] kwargs.setdefault("backend_hints", hints) + section = pin.get("deployment_backend") + if canonical in _SCREEN_ONLY_BACKENDS and section is not None: + read = section.get if isinstance(section, dict) else section.__getattribute__ + hints = { + key: read(key) + for key in ("rdp_window", "rdp_window_title", "rdp_readiness_text") + } + hints = {key: value for key, value in hints.items() if value} + if hints: + kwargs.setdefault("backend_hints", hints) kind = canonical opener = getattr(flow_authoring, "open_session", None) session_cls = getattr(flow_authoring, "AuthoringSession", None) diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index ef2ca3534..9790e57e6 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -36,6 +36,23 @@ __all__ = ["build_parser", "main"] +def _authoring_section(args: argparse.Namespace) -> object: + """The ``backend:`` section ``--config`` names, when authoring should pin it. + + Only a non-web section, and only when no other pin flag was given: a web + app is still pinned with ``--url --headed`` (a fresh browser window). + """ + + if not args.config or ( + args.url or args.windows_agent_url or args.remote_window or args.citrix + ): + return None + from openadapt_flow.deployment import load_deployment + + section = load_deployment(args.config).backend + return None if (section.kind or "web").strip().lower() == "web" else section + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="openadapt-agent", @@ -201,7 +218,11 @@ def build_parser() -> argparse.ArgumentParser: "--config", default=None, metavar="YAML", - help="Deployment config YAML forwarded to `openadapt-flow run --config`", + help=( + "Deployment config YAML forwarded to `openadapt-flow run --config`. " + "With --authoring and no other pin flag, its backend: section also " + "names the app to record (windows, macos, linux, rdp, citrix)." + ), ) p.add_argument( "--policy", @@ -657,6 +678,7 @@ def _cmd_serve(args: argparse.Namespace) -> int: remote_window=args.remote_window, remote_window_title=args.remote_window_title, citrix=args.citrix, + deployment_backend=_authoring_section(args), ), out_dir=authoring_dir, screens=args.authoring_screens, From 0b9ccb5f3d2b2a2ce9048f31612fbed6ec14232d Mon Sep 17 00:00:00 2001 From: abrichr Date: Sun, 18 Oct 2026 15:07:36 -0400 Subject: [PATCH 2/2] fix(agent): pin authoring from --config only when asked; refuse network RDP Review fixes: - --authoring-from-config opts in. A --config given only for runs no longer pins authoring. - Network RDP (rdp_host) is refused for authoring: each run's own session would disconnect it. - A Windows section without agent_token takes OAFLOW_AGENT_TOKEN, as --windows-agent-url does; --remote-window-title counts as another pin. - An unreadable deployment file is reported as an authoring error, not a traceback. Co-Authored-By: Claude Opus 5.5 --- packages/agent/README.md | 2 +- packages/agent/tests/test_authoring_native.py | 41 ++++++++++++++----- .../flow/openadapt_flow/agent/authoring.py | 25 +++++++++-- packages/flow/openadapt_flow/agent/cli.py | 41 ++++++++++++++----- 4 files changed, 84 insertions(+), 25 deletions(-) diff --git a/packages/agent/README.md b/packages/agent/README.md index ae6336b11..e6ec65b80 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -364,7 +364,7 @@ For a native Windows app, start Flow's Windows agent server in the signed-in des For an RDP or Citrix session, pass `--remote-window APP` (the client app, for example `"Microsoft Remote Desktop"`) or `--citrix`, and `--remote-window-title` when the app has more than one window. A remote session is screen-only: there's no DOM or UI Automation tree, so `observe` returns no nodes and your agent clicks x,y pixels in the client window and presses keys, the way a person would. To let your agent see that window, add `--authoring-screens scrubbed`, which turns on a `screen` tool that returns the frame with identifiers scrubbed and shares nothing if scrubbing fails. `--authoring-screens unscrubbed` skips scrubbing. Screens are off by default. -You can also name the app with a deployment file instead of these flags: `--authoring --config deploy.yaml`, where the file's `backend:` section names a Windows app (`agent_url`), one macOS window (`macos_app`, optionally `macos_window_title`), one Linux window (`linux_app` and `linux_window_title`), or an RDP or Citrix client window. It is the same section `openadapt-flow run --config` uses, so the recording and its runs drive the same app. This is how you record a Linux app, or a macOS app that isn't the frontmost window. +You can also name the app with a deployment file instead of these flags: `--authoring --config deploy.yaml --authoring-from-config`. The file's `backend:` section names a Windows app (`agent_url`), one macOS window (`macos_app`, optionally `macos_window_title`), one Linux window (`linux_app` and `linux_window_title`), or a local RDP or Citrix client window (`rdp_window`). It is the same section `openadapt-flow run --config` uses, so the recording and its runs drive the same app. This is how you record a Linux app, or a macOS app that isn't the frontmost window. Your agent clicks at window coordinates, so a Linux section also needs `linux_allow_physical_input: true`. Network RDP (`rdp_host`) is refused for authoring, because each run would open its own session and disconnect the recording's. Without `--authoring-from-config`, `--config` only configures runs. [Recording on Windows, RDP and Citrix](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RECORDING_ON_WINDOWS_RDP_CITRIX.md) explains the Windows agent server, why it's needed when the agent runs on another computer, how to start it, the token, and the RDP and Citrix pixel path. diff --git a/packages/agent/tests/test_authoring_native.py b/packages/agent/tests/test_authoring_native.py index 08edfe5a8..a02d416aa 100644 --- a/packages/agent/tests/test_authoring_native.py +++ b/packages/agent/tests/test_authoring_native.py @@ -258,9 +258,23 @@ def test_a_deployment_section_is_one_target(factory): pin_local_backend(deployment_backend=SECTIONS["linux"], url="https://example.invalid/") with pytest.raises(AuthoringError, match="names a web app"): pin_local_backend(deployment_backend={"kind": "web", "url": "https://x/"}) + with pytest.raises(AuthoringError, match="network RDP"): + pin_local_backend(deployment_backend={"kind": "rdp", "rdp_host": "10.0.0.5"}) + with pytest.raises(AuthoringError, match="pin one target"): + pin_local_backend(deployment_backend=SECTIONS["citrix"], remote_window_title="Clinic") assert factory == [] +def test_a_deployment_windows_section_keeps_the_env_agent_token(factory): + pin_local_backend(deployment_backend=SECTIONS["windows"], windows_agent_token="t") + assert factory[0][0].agent_token == "t" + pin_local_backend( + deployment_backend=dict(SECTIONS["windows"], agent_token="file"), + windows_agent_token="t", + ) + assert factory[1][0].agent_token == "file" + + @pytest.mark.parametrize("kind", ["linux", "citrix"]) def test_agent_records_the_app_a_deployment_file_names(factory, tmp_path, kind): session = open_authoring_session( @@ -285,22 +299,29 @@ def test_agent_records_the_app_a_deployment_file_names(factory, tmp_path, kind): } -def test_serve_pins_the_config_section_only_without_another_pin(tmp_path): +def test_serve_pins_the_config_section_only_when_asked(tmp_path): import argparse - from openadapt_flow.agent.cli import _authoring_section + from openadapt_flow.agent.cli import _authoring_section, build_parser linux = tmp_path / "linux.yaml" linux.write_text("backend:\n kind: linux\n linux_app: gedit\n linux_window_title: Notes\n") web = tmp_path / "web.yaml" web.write_text("backend:\n kind: web\n url: https://example.invalid/\n") + broken = tmp_path / "broken.yaml" + broken.write_text("backend:\n kind: [not, a, kind]\n") - def ns(**kw): - base = dict(config=None, url=None, windows_agent_url=None, remote_window=None, citrix=False) - base.update(kw) - return argparse.Namespace(**base) + def ns(*argv): + return build_parser().parse_args(["serve", "--authoring", *argv]) - assert _authoring_section(ns(config=str(linux))).kind == "linux" - assert _authoring_section(ns(config=str(web))) is None - assert _authoring_section(ns(config=str(linux), url="https://x/")) is None - assert _authoring_section(ns()) is None + # A --config given for runs alone never pins authoring. + assert _authoring_section(ns("--config", str(linux))) is None + asked = ns("--config", str(linux), "--authoring-from-config") + assert _authoring_section(asked).kind == "linux" + with pytest.raises(AuthoringError, match="names a web app"): + _authoring_section(ns("--config", str(web), "--authoring-from-config")) + with pytest.raises(AuthoringError, match="needs --config"): + _authoring_section(ns("--authoring-from-config")) + with pytest.raises(AuthoringError, match="could not read"): + _authoring_section(ns("--config", str(broken), "--authoring-from-config")) + assert isinstance(asked, argparse.Namespace) diff --git a/packages/flow/openadapt_flow/agent/authoring.py b/packages/flow/openadapt_flow/agent/authoring.py index 7d0cefedd..13e9b6c73 100644 --- a/packages/flow/openadapt_flow/agent/authoring.py +++ b/packages/flow/openadapt_flow/agent/authoring.py @@ -500,12 +500,14 @@ def pin_local_backend( kind = backend_kind or "web" return backend, kind, None if deployment_backend is not None: - if url or windows_agent_url or remote_window or citrix: + if url or windows_agent_url or remote_window or remote_window_title or citrix: raise AuthoringError( "pin one target: the deployment file's backend section, or " "--url / --windows-agent-url / --remote-window / --citrix" ) - return _pin_deployment_backend(deployment_backend) + return _pin_deployment_backend( + deployment_backend, windows_agent_token=windows_agent_token + ) pinned = _pin_desktop_driver( windows_agent_url=windows_agent_url, windows_agent_token=windows_agent_token, @@ -597,8 +599,14 @@ def _pin_desktop_driver( return backend, kind, getattr(backend, "close", None) -def _pin_deployment_backend(section: Any) -> tuple[Any, str, Any]: - """Build the app a deployment ``backend:`` section names (not ``web``).""" +def _pin_deployment_backend( + section: Any, *, windows_agent_token: Optional[str] = None +) -> tuple[Any, str, Any]: + """Build the app a deployment ``backend:`` section names (not ``web``). + + Network RDP is refused: authoring would hold one RDP session and every + run would open another, which usually disconnects the first. + """ try: from openadapt_flow.backends.factory import build_backend @@ -617,6 +625,15 @@ def _pin_deployment_backend(section: Any) -> tuple[Any, str, Any]: "the deployment file names a web app; pass --url URL --headed to " "record it in a new Chromium window" ) + if kind == "rdp" and config.rdp_host: + raise AuthoringError( + "authoring over network RDP (backend.rdp_host) is refused: it would " + "hold an RDP session that each run's own session disconnects. " + "Record inside the remote session, or name the local client window " + "(backend.rdp_window)" + ) + if kind == "windows" and not config.agent_token and windows_agent_token: + config = config.model_copy(update={"agent_token": windows_agent_token}) try: backend = build_backend(config) except (ValueError, RuntimeError, OSError) as exc: diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index 9790e57e6..baf64b9a4 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -37,20 +37,31 @@ def _authoring_section(args: argparse.Namespace) -> object: - """The ``backend:`` section ``--config`` names, when authoring should pin it. + """The ``backend:`` section to pin for authoring, with + ``--authoring-from-config``; otherwise None. - Only a non-web section, and only when no other pin flag was given: a web - app is still pinned with ``--url --headed`` (a fresh browser window). + A ``--config`` given only for runs never pins authoring by itself. A web + section is pinned with ``--url --headed`` instead (a fresh browser window). """ - if not args.config or ( - args.url or args.windows_agent_url or args.remote_window or args.citrix - ): + from openadapt_flow.agent.authoring import AuthoringError + + if not getattr(args, "authoring_from_config", False): return None + if not args.config: + raise AuthoringError("--authoring-from-config needs --config deploy.yaml") from openadapt_flow.deployment import load_deployment - section = load_deployment(args.config).backend - return None if (section.kind or "web").strip().lower() == "web" else section + try: + section = load_deployment(args.config).backend + except (OSError, ValueError) as exc: + raise AuthoringError(f"could not read {args.config}: {exc}") from exc + if (section.kind or "web").strip().lower() == "web": + raise AuthoringError( + "the deployment file names a web app; pass --url URL --headed to " + "record it in a new Chromium window" + ) + return section def build_parser() -> argparse.ArgumentParser: @@ -118,6 +129,16 @@ def build_parser() -> argparse.ArgumentParser: "tools. This process stays stdio and must not be served over HTTP." ), ) + p.add_argument( + "--authoring-from-config", + action="store_true", + help=( + "With --authoring and --config: record the app the deployment " + "file's backend: section names (windows, macos, linux, a local " + "rdp or citrix client window), the same app its runs drive. " + "Network RDP (rdp_host) is refused for authoring." + ), + ) p.add_argument( "--windows-agent-url", metavar="URL", @@ -220,8 +241,8 @@ def build_parser() -> argparse.ArgumentParser: metavar="YAML", help=( "Deployment config YAML forwarded to `openadapt-flow run --config`. " - "With --authoring and no other pin flag, its backend: section also " - "names the app to record (windows, macos, linux, rdp, citrix)." + "With --authoring-from-config, its backend: section also names the " + "app to record." ), ) p.add_argument(