diff --git a/packages/agent/README.md b/packages/agent/README.md index 29c07f103..e6ec65b80 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -364,6 +364,8 @@ For a native Windows app, start Flow's Windows agent server in the signed-in des For an RDP or Citrix session, pass `--remote-window APP` (the client app, for example `"Microsoft Remote Desktop"`) or `--citrix`, and `--remote-window-title` when the app has more than one window. A remote session is screen-only: there's no DOM or UI Automation tree, so `observe` returns no nodes and your agent clicks x,y pixels in the client window and presses keys, the way a person would. To let your agent see that window, add `--authoring-screens scrubbed`, which turns on a `screen` tool that returns the frame with identifiers scrubbed and shares nothing if scrubbing fails. `--authoring-screens unscrubbed` skips scrubbing. Screens are off by default. +You can also name the app with a deployment file instead of these flags: `--authoring --config deploy.yaml --authoring-from-config`. The file's `backend:` section names a Windows app (`agent_url`), one macOS window (`macos_app`, optionally `macos_window_title`), one Linux window (`linux_app` and `linux_window_title`), or a local RDP or Citrix client window (`rdp_window`). It is the same section `openadapt-flow run --config` uses, so the recording and its runs drive the same app. This is how you record a Linux app, or a macOS app that isn't the frontmost window. Your agent clicks at window coordinates, so a Linux section also needs `linux_allow_physical_input: true`. Network RDP (`rdp_host`) is refused for authoring, because each run would open its own session and disconnect the recording's. Without `--authoring-from-config`, `--config` only configures runs. + [Recording on Windows, RDP and Citrix](https://github.com/OpenAdaptAI/OpenAdapt/blob/main/packages/flow/docs/RECORDING_ON_WINDOWS_RDP_CITRIX.md) explains the Windows agent server, why it's needed when the agent runs on another computer, how to start it, the token, and the RDP and Citrix pixel path. The recording marks every step as the agent's or the person's (`actor` in `events.jsonl`, `demonstrated_by` in `meta.json`), and a person still accepts the result. On Windows with none of these flags, the tools can only tell the person what to click. diff --git a/packages/agent/tests/test_authoring_native.py b/packages/agent/tests/test_authoring_native.py index e7b2d6b50..a02d416aa 100644 --- a/packages/agent/tests/test_authoring_native.py +++ b/packages/agent/tests/test_authoring_native.py @@ -227,3 +227,101 @@ def test_desktop_pin_flags_need_authoring(): from openadapt_agent.cli import main assert main(["serve", "--citrix"]) == 2 + + +# -- the deployment file names the app (docs/BACKEND_PARITY.md) ----------------- + +SECTIONS = { + "windows": {"kind": "windows", "agent_url": "http://localhost:5001"}, + "macos": {"kind": "macos", "macos_app": "TextEdit"}, + "linux": {"kind": "linux", "linux_app": "gedit", "linux_window_title": "Notes"}, + "rdp": {"kind": "rdp", "rdp_window": "mstsc", "rdp_window_title": "Clinic"}, + "citrix": { + "kind": "citrix", + "rdp_window": "Citrix Viewer", + "rdp_readiness_text": "Patient Notes", + }, +} + + +@pytest.mark.parametrize("kind", sorted(SECTIONS)) +def test_a_deployment_section_pins_any_surface(factory, kind): + _backend, pinned, _close = pin_local_backend(deployment_backend=SECTIONS[kind]) + cfg, _built = factory[0] + assert pinned == kind and cfg.kind == kind + for key, value in SECTIONS[kind].items(): + assert getattr(cfg, key) == value + + +def test_a_deployment_section_is_one_target(factory): + with pytest.raises(AuthoringError, match="pin one target"): + pin_local_backend(deployment_backend=SECTIONS["linux"], url="https://example.invalid/") + with pytest.raises(AuthoringError, match="names a web app"): + pin_local_backend(deployment_backend={"kind": "web", "url": "https://x/"}) + with pytest.raises(AuthoringError, match="network RDP"): + pin_local_backend(deployment_backend={"kind": "rdp", "rdp_host": "10.0.0.5"}) + with pytest.raises(AuthoringError, match="pin one target"): + pin_local_backend(deployment_backend=SECTIONS["citrix"], remote_window_title="Clinic") + assert factory == [] + + +def test_a_deployment_windows_section_keeps_the_env_agent_token(factory): + pin_local_backend(deployment_backend=SECTIONS["windows"], windows_agent_token="t") + assert factory[0][0].agent_token == "t" + pin_local_backend( + deployment_backend=dict(SECTIONS["windows"], agent_token="file"), + windows_agent_token="t", + ) + assert factory[1][0].agent_token == "file" + + +@pytest.mark.parametrize("kind", ["linux", "citrix"]) +def test_agent_records_the_app_a_deployment_file_names(factory, tmp_path, kind): + session = open_authoring_session( + out_dir=tmp_path / "rec", + deployment_backend=SECTIONS[kind], + settle_interval_s=0.0, + settle_stable_frames=1, + settle_timeout_s=0.1, + ) + bridge = AuthoringBridge(session, out_dir=tmp_path) + assert bridge.dispatch("observe", {})["backend"] == kind + bridge.dispatch("start_record", {}) + bridge.dispatch("click", {"x": 40, "y": 50}) + bridge.dispatch("stop_record", {}) + meta = json.loads((session.recording_dir / "meta.json").read_text()) + assert meta["surface"] == kind and meta["demonstrated_by"] == "agent" + if kind == "citrix": + assert meta["backend_hints"] == { + "backend": "citrix", + "rdp_window": "Citrix Viewer", + "rdp_readiness_text": "Patient Notes", + } + + +def test_serve_pins_the_config_section_only_when_asked(tmp_path): + import argparse + + from openadapt_flow.agent.cli import _authoring_section, build_parser + + linux = tmp_path / "linux.yaml" + linux.write_text("backend:\n kind: linux\n linux_app: gedit\n linux_window_title: Notes\n") + web = tmp_path / "web.yaml" + web.write_text("backend:\n kind: web\n url: https://example.invalid/\n") + broken = tmp_path / "broken.yaml" + broken.write_text("backend:\n kind: [not, a, kind]\n") + + def ns(*argv): + return build_parser().parse_args(["serve", "--authoring", *argv]) + + # A --config given for runs alone never pins authoring. + assert _authoring_section(ns("--config", str(linux))) is None + asked = ns("--config", str(linux), "--authoring-from-config") + assert _authoring_section(asked).kind == "linux" + with pytest.raises(AuthoringError, match="names a web app"): + _authoring_section(ns("--config", str(web), "--authoring-from-config")) + with pytest.raises(AuthoringError, match="needs --config"): + _authoring_section(ns("--authoring-from-config")) + with pytest.raises(AuthoringError, match="could not read"): + _authoring_section(ns("--config", str(broken), "--authoring-from-config")) + assert isinstance(asked, argparse.Namespace) diff --git a/packages/flow/openadapt_flow/agent/authoring.py b/packages/flow/openadapt_flow/agent/authoring.py index 59b84c106..13e9b6c73 100644 --- a/packages/flow/openadapt_flow/agent/authoring.py +++ b/packages/flow/openadapt_flow/agent/authoring.py @@ -471,6 +471,7 @@ def pin_local_backend( remote_window: Optional[str] = None, remote_window_title: Optional[str] = None, citrix: bool = False, + deployment_backend: Any = None, ) -> tuple[Any, str, Any]: """Pin the local target. Titles never go to MCP. @@ -484,6 +485,10 @@ def pin_local_backend( operator already started (never started here). - ``remote_window`` / ``citrix`` → the RDP or Citrix client window, screen-only. + - ``deployment_backend`` (the ``backend:`` section of a deployment file) + → whatever app it names, built by the same factory ``openadapt-flow + run --config`` uses: a Windows app, one macOS or Linux window, or an + RDP or Citrix client window. - Windows with none of those → coach-only. - macOS / Linux → unique frontmost window via Flow backends when those constructors are importable (F1). Non-unique Linux titles are coach-only. @@ -494,6 +499,15 @@ def pin_local_backend( if backend is not None: kind = backend_kind or "web" return backend, kind, None + if deployment_backend is not None: + if url or windows_agent_url or remote_window or remote_window_title or citrix: + raise AuthoringError( + "pin one target: the deployment file's backend section, or " + "--url / --windows-agent-url / --remote-window / --citrix" + ) + return _pin_deployment_backend( + deployment_backend, windows_agent_token=windows_agent_token + ) pinned = _pin_desktop_driver( windows_agent_url=windows_agent_url, windows_agent_token=windows_agent_token, @@ -585,6 +599,48 @@ def _pin_desktop_driver( return backend, kind, getattr(backend, "close", None) +def _pin_deployment_backend( + section: Any, *, windows_agent_token: Optional[str] = None +) -> tuple[Any, str, Any]: + """Build the app a deployment ``backend:`` section names (not ``web``). + + Network RDP is refused: authoring would hold one RDP session and every + run would open another, which usually disconnects the first. + """ + + try: + from openadapt_flow.backends.factory import build_backend + from openadapt_flow.deployment import BackendConfig + except ImportError as exc: # pragma: no cover - Flow is a dependency + raise AuthoringError(f"openadapt-flow backends are unavailable: {exc}") from exc + config = ( + section + if isinstance(section, BackendConfig) + else BackendConfig.model_validate(dict(section)) + ) + kind = str(config.kind or "").strip().lower().replace("_", "-") + kind = "rdp" if kind == "remote-display" else kind + if kind == "web": + raise AuthoringError( + "the deployment file names a web app; pass --url URL --headed to " + "record it in a new Chromium window" + ) + if kind == "rdp" and config.rdp_host: + raise AuthoringError( + "authoring over network RDP (backend.rdp_host) is refused: it would " + "hold an RDP session that each run's own session disconnects. " + "Record inside the remote session, or name the local client window " + "(backend.rdp_window)" + ) + if kind == "windows" and not config.agent_token and windows_agent_token: + config = config.model_copy(update={"agent_token": windows_agent_token}) + try: + backend = build_backend(config) + except (ValueError, RuntimeError, OSError) as exc: + raise AuthoringError(f"could not pin the {kind} target: {exc}") from exc + return backend, kind, getattr(backend, "close", None) + + def _pin_web(url: str, *, headed: bool) -> tuple[Any, str, Any]: try: from openadapt_flow.backends.playwright_backend import PlaywrightBackend @@ -715,6 +771,7 @@ def open_authoring_session( remote_window: Optional[str] = None, remote_window_title: Optional[str] = None, citrix: bool = False, + deployment_backend: Any = None, **kwargs: Any, ) -> object: """Construct Flow's public authoring session when that module exists. @@ -750,6 +807,7 @@ def open_authoring_session( remote_window=remote_window, remote_window_title=remote_window_title, citrix=citrix, + deployment_backend=deployment_backend, ), **kwargs, ) @@ -818,6 +876,16 @@ def _open_pinned_session( if pin.get("remote_window_title"): hints["rdp_window_title"] = pin["remote_window_title"] kwargs.setdefault("backend_hints", hints) + section = pin.get("deployment_backend") + if canonical in _SCREEN_ONLY_BACKENDS and section is not None: + read = section.get if isinstance(section, dict) else section.__getattribute__ + hints = { + key: read(key) + for key in ("rdp_window", "rdp_window_title", "rdp_readiness_text") + } + hints = {key: value for key, value in hints.items() if value} + if hints: + kwargs.setdefault("backend_hints", hints) kind = canonical opener = getattr(flow_authoring, "open_session", None) session_cls = getattr(flow_authoring, "AuthoringSession", None) diff --git a/packages/flow/openadapt_flow/agent/cli.py b/packages/flow/openadapt_flow/agent/cli.py index ef2ca3534..baf64b9a4 100644 --- a/packages/flow/openadapt_flow/agent/cli.py +++ b/packages/flow/openadapt_flow/agent/cli.py @@ -36,6 +36,34 @@ __all__ = ["build_parser", "main"] +def _authoring_section(args: argparse.Namespace) -> object: + """The ``backend:`` section to pin for authoring, with + ``--authoring-from-config``; otherwise None. + + A ``--config`` given only for runs never pins authoring by itself. A web + section is pinned with ``--url --headed`` instead (a fresh browser window). + """ + + from openadapt_flow.agent.authoring import AuthoringError + + if not getattr(args, "authoring_from_config", False): + return None + if not args.config: + raise AuthoringError("--authoring-from-config needs --config deploy.yaml") + from openadapt_flow.deployment import load_deployment + + try: + section = load_deployment(args.config).backend + except (OSError, ValueError) as exc: + raise AuthoringError(f"could not read {args.config}: {exc}") from exc + if (section.kind or "web").strip().lower() == "web": + raise AuthoringError( + "the deployment file names a web app; pass --url URL --headed to " + "record it in a new Chromium window" + ) + return section + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="openadapt-agent", @@ -101,6 +129,16 @@ def build_parser() -> argparse.ArgumentParser: "tools. This process stays stdio and must not be served over HTTP." ), ) + p.add_argument( + "--authoring-from-config", + action="store_true", + help=( + "With --authoring and --config: record the app the deployment " + "file's backend: section names (windows, macos, linux, a local " + "rdp or citrix client window), the same app its runs drive. " + "Network RDP (rdp_host) is refused for authoring." + ), + ) p.add_argument( "--windows-agent-url", metavar="URL", @@ -201,7 +239,11 @@ def build_parser() -> argparse.ArgumentParser: "--config", default=None, metavar="YAML", - help="Deployment config YAML forwarded to `openadapt-flow run --config`", + help=( + "Deployment config YAML forwarded to `openadapt-flow run --config`. " + "With --authoring-from-config, its backend: section also names the " + "app to record." + ), ) p.add_argument( "--policy", @@ -657,6 +699,7 @@ def _cmd_serve(args: argparse.Namespace) -> int: remote_window=args.remote_window, remote_window_title=args.remote_window_title, citrix=args.citrix, + deployment_backend=_authoring_section(args), ), out_dir=authoring_dir, screens=args.authoring_screens,