feat(flow): compile_agent runs on Windows, macOS, Linux, RDP and Citrix apps

packages/flow/docs/JIT.md

@@ -50,6 +50,46 @@ Claude computer-use trajectory against the bundled MockMed app:
 python examples/jit/run_mockmed.py
 ```
 
+## Desktop and remote apps
+
+`WebApp(url=...)` is a browser app. `SurfaceApp` is any other app, named the
+way a deployment file names it: the `backend` section of a `deploy.yaml`.
+
+```python
+from openadapt_flow.jit import SurfaceApp
+
+app = SurfaceApp.from_deployment("deploy.yaml", record_reader=read_records)
+# or
+app = SurfaceApp({"kind": "windows", "agent_url": "http://127.0.0.1:5001"})
+```
+
+| `backend.kind` | The section names | The agent sees |
+| --- | --- | --- |
+| `windows` | `agent_url` (the in-guest agent) | screenshots; clicks keep the UI Automation locator |
+| `macos` | `macos_app`, optionally `macos_window_title` | screenshots; clicks keep the Accessibility locator |
+| `linux` | `linux_app`, `linux_window_title`, and `linux_allow_physical_input: true` (see below) | screenshots; clicks keep the AT-SPI locator |
+| `rdp` | `rdp_host`, or `rdp_window` for a local client window | pixels only |
+| `citrix` | nothing (the Citrix Workspace window) or `rdp_window`; runs need `rdp_readiness_text` | pixels only |
+
+An agent clicks at window coordinates. The Linux backend refuses
+coordinate input unless the section sets `linux_allow_physical_input: true`,
+which allows window-bound X11 input; without it the capture ends
+`capture_failed` with "coordinate input is disabled for native Linux".
+
+The agent drives the app through the same `AuthoringSession` and
+`CaptureComputer` the MCP recording tools use. Every canary and run opens a
+fresh backend from the same section. The app's origin is `<kind>://<target>`
+in lower case, for example `macos://textedit` or `rdp://10.0.0.5`. Window
+titles and credentials are left out: a title can carry a patient or document
+name. A program admitted on one origin is refused on any other, by the JIT
+wrapper and by `openadapt-flow run --config` alike. An admission policy names
+a desktop or remote app in `allowed_hosts` with its kind
+(`rdp://10.0.0.5`); a bare host there names a web app only, so it never
+covers the remote desktop on the same address.
+
+The Windows origin is the agent's host and port. Two VMs reached through the
+same tunnel port share it, so use a separate cache directory per environment.
+
 ## What each run does
 
 | State of the task | `run(**params)` does | `status` |
@@ -241,8 +281,13 @@ explores again.
 
 ## Limits
 
-- Web apps only. `AuthoringSession` can drive desktop, RDP and Citrix, but the
-  wrapper launches a Playwright browser for capture and for runs.
+- On RDP and Citrix the agent sees only pixels, and a pixel click carries no
+  verified record identity. The standard run gate refuses the canary
+  ("Identity coverage"), exactly as `run --profile standard` refuses any
+  pixel-only bundle that was not qualified. Nothing is sent to the window.
+- Citrix canaries and runs refuse without `backend.rdp_readiness_text`.
+- The `playwright_mcp` action format needs a browser page; a `SurfaceApp`
+  refuses it.
 - Without a `record_reader`, the candidate has no effect contract, admission
   refuses it, and no run can reach `VERIFIED`.
 - Secret inputs are not lifted. A secret is typed by a person at a pause

packages/flow/openadapt_flow/__main__.py

@@ -798,6 +798,7 @@ def _build_and_run_replayer(
     surface_override: bool = False,
     execution_origin: Optional[str] = None,
     execution_entry_url: Optional[str] = None,
+    execution_target_origin: Optional[str] = None,
     run_id: Optional[str] = None,
     approval_gate=None,
     qualification_fault_driver=None,
@@ -836,6 +837,7 @@ def _build_and_run_replayer(
         surface_override=surface_override,
         execution_origin=execution_origin,
         execution_entry_url=execution_entry_url,
+        execution_target_origin=execution_target_origin,
         run_id=(
             remote_delivery_run_id
             if delivery_authority_kind == "cloud_runner"
@@ -1086,6 +1088,7 @@ def _replay_desktop(
             qualification_campaign_guard=qualification_campaign_guard,
             runtime_config=runtime_config,
             execution_target_kind=_report_backend_kind(backend_cfg.kind),
+            execution_target_origin=_surface_target_origin_or_none(backend_cfg),
             surface_override=bool(getattr(args, "_surface_override", False)),
             run_id=run_id,
             approval_gate=approval_gate,
@@ -1103,6 +1106,18 @@ def _replay_desktop(
     )
 
 
+def _surface_target_origin_or_none(backend_cfg) -> Optional[str]:
+    """The non-web app's ``<kind>://<target>``, for an admitted JIT program's
+    canary match. A section that can't name one gives None, which that match
+    refuses; every other bundle ignores it."""
+    from openadapt_flow.surface_selection import surface_target_origin
+
+    try:
+        return surface_target_origin(backend_cfg)
+    except ValueError:
+        return None
+
+
 def _record_until_stop(args: argparse.Namespace):
     """Poll ``--until-cmd`` until it exits 0. Ctrl-C remains the fallback."""
 

packages/flow/openadapt_flow/jit/__init__.py

@@ -25,10 +25,13 @@
 )
 from openadapt_flow.jit.compile_agent import (
     RECORDING_WARNING,
+    SURFACE_KINDS,
+    App,
     CompiledAgent,
     ComputerUseAgent,
     JitResult,
     Recording,
+    SurfaceApp,
     TaskRequest,
     WebApp,
     compile_agent,
@@ -42,6 +45,7 @@
 from openadapt_flow.jit.signature import ParamError, TaskSignature
 
 __all__ = [
+    "App",
     "AdmissionPolicy",
     "AuthorityError",
     "Principal",
@@ -50,6 +54,7 @@
     "principal_from_key",
     "DIALECTS",
     "RECORDING_WARNING",
+    "SURFACE_KINDS",
     "AdmissionError",
     "BundleState",
     "CaptureComputer",
@@ -61,6 +66,7 @@
     "Observation",
     "ParamError",
     "Recording",
+    "SurfaceApp",
     "TaskRequest",
     "TaskSignature",
     "UnsupportedAction",

packages/flow/openadapt_flow/jit/authority.py

@@ -316,7 +316,8 @@ class AdmissionPolicy(BaseModel):
     signer: str = Field(pattern=_ID)
     signed_at: str = Field(min_length=10, max_length=64)
     expires_at: str = Field(min_length=10, max_length=64)
-    #: ``host`` or ``host:port`` of the target app's origin.
+    #: ``host`` or ``host:port`` of a web app's origin, or a desktop or
+    #: remote app's ``<kind>://<target>`` (``macos://textedit``).
     allowed_hosts: tuple[str, ...] = Field(min_length=1)
     #: Step actions (``openadapt_flow.ir.ActionKind`` values).
     allowed_actions: tuple[str, ...] = Field(min_length=1)
@@ -460,8 +461,17 @@ def scope_problems(
 
     problems: list[str] = []
     if target_origin is not None:
-        host = (urlsplit(target_origin).netloc or "").lower()
-        if host not in policy.allowed_hosts:
+        parts = urlsplit(target_origin)
+        if parts.scheme in ("http", "https"):
+            # A bare host names a web app only.
+            host = (parts.netloc or "").lower()
+            allowed = {h.lower() for h in policy.allowed_hosts if "://" not in h}
+        else:
+            # A desktop or remote app is named with its kind
+            # (``macos://textedit``), so a web host entry never covers it.
+            host = target_origin.lower()
+            allowed = {h.lower() for h in policy.allowed_hosts if "://" in h}
+        if host not in allowed:
             problems.append(f"the app at {host or target_origin!r} is not allowed")
     for step in workflow.steps:
         if step.action.value not in policy.allowed_actions: