test(ios): stream a leased physical iPhone's screen

docs/CONFIGURATION.md

@@ -39,7 +39,7 @@ edit yourself keeps the mode you give it until the next `config set`; run
 | `capabilities.agent-device.enabled`             | **Both sides.** On a worker: run one agent-device daemon on this worker, started and watched by Simlock. It sees Simlock's devices and no others, refuses `boot` and `shutdown`, and keeps its files in `~/.simlock/agent-device`. If agent-device is missing, older than 0.21.28, speaks another RPC protocol or needs a newer Node than Simlock runs on, it is not started and Simlock runs on without it. A daemon a killed Simlock left behind is stopped at the next start with `capabilities.agent-device.enabled` `true`; with it off, Simlock does not look for one. With `http.enabled` also `true`, the agent-device CLI reaches it at `/v1/agent-device` (see [HTTP-API.md](HTTP-API.md#v1agent-device)). On a gateway: serve `/v1/agent-device` and send each agent to a worker whose agent-device is available; the gateway runs no daemon of its own. | `false` |
 | `capabilities.agent-device.leaseTtlCapMs`       | **Both sides.** Caps the TTL of the leases the agent-device CLI makes and renews through Simlock: a longer `ttlMs` is lowered to it, and a lease that sends none gets it. A whole number of milliseconds from `5000` to `lease.maxTtlMs`; a value outside that range stops the daemon at start, on a gateway too. Used only when `capabilities.agent-device.enabled` is `true`. | unset (no cap) |
 | `capabilities.agent-device.command`             | **Worker side.** The agent-device executable: a name looked up on `PATH`, or a path. A gateway ignores it and logs a warning. Used only when `capabilities.agent-device.enabled` is `true`. | `agent-device` |
-| `capabilities.stream.enabled`                  | **Both sides.** Offer a live view of a leased simulator's or emulator's screen. On a worker: answer `GET /v1/stream/leases/{leaseId}` (see [HTTP-API.md](HTTP-API.md#get-v1streamleasesleaseid)) by capturing the screen of the device that lease holds. On a gateway: serve the same URL and relay each request to the worker that holds the lease. Off by default because a stream is a capture pinned to a leased device — with it off, nothing is running until someone asks to look, and nothing is running when they stop. Read at start; changing it needs a restart. | `false` |
+| `capabilities.stream.enabled`                  | **Both sides.** Offer a live view of a leased device's screen: a simulator, an emulator or a physical iPhone. On a worker: answer `GET /v1/stream/leases/{leaseId}` (see [HTTP-API.md](HTTP-API.md#get-v1streamleasesleaseid)) by capturing the screen of the device that lease holds. On a gateway: serve the same URL and relay each request to the worker that holds the lease. Off by default because a stream is a capture pinned to a leased device — with it off, nothing is running until someone asks to look, and nothing is running when they stop. Read at start; changing it needs a restart. | `false` |
 | `capabilities.stream.fps`                      | **Worker side.** The frame rate the encoder is asked for. A capture sends full frames only when the device's screen changes, and on a still screen only small keep-alive frames, so this is a ceiling and not a promise: an untouched home screen stays far below it, which is what keeps an idle stream cheap. A whole number from `1` to `60`. | `30` |
 | `capabilities.stream.maxDimension`             | **Worker side.** Cap on the encoded picture's longest side, in pixels; the other side follows the device's aspect ratio. This is the lever that costs the most bandwidth — a device that encodes at about 6 Mbit/s native is nearer 1.4 Mbit/s at `720`. A whole number from `96` to `4096`. | `720` |
 | `capabilities.stream.bitrateKbps`              | **Worker side.** Target bitrate in kilobits per second, handed to the encoder as an average rather than a ceiling; a frame that needs more takes it. A whole number from `100` to `40000`; the iOS encoder takes nothing lower. | `1500` |
@@ -78,6 +78,7 @@ edit yourself keeps the mode you give it until the next `config set`; run
 | `stalledTransition.minimumThresholdMs` | Floor under the multiplied estimate, for a driver whose estimate is near zero.                                                                                                                                | `1 minute`                                                        |
 | `drivers.ios.deviceRoot`          | The CoreSimulator device set Simlock owns and scopes every `simctl` call to. See [Device roots](#device-roots).                                                                                              | `${SIMLOCK_HOME}/devices/ios`                                    |
 | `drivers.ios.screenCaptureCommand` | The program that captures a simulator's screen for `capabilities.stream.enabled`. Unset, the one `pnpm build` put beside the daemon. | unset |
+| `drivers.ios.deviceCaptureCommand` | The program that captures a physical iPhone's screen for `capabilities.stream.enabled`. Unset, the one `pnpm build` put beside the daemon. | unset |
 | `drivers.android.deviceRoot`      | The AVD home Simlock owns; exported as `ANDROID_AVD_HOME` to every `avdmanager`/`emulator` call. See [Device roots](#device-roots).                                                                          | `${SIMLOCK_HOME}/devices/android`                                |
 | `drivers.android.adbServerPort`   | TCP port for Simlock's own adb server. Must not be the shared server's `5037`. Startup fails closed if it is occupied.                                                                                       | `5038`                                                            |
 | `ios.defaultMode`                 | The device mode an iOS lease request gets when it names none: `slim` or `full`. Every worker makes both kinds whatever this says. See [Device mode: slim and full](#device-mode-slim-and-full).                             | `full`                                                           |
@@ -607,16 +608,19 @@ sends almost nothing. `capabilities.stream.fps` is a ceiling rather than a rate
 reason. On a Mac an iOS capture costs about 50-80 MB of memory per device being
 watched.
 
-An iOS simulator's screen and an Android emulator's screen can be streamed. A
-lease on a physical iPhone or Android phone is refused with
-`STREAM_NOT_SUPPORTED`, naming that as the reason rather than failing for one
-the reader has to guess.
-
-- **iOS** needs the capture program `pnpm build` builds on a Mac with Xcode
-  selected. Without it an iOS stream is refused with
-  `STREAM_UNAVAILABLE`, saying why; `simlock status` says why too when no
-  platform on the worker can stream. `drivers.ios.screenCaptureCommand` names a
-  program at another path.
+An iOS simulator's screen, a physical iPhone's screen and an Android
+emulator's screen can be streamed. A lease on a physical Android phone is
+refused with `STREAM_NOT_SUPPORTED`, naming that as the reason rather than
+failing for one the reader has to guess.
+
+- **iOS** needs the capture programs `pnpm build` builds on a Mac with Xcode
+  selected: one for a simulator and one for a physical iPhone. Without the one
+  a device needs, that stream is refused with `STREAM_UNAVAILABLE`, saying why;
+  `simlock status` says why too when no platform on the worker can stream. A
+  physical iPhone additionally needs a worker on Xcode 27 — the device program
+  refuses a Mac whose CoreDevice major is another.
+  `drivers.ios.screenCaptureCommand` names another simulator program and
+  `drivers.ios.deviceCaptureCommand` another iPhone one.
 - **Android** needs nothing more than the SDK the emulators already use. Each
   stream copies a small screen server into the emulator, and removes it when
   the stream ends. An emulator encodes in software unless its GPU mode gives it

docs/CONSOLE.md

@@ -258,10 +258,10 @@ live on the page. What you are looking at is either moving or gone.
 
 The panel is on every lease's page, and streams wherever the worker the lease
 runs on has screen streaming turned on (see [`capabilities.stream.enabled`](CONFIGURATION.md)).
-Where it is off, the panel says so before you press anything, and the button
-cannot be pressed; where the lease is on a device a screen cannot be
-captured from — a physical iPhone or Android phone — it says that a live screen
-comes only from a simulator or an emulator. Either way it starts nothing.
+Simulators, emulators and physical iPhones all stream. Where it is off, the
+panel says so before you press anything, and the button cannot be pressed; it
+starts nothing. An iPhone streams on a worker running Xcode 27: on another
+worker, the panel reports that the screen cannot be captured there and why.
 
 The first picture can take a while on a slow emulator. The panel waits up to 90
 seconds for it, longer than a worker takes to give up, so a capture that cannot

docs/HTTP-API.md

@@ -942,7 +942,9 @@ worth asking for twice rather than holding open in advance.
 Authorization is the same as every other lease route: a token sees its own
 leases, an operator sees any, and anything else is a `404`. `STREAM_NOT_SUPPORTED`
 (415) means the device behind the lease can never produce a picture here — a
-physical phone. `STREAM_UNAVAILABLE` (503) means it could but cannot now: a
+physical Android phone. A physical iPhone is not refused that way: it streams
+like a simulator, on a worker that can capture it. `STREAM_UNAVAILABLE`
+(503) means it could but cannot now: a
 capture program that is not installed, or a device that painted nothing within
 30 seconds; asking again is the right move. A stream that stops receiving frames
 for 30 seconds ends with an error rather than freezing on its last picture. `WORKER_UNREACHABLE` on

docs/internal/VENDORED-CODE.md

@@ -15,7 +15,8 @@ license text(s) they require under `LICENSES/`, and a `NOTICE.md`.
 | What | Upstream | Version / commit | License | Where it lives | Consumed by | Re-verify with |
 |---|---|---|---|---|---|---|
 | `@agent-device/proxy` — the transport proxy in front of an agent-device daemon, packed from a branch of agent-device that is not on npm | `callstack/agent-device`, `simlock-integration` branch | commit `253dca023edf13410f7928c76e9e604e5e5871a0` | MIT (Callstack) | `vendor/agent-device-proxy-0.21.28-dev-253dca0.tgz`; license inside the archive at `package/LICENSE` | `package.json`'s `"@agent-device/proxy": "file:vendor/…tgz"`, which `src/agent-device/host/` imports | `tar -tzf vendor/agent-device-proxy-*.tgz \| grep LICENSE`; the commit is what `scripts/slow-e2e.sh` checks a local checkout against, so keep it the first 40-char hash in `vendor/README.md` |
-| serve-sim's capture and encode pipeline — CoreSimulator framebuffer capture and the H.264 encode path, patched by us, with four files rewritten by us that stay serve-sim's modified files | `EvanBacon/serve-sim` | commit `6d009686ed08f7d8bd19bca9f9f67a9237931f44` | Apache-2.0 for serve-sim's files and our rewrites of four of them; MIT for the three files that are ours entirely | `vendor/screen-capture/Sources/simlock-screen-capture/`, license at `vendor/screen-capture/LICENSES/serve-sim.APACHE-2.0.txt` | the `simlock-screen-capture` program `pnpm build` puts in dist/bin, run by `src/drivers/ios/screen-capture.ts` | the `curl … \| diff -u -` command in [`vendor/screen-capture/NOTICE.md`](../../vendor/screen-capture/NOTICE.md), per file |
+| serve-sim's capture and encode pipeline — CoreSimulator framebuffer capture and the H.264 encode path, patched by us, with four files rewritten by us that stay serve-sim's modified files | `EvanBacon/serve-sim` | commit `6d009686ed08f7d8bd19bca9f9f67a9237931f44` | Apache-2.0 for serve-sim's files and our rewrites of four of them; MIT for the files that are ours entirely | `vendor/screen-capture/Sources/`, licenses at `vendor/screen-capture/LICENSES/serve-sim.APACHE-2.0.txt` and `…/idevice.MIT.txt` | the `simlock-screen-capture` and `simlock-device-capture` programs `pnpm build` puts in dist/bin, run by `src/drivers/ios/screen-capture.ts` | the `curl … \| diff -u -` command in [`vendor/screen-capture/NOTICE.md`](../../vendor/screen-capture/NOTICE.md), per file |
+| the HEVC capability string the iPhone's media-stream offer advertises for payload 100 — one string inside one of our own files | `jkcoxson/idevice` | `master` at the time of taking; the string is what its receiver documents | MIT | `vendor/screen-capture/LICENSES/idevice.MIT.txt`, the string in `vendor/screen-capture/Sources/SimlockMediaBridge/SimlockMediaBridge.m` | `simlock-device-capture`, whose offer a physical iPhone accepts without painting torn pictures | the string in `Sources/SimlockMediaBridge/SimlockMediaBridge.m` against idevice's receiver, and its `LICENSE.txt` at `https://raw.githubusercontent.com/jkcoxson/idevice/master/LICENSE.txt` |
 | `scrcpy-server` — the Android screen-streaming server artifact, byte-identical to upstream's release asset | `Genymobile/scrcpy` | release `v5.0.1`; revision built `dc8a55a24c27ae78ae202b5a3288da681d0f141a` | Apache-2.0; the Kotlin standard library inside it adds BSD-3-Clause (ThreeTen-BP) and BSL-1.0 (Boost) code | `vendor/scrcpy/scrcpy-server-v5.0.1`, licenses at `vendor/scrcpy/LICENSES/scrcpy.APACHE-2.0.txt` and `vendor/scrcpy/LICENSES/threetenbp.BSD-3-Clause.txt`, digest in `vendor/scrcpy/SHA256SUMS.txt` | the Android driver's screen capture, which pushes it unmodified to a leased emulator; the client that drives it is ours, in `src/drivers/android/`, and is not vendored | `cd vendor/scrcpy && shasum -a 256 -c SHA256SUMS.txt`, and the same against upstream's own release `SHA256SUMS.txt`, whose GPG signature `vendor/scrcpy/NOTICE.md` says how to check |
 
 `vendor/scrcpy/NOTICE.md` names what is *inside* that artifact besides scrcpy's

docs/internal/agent-rules/toolchain.md

@@ -66,8 +66,11 @@ that goes through the reviews its diff's kind gets and the merge gate:
 `pnpm build` runs `scripts/build-capture.mjs`: it copies the vendored scrcpy
 server into `dist/vendor/scrcpy/` after checking its pinned SHA-256, and on a
 Mac with Xcode selected it builds `vendor/screen-capture/` (Swift) into
-`dist/bin/simlock-screen-capture`. Elsewhere the Swift build is skipped and iOS
-streams report unavailable.
+`dist/bin/simlock-screen-capture` and `dist/bin/simlock-device-capture`, one
+product per `swift build`, each whether or not the other built. Elsewhere the
+Swift build is skipped and iOS streams report unavailable. Neither build checks
+a CoreDevice version: the device program is the one place that enforces that
+pin.
 
 Neither runs in the unit, e2e or console lanes: both read a booted device's
 screen, which means real devices and the slow lane. The lanes cover what this
@@ -108,7 +111,7 @@ The machine can run a line only if:
 | ---------- | ---------------------------------------------------------- |
 | iOS        | `uname -s` is `Darwin`; `xcrun simctl list runtimes` lists one |
 | Android    | `command -v emulator adb` finds both                       |
-| Physical iOS | `uname -s` is `Darwin`; `SIMLOCK_PHYSICAL_IOS` is the UDID of an iPhone on USB (trusted, Developer Mode on, unlocked, Auto-Lock Never); `SIMLOCK_PHYSICAL_IOS_APP` is a `.app` signed for it. Run `scripts/slow-e2e.sh --physical e2e/slow-ios-physical.test.ts` |
+| Physical iOS | `uname -s` is `Darwin`; `SIMLOCK_PHYSICAL_IOS` is the UDID of an iPhone on USB (trusted, Developer Mode on, unlocked, Auto-Lock Never); `SIMLOCK_PHYSICAL_IOS_APP` is a `.app` signed for it. `/usr/libexec/PlistBuddy -c "Print :CFBundleShortVersionString" /Library/Developer/PrivateFrameworks/CoreDevice.framework/Resources/Info.plist` prints `642.` followed by a number (streaming a phone needs Xcode 27's CoreDevice; the framework is machine-wide and does not follow `xcode-select`). DeviceHub and Xcode are not running during the lane. Run `scripts/slow-e2e.sh --physical e2e/slow-ios-physical.test.ts` |
 | agent-device through simlock | `node -v` is 22.12.0 or later (agent-device's own minimum); the iOS check and `uname -m` is `arm64` (the fixture app is arm64 only), or the Android check; `SIMLOCK_E2E_AGENT_DEVICE` (in `.slow-lane.env`) is the absolute path of `bin/agent-device.mjs` in an install of the agent-device version `package.json` pins for `@agent-device/proxy` (`npm install -g agent-device@<version>`, or a checkout at tag `v<version>` built with `pnpm install && pnpm build && pnpm build:android`, the last building the Android snapshot helper); room for two devices: `node -p "os.totalmem() / 2**30"` prints at least 7 for iOS or 12 for Android (simlock's 4 GiB OS reserve plus two of its default 1.5 GiB iOS or 4 GiB Android per-device budgets). Run `scripts/slow-e2e.sh e2e/slow-agent-device.test.ts` or `scripts/slow-e2e.sh e2e/slow-agent-device-gateway.test.ts` |
 | Physical Android | `command -v adb` finds it; `adb devices -l` lists `$SIMLOCK_PHYSICAL_ANDROID` on a USB transport (a `usb:` field) in state `device`; `SIMLOCK_PHYSICAL_ANDROID_APK` names an APK. Run `scripts/slow-e2e.sh --physical e2e/slow-android-physical.test.ts` |
 | Physical Android, a person at the phone | the Physical Android checks above, and `SIMLOCK_PHYSICAL_PERSON` is `1`. Run `scripts/slow-e2e.sh --physical-manual e2e/slow-android-physical-manual.test.ts` |

e2e/helpers/index.ts

@@ -20,6 +20,8 @@ export { DEVICE_READY_MS, waitFor } from "./wait.js";
 export { events } from "./events.js";
 export type { RecordedEvent } from "./events.js";
 export { waitForDeviceState, waitForLeaseCount } from "./status.js";
+export { SCREEN_TAGS, readScreenFrames, type ScreenFrame } from "./screen-frames.js";
+export { REPO_ROOT } from "./repo-root.js";
 export type { CliResult } from "./cli.js";
 export {
   ANDROID,

e2e/helpers/screen-frames.ts

@@ -0,0 +1,35 @@
+// why: the stream's envelope is one contract both iOS stream lanes read -- a big-endian length, a
+// one-byte tag, then the payload, written by the capture programs and handed to a viewer byte for
+// byte by the stream module. A lane that claims a stream carries a key frame reads it through here,
+// so the two lanes cannot drift into checking two different formats.
+export const SCREEN_TAGS = { description: 0x01, keyframe: 0x02, meta: 0x05 } as const;
+
+export interface ScreenFrame {
+  readonly tag: number;
+  readonly payload: Uint8Array;
+}
+
+// why: the body is an endless stream, so this reads frames until it has what it came for and then
+// lets go, rather than reading to an end that never comes.
+export async function readScreenFrames(
+  reader: ReadableStreamDefaultReader<Uint8Array>,
+  enough: (frames: readonly ScreenFrame[]) => boolean,
+): Promise<ScreenFrame[]> {
+  const frames: ScreenFrame[] = [];
+  let buffer = new Uint8Array(0);
+  while (!enough(frames)) {
+    const step = await reader.read();
+    if (step.done) break;
+    const joined = new Uint8Array(buffer.length + step.value.length);
+    joined.set(buffer);
+    joined.set(step.value, buffer.length);
+    buffer = joined;
+    while (buffer.length >= 5) {
+      const length = new DataView(buffer.buffer, buffer.byteOffset).getUint32(0);
+      if (buffer.length < 4 + length) break;
+      frames.push({ payload: buffer.slice(5, 4 + length), tag: buffer[4]! });
+      buffer = buffer.slice(4 + length);
+    }
+  }
+  return frames;
+}