From 47852b2def5755bcaf628193489830013534be44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 16:13:40 +0200 Subject: [PATCH 01/12] test(ios): isolate smoke contracts and remove incidental URL handoffs --- .github/workflows/ios.yml | 12 ++-- .../AgentDeviceRunner/AgentDeviceRunnerApp.m | 13 +++++ ...unnerTests+SynthesizedTextEntryTests.swift | 6 +- .../RunnerTests+TextTypingTests.swift | 4 +- .../ios-smoke-contracts-2026-10-17.md | 22 ++++++++ examples/test-app/app/(tabs)/settings.tsx | 1 + .../test-app/src/screens/SettingsScreen.tsx | 9 +++ ...imulator-e2e-deep-link-destination.test.ts | 35 +++++++++--- .../ios-simulator-e2e/behavior-coverage.ts | 2 +- .../live-automation-scenario.ts | 49 ++++++++++------- .../live-deep-link-destination.ts | 30 +++------- .../ios-simulator-e2e/live-runner.ts | 55 ++++++++++++++++--- .../live-snapshot-depth-frontier.ts | 15 ++--- .../live-webview-remote-content.ts | 27 +++++---- .../ios-simulator-e2e/scenarios.ts | 2 + 15 files changed, 195 insertions(+), 87 deletions(-) create mode 100644 docs/evidence/ios-smoke-contracts-2026-10-17.md diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index acd8e4dcc2..2a2edadc8c 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -162,6 +162,9 @@ jobs: } ' + - name: Build CLI used by every smoke step + run: pnpm gate build + # Preflight and every step below deliberately share one AGENT_DEVICE_STATE_DIR: the prepared # runner lives in the daemon this step leaves running, and a per-step state dir would rebuild it # per step. That also means this daemon outlives the step and can be superseded while it lives, @@ -172,13 +175,13 @@ jobs: - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json # The replay reuses the prepared runner in the same daemon. --debug keeps every step's request # log, not only the failing one, so a stale tap can be traced to its pre-tap capture (#3367). - name: Run iOS Settings replay smoke test run: | - node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml - name: Fetch current fixture app id: fixture-app @@ -201,7 +204,7 @@ jobs: - name: Run gesture pan-duration smoke replay run: | xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" - node --experimental-strip-types src/bin.ts test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml + node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml - name: Run fixture-backed iOS simulator E2E smoke env: @@ -211,7 +214,6 @@ jobs: AGENT_DEVICE_IOS_E2E_TIER: smoke AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} run: | - pnpm gate build node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - name: Assert simulator automation preserved host focus @@ -228,7 +230,7 @@ jobs: IOS_UDID: ${{ vars.IOS_UDID }} run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml + node bin/agent-device.mjs test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml - name: Assert iOS XCTest selection was honored if: always() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index c9b6d61bd7..acc0d318ac 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -84,6 +84,7 @@ - (NSString *)accessibilityValue { @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, strong) UILabel *maskedToggleStatus; +@property(nonatomic, strong) UILabel *maskedDeliveryStatus; @property(nonatomic, assign) NSUInteger maskedTogglePresses; @property(nonatomic, strong) UILabel *alertActionStatus; @property(nonatomic, strong) UILabel *alertActivationBusyAnswer; @@ -466,6 +467,9 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) { static const CGFloat AgentDeviceTextEntryNeighbourGap = 16; - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { + if (self.maskedDeliveryStatus != nil) { + self.maskedDeliveryStatus.text = [textField.text hasSuffix:@"not-a-secret-123"] ? @"delivered" : @"pending"; + } // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst // typed faster than the app renders loses the characters that arrived while a render was in // flight, and the field settles stable short of the request. @@ -809,6 +813,15 @@ - (void)viewDidLoad { [self.maskedToggleStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], [self.maskedToggleStatus.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:12], ]]; + self.maskedDeliveryStatus = [[UILabel alloc] init]; + self.maskedDeliveryStatus.accessibilityIdentifier = @"agent-device-masked-delivery"; + self.maskedDeliveryStatus.text = @"pending"; + self.maskedDeliveryStatus.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:self.maskedDeliveryStatus]; + [NSLayoutConstraint activateConstraints:@[ + [self.maskedDeliveryStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [self.maskedDeliveryStatus.topAnchor constraintEqualToAnchor:self.maskedToggleStatus.bottomAnchor constant:8], + ]]; [self updateMaskedToggleStatus]; } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index ac0ac0e54d..f8a5431ba0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -166,7 +166,11 @@ extension RunnerTests { XCTAssertTrue(response.ok, "prefilled: \(prefilled) \(String(describing: response.error))") XCTAssertEqual(response.data?.textEntryRoute, "synthesized-first-responder-replacement") XCTAssertEqual(response.data?.message, "typed", "prefilled: \(prefilled)") - XCTAssertEqual(textField.value as? String, String(repeating: "\u{2022}", count: 8), "prefilled: \(prefilled)") + // An unreadable field cannot certify replacement or commit timing. Readable-field tests + // below pin exact replacement, including dropped select-alls; this case pins classification. + let delivered = app.staticTexts["agent-device-masked-delivery"] + let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) + XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed) XCTAssertEqual(togglePresses.label, "presses=0", "prefilled: \(prefilled)") } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 22ffe9d0e0..9bc6da9329 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -441,7 +441,9 @@ extension RunnerTests { XCTAssertTrue(fillResponse.ok, String(describing: fillResponse.error)) XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertEqual(fillResponse.data?.message, "typed") - XCTAssertEqual(textField.value as? String, String(repeating: "\u{2022}", count: 8)) + let delivered = app.staticTexts["agent-device-masked-delivery"] + let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) + XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed) XCTAssertEqual(togglePresses.label, "presses=0") } diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md new file mode 100644 index 0000000000..3cd783667e --- /dev/null +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -0,0 +1,22 @@ +# iOS smoke contracts + +The merge gate must detect agent-device regressions on a controlled app. A command exit alone is not evidence of success. Each scenario owns semantic assertions and a known starting screen. Device commands run serially. + +## Decisions + +- Use the built CLI for runner preparation, replays, and fixture scenarios. Build once before preflight. A source client and a built client have different daemon code signatures; alternating them tests daemon replacement instead of the command under test. The forced-backend SDK probe sends to the existing daemon through the transport owner and cannot start or replace it. +- Report each smoke scenario as a Node subtest. Relaunch the installed fixture and observe its home screen before each UI scenario. Keep coverage accounting across the run. A failed subtest stays failed while later scenarios can produce independent evidence. +- Test deep links and stored-URL clear-state replay in their own smoke scenario. Assert the route, event name, payload, removed data canary, and onward navigation. Do not use URL delivery to set up snapshot depth or WebView capture tests. Navigate to those fixtures through stable app controls. +- Repeat destination observation only when typed runtime evidence says observation was prevented. A readable absence, invalid selector, or non-retriable failure ends the test. Limit retries to one, including callers that return the last failure instead of throwing. Disable the harness retry inside this owner so policies cannot multiply attempts. +- Test masked input against its unverified contract. Require the app to acknowledge receipt of the entire requested suffix and require zero password-toggle presses. Poll the independent app canary because synthesized posting does not promise synchronous commit. Do not infer replacement or character count from bullets. Readable-field regressions retain exact replacement assertions, including deliberately dropped select-alls; policy tests retain the unreadable classification and bounded clear behavior. + +## Failure evidence + +- [Run 39029845932](https://github.com/callstack/agent-device/actions/runs/39029845932): source-CLI runner preflight exhausted the daemon's 15-second startup budget. Its uploaded daemon log was empty; the log alone cannot identify why startup was slow. The lane now exercises the packaged entry point throughout without changing the daemon timeout. +- [Run 39027567868](https://github.com/callstack/agent-device/actions/runs/39027567868): step 84 failed in CoreSimulator URL delivery before any depth assertion. The depth scenario now enters through the fixture's Settings screen. +- [Run 39030211303](https://github.com/callstack/agent-device/actions/runs/39030211303): five destination waits found no app process. The failure screenshot showed an unanswered “Open in” confirmation. Waiting for a destination cannot answer a system prompt. The dedicated deep-link scenario retains this signal; it no longer prevents unrelated interaction tests from running. +- The original masked XCTest reproduced locally on iOS 27.0: the prefilled field showed 13 bullets instead of the asserted 8. This is not proof of failed delivery: the route explicitly returns unverified. The replacement assertion was stronger than its production contract. The new app-owned delivery canary distinguishes successful posting from missing or truncated input without claiming the old value was cleared. + +## Local verification + +Run the checked-in harness entry point with a current Release fixture, the selected simulator UDID, a purpose-specific state directory, and `AGENT_DEVICE_IOS_E2E=1 AGENT_DEVICE_IOS_E2E_TIER=smoke`. Build the CLI and unit-test-enabled XCTest runner first. The fixture must contain current JavaScript; an installed bundle ID is insufficient. Preserve the emitted coverage report and per-scenario evidence. Do not count skipped device tests as passed. diff --git a/examples/test-app/app/(tabs)/settings.tsx b/examples/test-app/app/(tabs)/settings.tsx index 1b561aa662..7f7ca57180 100644 --- a/examples/test-app/app/(tabs)/settings.tsx +++ b/examples/test-app/app/(tabs)/settings.tsx @@ -20,6 +20,7 @@ export default function SettingsRoute() { onOpenAccessorySetup={() => router.push('/accessory-setup')} onOpenAutomationLab={() => router.push('/automation')} onOpenInertSurface={() => router.push('/inert')} + onOpenSnapshotDepth={() => router.push('/snapshot-depth')} onOpenWebViewLab={() => router.push('/webview')} onRetryDiagnostics={state.retryDiagnostics} onSetNotificationsEnabled={state.setNotificationsEnabled} diff --git a/examples/test-app/src/screens/SettingsScreen.tsx b/examples/test-app/src/screens/SettingsScreen.tsx index 4fb28d8991..c580c3f21a 100644 --- a/examples/test-app/src/screens/SettingsScreen.tsx +++ b/examples/test-app/src/screens/SettingsScreen.tsx @@ -21,6 +21,7 @@ export interface SettingsScreenProps { onOpenAccessorySetup: () => void; onOpenAutomationLab: () => void; onOpenInertSurface: () => void; + onOpenSnapshotDepth: () => void; onOpenWebViewLab: () => void; onRetryDiagnostics: () => void; onSetNotificationsEnabled: (value: boolean) => void; @@ -74,6 +75,14 @@ export function SettingsScreen(props: SettingsScreenProps) { /> + + + + ): CliJsonResult { - return result(1, { error: { code: 'COMMAND_FAILED', details } }); + return result(1, { error: { code: 'COMMAND_FAILED', retriable: true, details } }); } const LANDED = result(0, { success: true }); @@ -39,20 +39,37 @@ test('a destination seen on the first wait costs one wait', async () => { assert.equal(calls.length, 1); }); -test('a runner start and a slow foreground are waited out within the bounded waits', async () => { - const { wait, calls } = waits([RUNNER_START_TIMEOUT, SLOW_FOREGROUND, LANDED]); +test('an observation prevented by runner startup gets one repeat', async () => { + const { wait, calls } = waits([RUNNER_START_TIMEOUT, LANDED]); assert.equal(await retryDestinationWait(wait), LANDED); assert.deepEqual( calls.map((call) => call.allowFailure), - [true, true, true], + [true, false], ); }); -test('the fifth miss fails the wait instead of retrying again', async () => { - const { wait, calls } = waits([SLOW_FOREGROUND]); - await assert.rejects(retryDestinationWait(wait), /\(5\/5\) failed/); - assert.equal(calls.length, 5); - assert.equal(calls.at(-1)?.allowFailure, false); +test('a readable destination miss fails without repeating the wait', async () => { + const { wait, calls } = waits([SLOW_FOREGROUND, LANDED]); + await assert.rejects(retryDestinationWait(wait), /deep-link destination wait failed/); + assert.equal(calls.length, 1); +}); + +test('the second observation failure ends the wait', async () => { + const { wait, calls } = waits([RUNNER_START_TIMEOUT]); + await assert.rejects(retryDestinationWait(wait), /\(2\/2\) failed/); + assert.equal(calls.length, 2); +}); + +test('the final failed result cannot start an unbounded wait if the caller returns it', async () => { + let calls = 0; + await assert.rejects( + retryDestinationWait(async () => { + calls += 1; + return RUNNER_START_TIMEOUT; + }), + /deep-link destination wait failed/, + ); + assert.equal(calls, 2); }); test('a miss that is not a pending destination fails at once', async () => { diff --git a/test/integration/ios-simulator-e2e/behavior-coverage.ts b/test/integration/ios-simulator-e2e/behavior-coverage.ts index d157fc50f6..7e8fdf6b13 100644 --- a/test/integration/ios-simulator-e2e/behavior-coverage.ts +++ b/test/integration/ios-simulator-e2e/behavior-coverage.ts @@ -32,7 +32,7 @@ export const IOS_SIMULATOR_BEHAVIOR_COVERAGE = { assertion: 'launchApp(clearState) removes seeded app data, opens the stored deep route, renders its payload, and navigates onward', level: 'live', - owner: 'smoke:automation-input', + owner: 'smoke:deep-link-navigation', }, 'text-entry-keyboard-lifecycle': { assertion: diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index cf0465b30a..d2df25bfe9 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -83,26 +83,6 @@ export async function assertAutomationInput(context: LiveContext): Promise verifyCommand(context, C.snapshot, 'interactive fixture tree exposes its stable title'); verifyCommand(context, C.open, 'cold launch exposes the fixture UI through snapshot and wait'); - await openAutomationDeepLink(context, 'cold launch fixture through a deep link'); - await waitForDeepLinkDestination(context, ['text', 'Automation lab']); - await assertElementText(context, 'id="automation-event-name"', 'cold.start'); - await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); - await assertClearStateLaunchUrl(context); - await runStep(context, 'navigate onward from cold deep link', [ - 'click', - 'id="automation-continue-catalog"', - ]); - await runStep(context, 'wait for exact catalog destination', [ - 'wait', - 'id="catalog-title"', - '10000', - ]); - verifyBehavior( - context, - 'cold-start-deep-link-navigation', - 'launchApp(clearState) removed a seeded data canary, rendered the stored deep route and payload, and continued into the fixture catalog', - ); - await runStep(context, 'wait for settings tab target', ['wait', 'label="Settings"', '10000']); await runStep(context, 'open settings tab', ['click', 'label="Settings"']); await assertWaitText(context, 'Settings'); @@ -217,6 +197,35 @@ export async function assertAutomationInput(context: LiveContext): Promise verifyCommand(context, C.back, 'back returns from automation route to Settings'); } +export async function assertColdDeepLinkNavigation(context: LiveContext): Promise { + await openAutomationDeepLink(context, 'cold launch fixture through a deep link'); + await waitForDeepLinkDestination(context, ['text', 'Automation lab']); + await assertElementText(context, 'id="automation-event-name"', 'cold.start'); + await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); + await assertClearStateLaunchUrl(context); + await runStep(context, 'navigate onward from cold deep link', [ + 'click', + 'id="automation-continue-catalog"', + ]); + await runStep(context, 'wait for exact catalog destination', [ + 'wait', + 'id="catalog-title"', + '10000', + ]); + verifyBehavior( + context, + 'cold-start-deep-link-navigation', + 'launchApp(clearState) removed a seeded data canary, rendered the stored deep route and payload, and continued into the fixture catalog', + ); + + await runStep(context, 'restore home after deep-link contract', [ + 'open', + context.appId, + '--relaunch', + ]); + await assertWaitText(context, FIXTURE_HOME_TITLE); +} + async function assertClearStateLaunchUrl(context: LiveContext): Promise { const prepared = await runStep(context, 'prepare fixture data clear canary', [ 'settings', diff --git a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts index 321d543094..d4bc7c7ae6 100644 --- a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts +++ b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts @@ -1,27 +1,14 @@ import assert from 'node:assert/strict'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { WAIT_REASONS } from '@agent-device/contracts/wait'; +import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts'; import type { CliJsonResult } from '../cli-json.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; /** One destination wait; generous because the WebView lab took over 2.5 s to mount on cold CI. */ const DESTINATION_WAIT_MS = 15_000; -/** - * The answered launch usually reaches the foreground within 3 s, but a loaded host held it for - * 20.7 s (CI run 35991523779) and 27.4 s (a local run), and a relaunch can restart the runner, - * whose start outlasts one wait. - */ -const DESTINATION_WAITS = 5; - -/** Misses that say the destination was not observed yet, so another bounded wait may see it. */ -const NOT_YET_OBSERVED: ReadonlySet = new Set([ - WAIT_REASONS.deadlineExceeded, - WAIT_REASONS.captureStalled, - WAIT_REASONS.runnerRestartExhausted, - WAIT_REASONS.readinessExhausted, - WAIT_REASONS.targetAbsent, -]); +// One repeat is allowed only when the runtime could not make an observation. +const DESTINATION_WAITS = 2; /** One destination wait; the final one is run without `allowFailure`, so its failure throws. */ export type DestinationWait = ( @@ -31,8 +18,7 @@ export type DestinationWait = ( /** * Waits for a deep-link route's own first landmark after `open` answered any launch confirmation. - * Retries a miss that only says the destination was not observed yet, up to five 15 s waits; - * any other failure fails at once. + * Repeats only a typed, retriable observation failure. A readable miss fails immediately. */ export async function waitForDeepLinkDestination( context: LiveContext, @@ -50,7 +36,7 @@ export async function waitForDeepLinkDestination( String(DESTINATION_WAIT_MS), ...(options.debug ? ['--debug'] : []), ], - { allowFailure }, + { allowFailure, reattemptInfrastructureMiss: undefined }, ), ); verifyCommand( @@ -62,16 +48,16 @@ export async function waitForDeepLinkDestination( } export async function retryDestinationWait(wait: DestinationWait): Promise { - for (let attempt = 1; ; attempt += 1) { + for (let attempt = 1; attempt <= DESTINATION_WAITS; attempt += 1) { const arrived = await wait( `wait for the deep-link destination (${attempt}/${DESTINATION_WAITS})`, { allowFailure: attempt < DESTINATION_WAITS }, ); if (arrived.status === 0) return arrived; - const reason = arrived.json?.error?.details?.reason; assert.ok( - NOT_YET_OBSERVED.has(reason), + attempt < DESTINATION_WAITS && isObservationPreventedStepMiss(arrived), `deep-link destination wait failed: ${JSON.stringify(arrived.json)}`, ); } + throw new Error('deep-link destination observation exhausted'); } diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index d01b2f1025..57e56d7302 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -1,11 +1,14 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import type { TestContext } from 'node:test'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { sendToDaemon } from '../../../src/daemon-client/daemon-client.ts'; +import { readDaemonInfo } from '../../../src/daemon-client/daemon-client-metadata.ts'; +import { sendRequest } from '../../../src/daemon-client/daemon-client-transport.ts'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { assertPngFile } from '../provider-scenarios/assertions.ts'; import { assertFilesDiffer, @@ -15,7 +18,7 @@ import { capturePng, requireNodeRect, } from './live-assertions.ts'; -import { assertAutomationInput } from './live-automation-scenario.ts'; +import { assertAutomationInput, assertColdDeepLinkNavigation } from './live-automation-scenario.ts'; import { assertDeviceLifecycle } from './live-device-lifecycle.ts'; import { assertRegularVisibleDepthFrontier } from './live-snapshot-depth-frontier.ts'; import { assertWebViewRemoteContent } from './live-webview-remote-content.ts'; @@ -54,7 +57,22 @@ const sendToDaemonTransport: AgentDeviceDaemonTransport = async (request, contex if (request.session === undefined) { throw new Error('Snapshot conformance transport requires an explicit session.'); } - return await sendToDaemon({ ...request, session: request.session }, context); + const stateDir = request.flags?.stateDir; + assert.ok(typeof stateDir === 'string', 'backend conformance requires the smoke state directory'); + const paths = resolveDaemonPaths(stateDir, { env: process.env }); + const info = readDaemonInfo(paths.infoPath); + assert.ok(info, 'backend conformance requires the daemon prepared by the built CLI'); + return await sendRequest( + info, + { ...request, session: request.session, token: info.token }, + 'auto', + paths, + 90_000, + { + signal: context?.signal, + timeoutRecovery: 'none', + }, + ); }; async function loadBuiltAgentDeviceClient() { @@ -74,6 +92,7 @@ const LIVE_SCENARIOS = bindIosSimulatorScenarios({ await assertClose(context); }, deviceLifecycle: assertDeviceLifecycle, + deepLinkNavigation: assertColdDeepLinkNavigation, fixtureReplays: assertFixtureReplays, formInput: assertFormInput, inventoryInstall: assertInventoryAndInstall, @@ -83,12 +102,12 @@ const LIVE_SCENARIOS = bindIosSimulatorScenarios({ webviewRemoteContent: assertWebViewRemoteContent, }); -export async function runIosSimulatorE2E(): Promise { +export async function runIosSimulatorE2E(testContext?: TestContext): Promise { const context = createContext(); let primaryError: unknown; try { await prepareRunner(context); - await executeLiveScenarios(context); + await executeLiveScenarios(context, testContext); } catch (error) { primaryError = error; } @@ -121,11 +140,31 @@ async function prepareRunner(context: LiveContext): Promise { ); } -async function executeLiveScenarios(context: LiveContext): Promise { +async function executeLiveScenarios( + context: LiveContext, + testContext?: TestContext, +): Promise { let assertionCaptureVerified = false; for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) { - await runScenario(context, scenario); - if (!assertionCaptureVerified && context.sessionOpen) { + const run = async () => { + if (scenario.id !== 'smoke:inventory-install') { + const opened = await runStep(context, `reset fixture for ${scenario.id}`, [ + 'open', + context.appId, + '--relaunch', + ]); + context.runnerLogPath = opened.json?.data?.runnerLogPath; + await assertWaitText(context, 'Agent Device Tester', { timeoutMs: 60_000 }); + } + await runScenario(context, scenario); + }; + if (testContext) await testContext.test(scenario.id, run); + else await run(); + if ( + !assertionCaptureVerified && + context.sessionOpen && + context.completedScenarios.includes(scenario.id) + ) { await assertLiveAssertionCapture(context); assertionCaptureVerified = true; } diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index 027aa02ec0..75a4ce3bdb 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -7,25 +7,20 @@ import { type LiveSnapshotNode as SnapshotNode, snapshotNodes, } from './live-assertions.ts'; -import { waitForDeepLinkDestination } from './live-deep-link-destination.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; -const VISIBLE_DEPTH_DEEP_LINK = 'agent-device-test-app:///snapshot-depth'; const CHILD_ID = 'visible-depth-projected-child'; const MISSING_HITTABILITY_WARNING = 'iOS snapshot acquisition does not provide hittability evidence; regular snapshots omit unverified hittability while raw snapshots preserve supplied facts.'; export async function assertRegularVisibleDepthFrontier(context: LiveContext): Promise { + await runStep(context, 'open depth fixture settings', ['click', 'label="Settings"']); + await assertWaitText(context, 'Settings'); await runStep(context, 'open regular visible-depth fixture', [ - 'open', - context.appId, - '--relaunch', - '--launch-url', - VISIBLE_DEPTH_DEEP_LINK, - '--debug', + 'click', + 'id="open-snapshot-depth-lab"', ]); - // Wait for the target itself so the depth assertion is about the frontier, not route readiness. - await waitForDeepLinkDestination(context, [`id="${CHILD_ID}"`], { debug: true }); + await runStep(context, 'wait for visible-depth fixture', ['wait', `id="${CHILD_ID}"`, '15000']); const regular = await runStep(context, 'capture regular visible-depth frontier', [ 'snapshot', diff --git a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts index 0efd7e1089..d0bbe68971 100644 --- a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts +++ b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts @@ -1,10 +1,12 @@ import assert from 'node:assert/strict'; -import { assertWaitText, snapshotNodes } from './live-assertions.ts'; -import { waitForDeepLinkDestination } from './live-deep-link-destination.ts'; +import { + assertElementTextAfterScrolling, + assertWaitText, + snapshotNodes, +} from './live-assertions.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; -const WEBVIEW_LAB_DEEP_LINK = 'agent-device-test-app:///webview'; // Native chrome of the lab, rendered with the route and before the page: it proves the deep link // landed without asking the runner to query a screen whose web view is still loading. const LAB_CHROME_ID = 'close-webview-lab'; @@ -28,14 +30,19 @@ const XCTEST_FALLBACK_WARNING = * vanish from every snapshot and no ref can reach them. */ export async function assertWebViewRemoteContent(context: LiveContext): Promise { - await runStep(context, 'open WebView accessibility lab', [ - 'open', - context.appId, - '--relaunch', - '--launch-url', - WEBVIEW_LAB_DEEP_LINK, + await runStep(context, 'open WebView fixture settings', ['click', 'label="Settings"']); + await assertWaitText(context, 'Settings'); + await assertElementTextAfterScrolling( + context, + 'id="open-webview-lab"', + 'Open WebView accessibility lab', + ); + await runStep(context, 'open WebView accessibility lab', ['click', 'id="open-webview-lab"']); + await runStep(context, 'wait for WebView native chrome', [ + 'wait', + `id="${LAB_CHROME_ID}"`, + '15000', ]); - await waitForDeepLinkDestination(context, [`id="${LAB_CHROME_ID}"`]); // `wait` observes through the same route as `snapshot`: page content is reachable only once the // route has stopped publishing the bridge's page-less tree. await runStep(context, 'wait for the WebView page to expose its link', [ diff --git a/test/integration/ios-simulator-e2e/scenarios.ts b/test/integration/ios-simulator-e2e/scenarios.ts index 7418fdfd03..5ac889b238 100644 --- a/test/integration/ios-simulator-e2e/scenarios.ts +++ b/test/integration/ios-simulator-e2e/scenarios.ts @@ -6,6 +6,7 @@ export type IosSimulatorScenario = { type ScenarioRunnerKey = | 'automationInput' | 'captureClose' + | 'deepLinkNavigation' | 'deviceLifecycle' | 'fixtureReplays' | 'formInput' @@ -29,6 +30,7 @@ const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ tier: 'smoke', }, { id: 'smoke:webview-remote-content', runner: 'webviewRemoteContent', tier: 'smoke' }, + { id: 'smoke:deep-link-navigation', runner: 'deepLinkNavigation', tier: 'smoke' }, { id: 'smoke:capture-close', runner: 'captureClose', tier: 'smoke' }, { id: 'full:lifecycle-system', runner: 'lifecycleSystem', tier: 'full' }, { From fe3341ef30e0bfb2794891fceb51530c5a38959c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 16:18:31 +0200 Subject: [PATCH 02/12] test(ios): synchronize fixture readiness and explicit launch prompts --- .../RunnerTests+TextTypingTests.swift | 4 +- .../ios-smoke-contracts-2026-10-17.md | 4 +- ...imulator-e2e-deep-link-destination.test.ts | 45 ++++++++++++++++++- .../live-automation-scenario.ts | 22 ++++++++- .../live-deep-link-destination.ts | 20 +++++++++ .../ios-simulator-e2e/live-runner.ts | 26 ++++++----- 6 files changed, 104 insertions(+), 17 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 9bc6da9329..ef7475e197 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -442,8 +442,8 @@ extension RunnerTests { XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertEqual(fillResponse.data?.message, "typed") let delivered = app.staticTexts["agent-device-masked-delivery"] - let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) - XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed) + let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) + XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed) XCTAssertEqual(togglePresses.label, "presses=0") } diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index 3cd783667e..a6bc587821 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -5,8 +5,8 @@ The merge gate must detect agent-device regressions on a controlled app. A comma ## Decisions - Use the built CLI for runner preparation, replays, and fixture scenarios. Build once before preflight. A source client and a built client have different daemon code signatures; alternating them tests daemon replacement instead of the command under test. The forced-backend SDK probe sends to the existing daemon through the transport owner and cannot start or replace it. -- Report each smoke scenario as a Node subtest. Relaunch the installed fixture and observe its home screen before each UI scenario. Keep coverage accounting across the run. A failed subtest stays failed while later scenarios can produce independent evidence. -- Test deep links and stored-URL clear-state replay in their own smoke scenario. Assert the route, event name, payload, removed data canary, and onward navigation. Do not use URL delivery to set up snapshot depth or WebView capture tests. Navigate to those fixtures through stable app controls. +- Report each smoke scenario as a Node subtest. Relaunch the installed fixture and observe its home landmark and visible tab control before each UI scenario. A title alone can precede the React tab bar. Keep coverage accounting across the run. A failed subtest stays failed while later scenarios can produce independent evidence. +- Test deep links and stored-URL clear-state replay in their own smoke scenario. When open reports the launch is not observable and did not accept a confirmation, wait for the optional system alert and answer only the exact fixture confirmation. A typed absent alert needs no action; any other prompt or alert failure fails the scenario. Assert the route, event name, payload, removed data canary, and onward navigation. This journey tests explicit prompt handling; automatic acceptance remains owned by the launch-confirmation and lifecycle tests. Do not use URL delivery to set up snapshot depth or WebView capture tests. Navigate to those fixtures through stable app controls. - Repeat destination observation only when typed runtime evidence says observation was prevented. A readable absence, invalid selector, or non-retriable failure ends the test. Limit retries to one, including callers that return the last failure instead of throwing. Disable the harness retry inside this owner so policies cannot multiply attempts. - Test masked input against its unverified contract. Require the app to acknowledge receipt of the entire requested suffix and require zero password-toggle presses. Poll the independent app canary because synthesized posting does not promise synchronous commit. Do not infer replacement or character count from bullets. Readable-field regressions retain exact replacement assertions, including deliberately dropped select-alls; policy tests retain the unreadable classification and bounded clear behavior. diff --git a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts index 1164a16589..e469868316 100644 --- a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts +++ b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts @@ -3,7 +3,10 @@ import test from 'node:test'; import { WAIT_REASONS } from '@agent-device/contracts/wait'; import type { CliJsonResult } from './cli-json.ts'; -import { retryDestinationWait } from './ios-simulator-e2e/live-deep-link-destination.ts'; +import { + answerFixtureLaunchConfirmation, + retryDestinationWait, +} from './ios-simulator-e2e/live-deep-link-destination.ts'; function result(status: number, json?: unknown): CliJsonResult { return { json, status, stderr: '', stdout: '' }; @@ -77,3 +80,43 @@ test('a miss that is not a pending destination fails at once', async () => { await assert.rejects(retryDestinationWait(wait), /deep-link destination wait failed/); assert.equal(calls.length, 1); }); + +test('the fixture launch confirmation is answered, while an absent alert needs no action', async () => { + let accepts = 0; + const accept = async () => { + accepts += 1; + }; + await answerFixtureLaunchConfirmation( + async () => + result(0, { data: { message: 'Open in “Agent Device Tester”?', items: ['Cancel', 'Open'] } }), + accept, + ); + assert.equal(accepts, 1); + await answerFixtureLaunchConfirmation( + async () => result(1, { error: { details: { runnerErrorCode: 'ALERT_NOT_FOUND' } } }), + accept, + ); + assert.equal(accepts, 1); +}); + +test('an unrelated prompt or an unreadable alert is never accepted as a fixture launch', async () => { + let accepts = 0; + const accept = async () => { + accepts += 1; + }; + await assert.rejects( + answerFixtureLaunchConfirmation( + async () => + result(0, { data: { message: 'Allow microphone access?', items: ['Cancel', 'Open'] } }), + accept, + ), + ); + await assert.rejects( + answerFixtureLaunchConfirmation( + async () => result(1, { error: { message: 'ALERT_NOT_FOUND' } }), + accept, + ), + /launch confirmation observation failed/, + ); + assert.equal(accepts, 0); +}); diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index d2df25bfe9..6f6d83cf87 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -11,7 +11,10 @@ import { assertJsonContains, assertWaitText, } from './live-assertions.ts'; -import { waitForDeepLinkDestination } from './live-deep-link-destination.ts'; +import { + answerFixtureLaunchConfirmation, + waitForDeepLinkDestination, +} from './live-deep-link-destination.ts'; import { clearStateLaunchUrlMaestroFlow } from './live-fixtures.ts'; import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; @@ -263,7 +266,7 @@ async function assertClearStateLaunchUrl(context: LiveContext): Promise { async function openAutomationDeepLink(context: LiveContext, step: string): Promise { // --debug keeps the open's own request log, which is where a held launch URL's alert read, the // post-open observation verdict and any re-hand-off are visible when this step fails. - await runStep(context, step, [ + const opened = await runStep(context, step, [ 'open', context.appId, '--relaunch', @@ -271,4 +274,19 @@ async function openAutomationDeepLink(context: LiveContext, step: string): Promi AUTOMATION_DEEP_LINK, '--debug', ]); + if ( + opened.json?.data?.timing?.postOpenObservation !== 'observable' && + opened.json?.data?.launchConfirmation !== 'accepted' + ) { + await answerFixtureLaunchConfirmation( + () => + runStep( + context, + 'wait for optional fixture launch confirmation', + ['alert', 'wait', ALERT_WAIT_TIMEOUT], + { allowFailure: true }, + ), + () => runStep(context, 'accept fixture launch confirmation', ['alert', 'accept']), + ); + } } diff --git a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts index d4bc7c7ae6..d7a0851167 100644 --- a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts +++ b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; +import { ALERT_NOT_FOUND_RUNNER_CODE } from '@agent-device/contracts/alert-contract'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts'; import type { CliJsonResult } from '../cli-json.ts'; @@ -61,3 +62,22 @@ export async function retryDestinationWait(wait: DestinationWait): Promise Promise, + accept: () => Promise, +): Promise { + const result = await read(); + if (result.status !== 0) { + assert.equal( + result.json?.error?.details?.runnerErrorCode, + ALERT_NOT_FOUND_RUNNER_CODE, + `launch confirmation observation failed: ${JSON.stringify(result.json)}`, + ); + return; + } + assert.equal(result.json?.data?.message, 'Open in “Agent Device Tester”?'); + assert.deepEqual(result.json?.data?.items, ['Cancel', 'Open']); + await accept(); +} diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index 57e56d7302..3bd8459db1 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -147,16 +147,22 @@ async function executeLiveScenarios( let assertionCaptureVerified = false; for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) { const run = async () => { - if (scenario.id !== 'smoke:inventory-install') { - const opened = await runStep(context, `reset fixture for ${scenario.id}`, [ - 'open', - context.appId, - '--relaunch', - ]); - context.runnerLogPath = opened.json?.data?.runnerLogPath; - await assertWaitText(context, 'Agent Device Tester', { timeoutMs: 60_000 }); - } - await runScenario(context, scenario); + await runScenario(context, { + ...scenario, + run: async () => { + if (scenario.id !== 'smoke:inventory-install') { + const opened = await runStep(context, `reset fixture for ${scenario.id}`, [ + 'open', + context.appId, + '--relaunch', + ]); + context.runnerLogPath = opened.json?.data?.runnerLogPath; + await assertWaitSelector(context, 'id="home-title" visible=true'); + await assertWaitSelector(context, 'label="Form" visible=true'); + } + await scenario.run(context); + }, + }); }; if (testContext) await testContext.test(scenario.id, run); else await run(); From e8378575c4782890a3d7d456d74fb7432cc3c60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 16:22:59 +0200 Subject: [PATCH 03/12] test(ios): isolate session hints and observe before inspecting prompts --- .../ios-smoke-contracts-2026-10-17.md | 4 +-- ...imulator-e2e-deep-link-destination.test.ts | 22 ++++++++++++++ .../live-automation-scenario.ts | 22 ++------------ .../live-deep-link-destination.ts | 29 +++++++++++++++++-- .../ios-simulator-e2e/live-runner.ts | 11 ++++++- 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index a6bc587821..516d469af9 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -5,8 +5,8 @@ The merge gate must detect agent-device regressions on a controlled app. A comma ## Decisions - Use the built CLI for runner preparation, replays, and fixture scenarios. Build once before preflight. A source client and a built client have different daemon code signatures; alternating them tests daemon replacement instead of the command under test. The forced-backend SDK probe sends to the existing daemon through the transport owner and cannot start or replace it. -- Report each smoke scenario as a Node subtest. Relaunch the installed fixture and observe its home landmark and visible tab control before each UI scenario. A title alone can precede the React tab bar. Keep coverage accounting across the run. A failed subtest stays failed while later scenarios can produce independent evidence. -- Test deep links and stored-URL clear-state replay in their own smoke scenario. When open reports the launch is not observable and did not accept a confirmation, wait for the optional system alert and answer only the exact fixture confirmation. A typed absent alert needs no action; any other prompt or alert failure fails the scenario. Assert the route, event name, payload, removed data canary, and onward navigation. This journey tests explicit prompt handling; automatic acceptance remains owned by the launch-confirmation and lifecycle tests. Do not use URL delivery to set up snapshot depth or WebView capture tests. Navigate to those fixtures through stable app controls. +- Report each smoke scenario as a Node subtest. Close the preceding session and create a distinct session for each scenario. Relaunch the installed fixture and observe its home landmark and visible tab control before UI scenarios that do not own their launch. Session recreation clears stored launch URLs; relaunch alone deliberately retains them. A title alone can precede the React tab bar. Keep coverage accounting across the run. A failed subtest stays failed while later scenarios can produce independent evidence. +- Test deep links and stored-URL clear-state replay in their own smoke scenario. Observe the destination first. Only a typed `APP_NOT_RUNNING` result starts the optional system-alert wait; answer only the exact fixture confirmation. A typed absent alert needs no action; any other prompt or alert failure fails the scenario. Assert the route, event name, payload, removed data canary, and onward navigation. This journey tests explicit prompt handling; automatic acceptance remains owned by the launch-confirmation and lifecycle tests. Do not use URL delivery to set up snapshot depth or WebView capture tests. Navigate to those fixtures through stable app controls. - Repeat destination observation only when typed runtime evidence says observation was prevented. A readable absence, invalid selector, or non-retriable failure ends the test. Limit retries to one, including callers that return the last failure instead of throwing. Disable the harness retry inside this owner so policies cannot multiply attempts. - Test masked input against its unverified contract. Require the app to acknowledge receipt of the entire requested suffix and require zero password-toggle presses. Poll the independent app canary because synthesized posting does not promise synchronous commit. Do not infer replacement or character count from bullets. Readable-field regressions retain exact replacement assertions, including deliberately dropped select-alls; policy tests retain the unreadable classification and bounded clear behavior. diff --git a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts index e469868316..f6fcfba411 100644 --- a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts +++ b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts @@ -120,3 +120,25 @@ test('an unrelated prompt or an unreadable alert is never accepted as a fixture ); assert.equal(accepts, 0); }); + +test('a stopped app is checked for its launch confirmation before the destination is observed again', async () => { + const { wait, calls } = waits([miss({ runnerErrorCode: 'APP_NOT_RUNNING' }), LANDED]); + let answers = 0; + assert.equal( + await retryDestinationWait(wait, async () => { + answers += 1; + }), + LANDED, + ); + assert.equal(answers, 1); + assert.equal(calls.length, 2); +}); + +test('an already observed destination never probes for an optional alert', async () => { + const { wait } = waits([LANDED]); + let answers = 0; + await retryDestinationWait(wait, async () => { + answers += 1; + }); + assert.equal(answers, 0); +}); diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index 6f6d83cf87..d2df25bfe9 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -11,10 +11,7 @@ import { assertJsonContains, assertWaitText, } from './live-assertions.ts'; -import { - answerFixtureLaunchConfirmation, - waitForDeepLinkDestination, -} from './live-deep-link-destination.ts'; +import { waitForDeepLinkDestination } from './live-deep-link-destination.ts'; import { clearStateLaunchUrlMaestroFlow } from './live-fixtures.ts'; import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; @@ -266,7 +263,7 @@ async function assertClearStateLaunchUrl(context: LiveContext): Promise { async function openAutomationDeepLink(context: LiveContext, step: string): Promise { // --debug keeps the open's own request log, which is where a held launch URL's alert read, the // post-open observation verdict and any re-hand-off are visible when this step fails. - const opened = await runStep(context, step, [ + await runStep(context, step, [ 'open', context.appId, '--relaunch', @@ -274,19 +271,4 @@ async function openAutomationDeepLink(context: LiveContext, step: string): Promi AUTOMATION_DEEP_LINK, '--debug', ]); - if ( - opened.json?.data?.timing?.postOpenObservation !== 'observable' && - opened.json?.data?.launchConfirmation !== 'accepted' - ) { - await answerFixtureLaunchConfirmation( - () => - runStep( - context, - 'wait for optional fixture launch confirmation', - ['alert', 'wait', ALERT_WAIT_TIMEOUT], - { allowFailure: true }, - ), - () => runStep(context, 'accept fixture launch confirmation', ['alert', 'accept']), - ); - } } diff --git a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts index d7a0851167..d4b0b632a0 100644 --- a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts +++ b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts @@ -1,6 +1,9 @@ import assert from 'node:assert/strict'; -import { ALERT_NOT_FOUND_RUNNER_CODE } from '@agent-device/contracts/alert-contract'; +import { + ALERT_NOT_FOUND_RUNNER_CODE, + DEFAULT_ALERT_TIMEOUT_MS, +} from '@agent-device/contracts/alert-contract'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts'; import type { CliJsonResult } from '../cli-json.ts'; @@ -39,6 +42,17 @@ export async function waitForDeepLinkDestination( ], { allowFailure, reattemptInfrastructureMiss: undefined }, ), + () => + answerFixtureLaunchConfirmation( + () => + runStep( + context, + 'wait for optional fixture launch confirmation', + ['alert', 'wait', String(DEFAULT_ALERT_TIMEOUT_MS)], + { allowFailure: true }, + ), + () => runStep(context, 'accept fixture launch confirmation', ['alert', 'accept']), + ), ); verifyCommand( context, @@ -48,13 +62,24 @@ export async function waitForDeepLinkDestination( return arrived; } -export async function retryDestinationWait(wait: DestinationWait): Promise { +export async function retryDestinationWait( + wait: DestinationWait, + answerStoppedLaunch?: () => Promise, +): Promise { for (let attempt = 1; attempt <= DESTINATION_WAITS; attempt += 1) { const arrived = await wait( `wait for the deep-link destination (${attempt}/${DESTINATION_WAITS})`, { allowFailure: attempt < DESTINATION_WAITS }, ); if (arrived.status === 0) return arrived; + if ( + attempt < DESTINATION_WAITS && + answerStoppedLaunch && + arrived.json?.error?.details?.runnerErrorCode === 'APP_NOT_RUNNING' + ) { + await answerStoppedLaunch(); + continue; + } assert.ok( attempt < DESTINATION_WAITS && isObservationPreventedStepMiss(arrived), `deep-link destination wait failed: ${JSON.stringify(arrived.json)}`, diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index 3bd8459db1..5605a9dc91 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -145,12 +145,21 @@ async function executeLiveScenarios( testContext?: TestContext, ): Promise { let assertionCaptureVerified = false; + const sessionPrefix = context.session; for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) { const run = async () => { await runScenario(context, { ...scenario, run: async () => { - if (scenario.id !== 'smoke:inventory-install') { + if (await sessionExists(context)) { + await runStep(context, 'release preceding scenario session', ['close']); + } + context.session = `${sessionPrefix}-${scenario.id.replace(':', '-')}`; + if ( + scenario.id !== 'smoke:inventory-install' && + scenario.id !== 'smoke:automation-input' && + scenario.id !== 'smoke:deep-link-navigation' + ) { const opened = await runStep(context, `reset fixture for ${scenario.id}`, [ 'open', context.appId, From 560fb7ce186597b41314e3d110d308bc1a4eeb51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 16:24:42 +0200 Subject: [PATCH 04/12] test(ios): let scenario setup own fixture restoration --- .../ios-simulator-e2e/live-automation-scenario.ts | 7 ------- .../ios-simulator-e2e/live-snapshot-depth-frontier.ts | 6 ------ .../ios-simulator-e2e/live-webview-remote-content.ts | 6 ------ 3 files changed, 19 deletions(-) diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index d2df25bfe9..f5bd7992fc 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -217,13 +217,6 @@ export async function assertColdDeepLinkNavigation(context: LiveContext): Promis 'cold-start-deep-link-navigation', 'launchApp(clearState) removed a seeded data canary, rendered the stored deep route and payload, and continued into the fixture catalog', ); - - await runStep(context, 'restore home after deep-link contract', [ - 'open', - context.appId, - '--relaunch', - ]); - await assertWaitText(context, FIXTURE_HOME_TITLE); } async function assertClearStateLaunchUrl(context: LiveContext): Promise { diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index 75a4ce3bdb..e47896c8e3 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -82,12 +82,6 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P `raw --depth 1 exceeded the acquisition frontier: ${JSON.stringify(rawDepthOne)}`, ); - await runStep(context, 'restore fixture home after visible-depth capture', [ - 'open', - context.appId, - '--relaunch', - ]); - await assertWaitText(context, 'Agent Device Tester'); verifyBehavior( context, 'regular-visible-depth-frontier', diff --git a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts index d0bbe68971..05e582ea47 100644 --- a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts +++ b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts @@ -72,12 +72,6 @@ export async function assertWebViewRemoteContent(context: LiveContext): Promise< `WebView snapshot must disclose the XCTest fallback for remote content: ${JSON.stringify(snapshot)}`, ); - await runStep(context, 'restore fixture home after WebView lab', [ - 'open', - context.appId, - '--relaunch', - ]); - await assertWaitText(context, 'Agent Device Tester'); verifyBehavior( context, 'webview-remote-content', From 9d2fd527ec0ec7d7fe20ad3a65f946037622e708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 16:28:06 +0200 Subject: [PATCH 05/12] docs(ios): record smoke contracts and local verification evidence --- docs/evidence/ios-smoke-contracts-2026-10-17.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index 516d469af9..bd766e6b66 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -20,3 +20,15 @@ The merge gate must detect agent-device regressions on a controlled app. A comma ## Local verification Run the checked-in harness entry point with a current Release fixture, the selected simulator UDID, a purpose-specific state directory, and `AGENT_DEVICE_IOS_E2E=1 AGENT_DEVICE_IOS_E2E_TIER=smoke`. Build the CLI and unit-test-enabled XCTest runner first. The fixture must contain current JavaScript; an installed bundle ID is insufficient. Preserve the emitted coverage report and per-scenario evidence. Do not count skipped device tests as passed. + +### Verification results + +Device: iPhone 17 Pro, iOS 27.0, UDID `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`. Fixture: a fresh Release build of this checkout, with current JavaScript embedded. CLI and runner builds completed locally. CI still owns its iOS 26.2 environment. + +- Native source at `92db564a30ec0ff39aa248339b45b12901a1ead8`: the original masked XCTest pair produced one pass and one failure; the prefilled synthesized case observed 13 bullets versus the asserted 8. +- Revised native fixture and XCTest assertions: five focused tests passed, including both masked cases, exact readable replacement past dropped select-alls, shared unreadable classification, and bounded masked clearing. `pnpm check:xctest-selection` reached every declared regression. +- Harness source at `83af5f485e12e16ff23bcc5606282633cad4a1f9`: the isolated-session smoke command passed 22 tests, zero failures/skips, in 130 seconds. +- Final harness source at `31e6941b991d4f3421b8c306535472a61408ec36`: the same smoke command passed 22 tests, zero failures/skips, in 112 seconds. The seven subtests covered inventory/install, automation input, form input and forced-backend conformance, visible depth, WebView remote content, deep-link/clear-state navigation, and screenshot/session release. Each run closed its sessions and stopped its daemon. +- Focused harness and coverage checks passed 24 tests. `pnpm check:quick`, fixture TypeScript checking, gate-manifest checking, and XCTest-selection checking passed. +- At `31e6941b991d4f3421b8c306535472a61408ec36`, an isolated mutation restored the old policy's retries for `wait_deadline_exceeded`. The focused test `a readable destination miss fails without repeating the wait` changed from one pass to one failure (`Missing expected rejection`). The experiment checkout was removed; the implementation checkout remained unchanged. +- Independent read-only review found one unnecessary trailing deep-link relaunch. It was removed, along with the analogous depth/WebView restores; the final follow-up review found no remaining issues. From d0c9ece8163941a3f7ef4eebe4f39c3b73dd6ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 17:28:18 +0200 Subject: [PATCH 06/12] test(ios): declare smoke setup and consolidate authoring rules --- docs/agents/device-verification.md | 2 + docs/agents/ios-smoke.md | 68 +++++++++++++++++++ docs/agents/test-authoring.md | 2 + .../ios-smoke-contracts-2026-10-17.md | 16 +---- .../ios-simulator-e2e-cleanup.test.ts | 32 ++++++++- .../live-deep-link-destination.ts | 8 +-- .../ios-simulator-e2e/live-harness.ts | 28 +++----- .../ios-simulator-e2e/live-runner.ts | 10 +-- .../live-step-retry-policy.ts | 26 +------ .../ios-simulator-e2e/scenarios.ts | 41 +++++++---- 10 files changed, 151 insertions(+), 82 deletions(-) create mode 100644 docs/agents/ios-smoke.md diff --git a/docs/agents/device-verification.md b/docs/agents/device-verification.md index 1ed68c5746..147df6837e 100644 --- a/docs/agents/device-verification.md +++ b/docs/agents/device-verification.md @@ -2,6 +2,8 @@ Use this for Apple runner changes or manual device runs. For foldables, also read [foldable verification](device-foldable.md). +For iOS fixture smoke or native input regressions, read [iOS smoke authoring](ios-smoke.md). + ## Prepare 1. To build runtime changes, run `pnpm build`. diff --git a/docs/agents/ios-smoke.md b/docs/agents/ios-smoke.md new file mode 100644 index 0000000000..35f5ad22ca --- /dev/null +++ b/docs/agents/ios-smoke.md @@ -0,0 +1,68 @@ +# Author iOS smoke scenarios + +Use this for `test/integration/ios-simulator-e2e/` and its native input regressions. +Use [test authoring](test-authoring.md) for test boundaries and [device verification](device-verification.md) for builds and cleanup. + +## Declare the starting state + +Register the scenario in `scenarios.ts` and bind its implementation in `live-runner.ts`. +Declare `setup` on every smoke scenario: + +- `fixture-home`: the runner opens a fresh session, relaunches the fixture, and observes the home title and visible Form tab. +- `scenario-owned`: the scenario opens its own app or launch URL and observes its own first landmark. +- `no-session`: the scenario tests inventory or installation without opening an app. + +For a form interaction scenario, use the existing registration pattern: + +```ts +{ id: 'smoke:form-input', runner: 'formInput', tier: 'smoke', setup: 'fixture-home' } +``` + +The runner closes the preceding session and assigns a distinct session to each smoke scenario. +Session recreation clears stored launch URLs; relaunch alone retains them. +Keep assertions inside the scenario's Node subtest. +Keep command and behavior coverage in their existing coverage registries. +The full tier remains a sequential journey with scenario-owned transitions; smoke setup metadata does not apply to it. + +## Observe the behavior under test + +Navigate through stable fixture controls when navigation is setup for another feature. +Keep URL delivery in the dedicated deep-link scenario. +Assert the route, event payload, clear-state data canary, and onward navigation there. +Observe its destination before checking for a system confirmation. +If the result carries `APP_NOT_RUNNING`, use `waitForDeepLinkDestination` to handle the exact fixture confirmation. +A typed absent alert needs no action; an unexpected prompt or alert failure fails the scenario. +Automatic confirmation remains covered by the launch-confirmation and lifecycle tests. + +Use semantic readiness landmarks before interaction. +A title alone can precede the React tab controls. +Use the built CLI for preparation and execution after building once. +Use the built SDK with the existing-daemon transport for forced-backend probes. +Do not let a probe start or replace the daemon under test. + +## Keep recovery bounded + +Use `runStep` for the shared single-repeat observation policy. +Require both a retriable verdict and a typed prevented-observation reason. +Fail readable absence, exhausted readable waits, invalid selectors, and incorrect values. +If a helper owns retries, pass `reattemptInfrastructureMiss: undefined` on every attempt. +Keep cleanup retries in `retryCleanupStep`; return failed results with `allowFailure: true` so its typed already-clean handling applies on every attempt. +Report exhausted cleanup failures separately from scenario assertions. +Replay-suite attempt policies have their own owner; this step policy does not redefine them. + +## Match input assertions to the contract + +For masked unverified input, poll the app-owned delivery canary and assert zero password-toggle presses. +Require the canary to acknowledge the entire requested suffix. +Do not infer replacement or length from masking bullets. +For readable input, retain exact replacement assertions and deliberately dropped-select-all regressions. +Keep unreadable classification and bounded clearing in their owning policy tests. + +## Verify an extension + +Read the live entry point's environment contract before running it. +Build a current Release fixture with embedded JavaScript for simulator smoke verification. +Run the changed scenario through the checked-in smoke entry point and preserve its coverage and failure artifacts. +Record the source SHA, device, command, and semantic result under [device verification](device-verification.md). +For regressions, prove the test fails without the fix using [isolated experiments](test-experiments.md). +Do not count skipped device tests as passed. diff --git a/docs/agents/test-authoring.md b/docs/agents/test-authoring.md index 19800f795f..147bc72a4a 100644 --- a/docs/agents/test-authoring.md +++ b/docs/agents/test-authoring.md @@ -3,6 +3,8 @@ For checks, read [validation](testing.md). For deletions, read [test audits](test-audit.md). +For iOS fixture smoke or native input regressions, read [iOS smoke authoring](ios-smoke.md). + ## Choose the boundary Before you add a test, name: diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index bd766e6b66..c5454faf92 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -1,14 +1,6 @@ # iOS smoke contracts -The merge gate must detect agent-device regressions on a controlled app. A command exit alone is not evidence of success. Each scenario owns semantic assertions and a known starting screen. Device commands run serially. - -## Decisions - -- Use the built CLI for runner preparation, replays, and fixture scenarios. Build once before preflight. A source client and a built client have different daemon code signatures; alternating them tests daemon replacement instead of the command under test. The forced-backend SDK probe sends to the existing daemon through the transport owner and cannot start or replace it. -- Report each smoke scenario as a Node subtest. Close the preceding session and create a distinct session for each scenario. Relaunch the installed fixture and observe its home landmark and visible tab control before UI scenarios that do not own their launch. Session recreation clears stored launch URLs; relaunch alone deliberately retains them. A title alone can precede the React tab bar. Keep coverage accounting across the run. A failed subtest stays failed while later scenarios can produce independent evidence. -- Test deep links and stored-URL clear-state replay in their own smoke scenario. Observe the destination first. Only a typed `APP_NOT_RUNNING` result starts the optional system-alert wait; answer only the exact fixture confirmation. A typed absent alert needs no action; any other prompt or alert failure fails the scenario. Assert the route, event name, payload, removed data canary, and onward navigation. This journey tests explicit prompt handling; automatic acceptance remains owned by the launch-confirmation and lifecycle tests. Do not use URL delivery to set up snapshot depth or WebView capture tests. Navigate to those fixtures through stable app controls. -- Repeat destination observation only when typed runtime evidence says observation was prevented. A readable absence, invalid selector, or non-retriable failure ends the test. Limit retries to one, including callers that return the last failure instead of throwing. Disable the harness retry inside this owner so policies cannot multiply attempts. -- Test masked input against its unverified contract. Require the app to acknowledge receipt of the entire requested suffix and require zero password-toggle presses. Poll the independent app canary because synthesized posting does not promise synchronous commit. Do not infer replacement or character count from bullets. Readable-field regressions retain exact replacement assertions, including deliberately dropped select-alls; policy tests retain the unreadable classification and bounded clear behavior. +Authoring rules live in [iOS smoke authoring](../agents/ios-smoke.md). This file records the failures and verification behind PR #3786. ## Failure evidence @@ -17,11 +9,7 @@ The merge gate must detect agent-device regressions on a controlled app. A comma - [Run 39030211303](https://github.com/callstack/agent-device/actions/runs/39030211303): five destination waits found no app process. The failure screenshot showed an unanswered “Open in” confirmation. Waiting for a destination cannot answer a system prompt. The dedicated deep-link scenario retains this signal; it no longer prevents unrelated interaction tests from running. - The original masked XCTest reproduced locally on iOS 27.0: the prefilled field showed 13 bullets instead of the asserted 8. This is not proof of failed delivery: the route explicitly returns unverified. The replacement assertion was stronger than its production contract. The new app-owned delivery canary distinguishes successful posting from missing or truncated input without claiming the old value was cleared. -## Local verification - -Run the checked-in harness entry point with a current Release fixture, the selected simulator UDID, a purpose-specific state directory, and `AGENT_DEVICE_IOS_E2E=1 AGENT_DEVICE_IOS_E2E_TIER=smoke`. Build the CLI and unit-test-enabled XCTest runner first. The fixture must contain current JavaScript; an installed bundle ID is insufficient. Preserve the emitted coverage report and per-scenario evidence. Do not count skipped device tests as passed. - -### Verification results +## Verification results Device: iPhone 17 Pro, iOS 27.0, UDID `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`. Fixture: a fresh Release build of this checkout, with current JavaScript embedded. CLI and runner builds completed locally. CI still owns its iOS 26.2 environment. diff --git a/test/integration/ios-simulator-e2e-cleanup.test.ts b/test/integration/ios-simulator-e2e-cleanup.test.ts index 667779b121..5b9192342d 100644 --- a/test/integration/ios-simulator-e2e-cleanup.test.ts +++ b/test/integration/ios-simulator-e2e-cleanup.test.ts @@ -91,8 +91,7 @@ test('a different INVALID_ARGS still fails after exhausting retries', async (t) retryCleanupStep(OTHER_STEP, async (attempt) => { attempts += 1; if (attempt < 3) return invalidArgsResult(MIC_APPLESS_MESSAGE, sessionAppRequiredDetails()); - // Mirrors runStep(..., { allowFailure: false }) on the final attempt: it throws - // instead of returning a failed result. + // A thrown subprocess failure must also propagate on the final attempt. throw new Error(`cleanup: ${OTHER_STEP} (attempt 3) failed`); }), ); @@ -117,6 +116,35 @@ test('the appless message without its typed reason still fails after retries', a assert.equal(attempts, 3, 'should exhaust all three attempts'); }); +test('a returned cleanup failure on the third attempt is reported', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let attempts = 0; + const failure = await drainRetry( + t, + retryCleanupStep(OTHER_STEP, async () => { + attempts += 1; + return invalidArgsResult('unrelated refusal'); + }), + ); + assert.ok(failure instanceof Error); + assert.match(failure.message, /restore portrait orientation failed after 3 attempts/); + assert.equal(attempts, 3); +}); + +test('a session disappearing on the third cleanup attempt is already clean', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let attempts = 0; + const failure = await drainRetry( + t, + retryCleanupStep(OTHER_STEP, async () => { + attempts += 1; + return attempts === 3 ? sessionNotFoundResult() : invalidArgsResult('unrelated refusal'); + }), + ); + assert.equal(failure, undefined); + assert.equal(attempts, 3); +}); + // Deterministic regression for finalizeSessionCleanup: the other half of #1548, which // decides whether cleanup runs at all. A minimal LiveContext fixture; only sessionOpen // is read by the decision, the rest exists to satisfy the type. diff --git a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts index d4b0b632a0..43dd5b97f1 100644 --- a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts +++ b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts @@ -9,9 +9,9 @@ import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts'; import type { CliJsonResult } from '../cli-json.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; -/** One destination wait; generous because the WebView lab took over 2.5 s to mount on cold CI. */ +/** Allow the cold fixture route to mount before declaring a readable miss. */ const DESTINATION_WAIT_MS = 15_000; -// One repeat is allowed only when the runtime could not make an observation. +// One repeat follows a prevented observation or the stopped-app confirmation branch. const DESTINATION_WAITS = 2; /** One destination wait; the final one is run without `allowFailure`, so its failure throws. */ @@ -21,8 +21,8 @@ export type DestinationWait = ( ) => Promise; /** - * Waits for a deep-link route's own first landmark after `open` answered any launch confirmation. - * Repeats only a typed, retriable observation failure. A readable miss fails immediately. + * Observe the route before looking for a launch confirmation. + * A typed stopped-app result permits explicit confirmation handling; readable misses fail. */ export async function waitForDeepLinkDestination( context: LiveContext, diff --git a/test/integration/ios-simulator-e2e/live-harness.ts b/test/integration/ios-simulator-e2e/live-harness.ts index f448989f02..90169f777c 100644 --- a/test/integration/ios-simulator-e2e/live-harness.ts +++ b/test/integration/ios-simulator-e2e/live-harness.ts @@ -68,13 +68,7 @@ const harness = createLiveDeviceHarness({ writeCoverageReport, }); -/** - * Every iOS live step carries the observation-prevented re-issue policy (#2491): one runner - * restart or bridge rebuild re-issues its own wait instead of failing the job, while a readable - * miss (target absent, budget exhausted after a readable capture, wrong asserted value) fails at - * once exactly as before. Steps that already own a miss policy (`allowFailure` / `expectFailure`, - * like the destination-wait and cleanup retry loops) are exempt inside the harness itself. - */ +// Repeat at most one typed observation failure. Scenario-owned retry loops disable this policy. export function runStep( context: LiveContext, step: string, @@ -115,7 +109,8 @@ export async function cleanupSession(context: LiveContext): Promise { for (const [step, args] of cleanupSteps) { const failure = await retryCleanupStep(step, (attempt) => runStep(context, `cleanup: ${step} (attempt ${attempt})`, args, { - allowFailure: attempt < 3, + allowFailure: true, + reattemptInfrastructureMiss: undefined, }), ); if (failure !== undefined) failures.push(failure); @@ -126,28 +121,25 @@ export async function cleanupSession(context: LiveContext): Promise { throw new AggregateError(failures, `iOS E2E cleanup failed; details: ${errorPath}`); } -/** - * Runs one cleanup step's 3-attempt retry policy: success or an already-clean session - * stops immediately, anything else waits and retries. `runAttempt` mirrors the real - * `runStep(..., { allowFailure: attempt < 3 })` contract, including that the final - * attempt throws instead of returning a failed result. Exported so the retry/guard - * behavior is unit-testable without spawning the CLI (see - * `test/integration/ios-simulator-e2e-cleanup.test.ts`). - */ +/** Returns the failure after at most three attempts, or undefined when cleanup is complete. */ export async function retryCleanupStep( step: string, runAttempt: (attempt: number) => Promise, ): Promise { - for (let attempt = 1; attempt <= 3; attempt += 1) { + for (let attempt = 1; ; attempt += 1) { try { const result = await runAttempt(attempt); if (result.status === 0 || sessionAlreadyClean(step, result)) return undefined; + if (attempt === 3) { + return new Error( + `cleanup: ${step} failed after ${attempt} attempts: ${JSON.stringify(result)}`, + ); + } await new Promise((resolve) => setTimeout(resolve, 500)); } catch (error) { return error; } } - return undefined; } // SESSION_NOT_FOUND: nothing left to reset, any step. INVALID_ARGS with the published diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index 5605a9dc91..374053b628 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -76,9 +76,7 @@ const sendToDaemonTransport: AgentDeviceDaemonTransport = async (request, contex }; async function loadBuiltAgentDeviceClient() { - // The live harness drives the built CLI, so use the built SDK entry as well. Importing the - // source SDK here would intentionally take over the daemon on code-signature mismatch and make - // the forced-backend evidence come from a different runtime than the rest of the scenario. + // Exercise the same built SDK and CLI revision; the injected transport owns no daemon lifecycle. const builtSdk = (await import( pathToFileURL(path.resolve('dist/src/index.js')).href )) as AgentDeviceSdk; @@ -155,11 +153,7 @@ async function executeLiveScenarios( await runStep(context, 'release preceding scenario session', ['close']); } context.session = `${sessionPrefix}-${scenario.id.replace(':', '-')}`; - if ( - scenario.id !== 'smoke:inventory-install' && - scenario.id !== 'smoke:automation-input' && - scenario.id !== 'smoke:deep-link-navigation' - ) { + if (scenario.setup === 'fixture-home') { const opened = await runStep(context, `reset fixture for ${scenario.id}`, [ 'open', context.appId, diff --git a/test/integration/ios-simulator-e2e/live-step-retry-policy.ts b/test/integration/ios-simulator-e2e/live-step-retry-policy.ts index 9ad1f885f7..1edf380789 100644 --- a/test/integration/ios-simulator-e2e/live-step-retry-policy.ts +++ b/test/integration/ios-simulator-e2e/live-step-retry-policy.ts @@ -1,35 +1,15 @@ import { WAIT_REASONS } from '@agent-device/contracts/wait'; import type { CliJsonResult } from '../cli-json.ts'; -/** - * The wait verdicts that say the observation itself was prevented — the runner restarted, - * readiness work (runner start, bridge target discovery) ate the budget, or no capture was ever - * readable. Each is a lane condition on this host, not an answer about the screen: the product - * itself publishes all three with `retriable: true` and a "then retry" hint on the wire - * (`wait-polling.ts`), and they are the observed reds of the smoke lane (run 37356199982's - * `wait_runner_restart_exhausted` with `readableCaptures: 0`). - */ +// These wait verdicts indicate that capture or readiness prevented observation. const OBSERVATION_PREVENTED_WAIT_REASONS: ReadonlySet = new Set([ WAIT_REASONS.captureStalled, WAIT_REASONS.runnerRestartExhausted, WAIT_REASONS.readinessExhausted, ]); -/** - * Whether one failed E2E step's result says the DEVICE prevented the observation, so re-issuing - * the same step is a measurement of the same question rather than a second bite at a real defect. - * - * Keyed on the conjunction of two typed signals — the wire `retriable` verdict the daemon hoists - * from its throw sites AND `error.details.reason` from the wait taxonomy — never on error text. - * Either signal alone is too wide: `retriable` also rides failures the lane must not re-issue - * blind (e.g. an app the lane itself is expected to have launched via another route), and a reason - * set copied onto a response the product marks non-retriable would retry a verdict the product - * says cannot succeed unchanged. The pairing is pinned in - * `test/integration/ios-simulator-e2e-step-retry-policy.test.ts`. - * - * `wait_target_absent` (the screen was readable and simply did not hold the target), - * `wait_deadline_exceeded` (a readable capture consumed the budget), a wrong assertion value, a - * runner crash, and any non-wait failure are all non-retriable by this predicate. +/** Requires both a retriable verdict and a typed prevented-observation reason. + * Readable absence, deadline exhaustion, and non-wait failures cannot repeat here. */ export function isObservationPreventedStepMiss(result: CliJsonResult): boolean { const error = result.json?.error; diff --git a/test/integration/ios-simulator-e2e/scenarios.ts b/test/integration/ios-simulator-e2e/scenarios.ts index 5ac889b238..0d0f3314f6 100644 --- a/test/integration/ios-simulator-e2e/scenarios.ts +++ b/test/integration/ios-simulator-e2e/scenarios.ts @@ -1,7 +1,7 @@ -export type IosSimulatorScenario = { - id: string; - tier: 'smoke' | 'full'; -}; +export type IosSimulatorScenario = { id: string } & ( + | { tier: 'smoke'; setup: 'no-session' | 'scenario-owned' | 'fixture-home' } + | { tier: 'full' } +); type ScenarioRunnerKey = | 'automationInput' @@ -21,17 +21,33 @@ type ScenarioDefinition = IosSimulatorScenario & { }; const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ - { id: 'smoke:inventory-install', runner: 'inventoryInstall', tier: 'smoke' }, - { id: 'smoke:automation-input', runner: 'automationInput', tier: 'smoke' }, - { id: 'smoke:form-input', runner: 'formInput', tier: 'smoke' }, + { id: 'smoke:inventory-install', setup: 'no-session', runner: 'inventoryInstall', tier: 'smoke' }, + { + id: 'smoke:automation-input', + setup: 'scenario-owned', + runner: 'automationInput', + tier: 'smoke', + }, + { id: 'smoke:form-input', setup: 'fixture-home', runner: 'formInput', tier: 'smoke' }, { id: 'smoke:regular-visible-depth-frontier', + setup: 'fixture-home', runner: 'snapshotDepthFrontier', tier: 'smoke', }, - { id: 'smoke:webview-remote-content', runner: 'webviewRemoteContent', tier: 'smoke' }, - { id: 'smoke:deep-link-navigation', runner: 'deepLinkNavigation', tier: 'smoke' }, - { id: 'smoke:capture-close', runner: 'captureClose', tier: 'smoke' }, + { + id: 'smoke:webview-remote-content', + setup: 'fixture-home', + runner: 'webviewRemoteContent', + tier: 'smoke', + }, + { + id: 'smoke:deep-link-navigation', + setup: 'scenario-owned', + runner: 'deepLinkNavigation', + tier: 'smoke', + }, + { id: 'smoke:capture-close', setup: 'fixture-home', runner: 'captureClose', tier: 'smoke' }, { id: 'full:lifecycle-system', runner: 'lifecycleSystem', tier: 'full' }, { id: 'full:observability-artifacts', @@ -47,9 +63,8 @@ export const IOS_SIMULATOR_LIVE_SCENARIOS: readonly IosSimulatorScenario[] = SCE export function bindIosSimulatorScenarios( runners: Record Promise>, ): readonly (IosSimulatorScenario & { run: (context: Context) => Promise })[] { - return SCENARIO_DEFINITIONS.map(({ id, runner, tier }) => ({ - id, + return SCENARIO_DEFINITIONS.map(({ runner, ...scenario }) => ({ + ...scenario, run: runners[runner], - tier, })); } From ced8804dd11503c79c4c6aa2240ed6fab2beee1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 17:34:34 +0200 Subject: [PATCH 07/12] test(ios): assert keyboard readiness and the raw depth frontier --- docs/agents/ios-smoke.md | 3 +++ test/integration/ios-simulator-e2e/live-runner.ts | 1 + .../live-snapshot-depth-frontier.ts | 12 +++++++++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/agents/ios-smoke.md b/docs/agents/ios-smoke.md index 35f5ad22ca..b5e0674069 100644 --- a/docs/agents/ios-smoke.md +++ b/docs/agents/ios-smoke.md @@ -36,6 +36,9 @@ Automatic confirmation remains covered by the launch-confirmation and lifecycle Use semantic readiness landmarks before interaction. A title alone can precede the React tab controls. +Before testing keyboard dismissal, require the fixture's visible Done control. +Enable the simulator software keyboard for that lifecycle contract. +For intentionally truncated raw-depth assertions, validate the structural frontier; require semantic capture quality on regular and full-raw observations. Use the built CLI for preparation and execution after building once. Use the built SDK with the existing-daemon transport for forced-backend probes. Do not let a probe start or replace the daemon under test. diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index 374053b628..0ae003e864 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -352,6 +352,7 @@ async function assertFormInput(context: LiveContext): Promise { 'seeded email should be observable before dismiss', ); + await assertWaitSelector(context, 'label="Done" visible=true'); const keyboardVisiblePath = path.join(context.artifactDir, 'keyboard-visible.png'); await capturePng(context, 'capture visible input keyboard', keyboardVisiblePath); const keyboard = await runStep(context, 'dismiss input keyboard', ['keyboard', 'dismiss']); diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index e47896c8e3..c96fedf181 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -70,8 +70,18 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P '--depth', '1', ]); - assertSimulatorSnapshotAcquisition(rawDepthOne, 'raw depth-1 visible-depth snapshot'); const rawDepthOneNodes = snapshotNodes(rawDepthOne); + requireRoot(rawDepthOneNodes, 'raw depth-1 visible-depth snapshot'); + // Raw acquisition stops before semantic content; this structural frontier can be sparse. + if (rawDepthOne.json?.data?.snapshotQuality?.state === 'sparse') { + assert.equal(rawDepthOne.json.data.snapshotQuality.reasonCode, 'sparse-tree'); + assert.equal(rawDepthOne.json.data.truncated, true); + assert.ok( + rawDepthOneNodes.every((node) => node.type === 'Application' || node.type === 'Window'), + ); + } else { + assertSimulatorSnapshotAcquisition(rawDepthOne, 'raw depth-1 visible-depth snapshot'); + } assert.equal( rawDepthOneNodes.some((node) => node.identifier === CHILD_ID), false, From 8ac0760b686fd6a5963e398e69533efe41748f74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 17:36:28 +0200 Subject: [PATCH 08/12] test(ios): establish software keyboard setup for fixture smoke --- .github/workflows/ios.yml | 9 ++------- docs/agents/ios-smoke.md | 9 +++++++++ docs/evidence/ios-smoke-contracts-2026-10-17.md | 8 ++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 2a2edadc8c..2841edd259 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -165,13 +165,7 @@ jobs: - name: Build CLI used by every smoke step run: pnpm gate build - # Preflight and every step below deliberately share one AGENT_DEVICE_STATE_DIR: the prepared - # runner lives in the daemon this step leaves running, and a per-step state dir would rebuild it - # per step. That also means this daemon outlives the step and can be superseded while it lives, - # which used to make its 5-minute idle reap delete the metadata of the daemon then serving - # clients (#3087). It now removes daemon.json only when the file names its own pid and process - # start time, so a recycled pid cannot be mistaken for the owner, and the reap costs nothing - # but a daemon.log line. + # Reuse the prepared daemon and runner through the shared AGENT_DEVICE_STATE_DIR. - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon @@ -214,6 +208,7 @@ jobs: AGENT_DEVICE_IOS_E2E_TIER: smoke AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} run: | + xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - name: Assert simulator automation preserved host focus diff --git a/docs/agents/ios-smoke.md b/docs/agents/ios-smoke.md index b5e0674069..48bad452ad 100644 --- a/docs/agents/ios-smoke.md +++ b/docs/agents/ios-smoke.md @@ -65,6 +65,15 @@ Keep unreadable classification and bounded clearing in their owning policy tests Read the live entry point's environment contract before running it. Build a current Release fixture with embedded JavaScript for simulator smoke verification. +If a headless simulator minimizes its software keyboard, inspect its runtime setting: + +```sh +xcrun simctl spawn defaults read com.apple.keyboard.preferences AutomaticMinimizationEnabled +xcrun simctl spawn defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false +``` + +The iOS workflow sets this before fixture smoke. +For local verification, record the prior value and restore it afterward. Run the changed scenario through the checked-in smoke entry point and preserve its coverage and failure artifacts. Record the source SHA, device, command, and semantic result under [device verification](device-verification.md). For regressions, prove the test fails without the fix using [isolated experiments](test-experiments.md). diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index c5454faf92..a99ce16de3 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -20,3 +20,11 @@ Device: iPhone 17 Pro, iOS 27.0, UDID `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`. Fi - Focused harness and coverage checks passed 24 tests. `pnpm check:quick`, fixture TypeScript checking, gate-manifest checking, and XCTest-selection checking passed. - At `31e6941b991d4f3421b8c306535472a61408ec36`, an isolated mutation restored the old policy's retries for `wait_deadline_exceeded`. The focused test `a readable destination miss fails without repeating the wait` changed from one pass to one failure (`Missing expected rejection`). The experiment checkout was removed; the implementation checkout remained unchanged. - Independent read-only review found one unnecessary trailing deep-link relaunch. It was removed, along with the analogous depth/WebView restores; the final follow-up review found no remaining issues. + +## Authoring audit follow-up + +- At `87555e7543b8c55b27f66521808e4ebfd487f09e`, the simulator run exposed a keyboard-dismissal refusal and a raw depth-1 structural capture rejected as sparse. The unchanged harness at `5c7efc83eae318e78e1d637f0af2c0b41c59c9da` reproduced the same `UNSUPPORTED_OPERATION` at `dismiss input keyboard`. Its failure screenshot also showed no software keyboard. The selected simulator had `com.apple.keyboard.preferences AutomaticMinimizationEnabled=1`. +- With automatic minimization disabled, harness source `316052ba2e5c140ab54743212cbf79752851c7e1` passed the same simulator smoke command: **22 passed, zero failures/skips**, 102 seconds. The new visible-Done precondition, dismissal, refocus, and type assertions all executed. The workflow now establishes this keyboard setting before fixture smoke; local verification restores its prior value. +- The raw-depth assertion now accepts only a truncated `sparse-tree` containing Application/Window nodes. Root, depth-bound, and child-omission checks remain. Regular and full-raw captures still require semantic acquisition quality. Raw acquisition intentionally stops at the requested depth; structural sparsity there does not prove a failed capture of the full screen. +- Focused cleanup, destination, retry-policy, snapshot-acquisition, and coverage tests passed **49/49**. In a disposable checkout at `5c7efc83eae318e78e1d637f0af2c0b41c59c9da`, the new cleanup test `a returned cleanup failure on the third attempt is reported` failed because the old helper returned `undefined`. Applying the `live-harness.ts` fix changed the same command to **1 passed**. Both runs used `node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-name-pattern 'a returned cleanup failure on the third attempt is reported' test/integration/ios-simulator-e2e-cleanup.test.ts`. The experiment checkout was removed; the implementation checkout was unchanged by the experiment. +- Independent review accepted the authoring and declarative-setup changes. Its follow-up identified the raw-depth and keyboard-readiness assumptions; both were corrected and the next review found no remaining issues. From b274ace553898ad337dd75f1eb12bb67c91dd565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 17:39:37 +0200 Subject: [PATCH 09/12] docs(ios): keep smoke authoring guidance beside its harness --- docs/agents/device-verification.md | 2 -- docs/agents/test-authoring.md | 2 +- docs/evidence/ios-smoke-contracts-2026-10-17.md | 2 +- .../integration/ios-simulator-e2e/README.md | 6 +++--- 4 files changed, 5 insertions(+), 7 deletions(-) rename docs/agents/ios-smoke.md => test/integration/ios-simulator-e2e/README.md (93%) diff --git a/docs/agents/device-verification.md b/docs/agents/device-verification.md index 147df6837e..1ed68c5746 100644 --- a/docs/agents/device-verification.md +++ b/docs/agents/device-verification.md @@ -2,8 +2,6 @@ Use this for Apple runner changes or manual device runs. For foldables, also read [foldable verification](device-foldable.md). -For iOS fixture smoke or native input regressions, read [iOS smoke authoring](ios-smoke.md). - ## Prepare 1. To build runtime changes, run `pnpm build`. diff --git a/docs/agents/test-authoring.md b/docs/agents/test-authoring.md index 147bc72a4a..5cd970c3c0 100644 --- a/docs/agents/test-authoring.md +++ b/docs/agents/test-authoring.md @@ -3,7 +3,7 @@ For checks, read [validation](testing.md). For deletions, read [test audits](test-audit.md). -For iOS fixture smoke or native input regressions, read [iOS smoke authoring](ios-smoke.md). +For iOS fixture smoke or native input regressions, read [iOS smoke authoring](../../test/integration/ios-simulator-e2e/README.md). ## Choose the boundary diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index a99ce16de3..1a6bf83ac7 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -1,6 +1,6 @@ # iOS smoke contracts -Authoring rules live in [iOS smoke authoring](../agents/ios-smoke.md). This file records the failures and verification behind PR #3786. +Authoring rules live in [iOS smoke authoring](../../test/integration/ios-simulator-e2e/README.md). This file records the failures and verification behind PR #3786. ## Failure evidence diff --git a/docs/agents/ios-smoke.md b/test/integration/ios-simulator-e2e/README.md similarity index 93% rename from docs/agents/ios-smoke.md rename to test/integration/ios-simulator-e2e/README.md index 48bad452ad..e1baa7ae9d 100644 --- a/docs/agents/ios-smoke.md +++ b/test/integration/ios-simulator-e2e/README.md @@ -1,7 +1,7 @@ # Author iOS smoke scenarios Use this for `test/integration/ios-simulator-e2e/` and its native input regressions. -Use [test authoring](test-authoring.md) for test boundaries and [device verification](device-verification.md) for builds and cleanup. +Use [test authoring](../../../docs/agents/test-authoring.md) for test boundaries and [device verification](../../../docs/agents/device-verification.md) for builds and cleanup. ## Declare the starting state @@ -75,6 +75,6 @@ xcrun simctl spawn defaults write com.apple.keyboard.preferences Automati The iOS workflow sets this before fixture smoke. For local verification, record the prior value and restore it afterward. Run the changed scenario through the checked-in smoke entry point and preserve its coverage and failure artifacts. -Record the source SHA, device, command, and semantic result under [device verification](device-verification.md). -For regressions, prove the test fails without the fix using [isolated experiments](test-experiments.md). +Record the source SHA, device, command, and semantic result under [device verification](../../../docs/agents/device-verification.md). +For regressions, prove the test fails without the fix using [isolated experiments](../../../docs/agents/test-experiments.md). Do not count skipped device tests as passed. From 360ae21bfb41f3e54f73616669ab63b3d7fffecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 19:08:23 +0200 Subject: [PATCH 10/12] test(ios): address review diagnostics and preserve cleanup evidence --- ...RunnerTests+SynthesizedTextEntryTests.swift | 2 +- ...simulator-e2e-deep-link-destination.test.ts | 6 ++++++ test/integration/ios-simulator-e2e/README.md | 7 +++++-- .../ios-simulator-e2e/live-harness.ts | 6 +++++- .../live-webview-remote-content.ts | 3 +-- .../live-device-e2e-runtime.test.ts | 18 ++++++++++++++++++ test/integration/live-device-e2e/runtime.ts | 5 +++-- 7 files changed, 39 insertions(+), 8 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index f8a5431ba0..2c67a43489 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -170,7 +170,7 @@ extension RunnerTests { // below pin exact replacement, including dropped select-alls; this case pins classification. let delivered = app.staticTexts["agent-device-masked-delivery"] let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) - XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed) + XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed, "prefilled: \(prefilled)") XCTAssertEqual(togglePresses.label, "presses=0", "prefilled: \(prefilled)") } } diff --git a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts index f6fcfba411..8dc2639bff 100644 --- a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts +++ b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts @@ -110,6 +110,12 @@ test('an unrelated prompt or an unreadable alert is never accepted as a fixture result(0, { data: { message: 'Allow microphone access?', items: ['Cancel', 'Open'] } }), accept, ), + { + name: 'AssertionError', + code: 'ERR_ASSERTION', + operator: 'strictEqual', + message: /Allow microphone access/, + }, ); await assert.rejects( answerFixtureLaunchConfirmation( diff --git a/test/integration/ios-simulator-e2e/README.md b/test/integration/ios-simulator-e2e/README.md index e1baa7ae9d..26eb5116b1 100644 --- a/test/integration/ios-simulator-e2e/README.md +++ b/test/integration/ios-simulator-e2e/README.md @@ -18,6 +18,7 @@ For a form interaction scenario, use the existing registration pattern: { id: 'smoke:form-input', runner: 'formInput', tier: 'smoke', setup: 'fixture-home' } ``` +Run commands serially on the selected device. The runner closes the preceding session and assigns a distinct session to each smoke scenario. Session recreation clears stored launch URLs; relaunch alone retains them. Keep assertions inside the scenario's Node subtest. @@ -40,6 +41,7 @@ Before testing keyboard dismissal, require the fixture's visible Done control. Enable the simulator software keyboard for that lifecycle contract. For intentionally truncated raw-depth assertions, validate the structural frontier; require semantic capture quality on regular and full-raw observations. Use the built CLI for preparation and execution after building once. +Source and built clients have different daemon code signatures; alternating them can replace the daemon under test. Use the built SDK with the existing-daemon transport for forced-backend probes. Do not let a probe start or replace the daemon under test. @@ -50,7 +52,8 @@ Require both a retriable verdict and a typed prevented-observation reason. Fail readable absence, exhausted readable waits, invalid selectors, and incorrect values. If a helper owns retries, pass `reattemptInfrastructureMiss: undefined` on every attempt. Keep cleanup retries in `retryCleanupStep`; return failed results with `allowFailure: true` so its typed already-clean handling applies on every attempt. -Report exhausted cleanup failures separately from scenario assertions. +Capture exhausted cleanup failures through the shared harness before continuing cleanup. +Report them separately from scenario assertions. Replay-suite attempt policies have their own owner; this step policy does not redefine them. ## Match input assertions to the contract @@ -61,7 +64,7 @@ Do not infer replacement or length from masking bullets. For readable input, retain exact replacement assertions and deliberately dropped-select-all regressions. Keep unreadable classification and bounded clearing in their owning policy tests. -## Verify an extension +## Verify simulator smoke changes Read the live entry point's environment contract before running it. Build a current Release fixture with embedded JavaScript for simulator smoke verification. diff --git a/test/integration/ios-simulator-e2e/live-harness.ts b/test/integration/ios-simulator-e2e/live-harness.ts index 90169f777c..c6461b5069 100644 --- a/test/integration/ios-simulator-e2e/live-harness.ts +++ b/test/integration/ios-simulator-e2e/live-harness.ts @@ -96,6 +96,7 @@ export function verifyNestedReplayCommand( const MICROPHONE_PERMISSION_RESET_STEP = 'reset microphone permission'; export async function cleanupSession(context: LiveContext): Promise { + context.currentScenario = 'cleanup'; const failures: unknown[] = []; const cleanupSteps: Array<[string, string[]]> = []; if (context.tier === 'full') { @@ -113,7 +114,10 @@ export async function cleanupSession(context: LiveContext): Promise { reattemptInfrastructureMiss: undefined, }), ); - if (failure !== undefined) failures.push(failure); + if (failure !== undefined) { + await harness.recordFailure(context, failure); + failures.push(failure); + } } if (failures.length === 0) return; const errorPath = path.join(context.artifactDir, 'cleanup-error.txt'); diff --git a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts index 05e582ea47..d0d06449c5 100644 --- a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts +++ b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts @@ -7,8 +7,7 @@ import { } from './live-assertions.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; -// Native chrome of the lab, rendered with the route and before the page: it proves the deep link -// landed without asking the runner to query a screen whose web view is still loading. +// Native chrome proves Settings navigation reached the lab before its web page finishes loading. const LAB_CHROME_ID = 'close-webview-lab'; // The first WebContent process of the run spawns here; a cold CI simulator needs more than the // shared 10 s wait budget before the page's tree exists. diff --git a/test/integration/live-device-e2e-runtime.test.ts b/test/integration/live-device-e2e-runtime.test.ts index 8a092652ce..adcff789c4 100644 --- a/test/integration/live-device-e2e-runtime.test.ts +++ b/test/integration/live-device-e2e-runtime.test.ts @@ -69,6 +69,24 @@ test('a scenario assertion after allowed misses captures evidence before caller assert.equal(reports(), 1); }); +test('a caller-owned cleanup failure captures evidence without reissuing the command', async () => { + const { context, harness, calls } = fixture(); + context.currentScenario = 'cleanup'; + const result = await harness.runStep(context, 'close session', ['close'], { allowFailure: true }); + assert.equal(result.status, 1); + await harness.recordFailure(context, new Error('close session exhausted its retry budget')); + + assert.deepEqual( + calls.map((args) => args[0]), + ['close', 'screenshot', 'snapshot'], + ); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1.png'))); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1-snapshot.json'))); + const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'); + assert.match(report, /close session exhausted its retry budget/); + assert.match(report, /scenario: cleanup/); +}); + test('a scenario assertion before any command also captures evidence', async () => { const { context, harness, calls } = fixture(); const failure = new Error('fixture assertion'); diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index 6ff99abb59..c1c387befe 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -162,7 +162,7 @@ export function createLiveDeviceHarness< ); context.completedScenarios.push(scenario.id); } catch (error) { - await recordScenarioFailure(context, error); + await recordFailure(context, error); throw error; } finally { context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id }); @@ -170,7 +170,7 @@ export function createLiveDeviceHarness< } } - async function recordScenarioFailure(context: Context, error: unknown): Promise { + async function recordFailure(context: Context, error: unknown): Promise { if (error instanceof Error && reportedFailures.has(error)) return; try { const evidence = await captureFailedStepEvidence(context); @@ -363,6 +363,7 @@ export function createLiveDeviceHarness< } return { + recordFailure, runScenario, runStep, sessionExists, From 8e430b7c73cec3b5e31226817f0a27fa5769e12a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 19:10:19 +0200 Subject: [PATCH 11/12] docs(ios): record review regression proof and rebase evidence --- docs/evidence/ios-smoke-contracts-2026-10-17.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md index 1a6bf83ac7..e145f206d8 100644 --- a/docs/evidence/ios-smoke-contracts-2026-10-17.md +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -28,3 +28,11 @@ Device: iPhone 17 Pro, iOS 27.0, UDID `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`. Fi - The raw-depth assertion now accepts only a truncated `sparse-tree` containing Application/Window nodes. Root, depth-bound, and child-omission checks remain. Regular and full-raw captures still require semantic acquisition quality. Raw acquisition intentionally stops at the requested depth; structural sparsity there does not prove a failed capture of the full screen. - Focused cleanup, destination, retry-policy, snapshot-acquisition, and coverage tests passed **49/49**. In a disposable checkout at `5c7efc83eae318e78e1d637f0af2c0b41c59c9da`, the new cleanup test `a returned cleanup failure on the third attempt is reported` failed because the old helper returned `undefined`. Applying the `live-harness.ts` fix changed the same command to **1 passed**. Both runs used `node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-name-pattern 'a returned cleanup failure on the third attempt is reported' test/integration/ios-simulator-e2e-cleanup.test.ts`. The experiment checkout was removed; the implementation checkout was unchanged by the experiment. - Independent review accepted the authoring and declarative-setup changes. Its follow-up identified the raw-depth and keyboard-readiness assumptions; both were corrected and the next review found no remaining issues. + +## Review follow-up after rebase + +The published `b78ff238247774a1eb98afa74894340c78b26118` passed all 18 reported checks, including iOS 26.2 smoke, before this follow-up. Its local simulator result remains attributed to harness source `316052ba2e5c140ab54743212cbf79752851c7e1`; it is not a new simulator run at the rebased head. + +Rebased onto `6e8c2a046cebb4729aad92a63c55f8f53f16579d`. Range-diff retained the runtime/test changes and preserved the upstream device-verification instruction when resolving its documentation conflict. Follow-up source `360ae21bfb41f3e54f73616669ab63b3d7fffecf` passed **32/32** focused destination, cleanup, and shared-harness tests. Exhausted cleanup captures evidence through the shared recorder without reissuing its command or losing final-attempt typed already-clean handling. The native change only restores the prefilled-case assertion diagnostic. + +In a disposable checkout at the rebased pre-fix `b274ace553`, replacing the prompt-title assertion with an unrelated `Error` left the old unrelated-prompt test green (**1 passed**). Applying only the new matcher made the same mutation fail (**1 failed**) for the unexpected error name/message. Removing the mutation made the updated test pass (**1 passed**). All three runs used `node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-name-pattern 'an unrelated prompt or an unreadable alert' test/integration/ios-simulator-e2e-deep-link-destination.test.ts`. The disposable checkout was removed and the implementation checkout remained unchanged by the experiment. Independent read-only review found no remaining issues in the fixes or rebase range-diff. From 1067255afada175396fbbf98db710f80938f9d14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 17 Oct 2026 19:16:26 +0200 Subject: [PATCH 12/12] docs(agents): remove duplicate test routing after rebase --- docs/agents/test-authoring.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/agents/test-authoring.md b/docs/agents/test-authoring.md index 5cd970c3c0..09d7c4c3b1 100644 --- a/docs/agents/test-authoring.md +++ b/docs/agents/test-authoring.md @@ -1,9 +1,6 @@ # Write tests -For checks, read [validation](testing.md). -For deletions, read [test audits](test-audit.md). - -For iOS fixture smoke or native input regressions, read [iOS smoke authoring](../../test/integration/ios-simulator-e2e/README.md). +For iOS smoke, read [the guide](../../test/integration/ios-simulator-e2e/README.md). ## Choose the boundary