diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index acd8e4dcc2..2841edd259 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -162,23 +162,20 @@ jobs: } ' - # Preflight and every step below deliberately share one AGENT_DEVICE_STATE_DIR: the prepared - # runner lives in the daemon this step leaves running, and a per-step state dir would rebuild it - # per step. That also means this daemon outlives the step and can be superseded while it lives, - # which used to make its 5-minute idle reap delete the metadata of the daemon then serving - # clients (#3087). It now removes daemon.json only when the file names its own pid and process - # start time, so a recycled pid cannot be mistaken for the owner, and the reap costs nothing - # but a daemon.log line. + - name: Build CLI used by every smoke step + run: pnpm gate build + + # Reuse the prepared daemon and runner through the shared AGENT_DEVICE_STATE_DIR. - name: Preflight iOS runner through public CLI run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json + node bin/agent-device.mjs prepare ios-runner --platform ios --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout "$AGENT_DEVICE_IOS_PREPARE_TIMEOUT_MS" --json # The replay reuses the prepared runner in the same daemon. --debug keeps every step's request # log, not only the failing one, so a stale tap can be traced to its pre-tap capture (#3367). - name: Run iOS Settings replay smoke test run: | - node --experimental-strip-types src/bin.ts test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml + node bin/agent-device.mjs test test/integration/replays/ios/simulator/01-settings.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --retries 2 --debug --artifacts-dir test/artifacts/replays-ios-simulator-smoke --report-junit test/artifacts/replays-ios-simulator-smoke.junit.xml - name: Fetch current fixture app id: fixture-app @@ -201,7 +198,7 @@ jobs: - name: Run gesture pan-duration smoke replay run: | xcrun simctl install "${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.fixture-app.outputs.app-path }}" - node --experimental-strip-types src/bin.ts test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml + node bin/agent-device.mjs test examples/test-app/replays/gesture-pan-duration.ad --udid "${{ steps.ios-simulator.outputs.simulator-udid }}" --timeout 180000 --retries 2 --artifacts-dir test/artifacts/replays-ios-gesture-pan-duration --report-junit test/artifacts/replays-ios-gesture-pan-duration.junit.xml - name: Run fixture-backed iOS simulator E2E smoke env: @@ -211,7 +208,7 @@ jobs: AGENT_DEVICE_IOS_E2E_TIER: smoke AGENT_DEVICE_IOS_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} run: | - pnpm gate build + xcrun simctl spawn "$AGENT_DEVICE_IOS_UDID" defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false node --experimental-strip-types scripts/node-test-tmpdir.ts --test test/integration/smoke-ios-simulator-coverage.test.ts test/integration/smoke-ios-simulator.test.ts - name: Assert simulator automation preserved host focus @@ -228,7 +225,7 @@ jobs: IOS_UDID: ${{ vars.IOS_UDID }} run: | pnpm clean:daemon - node --experimental-strip-types src/bin.ts test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml + node bin/agent-device.mjs test test/integration/replays/ios/device/01-physical-lifecycle.ad --udid "$IOS_UDID" --retries 2 --artifacts-dir test/artifacts/replays-ios-device-smoke --report-junit test/artifacts/replays-ios-device-smoke.junit.xml - name: Assert iOS XCTest selection was honored if: always() diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m index c9b6d61bd7..acc0d318ac 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m @@ -84,6 +84,7 @@ - (NSString *)accessibilityValue { @interface AgentDeviceRunnerViewController : UIViewController @property(nonatomic, strong) UILabel *maskedToggleStatus; +@property(nonatomic, strong) UILabel *maskedDeliveryStatus; @property(nonatomic, assign) NSUInteger maskedTogglePresses; @property(nonatomic, strong) UILabel *alertActionStatus; @property(nonatomic, strong) UILabel *alertActivationBusyAnswer; @@ -466,6 +467,9 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) { static const CGFloat AgentDeviceTextEntryNeighbourGap = 16; - (void)agentDeviceTextEntryDidChange:(UITextField *)textField { + if (self.maskedDeliveryStatus != nil) { + self.maskedDeliveryStatus.text = [textField.text hasSuffix:@"not-a-secret-123"] ? @"delivered" : @"pending"; + } // A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst // typed faster than the app renders loses the characters that arrived while a render was in // flight, and the field settles stable short of the request. @@ -809,6 +813,15 @@ - (void)viewDidLoad { [self.maskedToggleStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], [self.maskedToggleStatus.topAnchor constraintEqualToAnchor:textField.bottomAnchor constant:12], ]]; + self.maskedDeliveryStatus = [[UILabel alloc] init]; + self.maskedDeliveryStatus.accessibilityIdentifier = @"agent-device-masked-delivery"; + self.maskedDeliveryStatus.text = @"pending"; + self.maskedDeliveryStatus.translatesAutoresizingMaskIntoConstraints = NO; + [self.view addSubview:self.maskedDeliveryStatus]; + [NSLayoutConstraint activateConstraints:@[ + [self.maskedDeliveryStatus.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor], + [self.maskedDeliveryStatus.topAnchor constraintEqualToAnchor:self.maskedToggleStatus.bottomAnchor constant:8], + ]]; [self updateMaskedToggleStatus]; } if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift index ac0ac0e54d..2c67a43489 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+SynthesizedTextEntryTests.swift @@ -166,7 +166,11 @@ extension RunnerTests { XCTAssertTrue(response.ok, "prefilled: \(prefilled) \(String(describing: response.error))") XCTAssertEqual(response.data?.textEntryRoute, "synthesized-first-responder-replacement") XCTAssertEqual(response.data?.message, "typed", "prefilled: \(prefilled)") - XCTAssertEqual(textField.value as? String, String(repeating: "\u{2022}", count: 8), "prefilled: \(prefilled)") + // An unreadable field cannot certify replacement or commit timing. Readable-field tests + // below pin exact replacement, including dropped select-alls; this case pins classification. + let delivered = app.staticTexts["agent-device-masked-delivery"] + let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) + XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed, "prefilled: \(prefilled)") XCTAssertEqual(togglePresses.label, "presses=0", "prefilled: \(prefilled)") } } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift index 22ffe9d0e0..ef7475e197 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextTypingTests.swift @@ -441,7 +441,9 @@ extension RunnerTests { XCTAssertTrue(fillResponse.ok, String(describing: fillResponse.error)) XCTAssertFalse(didRecordXCTestFailure(since: failureCountBefore)) XCTAssertEqual(fillResponse.data?.message, "typed") - XCTAssertEqual(textField.value as? String, String(repeating: "\u{2022}", count: 8)) + let delivered = app.staticTexts["agent-device-masked-delivery"] + let delivery = XCTNSPredicateExpectation(predicate: NSPredicate(format: "label == %@", "delivered"), object: delivered) + XCTAssertEqual(XCTWaiter.wait(for: [delivery], timeout: appExistenceTimeout), .completed) XCTAssertEqual(togglePresses.label, "presses=0") } diff --git a/docs/agents/test-authoring.md b/docs/agents/test-authoring.md index 19800f795f..09d7c4c3b1 100644 --- a/docs/agents/test-authoring.md +++ b/docs/agents/test-authoring.md @@ -1,7 +1,6 @@ # Write tests -For checks, read [validation](testing.md). -For deletions, read [test audits](test-audit.md). +For iOS smoke, read [the guide](../../test/integration/ios-simulator-e2e/README.md). ## Choose the boundary diff --git a/docs/evidence/ios-smoke-contracts-2026-10-17.md b/docs/evidence/ios-smoke-contracts-2026-10-17.md new file mode 100644 index 0000000000..e145f206d8 --- /dev/null +++ b/docs/evidence/ios-smoke-contracts-2026-10-17.md @@ -0,0 +1,38 @@ +# iOS smoke contracts + +Authoring rules live in [iOS smoke authoring](../../test/integration/ios-simulator-e2e/README.md). This file records the failures and verification behind PR #3786. + +## Failure evidence + +- [Run 39029845932](https://github.com/callstack/agent-device/actions/runs/39029845932): source-CLI runner preflight exhausted the daemon's 15-second startup budget. Its uploaded daemon log was empty; the log alone cannot identify why startup was slow. The lane now exercises the packaged entry point throughout without changing the daemon timeout. +- [Run 39027567868](https://github.com/callstack/agent-device/actions/runs/39027567868): step 84 failed in CoreSimulator URL delivery before any depth assertion. The depth scenario now enters through the fixture's Settings screen. +- [Run 39030211303](https://github.com/callstack/agent-device/actions/runs/39030211303): five destination waits found no app process. The failure screenshot showed an unanswered “Open in” confirmation. Waiting for a destination cannot answer a system prompt. The dedicated deep-link scenario retains this signal; it no longer prevents unrelated interaction tests from running. +- The original masked XCTest reproduced locally on iOS 27.0: the prefilled field showed 13 bullets instead of the asserted 8. This is not proof of failed delivery: the route explicitly returns unverified. The replacement assertion was stronger than its production contract. The new app-owned delivery canary distinguishes successful posting from missing or truncated input without claiming the old value was cleared. + +## Verification results + +Device: iPhone 17 Pro, iOS 27.0, UDID `BC54AC11-CD86-4C10-8C4E-5101C773FFEB`. Fixture: a fresh Release build of this checkout, with current JavaScript embedded. CLI and runner builds completed locally. CI still owns its iOS 26.2 environment. + +- Native source at `92db564a30ec0ff39aa248339b45b12901a1ead8`: the original masked XCTest pair produced one pass and one failure; the prefilled synthesized case observed 13 bullets versus the asserted 8. +- Revised native fixture and XCTest assertions: five focused tests passed, including both masked cases, exact readable replacement past dropped select-alls, shared unreadable classification, and bounded masked clearing. `pnpm check:xctest-selection` reached every declared regression. +- Harness source at `83af5f485e12e16ff23bcc5606282633cad4a1f9`: the isolated-session smoke command passed 22 tests, zero failures/skips, in 130 seconds. +- Final harness source at `31e6941b991d4f3421b8c306535472a61408ec36`: the same smoke command passed 22 tests, zero failures/skips, in 112 seconds. The seven subtests covered inventory/install, automation input, form input and forced-backend conformance, visible depth, WebView remote content, deep-link/clear-state navigation, and screenshot/session release. Each run closed its sessions and stopped its daemon. +- Focused harness and coverage checks passed 24 tests. `pnpm check:quick`, fixture TypeScript checking, gate-manifest checking, and XCTest-selection checking passed. +- At `31e6941b991d4f3421b8c306535472a61408ec36`, an isolated mutation restored the old policy's retries for `wait_deadline_exceeded`. The focused test `a readable destination miss fails without repeating the wait` changed from one pass to one failure (`Missing expected rejection`). The experiment checkout was removed; the implementation checkout remained unchanged. +- Independent read-only review found one unnecessary trailing deep-link relaunch. It was removed, along with the analogous depth/WebView restores; the final follow-up review found no remaining issues. + +## Authoring audit follow-up + +- At `87555e7543b8c55b27f66521808e4ebfd487f09e`, the simulator run exposed a keyboard-dismissal refusal and a raw depth-1 structural capture rejected as sparse. The unchanged harness at `5c7efc83eae318e78e1d637f0af2c0b41c59c9da` reproduced the same `UNSUPPORTED_OPERATION` at `dismiss input keyboard`. Its failure screenshot also showed no software keyboard. The selected simulator had `com.apple.keyboard.preferences AutomaticMinimizationEnabled=1`. +- With automatic minimization disabled, harness source `316052ba2e5c140ab54743212cbf79752851c7e1` passed the same simulator smoke command: **22 passed, zero failures/skips**, 102 seconds. The new visible-Done precondition, dismissal, refocus, and type assertions all executed. The workflow now establishes this keyboard setting before fixture smoke; local verification restores its prior value. +- The raw-depth assertion now accepts only a truncated `sparse-tree` containing Application/Window nodes. Root, depth-bound, and child-omission checks remain. Regular and full-raw captures still require semantic acquisition quality. Raw acquisition intentionally stops at the requested depth; structural sparsity there does not prove a failed capture of the full screen. +- Focused cleanup, destination, retry-policy, snapshot-acquisition, and coverage tests passed **49/49**. In a disposable checkout at `5c7efc83eae318e78e1d637f0af2c0b41c59c9da`, the new cleanup test `a returned cleanup failure on the third attempt is reported` failed because the old helper returned `undefined`. Applying the `live-harness.ts` fix changed the same command to **1 passed**. Both runs used `node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-name-pattern 'a returned cleanup failure on the third attempt is reported' test/integration/ios-simulator-e2e-cleanup.test.ts`. The experiment checkout was removed; the implementation checkout was unchanged by the experiment. +- Independent review accepted the authoring and declarative-setup changes. Its follow-up identified the raw-depth and keyboard-readiness assumptions; both were corrected and the next review found no remaining issues. + +## Review follow-up after rebase + +The published `b78ff238247774a1eb98afa74894340c78b26118` passed all 18 reported checks, including iOS 26.2 smoke, before this follow-up. Its local simulator result remains attributed to harness source `316052ba2e5c140ab54743212cbf79752851c7e1`; it is not a new simulator run at the rebased head. + +Rebased onto `6e8c2a046cebb4729aad92a63c55f8f53f16579d`. Range-diff retained the runtime/test changes and preserved the upstream device-verification instruction when resolving its documentation conflict. Follow-up source `360ae21bfb41f3e54f73616669ab63b3d7fffecf` passed **32/32** focused destination, cleanup, and shared-harness tests. Exhausted cleanup captures evidence through the shared recorder without reissuing its command or losing final-attempt typed already-clean handling. The native change only restores the prefilled-case assertion diagnostic. + +In a disposable checkout at the rebased pre-fix `b274ace553`, replacing the prompt-title assertion with an unrelated `Error` left the old unrelated-prompt test green (**1 passed**). Applying only the new matcher made the same mutation fail (**1 failed**) for the unexpected error name/message. Removing the mutation made the updated test pass (**1 passed**). All three runs used `node --experimental-strip-types scripts/node-test-tmpdir.ts --test --test-name-pattern 'an unrelated prompt or an unreadable alert' test/integration/ios-simulator-e2e-deep-link-destination.test.ts`. The disposable checkout was removed and the implementation checkout remained unchanged by the experiment. Independent read-only review found no remaining issues in the fixes or rebase range-diff. diff --git a/examples/test-app/app/(tabs)/settings.tsx b/examples/test-app/app/(tabs)/settings.tsx index 1b561aa662..7f7ca57180 100644 --- a/examples/test-app/app/(tabs)/settings.tsx +++ b/examples/test-app/app/(tabs)/settings.tsx @@ -20,6 +20,7 @@ export default function SettingsRoute() { onOpenAccessorySetup={() => router.push('/accessory-setup')} onOpenAutomationLab={() => router.push('/automation')} onOpenInertSurface={() => router.push('/inert')} + onOpenSnapshotDepth={() => router.push('/snapshot-depth')} onOpenWebViewLab={() => router.push('/webview')} onRetryDiagnostics={state.retryDiagnostics} onSetNotificationsEnabled={state.setNotificationsEnabled} diff --git a/examples/test-app/src/screens/SettingsScreen.tsx b/examples/test-app/src/screens/SettingsScreen.tsx index 4fb28d8991..c580c3f21a 100644 --- a/examples/test-app/src/screens/SettingsScreen.tsx +++ b/examples/test-app/src/screens/SettingsScreen.tsx @@ -21,6 +21,7 @@ export interface SettingsScreenProps { onOpenAccessorySetup: () => void; onOpenAutomationLab: () => void; onOpenInertSurface: () => void; + onOpenSnapshotDepth: () => void; onOpenWebViewLab: () => void; onRetryDiagnostics: () => void; onSetNotificationsEnabled: (value: boolean) => void; @@ -74,6 +75,14 @@ export function SettingsScreen(props: SettingsScreenProps) { /> + + + + { attempts += 1; if (attempt < 3) return invalidArgsResult(MIC_APPLESS_MESSAGE, sessionAppRequiredDetails()); - // Mirrors runStep(..., { allowFailure: false }) on the final attempt: it throws - // instead of returning a failed result. + // A thrown subprocess failure must also propagate on the final attempt. throw new Error(`cleanup: ${OTHER_STEP} (attempt 3) failed`); }), ); @@ -117,6 +116,35 @@ test('the appless message without its typed reason still fails after retries', a assert.equal(attempts, 3, 'should exhaust all three attempts'); }); +test('a returned cleanup failure on the third attempt is reported', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let attempts = 0; + const failure = await drainRetry( + t, + retryCleanupStep(OTHER_STEP, async () => { + attempts += 1; + return invalidArgsResult('unrelated refusal'); + }), + ); + assert.ok(failure instanceof Error); + assert.match(failure.message, /restore portrait orientation failed after 3 attempts/); + assert.equal(attempts, 3); +}); + +test('a session disappearing on the third cleanup attempt is already clean', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let attempts = 0; + const failure = await drainRetry( + t, + retryCleanupStep(OTHER_STEP, async () => { + attempts += 1; + return attempts === 3 ? sessionNotFoundResult() : invalidArgsResult('unrelated refusal'); + }), + ); + assert.equal(failure, undefined); + assert.equal(attempts, 3); +}); + // Deterministic regression for finalizeSessionCleanup: the other half of #1548, which // decides whether cleanup runs at all. A minimal LiveContext fixture; only sessionOpen // is read by the decision, the rest exists to satisfy the type. diff --git a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts index 30dd4eff9b..8dc2639bff 100644 --- a/test/integration/ios-simulator-e2e-deep-link-destination.test.ts +++ b/test/integration/ios-simulator-e2e-deep-link-destination.test.ts @@ -3,14 +3,17 @@ import test from 'node:test'; import { WAIT_REASONS } from '@agent-device/contracts/wait'; import type { CliJsonResult } from './cli-json.ts'; -import { retryDestinationWait } from './ios-simulator-e2e/live-deep-link-destination.ts'; +import { + answerFixtureLaunchConfirmation, + retryDestinationWait, +} from './ios-simulator-e2e/live-deep-link-destination.ts'; function result(status: number, json?: unknown): CliJsonResult { return { json, status, stderr: '', stdout: '' }; } function miss(details: Record): CliJsonResult { - return result(1, { error: { code: 'COMMAND_FAILED', details } }); + return result(1, { error: { code: 'COMMAND_FAILED', retriable: true, details } }); } const LANDED = result(0, { success: true }); @@ -39,20 +42,37 @@ test('a destination seen on the first wait costs one wait', async () => { assert.equal(calls.length, 1); }); -test('a runner start and a slow foreground are waited out within the bounded waits', async () => { - const { wait, calls } = waits([RUNNER_START_TIMEOUT, SLOW_FOREGROUND, LANDED]); +test('an observation prevented by runner startup gets one repeat', async () => { + const { wait, calls } = waits([RUNNER_START_TIMEOUT, LANDED]); assert.equal(await retryDestinationWait(wait), LANDED); assert.deepEqual( calls.map((call) => call.allowFailure), - [true, true, true], + [true, false], ); }); -test('the fifth miss fails the wait instead of retrying again', async () => { - const { wait, calls } = waits([SLOW_FOREGROUND]); - await assert.rejects(retryDestinationWait(wait), /\(5\/5\) failed/); - assert.equal(calls.length, 5); - assert.equal(calls.at(-1)?.allowFailure, false); +test('a readable destination miss fails without repeating the wait', async () => { + const { wait, calls } = waits([SLOW_FOREGROUND, LANDED]); + await assert.rejects(retryDestinationWait(wait), /deep-link destination wait failed/); + assert.equal(calls.length, 1); +}); + +test('the second observation failure ends the wait', async () => { + const { wait, calls } = waits([RUNNER_START_TIMEOUT]); + await assert.rejects(retryDestinationWait(wait), /\(2\/2\) failed/); + assert.equal(calls.length, 2); +}); + +test('the final failed result cannot start an unbounded wait if the caller returns it', async () => { + let calls = 0; + await assert.rejects( + retryDestinationWait(async () => { + calls += 1; + return RUNNER_START_TIMEOUT; + }), + /deep-link destination wait failed/, + ); + assert.equal(calls, 2); }); test('a miss that is not a pending destination fails at once', async () => { @@ -60,3 +80,71 @@ test('a miss that is not a pending destination fails at once', async () => { await assert.rejects(retryDestinationWait(wait), /deep-link destination wait failed/); assert.equal(calls.length, 1); }); + +test('the fixture launch confirmation is answered, while an absent alert needs no action', async () => { + let accepts = 0; + const accept = async () => { + accepts += 1; + }; + await answerFixtureLaunchConfirmation( + async () => + result(0, { data: { message: 'Open in “Agent Device Tester”?', items: ['Cancel', 'Open'] } }), + accept, + ); + assert.equal(accepts, 1); + await answerFixtureLaunchConfirmation( + async () => result(1, { error: { details: { runnerErrorCode: 'ALERT_NOT_FOUND' } } }), + accept, + ); + assert.equal(accepts, 1); +}); + +test('an unrelated prompt or an unreadable alert is never accepted as a fixture launch', async () => { + let accepts = 0; + const accept = async () => { + accepts += 1; + }; + await assert.rejects( + answerFixtureLaunchConfirmation( + async () => + result(0, { data: { message: 'Allow microphone access?', items: ['Cancel', 'Open'] } }), + accept, + ), + { + name: 'AssertionError', + code: 'ERR_ASSERTION', + operator: 'strictEqual', + message: /Allow microphone access/, + }, + ); + await assert.rejects( + answerFixtureLaunchConfirmation( + async () => result(1, { error: { message: 'ALERT_NOT_FOUND' } }), + accept, + ), + /launch confirmation observation failed/, + ); + assert.equal(accepts, 0); +}); + +test('a stopped app is checked for its launch confirmation before the destination is observed again', async () => { + const { wait, calls } = waits([miss({ runnerErrorCode: 'APP_NOT_RUNNING' }), LANDED]); + let answers = 0; + assert.equal( + await retryDestinationWait(wait, async () => { + answers += 1; + }), + LANDED, + ); + assert.equal(answers, 1); + assert.equal(calls.length, 2); +}); + +test('an already observed destination never probes for an optional alert', async () => { + const { wait } = waits([LANDED]); + let answers = 0; + await retryDestinationWait(wait, async () => { + answers += 1; + }); + assert.equal(answers, 0); +}); diff --git a/test/integration/ios-simulator-e2e/README.md b/test/integration/ios-simulator-e2e/README.md new file mode 100644 index 0000000000..26eb5116b1 --- /dev/null +++ b/test/integration/ios-simulator-e2e/README.md @@ -0,0 +1,83 @@ +# Author iOS smoke scenarios + +Use this for `test/integration/ios-simulator-e2e/` and its native input regressions. +Use [test authoring](../../../docs/agents/test-authoring.md) for test boundaries and [device verification](../../../docs/agents/device-verification.md) for builds and cleanup. + +## Declare the starting state + +Register the scenario in `scenarios.ts` and bind its implementation in `live-runner.ts`. +Declare `setup` on every smoke scenario: + +- `fixture-home`: the runner opens a fresh session, relaunches the fixture, and observes the home title and visible Form tab. +- `scenario-owned`: the scenario opens its own app or launch URL and observes its own first landmark. +- `no-session`: the scenario tests inventory or installation without opening an app. + +For a form interaction scenario, use the existing registration pattern: + +```ts +{ id: 'smoke:form-input', runner: 'formInput', tier: 'smoke', setup: 'fixture-home' } +``` + +Run commands serially on the selected device. +The runner closes the preceding session and assigns a distinct session to each smoke scenario. +Session recreation clears stored launch URLs; relaunch alone retains them. +Keep assertions inside the scenario's Node subtest. +Keep command and behavior coverage in their existing coverage registries. +The full tier remains a sequential journey with scenario-owned transitions; smoke setup metadata does not apply to it. + +## Observe the behavior under test + +Navigate through stable fixture controls when navigation is setup for another feature. +Keep URL delivery in the dedicated deep-link scenario. +Assert the route, event payload, clear-state data canary, and onward navigation there. +Observe its destination before checking for a system confirmation. +If the result carries `APP_NOT_RUNNING`, use `waitForDeepLinkDestination` to handle the exact fixture confirmation. +A typed absent alert needs no action; an unexpected prompt or alert failure fails the scenario. +Automatic confirmation remains covered by the launch-confirmation and lifecycle tests. + +Use semantic readiness landmarks before interaction. +A title alone can precede the React tab controls. +Before testing keyboard dismissal, require the fixture's visible Done control. +Enable the simulator software keyboard for that lifecycle contract. +For intentionally truncated raw-depth assertions, validate the structural frontier; require semantic capture quality on regular and full-raw observations. +Use the built CLI for preparation and execution after building once. +Source and built clients have different daemon code signatures; alternating them can replace the daemon under test. +Use the built SDK with the existing-daemon transport for forced-backend probes. +Do not let a probe start or replace the daemon under test. + +## Keep recovery bounded + +Use `runStep` for the shared single-repeat observation policy. +Require both a retriable verdict and a typed prevented-observation reason. +Fail readable absence, exhausted readable waits, invalid selectors, and incorrect values. +If a helper owns retries, pass `reattemptInfrastructureMiss: undefined` on every attempt. +Keep cleanup retries in `retryCleanupStep`; return failed results with `allowFailure: true` so its typed already-clean handling applies on every attempt. +Capture exhausted cleanup failures through the shared harness before continuing cleanup. +Report them separately from scenario assertions. +Replay-suite attempt policies have their own owner; this step policy does not redefine them. + +## Match input assertions to the contract + +For masked unverified input, poll the app-owned delivery canary and assert zero password-toggle presses. +Require the canary to acknowledge the entire requested suffix. +Do not infer replacement or length from masking bullets. +For readable input, retain exact replacement assertions and deliberately dropped-select-all regressions. +Keep unreadable classification and bounded clearing in their owning policy tests. + +## Verify simulator smoke changes + +Read the live entry point's environment contract before running it. +Build a current Release fixture with embedded JavaScript for simulator smoke verification. +If a headless simulator minimizes its software keyboard, inspect its runtime setting: + +```sh +xcrun simctl spawn defaults read com.apple.keyboard.preferences AutomaticMinimizationEnabled +xcrun simctl spawn defaults write com.apple.keyboard.preferences AutomaticMinimizationEnabled -bool false +``` + +The iOS workflow sets this before fixture smoke. +For local verification, record the prior value and restore it afterward. +Run the changed scenario through the checked-in smoke entry point and preserve its coverage and failure artifacts. +Record the source SHA, device, command, and semantic result under [device verification](../../../docs/agents/device-verification.md). +For regressions, prove the test fails without the fix using [isolated experiments](../../../docs/agents/test-experiments.md). +Do not count skipped device tests as passed. diff --git a/test/integration/ios-simulator-e2e/behavior-coverage.ts b/test/integration/ios-simulator-e2e/behavior-coverage.ts index d157fc50f6..7e8fdf6b13 100644 --- a/test/integration/ios-simulator-e2e/behavior-coverage.ts +++ b/test/integration/ios-simulator-e2e/behavior-coverage.ts @@ -32,7 +32,7 @@ export const IOS_SIMULATOR_BEHAVIOR_COVERAGE = { assertion: 'launchApp(clearState) removes seeded app data, opens the stored deep route, renders its payload, and navigates onward', level: 'live', - owner: 'smoke:automation-input', + owner: 'smoke:deep-link-navigation', }, 'text-entry-keyboard-lifecycle': { assertion: diff --git a/test/integration/ios-simulator-e2e/live-automation-scenario.ts b/test/integration/ios-simulator-e2e/live-automation-scenario.ts index cf0465b30a..f5bd7992fc 100644 --- a/test/integration/ios-simulator-e2e/live-automation-scenario.ts +++ b/test/integration/ios-simulator-e2e/live-automation-scenario.ts @@ -83,26 +83,6 @@ export async function assertAutomationInput(context: LiveContext): Promise verifyCommand(context, C.snapshot, 'interactive fixture tree exposes its stable title'); verifyCommand(context, C.open, 'cold launch exposes the fixture UI through snapshot and wait'); - await openAutomationDeepLink(context, 'cold launch fixture through a deep link'); - await waitForDeepLinkDestination(context, ['text', 'Automation lab']); - await assertElementText(context, 'id="automation-event-name"', 'cold.start'); - await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); - await assertClearStateLaunchUrl(context); - await runStep(context, 'navigate onward from cold deep link', [ - 'click', - 'id="automation-continue-catalog"', - ]); - await runStep(context, 'wait for exact catalog destination', [ - 'wait', - 'id="catalog-title"', - '10000', - ]); - verifyBehavior( - context, - 'cold-start-deep-link-navigation', - 'launchApp(clearState) removed a seeded data canary, rendered the stored deep route and payload, and continued into the fixture catalog', - ); - await runStep(context, 'wait for settings tab target', ['wait', 'label="Settings"', '10000']); await runStep(context, 'open settings tab', ['click', 'label="Settings"']); await assertWaitText(context, 'Settings'); @@ -217,6 +197,28 @@ export async function assertAutomationInput(context: LiveContext): Promise verifyCommand(context, C.back, 'back returns from automation route to Settings'); } +export async function assertColdDeepLinkNavigation(context: LiveContext): Promise { + await openAutomationDeepLink(context, 'cold launch fixture through a deep link'); + await waitForDeepLinkDestination(context, ['text', 'Automation lab']); + await assertElementText(context, 'id="automation-event-name"', 'cold.start'); + await assertElementText(context, 'id="automation-event-payload"', '{"source":"deep-link"}'); + await assertClearStateLaunchUrl(context); + await runStep(context, 'navigate onward from cold deep link', [ + 'click', + 'id="automation-continue-catalog"', + ]); + await runStep(context, 'wait for exact catalog destination', [ + 'wait', + 'id="catalog-title"', + '10000', + ]); + verifyBehavior( + context, + 'cold-start-deep-link-navigation', + 'launchApp(clearState) removed a seeded data canary, rendered the stored deep route and payload, and continued into the fixture catalog', + ); +} + async function assertClearStateLaunchUrl(context: LiveContext): Promise { const prepared = await runStep(context, 'prepare fixture data clear canary', [ 'settings', diff --git a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts index 321d543094..43dd5b97f1 100644 --- a/test/integration/ios-simulator-e2e/live-deep-link-destination.ts +++ b/test/integration/ios-simulator-e2e/live-deep-link-destination.ts @@ -1,27 +1,18 @@ import assert from 'node:assert/strict'; +import { + ALERT_NOT_FOUND_RUNNER_CODE, + DEFAULT_ALERT_TIMEOUT_MS, +} from '@agent-device/contracts/alert-contract'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { WAIT_REASONS } from '@agent-device/contracts/wait'; +import { isObservationPreventedStepMiss } from './live-step-retry-policy.ts'; import type { CliJsonResult } from '../cli-json.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; -/** One destination wait; generous because the WebView lab took over 2.5 s to mount on cold CI. */ +/** Allow the cold fixture route to mount before declaring a readable miss. */ const DESTINATION_WAIT_MS = 15_000; -/** - * The answered launch usually reaches the foreground within 3 s, but a loaded host held it for - * 20.7 s (CI run 35991523779) and 27.4 s (a local run), and a relaunch can restart the runner, - * whose start outlasts one wait. - */ -const DESTINATION_WAITS = 5; - -/** Misses that say the destination was not observed yet, so another bounded wait may see it. */ -const NOT_YET_OBSERVED: ReadonlySet = new Set([ - WAIT_REASONS.deadlineExceeded, - WAIT_REASONS.captureStalled, - WAIT_REASONS.runnerRestartExhausted, - WAIT_REASONS.readinessExhausted, - WAIT_REASONS.targetAbsent, -]); +// One repeat follows a prevented observation or the stopped-app confirmation branch. +const DESTINATION_WAITS = 2; /** One destination wait; the final one is run without `allowFailure`, so its failure throws. */ export type DestinationWait = ( @@ -30,9 +21,8 @@ export type DestinationWait = ( ) => Promise; /** - * Waits for a deep-link route's own first landmark after `open` answered any launch confirmation. - * Retries a miss that only says the destination was not observed yet, up to five 15 s waits; - * any other failure fails at once. + * Observe the route before looking for a launch confirmation. + * A typed stopped-app result permits explicit confirmation handling; readable misses fail. */ export async function waitForDeepLinkDestination( context: LiveContext, @@ -50,7 +40,18 @@ export async function waitForDeepLinkDestination( String(DESTINATION_WAIT_MS), ...(options.debug ? ['--debug'] : []), ], - { allowFailure }, + { allowFailure, reattemptInfrastructureMiss: undefined }, + ), + () => + answerFixtureLaunchConfirmation( + () => + runStep( + context, + 'wait for optional fixture launch confirmation', + ['alert', 'wait', String(DEFAULT_ALERT_TIMEOUT_MS)], + { allowFailure: true }, + ), + () => runStep(context, 'accept fixture launch confirmation', ['alert', 'accept']), ), ); verifyCommand( @@ -61,17 +62,47 @@ export async function waitForDeepLinkDestination( return arrived; } -export async function retryDestinationWait(wait: DestinationWait): Promise { - for (let attempt = 1; ; attempt += 1) { +export async function retryDestinationWait( + wait: DestinationWait, + answerStoppedLaunch?: () => Promise, +): Promise { + for (let attempt = 1; attempt <= DESTINATION_WAITS; attempt += 1) { const arrived = await wait( `wait for the deep-link destination (${attempt}/${DESTINATION_WAITS})`, { allowFailure: attempt < DESTINATION_WAITS }, ); if (arrived.status === 0) return arrived; - const reason = arrived.json?.error?.details?.reason; + if ( + attempt < DESTINATION_WAITS && + answerStoppedLaunch && + arrived.json?.error?.details?.runnerErrorCode === 'APP_NOT_RUNNING' + ) { + await answerStoppedLaunch(); + continue; + } assert.ok( - NOT_YET_OBSERVED.has(reason), + attempt < DESTINATION_WAITS && isObservationPreventedStepMiss(arrived), `deep-link destination wait failed: ${JSON.stringify(arrived.json)}`, ); } + throw new Error('deep-link destination observation exhausted'); +} + +/** The fixture owns this scheme; only its exact English launch confirmation may be answered. */ +export async function answerFixtureLaunchConfirmation( + read: () => Promise, + accept: () => Promise, +): Promise { + const result = await read(); + if (result.status !== 0) { + assert.equal( + result.json?.error?.details?.runnerErrorCode, + ALERT_NOT_FOUND_RUNNER_CODE, + `launch confirmation observation failed: ${JSON.stringify(result.json)}`, + ); + return; + } + assert.equal(result.json?.data?.message, 'Open in “Agent Device Tester”?'); + assert.deepEqual(result.json?.data?.items, ['Cancel', 'Open']); + await accept(); } diff --git a/test/integration/ios-simulator-e2e/live-harness.ts b/test/integration/ios-simulator-e2e/live-harness.ts index f448989f02..c6461b5069 100644 --- a/test/integration/ios-simulator-e2e/live-harness.ts +++ b/test/integration/ios-simulator-e2e/live-harness.ts @@ -68,13 +68,7 @@ const harness = createLiveDeviceHarness({ writeCoverageReport, }); -/** - * Every iOS live step carries the observation-prevented re-issue policy (#2491): one runner - * restart or bridge rebuild re-issues its own wait instead of failing the job, while a readable - * miss (target absent, budget exhausted after a readable capture, wrong asserted value) fails at - * once exactly as before. Steps that already own a miss policy (`allowFailure` / `expectFailure`, - * like the destination-wait and cleanup retry loops) are exempt inside the harness itself. - */ +// Repeat at most one typed observation failure. Scenario-owned retry loops disable this policy. export function runStep( context: LiveContext, step: string, @@ -102,6 +96,7 @@ export function verifyNestedReplayCommand( const MICROPHONE_PERMISSION_RESET_STEP = 'reset microphone permission'; export async function cleanupSession(context: LiveContext): Promise { + context.currentScenario = 'cleanup'; const failures: unknown[] = []; const cleanupSteps: Array<[string, string[]]> = []; if (context.tier === 'full') { @@ -115,10 +110,14 @@ export async function cleanupSession(context: LiveContext): Promise { for (const [step, args] of cleanupSteps) { const failure = await retryCleanupStep(step, (attempt) => runStep(context, `cleanup: ${step} (attempt ${attempt})`, args, { - allowFailure: attempt < 3, + allowFailure: true, + reattemptInfrastructureMiss: undefined, }), ); - if (failure !== undefined) failures.push(failure); + if (failure !== undefined) { + await harness.recordFailure(context, failure); + failures.push(failure); + } } if (failures.length === 0) return; const errorPath = path.join(context.artifactDir, 'cleanup-error.txt'); @@ -126,28 +125,25 @@ export async function cleanupSession(context: LiveContext): Promise { throw new AggregateError(failures, `iOS E2E cleanup failed; details: ${errorPath}`); } -/** - * Runs one cleanup step's 3-attempt retry policy: success or an already-clean session - * stops immediately, anything else waits and retries. `runAttempt` mirrors the real - * `runStep(..., { allowFailure: attempt < 3 })` contract, including that the final - * attempt throws instead of returning a failed result. Exported so the retry/guard - * behavior is unit-testable without spawning the CLI (see - * `test/integration/ios-simulator-e2e-cleanup.test.ts`). - */ +/** Returns the failure after at most three attempts, or undefined when cleanup is complete. */ export async function retryCleanupStep( step: string, runAttempt: (attempt: number) => Promise, ): Promise { - for (let attempt = 1; attempt <= 3; attempt += 1) { + for (let attempt = 1; ; attempt += 1) { try { const result = await runAttempt(attempt); if (result.status === 0 || sessionAlreadyClean(step, result)) return undefined; + if (attempt === 3) { + return new Error( + `cleanup: ${step} failed after ${attempt} attempts: ${JSON.stringify(result)}`, + ); + } await new Promise((resolve) => setTimeout(resolve, 500)); } catch (error) { return error; } } - return undefined; } // SESSION_NOT_FOUND: nothing left to reset, any step. INVALID_ARGS with the published diff --git a/test/integration/ios-simulator-e2e/live-runner.ts b/test/integration/ios-simulator-e2e/live-runner.ts index d01b2f1025..0ae003e864 100644 --- a/test/integration/ios-simulator-e2e/live-runner.ts +++ b/test/integration/ios-simulator-e2e/live-runner.ts @@ -1,11 +1,14 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; +import type { TestContext } from 'node:test'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import type { AgentDeviceDaemonTransport } from '@agent-device/contracts/client'; import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; -import { sendToDaemon } from '../../../src/daemon-client/daemon-client.ts'; +import { readDaemonInfo } from '../../../src/daemon-client/daemon-client-metadata.ts'; +import { sendRequest } from '../../../src/daemon-client/daemon-client-transport.ts'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { assertPngFile } from '../provider-scenarios/assertions.ts'; import { assertFilesDiffer, @@ -15,7 +18,7 @@ import { capturePng, requireNodeRect, } from './live-assertions.ts'; -import { assertAutomationInput } from './live-automation-scenario.ts'; +import { assertAutomationInput, assertColdDeepLinkNavigation } from './live-automation-scenario.ts'; import { assertDeviceLifecycle } from './live-device-lifecycle.ts'; import { assertRegularVisibleDepthFrontier } from './live-snapshot-depth-frontier.ts'; import { assertWebViewRemoteContent } from './live-webview-remote-content.ts'; @@ -54,13 +57,26 @@ const sendToDaemonTransport: AgentDeviceDaemonTransport = async (request, contex if (request.session === undefined) { throw new Error('Snapshot conformance transport requires an explicit session.'); } - return await sendToDaemon({ ...request, session: request.session }, context); + const stateDir = request.flags?.stateDir; + assert.ok(typeof stateDir === 'string', 'backend conformance requires the smoke state directory'); + const paths = resolveDaemonPaths(stateDir, { env: process.env }); + const info = readDaemonInfo(paths.infoPath); + assert.ok(info, 'backend conformance requires the daemon prepared by the built CLI'); + return await sendRequest( + info, + { ...request, session: request.session, token: info.token }, + 'auto', + paths, + 90_000, + { + signal: context?.signal, + timeoutRecovery: 'none', + }, + ); }; async function loadBuiltAgentDeviceClient() { - // The live harness drives the built CLI, so use the built SDK entry as well. Importing the - // source SDK here would intentionally take over the daemon on code-signature mismatch and make - // the forced-backend evidence come from a different runtime than the rest of the scenario. + // Exercise the same built SDK and CLI revision; the injected transport owns no daemon lifecycle. const builtSdk = (await import( pathToFileURL(path.resolve('dist/src/index.js')).href )) as AgentDeviceSdk; @@ -74,6 +90,7 @@ const LIVE_SCENARIOS = bindIosSimulatorScenarios({ await assertClose(context); }, deviceLifecycle: assertDeviceLifecycle, + deepLinkNavigation: assertColdDeepLinkNavigation, fixtureReplays: assertFixtureReplays, formInput: assertFormInput, inventoryInstall: assertInventoryAndInstall, @@ -83,12 +100,12 @@ const LIVE_SCENARIOS = bindIosSimulatorScenarios({ webviewRemoteContent: assertWebViewRemoteContent, }); -export async function runIosSimulatorE2E(): Promise { +export async function runIosSimulatorE2E(testContext?: TestContext): Promise { const context = createContext(); let primaryError: unknown; try { await prepareRunner(context); - await executeLiveScenarios(context); + await executeLiveScenarios(context, testContext); } catch (error) { primaryError = error; } @@ -121,11 +138,42 @@ async function prepareRunner(context: LiveContext): Promise { ); } -async function executeLiveScenarios(context: LiveContext): Promise { +async function executeLiveScenarios( + context: LiveContext, + testContext?: TestContext, +): Promise { let assertionCaptureVerified = false; + const sessionPrefix = context.session; for (const scenario of LIVE_SCENARIOS.filter((candidate) => candidate.tier === 'smoke')) { - await runScenario(context, scenario); - if (!assertionCaptureVerified && context.sessionOpen) { + const run = async () => { + await runScenario(context, { + ...scenario, + run: async () => { + if (await sessionExists(context)) { + await runStep(context, 'release preceding scenario session', ['close']); + } + context.session = `${sessionPrefix}-${scenario.id.replace(':', '-')}`; + if (scenario.setup === 'fixture-home') { + const opened = await runStep(context, `reset fixture for ${scenario.id}`, [ + 'open', + context.appId, + '--relaunch', + ]); + context.runnerLogPath = opened.json?.data?.runnerLogPath; + await assertWaitSelector(context, 'id="home-title" visible=true'); + await assertWaitSelector(context, 'label="Form" visible=true'); + } + await scenario.run(context); + }, + }); + }; + if (testContext) await testContext.test(scenario.id, run); + else await run(); + if ( + !assertionCaptureVerified && + context.sessionOpen && + context.completedScenarios.includes(scenario.id) + ) { await assertLiveAssertionCapture(context); assertionCaptureVerified = true; } @@ -304,6 +352,7 @@ async function assertFormInput(context: LiveContext): Promise { 'seeded email should be observable before dismiss', ); + await assertWaitSelector(context, 'label="Done" visible=true'); const keyboardVisiblePath = path.join(context.artifactDir, 'keyboard-visible.png'); await capturePng(context, 'capture visible input keyboard', keyboardVisiblePath); const keyboard = await runStep(context, 'dismiss input keyboard', ['keyboard', 'dismiss']); diff --git a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts index 027aa02ec0..c96fedf181 100644 --- a/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts +++ b/test/integration/ios-simulator-e2e/live-snapshot-depth-frontier.ts @@ -7,25 +7,20 @@ import { type LiveSnapshotNode as SnapshotNode, snapshotNodes, } from './live-assertions.ts'; -import { waitForDeepLinkDestination } from './live-deep-link-destination.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; -const VISIBLE_DEPTH_DEEP_LINK = 'agent-device-test-app:///snapshot-depth'; const CHILD_ID = 'visible-depth-projected-child'; const MISSING_HITTABILITY_WARNING = 'iOS snapshot acquisition does not provide hittability evidence; regular snapshots omit unverified hittability while raw snapshots preserve supplied facts.'; export async function assertRegularVisibleDepthFrontier(context: LiveContext): Promise { + await runStep(context, 'open depth fixture settings', ['click', 'label="Settings"']); + await assertWaitText(context, 'Settings'); await runStep(context, 'open regular visible-depth fixture', [ - 'open', - context.appId, - '--relaunch', - '--launch-url', - VISIBLE_DEPTH_DEEP_LINK, - '--debug', + 'click', + 'id="open-snapshot-depth-lab"', ]); - // Wait for the target itself so the depth assertion is about the frontier, not route readiness. - await waitForDeepLinkDestination(context, [`id="${CHILD_ID}"`], { debug: true }); + await runStep(context, 'wait for visible-depth fixture', ['wait', `id="${CHILD_ID}"`, '15000']); const regular = await runStep(context, 'capture regular visible-depth frontier', [ 'snapshot', @@ -75,8 +70,18 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P '--depth', '1', ]); - assertSimulatorSnapshotAcquisition(rawDepthOne, 'raw depth-1 visible-depth snapshot'); const rawDepthOneNodes = snapshotNodes(rawDepthOne); + requireRoot(rawDepthOneNodes, 'raw depth-1 visible-depth snapshot'); + // Raw acquisition stops before semantic content; this structural frontier can be sparse. + if (rawDepthOne.json?.data?.snapshotQuality?.state === 'sparse') { + assert.equal(rawDepthOne.json.data.snapshotQuality.reasonCode, 'sparse-tree'); + assert.equal(rawDepthOne.json.data.truncated, true); + assert.ok( + rawDepthOneNodes.every((node) => node.type === 'Application' || node.type === 'Window'), + ); + } else { + assertSimulatorSnapshotAcquisition(rawDepthOne, 'raw depth-1 visible-depth snapshot'); + } assert.equal( rawDepthOneNodes.some((node) => node.identifier === CHILD_ID), false, @@ -87,12 +92,6 @@ export async function assertRegularVisibleDepthFrontier(context: LiveContext): P `raw --depth 1 exceeded the acquisition frontier: ${JSON.stringify(rawDepthOne)}`, ); - await runStep(context, 'restore fixture home after visible-depth capture', [ - 'open', - context.appId, - '--relaunch', - ]); - await assertWaitText(context, 'Agent Device Tester'); verifyBehavior( context, 'regular-visible-depth-frontier', diff --git a/test/integration/ios-simulator-e2e/live-step-retry-policy.ts b/test/integration/ios-simulator-e2e/live-step-retry-policy.ts index 9ad1f885f7..1edf380789 100644 --- a/test/integration/ios-simulator-e2e/live-step-retry-policy.ts +++ b/test/integration/ios-simulator-e2e/live-step-retry-policy.ts @@ -1,35 +1,15 @@ import { WAIT_REASONS } from '@agent-device/contracts/wait'; import type { CliJsonResult } from '../cli-json.ts'; -/** - * The wait verdicts that say the observation itself was prevented — the runner restarted, - * readiness work (runner start, bridge target discovery) ate the budget, or no capture was ever - * readable. Each is a lane condition on this host, not an answer about the screen: the product - * itself publishes all three with `retriable: true` and a "then retry" hint on the wire - * (`wait-polling.ts`), and they are the observed reds of the smoke lane (run 37356199982's - * `wait_runner_restart_exhausted` with `readableCaptures: 0`). - */ +// These wait verdicts indicate that capture or readiness prevented observation. const OBSERVATION_PREVENTED_WAIT_REASONS: ReadonlySet = new Set([ WAIT_REASONS.captureStalled, WAIT_REASONS.runnerRestartExhausted, WAIT_REASONS.readinessExhausted, ]); -/** - * Whether one failed E2E step's result says the DEVICE prevented the observation, so re-issuing - * the same step is a measurement of the same question rather than a second bite at a real defect. - * - * Keyed on the conjunction of two typed signals — the wire `retriable` verdict the daemon hoists - * from its throw sites AND `error.details.reason` from the wait taxonomy — never on error text. - * Either signal alone is too wide: `retriable` also rides failures the lane must not re-issue - * blind (e.g. an app the lane itself is expected to have launched via another route), and a reason - * set copied onto a response the product marks non-retriable would retry a verdict the product - * says cannot succeed unchanged. The pairing is pinned in - * `test/integration/ios-simulator-e2e-step-retry-policy.test.ts`. - * - * `wait_target_absent` (the screen was readable and simply did not hold the target), - * `wait_deadline_exceeded` (a readable capture consumed the budget), a wrong assertion value, a - * runner crash, and any non-wait failure are all non-retriable by this predicate. +/** Requires both a retriable verdict and a typed prevented-observation reason. + * Readable absence, deadline exhaustion, and non-wait failures cannot repeat here. */ export function isObservationPreventedStepMiss(result: CliJsonResult): boolean { const error = result.json?.error; diff --git a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts index 0efd7e1089..d0d06449c5 100644 --- a/test/integration/ios-simulator-e2e/live-webview-remote-content.ts +++ b/test/integration/ios-simulator-e2e/live-webview-remote-content.ts @@ -1,12 +1,13 @@ import assert from 'node:assert/strict'; -import { assertWaitText, snapshotNodes } from './live-assertions.ts'; -import { waitForDeepLinkDestination } from './live-deep-link-destination.ts'; +import { + assertElementTextAfterScrolling, + assertWaitText, + snapshotNodes, +} from './live-assertions.ts'; import { type LiveContext, runStep, verifyBehavior } from './live-harness.ts'; -const WEBVIEW_LAB_DEEP_LINK = 'agent-device-test-app:///webview'; -// Native chrome of the lab, rendered with the route and before the page: it proves the deep link -// landed without asking the runner to query a screen whose web view is still loading. +// Native chrome proves Settings navigation reached the lab before its web page finishes loading. const LAB_CHROME_ID = 'close-webview-lab'; // The first WebContent process of the run spawns here; a cold CI simulator needs more than the // shared 10 s wait budget before the page's tree exists. @@ -28,14 +29,19 @@ const XCTEST_FALLBACK_WARNING = * vanish from every snapshot and no ref can reach them. */ export async function assertWebViewRemoteContent(context: LiveContext): Promise { - await runStep(context, 'open WebView accessibility lab', [ - 'open', - context.appId, - '--relaunch', - '--launch-url', - WEBVIEW_LAB_DEEP_LINK, + await runStep(context, 'open WebView fixture settings', ['click', 'label="Settings"']); + await assertWaitText(context, 'Settings'); + await assertElementTextAfterScrolling( + context, + 'id="open-webview-lab"', + 'Open WebView accessibility lab', + ); + await runStep(context, 'open WebView accessibility lab', ['click', 'id="open-webview-lab"']); + await runStep(context, 'wait for WebView native chrome', [ + 'wait', + `id="${LAB_CHROME_ID}"`, + '15000', ]); - await waitForDeepLinkDestination(context, [`id="${LAB_CHROME_ID}"`]); // `wait` observes through the same route as `snapshot`: page content is reachable only once the // route has stopped publishing the bridge's page-less tree. await runStep(context, 'wait for the WebView page to expose its link', [ @@ -65,12 +71,6 @@ export async function assertWebViewRemoteContent(context: LiveContext): Promise< `WebView snapshot must disclose the XCTest fallback for remote content: ${JSON.stringify(snapshot)}`, ); - await runStep(context, 'restore fixture home after WebView lab', [ - 'open', - context.appId, - '--relaunch', - ]); - await assertWaitText(context, 'Agent Device Tester'); verifyBehavior( context, 'webview-remote-content', diff --git a/test/integration/ios-simulator-e2e/scenarios.ts b/test/integration/ios-simulator-e2e/scenarios.ts index 7418fdfd03..0d0f3314f6 100644 --- a/test/integration/ios-simulator-e2e/scenarios.ts +++ b/test/integration/ios-simulator-e2e/scenarios.ts @@ -1,11 +1,12 @@ -export type IosSimulatorScenario = { - id: string; - tier: 'smoke' | 'full'; -}; +export type IosSimulatorScenario = { id: string } & ( + | { tier: 'smoke'; setup: 'no-session' | 'scenario-owned' | 'fixture-home' } + | { tier: 'full' } +); type ScenarioRunnerKey = | 'automationInput' | 'captureClose' + | 'deepLinkNavigation' | 'deviceLifecycle' | 'fixtureReplays' | 'formInput' @@ -20,16 +21,33 @@ type ScenarioDefinition = IosSimulatorScenario & { }; const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ - { id: 'smoke:inventory-install', runner: 'inventoryInstall', tier: 'smoke' }, - { id: 'smoke:automation-input', runner: 'automationInput', tier: 'smoke' }, - { id: 'smoke:form-input', runner: 'formInput', tier: 'smoke' }, + { id: 'smoke:inventory-install', setup: 'no-session', runner: 'inventoryInstall', tier: 'smoke' }, + { + id: 'smoke:automation-input', + setup: 'scenario-owned', + runner: 'automationInput', + tier: 'smoke', + }, + { id: 'smoke:form-input', setup: 'fixture-home', runner: 'formInput', tier: 'smoke' }, { id: 'smoke:regular-visible-depth-frontier', + setup: 'fixture-home', runner: 'snapshotDepthFrontier', tier: 'smoke', }, - { id: 'smoke:webview-remote-content', runner: 'webviewRemoteContent', tier: 'smoke' }, - { id: 'smoke:capture-close', runner: 'captureClose', tier: 'smoke' }, + { + id: 'smoke:webview-remote-content', + setup: 'fixture-home', + runner: 'webviewRemoteContent', + tier: 'smoke', + }, + { + id: 'smoke:deep-link-navigation', + setup: 'scenario-owned', + runner: 'deepLinkNavigation', + tier: 'smoke', + }, + { id: 'smoke:capture-close', setup: 'fixture-home', runner: 'captureClose', tier: 'smoke' }, { id: 'full:lifecycle-system', runner: 'lifecycleSystem', tier: 'full' }, { id: 'full:observability-artifacts', @@ -45,9 +63,8 @@ export const IOS_SIMULATOR_LIVE_SCENARIOS: readonly IosSimulatorScenario[] = SCE export function bindIosSimulatorScenarios( runners: Record Promise>, ): readonly (IosSimulatorScenario & { run: (context: Context) => Promise })[] { - return SCENARIO_DEFINITIONS.map(({ id, runner, tier }) => ({ - id, + return SCENARIO_DEFINITIONS.map(({ runner, ...scenario }) => ({ + ...scenario, run: runners[runner], - tier, })); } diff --git a/test/integration/live-device-e2e-runtime.test.ts b/test/integration/live-device-e2e-runtime.test.ts index 8a092652ce..adcff789c4 100644 --- a/test/integration/live-device-e2e-runtime.test.ts +++ b/test/integration/live-device-e2e-runtime.test.ts @@ -69,6 +69,24 @@ test('a scenario assertion after allowed misses captures evidence before caller assert.equal(reports(), 1); }); +test('a caller-owned cleanup failure captures evidence without reissuing the command', async () => { + const { context, harness, calls } = fixture(); + context.currentScenario = 'cleanup'; + const result = await harness.runStep(context, 'close session', ['close'], { allowFailure: true }); + assert.equal(result.status, 1); + await harness.recordFailure(context, new Error('close session exhausted its retry budget')); + + assert.deepEqual( + calls.map((args) => args[0]), + ['close', 'screenshot', 'snapshot'], + ); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1.png'))); + assert.ok(fs.existsSync(path.join(context.artifactDir, 'failed-step-1-snapshot.json'))); + const report = fs.readFileSync(path.join(context.artifactDir, 'failed-step.txt'), 'utf8'); + assert.match(report, /close session exhausted its retry budget/); + assert.match(report, /scenario: cleanup/); +}); + test('a scenario assertion before any command also captures evidence', async () => { const { context, harness, calls } = fixture(); const failure = new Error('fixture assertion'); diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts index 6ff99abb59..c1c387befe 100644 --- a/test/integration/live-device-e2e/runtime.ts +++ b/test/integration/live-device-e2e/runtime.ts @@ -162,7 +162,7 @@ export function createLiveDeviceHarness< ); context.completedScenarios.push(scenario.id); } catch (error) { - await recordScenarioFailure(context, error); + await recordFailure(context, error); throw error; } finally { context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id }); @@ -170,7 +170,7 @@ export function createLiveDeviceHarness< } } - async function recordScenarioFailure(context: Context, error: unknown): Promise { + async function recordFailure(context: Context, error: unknown): Promise { if (error instanceof Error && reportedFailures.has(error)) return; try { const evidence = await captureFailedStepEvidence(context); @@ -363,6 +363,7 @@ export function createLiveDeviceHarness< } return { + recordFailure, runScenario, runStep, sessionExists,