From e7a02a1f43783552c14a2e635d818d43fab1e3c8 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:35:00 +0200 Subject: [PATCH 1/2] fix(mobilerun): bound the state snapshot so a stuck app can't hang it GET /mobilerun/state took one snapshot through XCTest, which waits up to its 60s accessibility timeout per request and retries on top. On an iPhone X / iOS 16 TikTok never finished snapshotting, so each state request held the runner for minutes and every request queued behind it, Home Screen included, timed out in mobilerun-ios and the CLI (DROID-305). Snapshots are now requested directly, without XCTest's resolution retries, under a temporary accessibility timeout: 10s for the full tree, then 5s for a shallow one (15 levels, 20 children per element). If neither finishes, the state still names the foreground app, which is known without a snapshot, with the app root alone as the tree. 'truncated' says when the tree is incomplete. The request now always answers 200, so a failed snapshot can't read as a missing route either. Co-Authored-By: Claude Opus 5.5 --- WebDriverAgent.xcodeproj/project.pbxproj | 4 + .../Categories/XCUIApplication+FBHelpers.h | 32 +++- .../Categories/XCUIApplication+FBHelpers.m | 69 ++++++++- .../Commands/FBMobilerunA11yCommands.h | 58 +++++-- .../Commands/FBMobilerunA11yCommands.m | 91 +++++++---- .../Utilities/FBXCAXClientProxy.h | 17 +++ .../Utilities/FBXCAXClientProxy.m | 24 ++- .../FBMobilerunA11yIntegrationTests.m | 122 +++++++++++++++ .../UnitTests/FBMobilerunA11yCommandsTests.m | 141 +++++++++++------- docs/request-dispatch.md | 5 +- 10 files changed, 446 insertions(+), 117 deletions(-) create mode 100644 WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 1fa113b44..1141c4545 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -1290,6 +1290,7 @@ FBCAFE000000000000003105 /* FBMobilerunActionsCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000003104 /* FBMobilerunActionsCommands.m */; }; FBCAFE000000000000003106 /* FBMobilerunActionsCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000003104 /* FBMobilerunActionsCommands.m */; }; FBCAFE000000000000003202 /* FBMobilerunActionsIntegrationTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000003201 /* FBMobilerunActionsIntegrationTests.m */; }; + FBCAFE000000000000003402 /* FBMobilerunA11yIntegrationTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000003401 /* FBMobilerunA11yIntegrationTests.m */; }; FBCAFE000000000000004002 /* FBVideoStreamSessionTests.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000004001 /* FBVideoStreamSessionTests.m */; }; FBCAFE000000000000005103 /* FBBroadcastSampleHandler.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005102 /* FBBroadcastSampleHandler.m */; }; FBCAFE000000000000005106 /* FBExtBroadcastClient.m in Sources */ = {isa = PBXBuildFile; fileRef = FBCAFE000000000000005105 /* FBExtBroadcastClient.m */; }; @@ -2082,6 +2083,7 @@ FBCAFE000000000000003101 /* FBMobilerunActionsCommands.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FBMobilerunActionsCommands.h; sourceTree = ""; }; FBCAFE000000000000003104 /* FBMobilerunActionsCommands.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunActionsCommands.m; sourceTree = ""; }; FBCAFE000000000000003201 /* FBMobilerunActionsIntegrationTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunActionsIntegrationTests.m; sourceTree = ""; }; + FBCAFE000000000000003401 /* FBMobilerunA11yIntegrationTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBMobilerunA11yIntegrationTests.m; sourceTree = ""; }; FBCAFE000000000000004001 /* FBVideoStreamSessionTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FBVideoStreamSessionTests.m; sourceTree = ""; }; FBCAFE000000000000005002 /* WebDriverAgentBroadcast.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = WebDriverAgentBroadcast.appex; sourceTree = BUILT_PRODUCTS_DIR; }; FBCAFE00000000000000500C /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -2908,6 +2910,7 @@ A1B2C3D41F001A00A1B0003 /* FBVoiceOverTests.m */, 71241D7D1FAF084E00B9559F /* FBW3CTouchActionsIntegrationTests.m */, FBCAFE000000000000003201 /* FBMobilerunActionsIntegrationTests.m */, + FBCAFE000000000000003401 /* FBMobilerunA11yIntegrationTests.m */, FBCAFE000000000000007330 /* FBMobilerunSocks5IntegrationTests.m */, 71241D7F1FAF087500B9559F /* FBW3CMultiTouchActionsIntegrationTests.m */, 7136C0F8243A182400921C76 /* FBW3CTypeActionsTests.m */, @@ -5206,6 +5209,7 @@ A1B2C3D41F001A00A1B0008 /* FBVoiceOverTests.m in Sources */, 71241D7E1FAF084E00B9559F /* FBW3CTouchActionsIntegrationTests.m in Sources */, FBCAFE000000000000003202 /* FBMobilerunActionsIntegrationTests.m in Sources */, + FBCAFE000000000000003402 /* FBMobilerunA11yIntegrationTests.m in Sources */, FBCAFE000000000000007331 /* FBMobilerunSocks5IntegrationTests.m in Sources */, 63FD950221F9D06100A3E356 /* FBImageProcessorTests.m in Sources */, 719CD8FF2126C90200C7D0C2 /* FBAutoAlertsHandlerTests.m in Sources */, diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h index 0f1ba0569..b9059d4d4 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.h @@ -43,19 +43,39 @@ NS_ASSUME_NONNULL_BEGIN */ - (NSDictionary *)fb_accessibilityTree; +/** + Takes one snapshot of the application that gives up after 'timeout' seconds. + Unlike -snapshotWithError: it skips XCTest's element resolution and its + retries, each of which waits up to XCTest's 60s accessibility timeout. + + @param timeout The accessibility timeout for this request, in seconds. + @param parameters Snapshot request parameters overriding the defaults, e.g. + a smaller FBSnapshotMaxDepthKey. + @param error Set when the snapshot failed or timed out. + @return The snapshot, or nil on failure. + */ +- (nullable id)fb_snapshotWithTimeout:(NSTimeInterval)timeout + parameters:(NSDictionary *)parameters + error:(NSError **)error; + /** Returns a compact accessibility state tailored for mobilerun: a recursive accessibility tree (Android-style `A11YNode` shape) plus a `phone_state` - summary. It is built from a single standard snapshot and only reads - snapshot-resident attributes, so it deliberately avoids the expensive - visibility/accessibility/hittable resolution performed by `fb_tree`. + summary. It only reads snapshot-resident attributes, so it deliberately + avoids the expensive visibility/accessibility/hittable resolution performed + by `fb_tree`, and makes no accessibility requests of its own. + @param snapshot The application's snapshot, or nil when none could be taken: + the tree is then the application root alone, spanning the screen. @param scale The factor the logical-point element bounds are multiplied by. Pass the native screen scale for device pixels or 1 for plain points. - @return Dictionary with `a11y_tree` (nested `A11YNode` dictionaries) and - `phone_state` entries. + @param truncated Whether the snapshot is incomplete, reported as `truncated`. + @return Dictionary with `a11y_tree` (nested `A11YNode` dictionaries), + `phone_state` and `truncated` entries. */ -- (NSDictionary *)fb_mobilerunA11yStateWithScale:(CGFloat)scale; +- (NSDictionary *)fb_mobilerunA11yStateWithSnapshot:(nullable id)snapshot + scale:(CGFloat)scale + truncated:(BOOL)truncated; /** Return application elements tree in a form of xml string diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 6abae1cba..94a1fce41 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -411,18 +411,43 @@ static BOOL FBMobilerunIsCheckableType(XCUIElementType type) } } -- (NSDictionary *)fb_mobilerunA11yStateWithScale:(CGFloat)scale +- (id)fb_snapshotWithTimeout:(NSTimeInterval)timeout + parameters:(NSDictionary *)parameters + error:(NSError **)error +{ + FBXCAXClientProxy *client = FBXCAXClientProxy.sharedClient; + // The timeout applies to every accessibility request of the runner, so it is put back right + // after this one. AXTimeout keeps reporting XCTest's initial value, which is what to restore. + NSTimeInterval defaultTimeout = client.AXTimeout; + NSError *timeoutError; + if (![client setAXTimeout:timeout error:&timeoutError]) { + [FBLogger logFmt:@"Could not limit the snapshot to %.1fs: %@", timeout, timeoutError]; + } + id snapshot = [client snapshotForElement:self.accessibilityElement + attributes:FBStandardAttributeNames() + parameters:parameters + error:error]; + if (![client setAXTimeout:defaultTimeout error:&timeoutError]) { + [FBLogger logFmt:@"Could not restore the %.1fs accessibility timeout: %@", defaultTimeout, timeoutError]; + } + return snapshot; +} + +- (NSDictionary *)fb_mobilerunA11yStateWithSnapshot:(id)snapshot + scale:(CGFloat)scale + truncated:(BOOL)truncated { - id snapshot = [self fb_standardSnapshot]; NSString *bundleId = self.bundleID ?: @""; // Side-channel populated during the single tree walk so `phone_state` needs no // extra snapshots or attribute queries. NSMutableDictionary *context = [NSMutableDictionary dictionary]; - NSDictionary *tree = [self.class fb_a11yNodeForSnapshot:snapshot - scale:scale - packageName:bundleId - context:context]; + NSDictionary *tree = nil == snapshot + ? [self.class fb_a11yRootNodeWithScale:scale packageName:bundleId] + : [self.class fb_a11yNodeForSnapshot:snapshot + scale:scale + packageName:bundleId + context:context]; NSMutableDictionary *phoneState = [NSMutableDictionary dictionary]; NSDictionary *focusedElement = context[@"focusedElement"]; @@ -442,8 +467,38 @@ - (NSDictionary *)fb_mobilerunA11yStateWithScale:(CGFloat)scale phoneState[@"isEditable"] = @([context[@"isEditable"] boolValue]); return @{ - @"a11y_tree": tree ?: [NSNull null], + @"a11y_tree": tree, @"phone_state": phoneState.copy, + @"truncated": @(truncated), + }; +} + +// Stands in for the tree when no snapshot could be taken: clients reject a null tree, and this +// still names the application. Its frame is unknown without a snapshot, and so is the interface +// orientation, so the root spans a square covering the screen either way round - clients drop a +// childless root that lies mostly off-screen. ++ (NSDictionary *)fb_a11yRootNodeWithScale:(CGFloat)scale packageName:(NSString *)bundleId +{ + CGSize screenSize = UIScreen.mainScreen.bounds.size; + CGFloat side = MAX(screenSize.width, screenSize.height); + return @{ + @"boundsInScreen": FBMobilerunBoundsFromFrame(CGRectMake(0, 0, side, side), scale), + @"className": [FBElementTypeTransformer stringWithElementType:XCUIElementTypeApplication], + @"resourceId": @"", + @"text": @"", + @"contentDescription": @"", + @"packageName": bundleId, + @"isCheckable": @NO, + @"isChecked": @NO, + @"isClickable": @NO, + @"isEnabled": @YES, + @"isFocusable": @NO, + @"isFocused": @NO, + @"isScrollable": @NO, + @"isLongClickable": @NO, + @"isPassword": @NO, + @"isSelected": @NO, + @"children": @[], }; } diff --git a/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.h b/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.h index 62210e19c..af5fbcdeb 100644 --- a/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.h +++ b/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.h @@ -12,24 +12,60 @@ NS_ASSUME_NONNULL_BEGIN -@class FBCommandStatus; +@class FBSession; + +/** + How long one full accessibility snapshot of the active app may take, in seconds. XCTest's own + limit is 60s per request and it retries; an app that is stuck or has a huge hierarchy (TikTok on + older iPhones) otherwise holds the request - and every request queued behind it - for minutes. + */ +extern const NSTimeInterval FBMobilerunStateSnapshotTimeout; + +/** The limit for the shallow snapshot taken when the full one did not finish. */ +extern const NSTimeInterval FBMobilerunStateShallowSnapshotTimeout; + +/** How far the shallow fallback snapshot descends: tree levels below the app, children per element. */ +extern const int FBMobilerunStateShallowSnapshotMaxDepth; +extern const int FBMobilerunStateShallowSnapshotMaxChildren; + +typedef NS_ENUM(NSUInteger, FBMobilerunSnapshotDepth) { + FBMobilerunSnapshotDepthFull, + FBMobilerunSnapshotDepthShallow, +}; + +/** What the state assembly needs to know about an application. XCUIApplication conforms. */ +@protocol FBMobilerunStateApplication +@property (nonatomic, readonly) int processID; +@end @interface FBMobilerunA11yCommands : NSObject /** - Builds the GET /mobilerun/state value by running 'attempt' - one snapshot walk of the active - application - and retrying it once when the snapshot fails (FBStaleElementException). + Builds the GET /mobilerun/state value for the active application within a bounded time: + a full snapshot, else a shallow one, else the application root alone. 'truncated' says whether + the tree is incomplete; 'phone_state.packageName' names the active application either way. + + @param scale the factor logical-point bounds are multiplied by + @param session the session whose active application to use, or nil to detect it + */ ++ (NSDictionary *)stateOfActiveApplicationWithScale:(CGFloat)scale session:(nullable FBSession *)session; + +/** + The decision logic behind 'stateOfActiveApplicationWithScale:session:', exposed for unit testing. - Exposed for unit testing; the route handler passes a block that resolves the active application - anew on every call. + Snapshots the active application in full. When that fails and another application has taken over + meanwhile (right after a Home press, say), snapshots that one in full instead; never the same + application twice, since a timed-out snapshot would only time out again. Then falls back to a + shallow snapshot, and finally to no snapshot at all. - @param attempt returns the state dictionary; may throw - @param failure set when both attempts failed to snapshot the application - @return the state dictionary, or nil with 'failure' set. Exceptions other than a failed snapshot - are rethrown unchanged. + @param activeApplication resolves the active application; called at most twice + @param snapshot takes a snapshot of the given depth, or returns nil when it failed + @param build assembles the state from the application, its snapshot (nil when none succeeded) and + whether the snapshot is incomplete */ -+ (nullable NSDictionary *)stateWithSnapshotAttempt:(NSDictionary *(NS_NOESCAPE ^)(void))attempt - failure:(FBCommandStatus *_Nullable *_Nullable)failure; ++ (NSDictionary *)stateWithActiveApplication:(id (NS_NOESCAPE ^)(void))activeApplication + snapshot:(id _Nullable (NS_NOESCAPE ^)(id application, FBMobilerunSnapshotDepth depth))snapshot + build:(NSDictionary *(NS_NOESCAPE ^)(id application, id _Nullable snapshot, BOOL truncated))build; @end diff --git a/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.m b/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.m index ea9d98d88..d2727d6fd 100644 --- a/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.m +++ b/WebDriverAgentLib/Commands/FBMobilerunA11yCommands.m @@ -9,7 +9,7 @@ #import "FBMobilerunA11yCommands.h" #import "FBCommandStatus.h" -#import "FBExceptions.h" +#import "FBLogger.h" #import "FBResponsePayload.h" #import "FBRoute.h" #import "FBRouteRequest.h" @@ -17,6 +17,7 @@ #import "FBSession.h" #import "XCUIApplication.h" #import "XCUIApplication+FBHelpers.h" +#import "XCAXClient_iOS+FBSnapshotReqParams.h" // Parses the optional 'scale' query parameter shared by the mobilerun endpoints. Response/request // coordinates are logical points multiplied by this scale; it defaults to the native screen scale @@ -38,6 +39,17 @@ static BOOL FBMobilerunScaleFromRequest(FBRouteRequest *request, CGFloat *scale) return YES; } +const NSTimeInterval FBMobilerunStateSnapshotTimeout = 10.; +const NSTimeInterval FBMobilerunStateShallowSnapshotTimeout = 5.; +const int FBMobilerunStateShallowSnapshotMaxDepth = 15; +const int FBMobilerunStateShallowSnapshotMaxChildren = 20; + +@interface XCUIApplication (FBMobilerunStateApplication) +@end + +@implementation XCUIApplication (FBMobilerunStateApplication) +@end + @implementation FBMobilerunA11yCommands #pragma mark - @@ -53,29 +65,56 @@ + (NSArray *)routes #pragma mark - Commands -+ (NSDictionary *)stateWithSnapshotAttempt:(NSDictionary *(NS_NOESCAPE ^)(void))attempt - failure:(FBCommandStatus **)failure ++ (NSDictionary *)stateWithActiveApplication:(id (NS_NOESCAPE ^)(void))activeApplication + snapshot:(id _Nullable (NS_NOESCAPE ^)(id application, FBMobilerunSnapshotDepth depth))snapshot + build:(NSDictionary *(NS_NOESCAPE ^)(id application, id _Nullable snapshot, BOOL truncated))build { - NSException *snapshotFailure = nil; - for (NSUInteger attemptIndex = 0; attemptIndex < 2; attemptIndex++) { - @try { - return attempt(); - } @catch (NSException *exception) { - if (![exception.name isEqualToString:FBStaleElementException]) { - @throw; - } - snapshotFailure = exception; - } + id application = activeApplication(); + id result = snapshot(application, FBMobilerunSnapshotDepthFull); + if (nil != result) { + return build(application, result, NO); } - // Left to the global exception mapping, a failed snapshot answers 404 (stale element reference), - // which is also what a stock WebDriverAgent answers for this missing route - so clients take it - // as "no /mobilerun/state here" and switch to the far slower /source for good. A 5xx would read - // as a wedged runner instead. The app was just not in a state that can be snapshotted. - if (failure) { - *failure = [FBCommandStatus invalidElementStateErrorWithMessage:snapshotFailure.reason - traceback:nil]; + // Right after a Home press the app resolved at the start of the request can be the one leaving + // the foreground, whose snapshot then fails; the app that took over (SpringBoard) is still worth + // a full snapshot. + id currentApplication = activeApplication(); + if (currentApplication.processID != application.processID) { + application = currentApplication; + result = snapshot(application, FBMobilerunSnapshotDepthFull); + if (nil != result) { + return build(application, result, NO); + } } - return nil; + // A shallow snapshot can still come back when the full one is too big to finish in time. + return build(application, snapshot(application, FBMobilerunSnapshotDepthShallow), YES); +} + ++ (NSDictionary *)stateOfActiveApplicationWithScale:(CGFloat)scale session:(FBSession *)session +{ + return [self stateWithActiveApplication:^id{ + // Detected without a snapshot, so the state names the app even when no snapshot succeeds. + return session.activeApplication ?: XCUIApplication.fb_activeApplication; + } snapshot:^id(id application, FBMobilerunSnapshotDepth depth) { + BOOL isFull = FBMobilerunSnapshotDepthFull == depth; + NSTimeInterval timeout = isFull ? FBMobilerunStateSnapshotTimeout : FBMobilerunStateShallowSnapshotTimeout; + NSDictionary *parameters = isFull ? @{} : @{ + FBSnapshotMaxDepthKey: @(FBMobilerunStateShallowSnapshotMaxDepth), + FBSnapshotMaxChildrenKey: @(FBMobilerunStateShallowSnapshotMaxChildren), + }; + NSError *error; + id snapshot = [(XCUIApplication *)application fb_snapshotWithTimeout:timeout + parameters:parameters + error:&error]; + if (nil == snapshot) { + [FBLogger logFmt:@"The %@ snapshot of %@ failed within %.0fs: %@", isFull ? @"full" : @"shallow", + ((XCUIApplication *)application).bundleID, timeout, error.localizedDescription]; + } + return snapshot; + } build:^NSDictionary *(id application, id snapshot, BOOL truncated) { + return [(XCUIApplication *)application fb_mobilerunA11yStateWithSnapshot:snapshot + scale:scale + truncated:truncated]; + }]; } + (id)handleGetState:(FBRouteRequest *)request @@ -85,15 +124,7 @@ + (NSDictionary *)stateWithSnapshotAttempt:(NSDictionary *(NS_NOESCAPE ^)(void)) return FBResponseWithStatus([FBCommandStatus invalidArgumentErrorWithMessage:@"'scale' must be a positive number" traceback:nil]); } - FBCommandStatus *failure = nil; - NSDictionary *state = [self stateWithSnapshotAttempt:^NSDictionary *{ - // Resolved on every attempt: right after a Home press the app picked at the start of the - // request can be the one leaving the foreground, whose snapshot then fails. The retry resolves - // the app that took over (SpringBoard). - XCUIApplication *app = request.session.activeApplication ?: XCUIApplication.fb_activeApplication; - return [app fb_mobilerunA11yStateWithScale:scale]; - } failure:&failure]; - return nil == state ? FBResponseWithStatus(failure) : FBResponseWithObject(state); + return FBResponseWithObject([self stateOfActiveApplicationWithScale:scale session:request.session]); } @end diff --git a/WebDriverAgentLib/Utilities/FBXCAXClientProxy.h b/WebDriverAgentLib/Utilities/FBXCAXClientProxy.h index 208672af7..66dc95c9c 100644 --- a/WebDriverAgentLib/Utilities/FBXCAXClientProxy.h +++ b/WebDriverAgentLib/Utilities/FBXCAXClientProxy.h @@ -24,11 +24,28 @@ NS_ASSUME_NONNULL_BEGIN - (BOOL)setAXTimeout:(NSTimeInterval)timeout error:(NSError **)error; +/** + The timeout of a single accessibility request, 60s by default. Reads the value XCTest started + with: it does not follow 'setAXTimeout:error:'. + */ +- (NSTimeInterval)AXTimeout; + - (nullable id)snapshotForElement:(id)element attributes:(nullable NSArray *)attributes inDepth:(BOOL)inDepth error:(NSError **)error; +/** + Requests one snapshot directly, without XCTest's element resolution and its retries. + + @param parameters snapshot request parameters (e.g. FBSnapshotMaxDepthKey) that override the + defaults for this request only + */ +- (nullable id)snapshotForElement:(id)element + attributes:(nullable NSArray *)attributes + parameters:(NSDictionary *)parameters + error:(NSError **)error; + - (NSArray> *)activeApplications; - (id)systemApplication; diff --git a/WebDriverAgentLib/Utilities/FBXCAXClientProxy.m b/WebDriverAgentLib/Utilities/FBXCAXClientProxy.m index fc90d43f9..6f195f73c 100644 --- a/WebDriverAgentLib/Utilities/FBXCAXClientProxy.m +++ b/WebDriverAgentLib/Utilities/FBXCAXClientProxy.m @@ -42,19 +42,33 @@ - (BOOL)setAXTimeout:(NSTimeInterval)timeout error:(NSError **)error return [FBAXClient _setAXTimeout:timeout error:error]; } +- (NSTimeInterval)AXTimeout +{ + return [FBAXClient AXTimeout]; +} + - (id)snapshotForElement:(id)element attributes:(NSArray *)attributes inDepth:(BOOL)inDepth error:(NSError **)error { - NSMutableDictionary *parameters = [NSMutableDictionary dictionaryWithDictionary:self.defaultParameters]; - if (!inDepth) { - parameters[FBSnapshotMaxDepthKey] = @1; - } + return [self snapshotForElement:element + attributes:attributes + parameters:inDepth ? @{} : @{FBSnapshotMaxDepthKey: @1} + error:error]; +} + +- (id)snapshotForElement:(id)element + attributes:(NSArray *)attributes + parameters:(NSDictionary *)parameters + error:(NSError **)error +{ + NSMutableDictionary *requestParameters = [NSMutableDictionary dictionaryWithDictionary:self.defaultParameters]; + [requestParameters addEntriesFromDictionary:parameters]; id result = [FBAXClient requestSnapshotForElement:element attributes:attributes - parameters:[parameters copy] + parameters:[requestParameters copy] error:error]; id snapshot = [result valueForKey:@"_rootElementSnapshot"]; return nil == snapshot ? result : snapshot; diff --git a/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m new file mode 100644 index 000000000..6fb902abd --- /dev/null +++ b/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m @@ -0,0 +1,122 @@ +/** + * Copyright (c) 2026-present, Droidrun. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBIntegrationTestCase.h" +#import "FBMobilerunA11yCommands.h" +#import "FBTestMacros.h" +#import "FBXCAXClientProxy.h" +#import "XCTestPrivateSymbols.h" +#import "XCUIApplication.h" +#import "XCUIApplication+FBHelpers.h" +#import "XCUIElement.h" +#import "XCUIElement+FBIsVisible.h" + +@interface FBMobilerunA11yIntegrationTests : FBIntegrationTestCase +@end + +@implementation FBMobilerunA11yIntegrationTests + +- (void)setUp +{ + [super setUp]; + [self launchApplication]; +} + +- (void)tearDown +{ + [self.testedApplication terminate]; + [super tearDown]; +} + +- (void)testStateOfResponsiveAppIsComplete +{ + NSDate *start = [NSDate date]; + NSDictionary *state = [FBMobilerunA11yCommands stateOfActiveApplicationWithScale:1 session:nil]; + NSLog(@"Responsive app state took %.2fs", -start.timeIntervalSinceNow); + + XCTAssertEqualObjects(state[@"truncated"], @NO); + XCTAssertEqualObjects(state[@"phone_state"][@"packageName"], self.testedApplication.bundleID); + XCTAssertEqualObjects(state[@"a11y_tree"][@"className"], @"XCUIElementTypeApplication"); + XCTAssertGreaterThan([state[@"a11y_tree"][@"children"] count], 0u); + NSDictionary *button = [self nodeIn:state[@"a11y_tree"] withContentDescription:@"Deadlock app"]; + XCTAssertEqualObjects(button[@"className"], @"XCUIElementTypeButton"); + XCTAssertEqualObjects(button[@"isClickable"], @YES); + XCTAssertGreaterThan([button[@"boundsInScreen"][@"right"] integerValue], + [button[@"boundsInScreen"][@"left"] integerValue]); +} + +- (void)testSnapshotTimeoutDoesNotOutliveTheSnapshot +{ + if (nil != NSProcessInfo.processInfo.environment[@"CI"]) { + XCTSkip(@"Deliberately freezes the app for several seconds, too slow/flaky for CI"); + } + + // The accessibility timeout applies to every request of the runner; left lowered, it would cut + // short the snapshots of all other commands. + NSError *error; + XCTAssertNotNil([self.testedApplication fb_snapshotWithTimeout:1 parameters:@{} error:&error], @"%@", error); + + XCUIElement *deadlockButton = self.testedApplication.buttons[@"Deadlock app"]; + FBAssertWaitTillBecomesTrue(deadlockButton.fb_isVisible); + [deadlockButton tap]; + + NSDate *start = [NSDate date]; + [FBXCAXClientProxy.sharedClient snapshotForElement:self.testedApplication.accessibilityElement + attributes:FBStandardAttributeNames() + inDepth:YES + error:&error]; + NSTimeInterval elapsed = -start.timeIntervalSinceNow; + NSLog(@"Unbounded snapshot of the frozen app took %.2fs", elapsed); + XCTAssertGreaterThan(elapsed, 5.0); +} + +- (NSDictionary *)nodeIn:(NSDictionary *)node withContentDescription:(NSString *)contentDescription +{ + if ([node[@"contentDescription"] isEqualToString:contentDescription]) { + return node; + } + for (NSDictionary *child in node[@"children"]) { + NSDictionary *match = [self nodeIn:child withContentDescription:contentDescription]; + if (nil != match) { + return match; + } + } + return nil; +} + +- (void)testStateOfFrozenAppIsBoundedAndNamesTheApp +{ + if (nil != NSProcessInfo.processInfo.environment[@"CI"]) { + XCTSkip(@"Deliberately freezes the app for several seconds, too slow/flaky for CI"); + } + + XCUIElement *deadlockButton = self.testedApplication.buttons[@"Deadlock app"]; + FBAssertWaitTillBecomesTrue(deadlockButton.fb_isVisible); + // Freezes the app's main thread for 20s - see -[ViewController deadlockApp:]. Without a bound + // the snapshot waits for the app to answer, up to XCTest's 60s accessibility timeout. + [deadlockButton tap]; + + NSDate *start = [NSDate date]; + NSDictionary *state = [FBMobilerunA11yCommands stateOfActiveApplicationWithScale:1 session:nil]; + NSTimeInterval elapsed = -start.timeIntervalSinceNow; + NSLog(@"Frozen app state took %.2fs", elapsed); + + XCTAssertLessThan(elapsed, FBMobilerunStateSnapshotTimeout + FBMobilerunStateShallowSnapshotTimeout + 2); + XCTAssertEqualObjects(state[@"truncated"], @YES); + XCTAssertEqualObjects(state[@"phone_state"][@"packageName"], self.testedApplication.bundleID); + // Clients reject a null tree, so the app itself still stands in as the root. + NSDictionary *root = state[@"a11y_tree"]; + XCTAssertEqualObjects(root[@"className"], @"XCUIElementTypeApplication"); + XCTAssertEqualObjects(root[@"packageName"], self.testedApplication.bundleID); + XCTAssertGreaterThan([root[@"boundsInScreen"][@"right"] integerValue], 0); + XCTAssertGreaterThan([root[@"boundsInScreen"][@"bottom"] integerValue], 0); +} + +@end diff --git a/WebDriverAgentTests/UnitTests/FBMobilerunA11yCommandsTests.m b/WebDriverAgentTests/UnitTests/FBMobilerunA11yCommandsTests.m index d674a9e9f..a9bc827d7 100644 --- a/WebDriverAgentTests/UnitTests/FBMobilerunA11yCommandsTests.m +++ b/WebDriverAgentTests/UnitTests/FBMobilerunA11yCommandsTests.m @@ -8,84 +8,111 @@ #import -#import "FBCommandStatus.h" -#import "FBExceptions.h" -#import "FBHTTPStatusCodes.h" #import "FBMobilerunA11yCommands.h" +@interface FBFakeStateApplication : NSObject +@property (nonatomic) int processID; +@end + +@implementation FBFakeStateApplication + ++ (instancetype)withProcessID:(int)processID +{ + FBFakeStateApplication *application = [self new]; + application.processID = processID; + return application; +} + +@end + @interface FBMobilerunA11yCommandsTests : XCTestCase +// Every snapshot request, as ":full" or ":shallow", in call order. +@property (nonatomic) NSMutableArray *snapshotRequests; @end @implementation FBMobilerunA11yCommandsTests -static NSException *FBStaleSnapshotException(void) +- (void)setUp { - return [NSException exceptionWithName:FBStaleElementException reason:@"snapshot failed" userInfo:nil]; + [super setUp]; + self.snapshotRequests = [NSMutableArray array]; } -- (void)testStateIsReturnedWhenTheFirstSnapshotSucceeds +// Runs the state assembly against the given active applications (one per resolution, the last +// one repeating) and snapshot results keyed like 'snapshotRequests'; a missing key is a failed +// snapshot. Returns what the build block received. +- (NSDictionary *)stateWithActiveApplications:(NSArray *)applications + snapshots:(NSDictionary *)snapshots { - __block NSUInteger attempts = 0; - FBCommandStatus *failure = nil; - NSDictionary *state = [FBMobilerunA11yCommands stateWithSnapshotAttempt:^NSDictionary *{ - attempts++; - return @{@"phone_state": @{}}; - } failure:&failure]; - - XCTAssertEqualObjects(state, @{@"phone_state": @{}}); - XCTAssertNil(failure); - XCTAssertEqual(attempts, 1u); + __block NSUInteger resolutions = 0; + return [FBMobilerunA11yCommands stateWithActiveApplication:^id{ + FBFakeStateApplication *application = applications[MIN(resolutions, applications.count - 1)]; + resolutions++; + return application; + } snapshot:^id(id application, FBMobilerunSnapshotDepth depth) { + NSString *request = [NSString stringWithFormat:@"%d:%@", application.processID, + FBMobilerunSnapshotDepthFull == depth ? @"full" : @"shallow"]; + [self.snapshotRequests addObject:request]; + return snapshots[request]; + } build:^NSDictionary *(id application, id snapshot, BOOL truncated) { + return @{ + @"pid": @(application.processID), + @"snapshot": snapshot ?: [NSNull null], + @"truncated": @(truncated), + }; + }]; } -- (void)testFailedSnapshotIsRetriedOnce +- (void)testCompleteSnapshotIsTakenOnce +{ + NSDictionary *state = [self stateWithActiveApplications:@[[FBFakeStateApplication withProcessID:1]] + snapshots:@{@"1:full": @"tree"}]; + + XCTAssertEqualObjects(state, (@{@"pid": @1, @"snapshot": @"tree", @"truncated": @NO})); + XCTAssertEqualObjects(self.snapshotRequests, @[@"1:full"]); +} + +- (void)testAppThatTookOverIsSnapshottedInFull { // After a Home press the app resolved at the start of the request can be the one that is - // leaving the foreground; its snapshot fails, and a second attempt resolves SpringBoard. - __block NSUInteger attempts = 0; - FBCommandStatus *failure = nil; - NSDictionary *state = [FBMobilerunA11yCommands stateWithSnapshotAttempt:^NSDictionary *{ - if (0 == attempts++) { - @throw FBStaleSnapshotException(); - } - return @{@"phone_state": @{@"packageName": @"com.apple.springboard"}}; - } failure:&failure]; - - XCTAssertEqualObjects(state[@"phone_state"][@"packageName"], @"com.apple.springboard"); - XCTAssertNil(failure); - XCTAssertEqual(attempts, 2u); + // leaving the foreground; its snapshot fails, and SpringBoard has taken over by then. + NSDictionary *state = [self stateWithActiveApplications:@[[FBFakeStateApplication withProcessID:1], + [FBFakeStateApplication withProcessID:2]] + snapshots:@{@"2:full": @"springboard"}]; + + XCTAssertEqualObjects(state, (@{@"pid": @2, @"snapshot": @"springboard", @"truncated": @NO})); + XCTAssertEqualObjects(self.snapshotRequests, (@[@"1:full", @"2:full"])); } -- (void)testPersistentSnapshotFailureIsNotReportedAsNotFound +- (void)testSameAppIsNotSnapshottedInFullTwice { - // A 404 is what a stock WebDriverAgent answers for the missing route, and clients take it as - // "this runner has no /mobilerun/state" for good. A failed snapshot must say something else, - // and must not be a 5xx either: that reads as a wedged runner and triggers a relaunch. - __block NSUInteger attempts = 0; - FBCommandStatus *failure = nil; - NSDictionary *state = [FBMobilerunA11yCommands stateWithSnapshotAttempt:^NSDictionary *{ - attempts++; - @throw FBStaleSnapshotException(); - } failure:&failure]; - - XCTAssertNil(state); - XCTAssertEqual(attempts, 2u); - XCTAssertNotNil(failure); - XCTAssertNotEqual(failure.statusCode, kHTTPStatusCodeNotFound); - XCTAssertGreaterThanOrEqual(failure.statusCode, 400); - XCTAssertLessThan(failure.statusCode, 500); - XCTAssertTrue([failure.message containsString:@"snapshot failed"]); + // A full snapshot that timed out would time out again; each attempt costs the whole budget. + NSDictionary *state = [self stateWithActiveApplications:@[[FBFakeStateApplication withProcessID:1]] + snapshots:@{@"1:shallow": @"top"}]; + + XCTAssertEqualObjects(state, (@{@"pid": @1, @"snapshot": @"top", @"truncated": @YES})); + XCTAssertEqualObjects(self.snapshotRequests, (@[@"1:full", @"1:shallow"])); } -- (void)testOtherExceptionsAreNotSwallowed +- (void)testAppThatTookOverFallsBackToShallowSnapshot { - // Anything but a failed snapshot keeps the global exception mapping (a deadlocked app, say). - __block NSUInteger attempts = 0; - FBCommandStatus *failure = nil; - XCTAssertThrowsSpecificNamed(([FBMobilerunA11yCommands stateWithSnapshotAttempt:^NSDictionary *{ - attempts++; - @throw [NSException exceptionWithName:FBApplicationDeadlockDetectedException reason:@"" userInfo:nil]; - } failure:&failure]), NSException, FBApplicationDeadlockDetectedException); - XCTAssertEqual(attempts, 1u); + NSDictionary *state = [self stateWithActiveApplications:@[[FBFakeStateApplication withProcessID:1], + [FBFakeStateApplication withProcessID:2]] + snapshots:@{@"2:shallow": @"top"}]; + + XCTAssertEqualObjects(state, (@{@"pid": @2, @"snapshot": @"top", @"truncated": @YES})); + XCTAssertEqualObjects(self.snapshotRequests, (@[@"1:full", @"2:full", @"2:shallow"])); +} + +- (void)testStateNamesTheAppWhenNoSnapshotSucceeds +{ + // The foreground app is known without any snapshot, which is what clients need most; a failed + // request instead leaves them retrying behind the stuck app for minutes. + NSDictionary *state = [self stateWithActiveApplications:@[[FBFakeStateApplication withProcessID:1]] + snapshots:@{}]; + + XCTAssertEqualObjects(state, (@{@"pid": @1, @"snapshot": [NSNull null], @"truncated": @YES})); + XCTAssertEqualObjects(self.snapshotRequests, (@[@"1:full", @"1:shallow"])); } @end diff --git a/docs/request-dispatch.md b/docs/request-dispatch.md index eaa221037..e6bfcbbb3 100644 --- a/docs/request-dispatch.md +++ b/docs/request-dispatch.md @@ -69,7 +69,10 @@ Deliberately **not** control routes: - the broadcast start/stop endpoints (drive the system broadcast picker through XCUI), - `/mobilerun/state` — it is the intended **liveness probe**: because it runs on the automation queue and does real accessibility work, it reflects a wedged automation queue - by timing out, while `/status` stays green as a process-liveness signal. + by timing out, while `/status` stays green as a process-liveness signal. A stuck or + oversized *app* does not read as a wedged queue: its snapshots are bounded (10s full, + then 5s shallow), and the answer degrades to a `truncated` tree that still names the + foreground app in `phone_state.packageName`. ### Thread-safety notes for control handlers From f7e46231d1fa35ffdd03274e26e586f4b6c8e888 Mon Sep 17 00:00:00 2001 From: timo <44401485+Timo972@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:27:49 +0200 Subject: [PATCH 2/2] fix(mobilerun): size the fallback root from SpringBoard, not the runner's screen On a device the runner app has no launch screen, so iOS runs it in the letterboxed compatibility mode: its UIScreen bounds and nativeBounds both describe a 320x480 screen. The root-only state for an app that could not be snapshotted came out 480x480 on an iPhone XR (414x896). SpringBoard's frame from a depth-1 snapshot (2s limit) gives the real screen, and it answers while the app is stuck. Verified on an iPhone XR (iOS 18.7.7): right after a cold start TikTok stops answering accessibility requests ("Error getting main window kAXErrorIPCTimeout"). /mobilerun/state now answers in 15.1s with packageName com.zhiliaoapp.musically and an 896x896 root. Co-Authored-By: Claude Opus 5.5 --- .../Categories/XCUIApplication+FBHelpers.m | 13 ++++++++++++- .../FBMobilerunA11yIntegrationTests.m | 8 ++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m index 94a1fce41..797980afc 100644 --- a/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIApplication+FBHelpers.m @@ -23,6 +23,7 @@ #import "FBXCElementSnapshotWrapper+Helpers.h" #import "FBXCAXClientProxy.h" #import "FBXMLGenerationOptions.h" +#import "XCAXClient_iOS+FBSnapshotReqParams.h" #import "XCTMessagingChannel_RunnerToDaemon-Protocol.h" #import "XCTestPrivateSymbols.h" #import "XCTRunnerDaemonSession.h" @@ -350,6 +351,9 @@ + (NSDictionary *)accessibilityInfoForElement:(id)snapshot #pragma mark - mobilerun accessibility state +// How long the root-only fallback waits for SpringBoard's frame, in seconds. +static const NSTimeInterval FBMobilerunSystemFrameTimeout = 2.; + // Converts a logical-point frame into integer device-pixel bounds matching the // `A11YBounds` shape (left/top/right/bottom) expected by mobilerun. static NSDictionary *FBMobilerunBoundsFromFrame(CGRect frame, CGFloat scale) @@ -479,7 +483,14 @@ - (NSDictionary *)fb_mobilerunA11yStateWithSnapshot:(id)sna // childless root that lies mostly off-screen. + (NSDictionary *)fb_a11yRootNodeWithScale:(CGFloat)scale packageName:(NSString *)bundleId { - CGSize screenSize = UIScreen.mainScreen.bounds.size; + // SpringBoard spans the screen and answers even while the app is stuck. The runner's own + // UIScreen is no substitute on a device: without a launch screen the runner runs in the + // letterboxed compatibility mode, where bounds and nativeBounds both describe a 320x480 screen. + id systemSnapshot = [XCUIApplication.fb_systemApplication + fb_snapshotWithTimeout:FBMobilerunSystemFrameTimeout + parameters:@{FBSnapshotMaxDepthKey: @1} + error:nil]; + CGSize screenSize = nil == systemSnapshot ? UIScreen.mainScreen.bounds.size : systemSnapshot.frame.size; CGFloat side = MAX(screenSize.width, screenSize.height); return @{ @"boundsInScreen": FBMobilerunBoundsFromFrame(CGRectMake(0, 0, side, side), scale), diff --git a/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m index 6fb902abd..0f3ef630f 100644 --- a/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBMobilerunA11yIntegrationTests.m @@ -97,6 +97,7 @@ - (void)testStateOfFrozenAppIsBoundedAndNamesTheApp XCTSkip(@"Deliberately freezes the app for several seconds, too slow/flaky for CI"); } + CGSize screenSize = self.testedApplication.frame.size; XCUIElement *deadlockButton = self.testedApplication.buttons[@"Deadlock app"]; FBAssertWaitTillBecomesTrue(deadlockButton.fb_isVisible); // Freezes the app's main thread for 20s - see -[ViewController deadlockApp:]. Without a bound @@ -115,8 +116,11 @@ - (void)testStateOfFrozenAppIsBoundedAndNamesTheApp NSDictionary *root = state[@"a11y_tree"]; XCTAssertEqualObjects(root[@"className"], @"XCUIElementTypeApplication"); XCTAssertEqualObjects(root[@"packageName"], self.testedApplication.bundleID); - XCTAssertGreaterThan([root[@"boundsInScreen"][@"right"] integerValue], 0); - XCTAssertGreaterThan([root[@"boundsInScreen"][@"bottom"] integerValue], 0); + // Covers the screen in either orientation; on a device the runner's own UIScreen bounds are the + // letterboxed 320x480 compatibility screen, which falls short of it. + NSInteger side = (NSInteger)MAX(screenSize.width, screenSize.height); + XCTAssertGreaterThanOrEqual([root[@"boundsInScreen"][@"right"] integerValue], side); + XCTAssertGreaterThanOrEqual([root[@"boundsInScreen"][@"bottom"] integerValue], side); } @end