From 0f44265a58d551b15de739bf85b1c41e90114e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 9 Sep 2026 17:26:43 +0200 Subject: [PATCH] test(ios): run the shared AX recovery fixture through the runner and characterize its depth memory Replay every recovery case of contracts/fixtures/ios-ax-recovery-conformance.json through the runner's real private AX bridge, depth ladder, frontier extension, and completeness verdict, asserting the delivered tree's canonical signature and node count alongside outcome and request accounting. To make that possible RunnerAXSnapshotBridge gains a resolved client/target capture seam and the ladder loop becomes a platform-neutral function; behaviour is unchanged. Add the fixture's hint cases as the runner's accepted-depth memory characterization: each step is a real ladder capture against a client that rejects above the accepted rung, bounded by node budget when the step is not complete and rejected at every depth when it fails, whose observed rejections, accepted rung, and boundedness are asserted before the runner's own learning step records or skips it. --- .../RunnerAXSnapshotBridge.h | 12 + .../RunnerAXSnapshotBridge.m | 20 + .../RunnerTests+AXSnapshotFallback.swift | 115 +++-- ...nnerTests+AXRecoveryConformanceTests.swift | 477 ++++++++++++++++++ apple/snapshot-bridge/README.md | 8 +- .../fixtures/ios-ax-recovery-conformance.json | 381 +++++++++++++- .../adr/0004-ios-snapshot-backend-strategy.md | 7 +- 7 files changed, 970 insertions(+), 50 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h index 56e31e0972..7335927746 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h @@ -49,6 +49,18 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey; customActionLimit:(NSInteger)customActionLimit deadline:(nullable NSDate *)deadline; +/// The capture behind `snapshotTreeForApplication:`, for an AX client and target +/// application element that are already resolved. Every request, extension, and +/// custom-action decision happens here; the application entry point only resolves +/// the two identities. ++ (NSDictionary *)snapshotTreeWithClient:(id)axClient + target:(id)target + maxDepth:(NSInteger)maxDepth + maxNodes:(NSInteger)maxNodes + deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit + customActionLimit:(NSInteger)customActionLimit + deadline:(nullable NSDate *)deadline; + /// Names of the element's UIAccessibilityCustomActions, or nil when it has /// none. /// diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m index 56ed35debe..77fd4f42cd 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m @@ -104,7 +104,27 @@ @implementation RunnerAXSnapshotBridge if (nil == target) { return [self failure:@"Could not match active AX application for XCTest application"]; } + return [self snapshotTreeWithClient:axClient + target:target + maxDepth:maxDepth + maxNodes:maxNodes + deepExtensionCallLimit:deepExtensionCallLimit + customActionLimit:customActionLimit + deadline:deadline]; + } @catch (NSException *exception) { + return [self failure:exception.reason ?: exception.name ?: @"AX snapshot bridge exception"]; + } +} ++ (NSDictionary *)snapshotTreeWithClient:(id)axClient + target:(id)target + maxDepth:(NSInteger)maxDepth + maxNodes:(NSInteger)maxNodes + deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit + customActionLimit:(NSInteger)customActionLimit + deadline:(nullable NSDate *)deadline +{ + @try { NSArray *attributes = [self snapshotAttributes]; NSError *error = nil; id root = [self requestSnapshotFromClient:axClient diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index 194389ca0c..d0e12be83c 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -8,7 +8,7 @@ extension RunnerTests { /// depth-capped Bluesky-class tree resolves in 1-3 chained requests (~100-300ms /// each); the bound exists so a pathological tree cannot stack requests past /// the capture-plan deadline, which is also enforced per call. - private static let privateAXDeepExtensionCallLimit = 8 + static let privateAXDeepExtensionCallLimit = 8 /// Upper bound on per-element custom-action reads per capture. Each is its own /// AX round trip (~100ms on an idle simulator), so this caps the opt-in cost @@ -67,6 +67,65 @@ extension RunnerTests { read: read, candidates: candidates, truncated: truncated, blocked: blocked) } + struct PrivateAXLadderOutcome { + let response: [String: Any] + let effectiveDepth: Int + let deadlineSpent: Bool + let lastError: String + + var succeeded: Bool { response["ok"] as? Bool == true } + } + + /// Walks the ladder rungs until one capture succeeds. The first rung always runs (the plan + /// gated entry on its own budget); later rungs stop when the capture-plan deadline is spent so + /// ladder retries can never stack past the runner's main-thread watchdog (#1105). + static func privateAXLadderCapture( + attemptDepths: [Int], + deadline: Date, + capture: (Int) -> [String: Any] + ) -> PrivateAXLadderOutcome { + var response: [String: Any] = [:] + var effectiveDepth = attemptDepths.first ?? 0 + var lastError = "unknown private AX snapshot failure" + for depth in attemptDepths { + if depth != attemptDepths.first, Date() >= deadline { + NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_BUDGET_EXHAUSTED depth=%ld", depth) + return PrivateAXLadderOutcome( + response: response, effectiveDepth: effectiveDepth, deadlineSpent: true, + lastError: lastError) + } + response = capture(depth) + if response["ok"] as? Bool == true { + effectiveDepth = depth + break + } + lastError = response["error"] as? String ?? lastError + NSLog( + "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_DEPTH_RETRY depth=%ld error=%@", + depth, + lastError + ) + } + return PrivateAXLadderOutcome( + response: response, effectiveDepth: effectiveDepth, deadlineSpent: false, lastError: lastError) + } + + /// Only a capture that actually descended records memory: a first-rung success on a + /// remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full + /// requested depth once per window instead of capping this screen class forever. + func recordPrivateAXAcceptedDepth( + exactDepthRequested: Bool, + effectiveDepth: Int, + attemptDepths: [Int] + ) { + guard !exactDepthRequested, effectiveDepth != attemptDepths.first else { return } + rememberPrivateAXAcceptedDepth( + bundleId: currentBundleId, + processIdentifier: currentAppProcessIdentifier, + depth: effectiveDepth + ) + } + func rememberPrivateAXAcceptedDepth(bundleId: String?, processIdentifier: Int?, depth: Int) { // No PID means no way to notice a relaunch later; record nothing rather than risk serving // a stale shallow rung to a fresh process. @@ -125,22 +184,13 @@ extension RunnerTests { requestedDepth: requestedDepth, rememberedDepth: rememberedDepth ) - var response: [String: Any] = [:] - var effectiveDepth = requestedDepth - var lastError = "unknown private AX snapshot failure" - for depth in attemptDepths { - // The first rung always runs (the plan gated entry on its own budget); later rungs - // stop when the capture-plan deadline is spent so ladder retries can never stack - // past the runner's main-thread watchdog (#1105). - if depth != attemptDepths.first, Date() >= deadline { - NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_BUDGET_EXHAUSTED depth=%ld", depth) - break - } - // Declared residue (#1797): the bridge caps the tree at 5000 nodes while serializing, - // BEFORE either projection exists, so a raw capture of a huge screen is bounded rather - // than failing the way the tree backend's own raw cap does. The cap is disclosed as - // `truncated`, and it applied to the acquired tree before this projection split too. - response = RunnerAXSnapshotBridge.snapshotTree( + // Declared residue (#1797): the bridge caps the tree at 5000 nodes while serializing, + // BEFORE either projection exists, so a raw capture of a huge screen is bounded rather + // than failing the way the tree backend's own raw cap does. The cap is disclosed as + // `truncated`, and it applied to the acquired tree before this projection split too. + let ladder = Self.privateAXLadderCapture(attemptDepths: attemptDepths, deadline: deadline) { + depth in + RunnerAXSnapshotBridge.snapshotTree( for: app, maxDepth: depth, maxNodes: Self.privateAXSnapshotMaxNodes, @@ -148,31 +198,18 @@ extension RunnerTests { customActionLimit: hint.customActions ? Self.privateAXCustomActionLimit : 0, deadline: deadline ) - if response["ok"] as? Bool == true { - effectiveDepth = depth - break - } - lastError = response["error"] as? String ?? lastError - NSLog( - "AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_DEPTH_RETRY depth=%ld error=%@", - depth, - lastError - ) } - guard response["ok"] as? Bool == true else { - NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=%@", lastError) + let response = ladder.response + let effectiveDepth = ladder.effectiveDepth + guard ladder.succeeded else { + NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=%@", ladder.lastError) return nil } - // Only a capture that actually descended records memory: a first-rung success on a - // remembered depth deliberately does NOT refresh the TTL, so expiry re-probes the full - // requested depth once per window instead of capping this screen class forever. - if !exactDepthRequested, effectiveDepth != attemptDepths.first { - rememberPrivateAXAcceptedDepth( - bundleId: currentBundleId, - processIdentifier: currentAppProcessIdentifier, - depth: effectiveDepth - ) - } + recordPrivateAXAcceptedDepth( + exactDepthRequested: exactDepthRequested, + effectiveDepth: effectiveDepth, + attemptDepths: attemptDepths + ) guard let root = response["root"] as? [String: Any] else { NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_FAILED=missing root") return nil diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift new file mode 100644 index 0000000000..6e6364c310 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXRecoveryConformanceTests.swift @@ -0,0 +1,477 @@ +import XCTest + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +// MARK: - Shared AX recovery conformance (runner adapter) + +/// Runner-side adapter for `contracts/fixtures/ios-ax-recovery-conformance.json`. The fixture's +/// synthetic native world is replayed through a fake AX client against the real bridge capture, +/// depth ladder, completeness verdict, and accepted-depth memory; the fixture's `runner` column +/// holds the expectations, and its `differences` section explains where the host bridge answers +/// differently on purpose. +private struct AXRecoveryFixture: Decodable { + struct Fan: Decodable { + let at: Int + let count: Int + let chain: Int + } + + struct Tree: Decodable { + let chain: Int + let fan: Fan? + } + + struct Request: Decodable { + let traversalDepth: Int + let explicitDepth: Bool + let nodeBudget: Int + let hint: [String: Int?]? + } + + struct Native: Decodable { + let tree: Tree + let rejectLevelsAbove: Int? + let vanishAtFrontier: Bool? + let deadlineAfterRequests: Int? + } + + struct Expected: Decodable { + let outcome: String + let failure: String? + let requests: Int? + let rejected: Int? + let continuations: Int? + let deepestLevel: Int? + let tree: String? + let nodes: Int? + } + + struct RecoveryCase: Decodable { + let name: String + let request: Request + let native: Native + let expected: [String: Expected] + } + + struct Target: Decodable { + let id: String + let generation: String + } + + struct Outcome: Decodable { + let failure: String? + let rejected: [String: Int] + let acceptedLevels: [String: Int]? + let complete: Bool? + } + + struct HintStep: Decodable { + let expire: Bool? + let target: Target? + let explicitDepth: Bool? + let expectHintBefore: [String: Int?]? + let outcome: Outcome? + let expectHintAfter: [String: Int?]? + let expectRenewed: Bool? + } + + struct HintCase: Decodable { + let name: String + let steps: [HintStep] + } + + let version: Int + let recoveryCases: [RecoveryCase] + let hintCases: [HintCase] +} + +private final class AXFixtureNode { + let level: Int + let identity: String + private(set) weak var parent: AXFixtureNode? + var children: [AXFixtureNode] = [] + + init(level: Int, identity: String, parent: AXFixtureNode?) { + self.level = level + self.identity = identity + self.parent = parent + } + + var branch: String { identity.split(separator: ".").count > 1 ? String(identity.split(separator: ".")[1]) : "" } + + static func build(_ tree: AXRecoveryFixture.Tree) -> AXFixtureNode { + let root = AXFixtureNode(level: 0, identity: "0", parent: nil) + let tail = root.extend(by: tree.chain - 1, branch: nil) + if let fan = tree.fan { + precondition(fan.at == tail.level, "the fan must hang off the last chain node") + for branch in 0.. [String: AXFixtureNode] { + var result: [String: AXFixtureNode] = [:] + var queue = [self] + while let node = queue.first { + queue.removeFirst() + result[node.identity] = node + queue.append(contentsOf: node.children) + } + return result + } + + private func extend(by count: Int, branch: String?) -> AXFixtureNode { + var current = self + for _ in 0.. AXFixtureSnapshot { + let boundary = levels <= 1 + let children = boundary + ? [] + : node.children.map { fragment($0, levels: levels - 1, vanishAtFrontier: vanishAtFrontier) } + return AXFixtureSnapshot( + node: node, + children: children, + element: boundary && vanishAtFrontier ? nil : AXFixtureElement(node: node)) + } +} + +/// Stands in for the private AX client: answers `requestSnapshotForElement:` from the synthetic +/// tree and rejects requests deeper than the fixture's native limit the way the AX server does. +private final class AXFixtureClient: NSObject { + private let rejectLevelsAbove: Int? + private let vanishAtFrontier: Bool + private(set) var requests = 0 + private(set) var rejected = 0 + + init(rejectLevelsAbove: Int?, vanishAtFrontier: Bool) { + self.rejectLevelsAbove = rejectLevelsAbove + self.vanishAtFrontier = vanishAtFrontier + } + + @objc(requestSnapshotForElement:attributes:parameters:error:) + func requestSnapshot( + forElement element: Any, attributes: Any, parameters: [String: Any], error: NSErrorPointer + ) -> Any? { + requests += 1 + let levels = (parameters["maxDepth"] as? NSNumber)?.intValue ?? 0 + if let limit = rejectLevelsAbove, levels > limit { + rejected += 1 + error?.pointee = NSError( + domain: "AX", code: -25201, + userInfo: [NSLocalizedDescriptionKey: "Error kAXErrorIllegalArgument"]) + return nil + } + guard let element = element as? AXFixtureElement else { return nil } + return AXFixtureSnapshot.fragment(element.node, levels: levels, vanishAtFrontier: vanishAtFrontier) + } +} + +private struct AXRecoveryObservation { + var outcome = "" + var failure: String? + var requests = 0 + var rejected = 0 + var continuations = 0 + var deepestLevel: Int? + var tree: String? + var nodes: Int? +} + +/// One ladder capture of the synthetic world through the real bridge: what the memory +/// characterization observes before it hands the result to the runner's own learning step. +private struct AXLadderObservation { + let succeeded: Bool + let rejected: Int + let effectiveDepth: Int + let truncated: Bool + let attemptDepths: [Int] +} + +extension RunnerTests { + private static func loadAXRecoveryFixture() throws -> AXRecoveryFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // UnitTests + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("ios-ax-recovery-conformance.json") + let fixture = try JSONDecoder().decode(AXRecoveryFixture.self, from: Data(contentsOf: fixtureURL)) + XCTAssertEqual(fixture.version, 1) + return fixture + } + + private static func fixtureHint(_ hints: [String: Int?]?, _ producer: String) -> Int? { + hints?[producer] ?? nil + } + + private static func identityLevel(_ identity: String) -> Int { + Int(identity.split(separator: ".").first ?? "") ?? 0 + } + + private static func identityBranch(_ identity: String) -> String { + let parts = identity.split(separator: ".") + return parts.count > 1 ? String(parts[1]) : "" + } + + /// The canonical signature described by the fixture's `nativeModel.signature`: preorder + /// identities, ` (tree: String, nodes: Int)? { + guard let root else { return nil } + var preorder: [(identity: String, parent: String)] = [] + func collect(_ node: [String: Any], parent: String) { + guard preorder.count < 10_000 else { return } + let identity = node["label"] as? String ?? "" + preorder.append((identity, parent)) + for child in node["children"] as? [[String: Any]] ?? [] { collect(child, parent: identity) } + } + collect(root, parent: "") + var tokens: [String] = [] + var runStart: String? + var runLast: String? + func flush() { + if let start = runStart, let last = runLast { tokens.append(start == last ? start : "\(start)-\(last)") } + runStart = nil + runLast = nil + } + for entry in preorder { + let canonicalParent = index[entry.identity]?.parent?.identity ?? "" + if entry.parent != canonicalParent { + flush() + tokens.append("\(entry.identity)<\(entry.parent)") + continue + } + if let last = runLast, entry.parent == last, identityBranch(entry.identity) == identityBranch(last), + identityLevel(entry.identity) == identityLevel(last) + 1 + { + runLast = entry.identity + continue + } + flush() + runStart = entry.identity + runLast = entry.identity + } + flush() + return (tokens.joined(separator: ","), preorder.count) + } + + private static func deepestLevel(_ node: [String: Any]?) -> Int? { + guard let node else { return nil } + let own = (node["label"] as? String).map(identityLevel) + let children = (node["children"] as? [[String: Any]] ?? []).compactMap(deepestLevel) + return ([own].compactMap { $0 } + children).max() + } + + private func observePrivateAXRecovery(_ recoveryCase: AXRecoveryFixture.RecoveryCase) + -> AXRecoveryObservation + { + let request = recoveryCase.request + let native = recoveryCase.native + let rootNode = AXFixtureNode.build(native.tree) + let root = AXFixtureElement(node: rootNode) + let client = AXFixtureClient( + rejectLevelsAbove: native.rejectLevelsAbove, vanishAtFrontier: native.vanishAtFrontier ?? false) + let explicit = request.explicitDepth + let attemptDepths = Self.privateAXAttemptDepths( + requestedDepth: request.traversalDepth, + rememberedDepth: explicit ? nil : Self.fixtureHint(request.hint, "runner")) + XCTAssertTrue( + native.deadlineAfterRequests == nil || native.deadlineAfterRequests == 1, + "\(recoveryCase.name): the runner adapter models a deadline spent after the first request") + let deadline: Date = native.deadlineAfterRequests == 1 ? Date() : .distantFuture + let ladder = Self.privateAXLadderCapture(attemptDepths: attemptDepths, deadline: deadline) { depth in + RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: root, + maxDepth: depth, + maxNodes: request.nodeBudget, + deepExtensionCallLimit: explicit ? 0 : Self.privateAXDeepExtensionCallLimit, + customActionLimit: 0, + deadline: deadline) + } + + var observation = AXRecoveryObservation() + observation.requests = client.requests + observation.rejected = client.rejected + let extension_ = ladder.response[RunnerAXSnapshotDeepExtensionKey] as? [String: Any] + observation.continuations = extension_?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int ?? 0 + guard ladder.succeeded else { + observation.outcome = "failed" + observation.failure = ladder.deadlineSpent ? "deadline" : "rejected" + return observation + } + let depthLimited = Self.privateAXDepthLimited( + effectiveDepth: ladder.effectiveDepth, + requestedDepth: request.traversalDepth, + pendingFrontiers: extension_?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, + missedFrontiers: extension_?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int) + let truncated = ladder.response["truncated"] as? Bool == true + observation.outcome = depthLimited || truncated ? "incomplete" : "complete" + let root_ = ladder.response["root"] as? [String: Any] + observation.deepestLevel = Self.deepestLevel(root_) + if let signature = Self.treeSignature(root_, index: rootNode.index()) { + observation.tree = signature.tree + observation.nodes = signature.nodes + } + return observation + } + + /// Every recovery case of the shared fixture, replayed through the real ladder, bridge + /// capture, frontier extension, and completeness verdict, down to the delivered tree. + func testPrivateAXRecoveryMatchesSharedFixture() throws { + let fixture = try Self.loadAXRecoveryFixture() + XCTAssertFalse(fixture.recoveryCases.isEmpty) + for recoveryCase in fixture.recoveryCases { + let expected = try XCTUnwrap(recoveryCase.expected["runner"], recoveryCase.name) + if expected.outcome == "not-applicable" { continue } + let observed = observePrivateAXRecovery(recoveryCase) + let name = recoveryCase.name + XCTAssertEqual(observed.outcome, expected.outcome, "\(name): outcome") + XCTAssertEqual(observed.failure, expected.failure, "\(name): failure") + XCTAssertEqual(observed.requests, expected.requests, "\(name): requests") + XCTAssertEqual(observed.rejected, expected.rejected, "\(name): rejected") + XCTAssertEqual(observed.continuations, expected.continuations, "\(name): continuations") + XCTAssertEqual(observed.deepestLevel, expected.deepestLevel, "\(name): deepestLevel") + XCTAssertEqual(observed.tree, expected.tree, "\(name): tree") + XCTAssertEqual(observed.nodes, expected.nodes, "\(name): nodes") + } + } + + /// One hint step's capture: a chain that fits the accepted rung when the step is complete, or + /// a longer chain under a node budget of that rung when it is bounded, captured through the + /// real ladder against a client that rejects anything deeper than the accepted rung. + private func observePrivateAXHintCapture( + _ outcome: AXRecoveryFixture.Outcome, remembered: Int?, explicit: Bool + ) throws -> AXLadderObservation { + // A failing step is rejected at every depth; otherwise the chain fits the accepted rung, or + // overflows a node budget of that rung when the step is bounded. + let failing = outcome.failure != nil + let accepted = failing ? 0 : try XCTUnwrap(outcome.acceptedLevels?["runner"]) + let complete = outcome.complete ?? true + let tree = AXRecoveryFixture.Tree(chain: failing ? 5 : complete ? max(1, accepted - 1) : accepted + 5, fan: nil) + let root = AXFixtureElement(node: AXFixtureNode.build(tree)) + let client = AXFixtureClient(rejectLevelsAbove: accepted, vanishAtFrontier: false) + let attemptDepths = Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: remembered) + let ladder = Self.privateAXLadderCapture(attemptDepths: attemptDepths, deadline: .distantFuture) { depth in + RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: root, + maxDepth: depth, + maxNodes: complete ? 1_500 : max(1, accepted), + deepExtensionCallLimit: explicit ? 0 : Self.privateAXDeepExtensionCallLimit, + customActionLimit: 0, + deadline: .distantFuture) + } + return AXLadderObservation( + succeeded: ladder.succeeded, + rejected: client.rejected, + effectiveDepth: ladder.effectiveDepth, + truncated: ladder.response["truncated"] as? Bool == true, + attemptDepths: attemptDepths) + } + + /// Every hint case of the shared fixture, replayed as real captures whose outcome feeds the + /// runner's own learning step: the fixture's target id is the bundle id and its generation is + /// the process identifier. + func testPrivateAXAcceptedDepthMemoryMatchesSharedFixture() throws { + let fixture = try Self.loadAXRecoveryFixture() + XCTAssertFalse(fixture.hintCases.isEmpty) + defer { + clearPrivateAXAcceptedDepth(reason: "test-cleanup") + currentBundleId = nil + currentAppProcessIdentifier = nil + } + for hintCase in fixture.hintCases { + clearPrivateAXAcceptedDepth(reason: "fixture-case") + for step in hintCase.steps { + if step.expire == true { + privateAXAcceptedDepthUntil = Date(timeIntervalSinceNow: -1) + continue + } + let name = hintCase.name + let target = try XCTUnwrap(step.target, name) + let processIdentifier = try XCTUnwrap( + Int(target.generation.filter(\.isNumber)), "\(name): generation must end in digits") + currentBundleId = target.id + currentAppProcessIdentifier = processIdentifier + let explicit = step.explicitDepth ?? false + let remembered = + explicit + ? nil + : rememberedPrivateAXAcceptedDepth(bundleId: target.id, processIdentifier: processIdentifier) + XCTAssertEqual(remembered, Self.fixtureHint(step.expectHintBefore, "runner"), "\(name): before") + + let outcome = try XCTUnwrap(step.outcome, name) + let capture = try observePrivateAXHintCapture(outcome, remembered: remembered, explicit: explicit) + XCTAssertEqual(capture.succeeded, outcome.failure == nil, "\(name): capture") + XCTAssertEqual(capture.rejected, outcome.rejected["runner"], "\(name): rejected") + let expiryBefore = privateAXAcceptedDepthUntil + if capture.succeeded { + XCTAssertEqual(capture.effectiveDepth, outcome.acceptedLevels?["runner"], "\(name): accepted rung") + XCTAssertEqual(capture.truncated, outcome.complete == false, "\(name): bounded") + // The production path records only after a successful ladder, exactly like this. + recordPrivateAXAcceptedDepth( + exactDepthRequested: explicit, + effectiveDepth: capture.effectiveDepth, + attemptDepths: capture.attemptDepths) + } + XCTAssertEqual( + rememberedPrivateAXAcceptedDepth(bundleId: target.id, processIdentifier: processIdentifier), + Self.fixtureHint(step.expectHintAfter, "runner"), + "\(name): after") + if step.expectRenewed == false { + XCTAssertEqual(privateAXAcceptedDepthUntil, expiryBefore, "\(name): renewed") + } + } + } + } +} +#endif diff --git a/apple/snapshot-bridge/README.md b/apple/snapshot-bridge/README.md index 676cbd6b6b..589d4973b3 100644 --- a/apple/snapshot-bridge/README.md +++ b/apple/snapshot-bridge/README.md @@ -69,11 +69,9 @@ continuation evidence. Unchanged native dictionaries and child arrays are reused `contracts/fixtures/ios-ax-recovery-conformance.json` is the shared, executable recovery contract for this bridge and the XCTest runner's private AX bridge. `packages/platform-apple/src/snapshot-source/fixtures/recovery-conformance.m` -replays each case through `captureSnapshotTree`; the runner adapter that -replays the same cases through the XCTest runner's private AX bridge is -pending in [#2428](https://github.com/callstack/agent-device/pull/2428), so at -this revision only the fixture's `host-bridge` column is executed. Each -expectation names the +replays each case through `captureSnapshotTree`; the runner replays the same +cases through its own bridge in +`RunnerTests+AXRecoveryConformanceTests.swift`. Each expectation names the outcome, the native request accounting, and the delivered tree as a canonical preorder signature with its retained node count, so a producer that drops, duplicates, reorders, or re-parents nodes cannot pass as complete. The fixture diff --git a/contracts/fixtures/ios-ax-recovery-conformance.json b/contracts/fixtures/ios-ax-recovery-conformance.json index 85df4ed565..212c2c28f5 100644 --- a/contracts/fixtures/ios-ax-recovery-conformance.json +++ b/contracts/fixtures/ios-ax-recovery-conformance.json @@ -8,7 +8,8 @@ "vanishAtFrontier": "The deepest level of every returned fragment has no live accessibility element, so no continuation can be rooted there.", "ownerChangesAfterRequests": "The primary foreground owner changes after that many native requests (host only).", "deadlineAfterRequests": "The capture deadline is spent after that many native requests (runner only; the host deadline is the guest watchdog and the host request timeout).", - "signature": "`tree` is the delivered tree in preorder: every node's identity, with `