From 043c41b46bccd258f08110af7f8df05613498519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 14:10:02 +0200 Subject: [PATCH 1/4] fix(ios): clip a scroll's swipe above the keyboard, refuse when it cannot The runner owns the live keyboard frame, so it does the clip and reports what it left: a scroll answers with `keyboardAvoided` and `keyboardMinY` beside its plan, and refuses with `SCROLL_KEYBOARD_OCCLUDES_SURFACE` when the keys leave too little band to swipe in instead of flinging into them. It never dismisses the keyboard, which would drop focus and mutate state that session-action provenance does not record. Scroll's keyboard policy moves to `requiredWhenAvailable`. The probe costs a live AX fetch, but gating it on a healthy tree left the first scroll of a session swiping under the keys, which is the failure this is for. Every scroll logs its decision, including the two ways it avoids reading the keyboard at all. Scroll no longer shares `frameAvoidingKeyboard`, whose 25% fail-open was a tap-reference-frame rule; that path is unchanged for its remaining callers. --- .../RunnerScrollViewportPolicy.swift | 256 ++++++++++++++++++ .../RunnerTests+CommandExecution.swift | 76 ++++-- .../RunnerTests+Interaction.swift | 12 - .../RunnerTests+Models.swift | 4 + ...RunnerTests+SynthesizedGesturePolicy.swift | 14 +- 5 files changed, 330 insertions(+), 32 deletions(-) create mode 100644 apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift new file mode 100644 index 0000000000..07b062c8d0 --- /dev/null +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -0,0 +1,256 @@ +import XCTest + +// The scroll viewport rule the runner shares with the TS runtime (#2500). +// +// RULE: a directional scroll centres its swipe, so a focused field puts the swipe's lower endpoint +// under the keyboard — the gesture lands on keys, the surface never moves, and the edge loop reads a +// stuck container (#2499) rather than a refusal. Clipping the viewport to the band above the +// keyboard BEFORE the gesture planner runs keeps the swipe in what is visible, and when that band is +// too thin to hold one the rule REFUSES instead of handing back the full frame: a keyboard-struck +// swipe and a tiny clipped swipe both read as "stuck", so failing open is what hides the failure. +// +// The pure rule below is geometry on purpose — no XCUIApplication — so its exact decision is proven +// against the golden table shared with its TS twin, `clipScrollViewportAboveKeyboard` in +// packages/contracts/src/scroll-gesture.ts, asserted in that file's test beside this one. The table +// carries only frames representable in both languages: `CGRect` standardizes a negative extent into a +// positive height at a moved origin, so a negative `height` is tested on the TS side alone. +// +// The `extension RunnerTests` below is the one impure caller: it reads the runner's own live keyboard +// frame, because a frame threaded from the daemon would predate the keyboard. + +/** What an on-screen keyboard leaves of a scroll viewport. */ +enum RunnerScrollKeyboardClip: Equatable { + /** No keyboard, or one that does not own this surface: swipe the whole viewport. */ + case unobstructed + /** The viewport trimmed above the keyboard. Report the reduced reference height honestly. */ + case avoided(frame: CGRect, keyboardMinY: Double) + /** Too little surface left to swipe. The caller refuses; it never swipes under the keys. */ + case occluded(keyboardMinY: Double, visibleHeight: Double) +} + +/** Where one directional scroll may place its swipe, once the keyboard has taken its share. */ +enum RunnerScrollViewport { + /** The frame to plan inside, plus the keyboard top when the swipe was clipped for one. */ + case swipe(frame: CGRect, keyboardMinY: Double?) + /** Nothing to swipe. The caller answers `occlusionRunnerCode` and performs no gesture. */ + case occluded(keyboardMinY: Double, visibleHeight: Double) +} + +enum ScrollViewportPolicy { + /** Below this fraction of the viewport, the clipped band cannot hold a reliable swipe. */ + static let minVisibleFraction: Double = 0.15 + /** + * A fixed allowance kept above the keyboard's top edge, in points. `keyboard.frame` reports the + * key plane, not the input accessory or composer bar riding above it, so a swipe ending exactly + * at the reported edge can still land on a bar. + */ + static let accessoryAllowance: Double = 12 + + /// The runner's own wire vocabulary, not a shared policy constant: the host keeps it + /// `COMMAND_FAILED` and reads it back from `details.runnerErrorCode`. + static let occlusionRunnerCode = "SCROLL_KEYBOARD_OCCLUDES_SURFACE" + + /// Clips a scroll viewport to the band above an occluding keyboard, failing open on a frame the + /// runner cannot measure: a missing keyboard query is not evidence that the surface is blocked. + static func clip(viewport: CGRect, keyboard: CGRect) -> RunnerScrollKeyboardClip { + guard isUsable(viewport), isUsable(keyboard) else { + return .unobstructed + } + // A vertical swipe runs along the viewport's centre line, which is the only part of the width + // the keyboard has to reach to be struck: a 320pt keyboard centred in an 834pt viewport is 38% + // of the width and sits exactly in the path. + let swipeCenterX = viewport.minX + viewport.width / 2 + if swipeCenterX < keyboard.minX || swipeCenterX >= keyboard.maxX { + return .unobstructed + } + let keyboardMinY = keyboard.minY + if keyboardMinY >= viewport.maxY || keyboard.maxY <= viewport.minY { + return .unobstructed + } + let visibleHeight = max(0, keyboardMinY - accessoryAllowance - viewport.minY) + if visibleHeight < minVisibleFraction * viewport.height { + return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) + } + return .avoided( + frame: CGRect( + x: viewport.minX, + y: viewport.minY, + width: viewport.width, + height: visibleHeight + ), + keyboardMinY: keyboardMinY + ) + } + + private static func isUsable(_ rect: CGRect) -> Bool { + return [rect.minX, rect.minY, rect.width, rect.height].allSatisfy(\.isFinite) + && rect.width > 0 && rect.height > 0 + } +} + +extension RunnerTests { + /// Resolves the frame one directional scroll places its swipe in, and what the keyboard leaves of + /// it. Never dismisses: a dismiss drops focus, breaks a `type`/`scroll`/`type` loop, and mutates + /// state session-action provenance does not record, so `keyboard dismiss` stays an explicit + /// command and this path only ever reduces the space it swipes in. + func resolvedScrollViewport( + app: XCUIApplication, + context: SynthesizedCoordinateContext + ) -> RunnerScrollViewport { +#if os(iOS) + // Every scroll reports its decision, including the two ways it avoids reading the keyboard at + // all: a policy that forbids the probe, and a probe that finds no keyboard. + guard context.allowsKeyboardProbe else { + logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) + return .swipe(frame: context.referenceFrame, keyboardMinY: nil) + } + guard let keyboardFrame = visibleKeyboardFrame(app: app) else { + logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) + return .swipe(frame: context.referenceFrame, keyboardMinY: nil) + } + switch ScrollViewportPolicy.clip(viewport: context.referenceFrame, keyboard: keyboardFrame) { + case .unobstructed: + logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) + return .swipe(frame: context.referenceFrame, keyboardMinY: nil) + case .avoided(let frame, let keyboardMinY): + logScrollViewport( + decision: "avoided", + keyboardMinY: keyboardMinY, + swipeHeight: frame.height, + context: context + ) + return .swipe(frame: frame, keyboardMinY: keyboardMinY) + case .occluded(let keyboardMinY, let visibleHeight): + logScrollViewport( + decision: "occluded", + keyboardMinY: keyboardMinY, + swipeHeight: visibleHeight, + context: context + ) + return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) + } +#else + return .swipe(frame: resolvedTouchReferenceFrame(app: app, appFrame: app.frame), keyboardMinY: nil) +#endif + } + +#if os(iOS) + /// The #2500 diagnostic for a scroll that reports no travel: whether the swipe was clipped, and + /// whether the keyboard probe was even permitted. `axHealth` is the first thing to read, because a + /// policy that skipped the probe looks exactly like a keyboard that was never found. + private func logScrollViewport( + decision: String, + keyboardMinY: Double?, + swipeHeight: Double, + context: SynthesizedCoordinateContext + ) { + NSLog( + "AGENT_DEVICE_RUNNER_SCROLL_VIEWPORT kind=scroll axHealth=%@ keyboardPolicy=%@ decision=%@ keyboardMinY=%@ swipeHeight=%.1f", + context.accessibilityHealth.rawValue, + context.keyboardPolicy.rawValue, + decision, + keyboardMinY.map { String(format: "%.1f", $0) } ?? "none", + swipeHeight + ) + } +#endif +} + +#if AGENT_DEVICE_RUNNER_UNIT_TESTS +private struct ScrollViewportPolicyFixture: Decodable { + struct Frame: Decodable { + let x: Double + let y: Double + let width: Double + let height: Double + + var cgRect: CGRect { + CGRect(x: x, y: y, width: width, height: height) + } + } + + struct Constants: Decodable { + let minVisibleFraction: Double + let accessoryAllowance: Double + } + + struct Expected: Decodable { + let kind: String + let viewport: Frame? + let keyboardMinY: Double? + let visibleHeight: Double? + } + + struct TestCase: Decodable { + let name: String + let viewport: Frame + let keyboard: Frame + let expected: Expected + } + + let constants: Constants + let cases: [TestCase] +} + +extension RunnerTests { + /// Golden parity table (#2500): every case in contracts/fixtures/scroll-keyboard-policy.json must + /// agree with the vitest twin. Add cases there, never fork the rule. + func testScrollViewportKeyboardClipMatchesGoldenParityTable() throws { + let fixture = try loadScrollViewportPolicyFixture() + XCTAssertFalse(fixture.cases.isEmpty, "parity table must not be empty") + for testCase in fixture.cases { + let clip = ScrollViewportPolicy.clip( + viewport: testCase.viewport.cgRect, + keyboard: testCase.keyboard.cgRect + ) + switch testCase.expected.kind { + case "unobstructed": + XCTAssertEqual(clip, .unobstructed, testCase.name) + case "avoided": + let expectedFrame = try XCTUnwrap(testCase.expected.viewport, testCase.name).cgRect + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + XCTAssertEqual( + clip, + .avoided(frame: expectedFrame, keyboardMinY: expectedMinY), + testCase.name + ) + case "occluded": + let expectedMinY = try XCTUnwrap(testCase.expected.keyboardMinY, testCase.name) + let expectedVisibleHeight = try XCTUnwrap(testCase.expected.visibleHeight, testCase.name) + XCTAssertEqual( + clip, + .occluded(keyboardMinY: expectedMinY, visibleHeight: expectedVisibleHeight), + testCase.name + ) + default: + XCTFail("unknown expected kind `\(testCase.expected.kind)` in \(testCase.name)") + } + } + } + + /// The thresholds are the table's, not this file's. The refusal reason and the runner code are + /// each one side's own vocabulary: the reason is what the host publishes, the code is what this + /// runner answers with, and neither is a shared clip constant. + func testScrollViewportPolicyUsesParityTableConstants() throws { + let constants = try loadScrollViewportPolicyFixture().constants + XCTAssertEqual(constants.minVisibleFraction, ScrollViewportPolicy.minVisibleFraction) + XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) + } + + private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { + let fixtureURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() // AgentDeviceRunnerUITests + .deletingLastPathComponent() // AgentDeviceRunner + .deletingLastPathComponent() // runner + .deletingLastPathComponent() // apple + .deletingLastPathComponent() // repo root + .appendingPathComponent("contracts") + .appendingPathComponent("fixtures") + .appendingPathComponent("scroll-keyboard-policy.json") + return try JSONDecoder().decode( + ScrollViewportPolicyFixture.self, + from: Data(contentsOf: fixtureURL) + ) + } +} +#endif diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 6f54fe6cf3..ec7bbc3950 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1953,7 +1953,20 @@ extension RunnerTests { error: ErrorPayload(message: "scroll could not resolve a usable interaction frame") ) } - let frame = scrollReferenceFrame(app: activeApp, context: scrollContext) + let viewport = resolvedScrollViewport(app: activeApp, context: scrollContext) + let frame: CGRect + let keyboardMinY: Double? + switch viewport { + case .occluded(let occlusionKeyboardMinY, let visibleHeight): + return scrollKeyboardOccludedResponse( + direction: direction.rawValue, + keyboardMinY: occlusionKeyboardMinY, + visibleHeight: visibleHeight + ) + case .swipe(let swipeFrame, let clippedAboveKeyboardMinY): + frame = swipeFrame + keyboardMinY = clippedAboveKeyboardMinY + } guard frame.width > 0, frame.height > 0 else { return Response( ok: false, @@ -1979,16 +1992,19 @@ extension RunnerTests { guard scrollDurationIsValid(command.durationMs) else { return invalidScrollDurationResponse(commandName: "scroll") } - return executeScrollDragGesture( - activeApp: activeApp, - x: frame.minX + plan.x1, - y: frame.minY + plan.y1, - x2: frame.minX + plan.x2, - y2: frame.minY + plan.y2, - durationMs: defaults.durationMs, - message: "scrolled", - context: scrollContext.withReferenceFrame(frame), - releaseBehavior: command.scrollReleaseBehavior + return attachingScrollViewportEvidence( + executeScrollDragGesture( + activeApp: activeApp, + x: frame.minX + plan.x1, + y: frame.minY + plan.y1, + x2: frame.minX + plan.x2, + y2: frame.minY + plan.y2, + durationMs: defaults.durationMs, + message: "scrolled", + context: scrollContext.withReferenceFrame(frame), + releaseBehavior: command.scrollReleaseBehavior + ), + keyboardMinY: keyboardMinY ) case .desktopScroll: guard let rawDirection = command.direction, @@ -2564,12 +2580,38 @@ extension RunnerTests { ) } - private func scrollReferenceFrame(app: XCUIApplication, context: SynthesizedCoordinateContext) -> CGRect { -#if os(iOS) - return synthesizedFrameAvoidingKeyboardWhenAllowed(app: app, context: context) -#else - return resolvedTouchReferenceFrame(app: app, appFrame: app.frame) -#endif + /// Adds the #2500 avoidance evidence to a scroll response. Only the frame resolver knows whether + /// it trimmed the swipe for a keyboard, and only `scroll` has this evidence to carry, so it is + /// attached where the frame was resolved rather than threaded through every gesture response. + private func attachingScrollViewportEvidence(_ response: Response, keyboardMinY: Double?) -> Response { + guard response.ok, let keyboardMinY else { return response } + var payload = response.data ?? DataPayload() + payload.keyboardAvoided = true + payload.keyboardMinY = keyboardMinY + return Response(ok: response.ok, data: payload, error: response.error) + } + + /// The refusal a keyboard forces. It performs no gesture: swiping into the keys would leave the + /// surface where it was, which the daemon's no-progress fingerprint reads as a stuck container + /// (#2499) and an agent reads as a broken scroll. The TS owner maps the code to the + /// `scroll_keyboard_occludes_surface` reason and the "dismiss the keyboard" hint. + private func scrollKeyboardOccludedResponse( + direction: String, + keyboardMinY: Double, + visibleHeight: Double + ) -> Response { + return Response( + ok: false, + error: ErrorPayload( + code: ScrollViewportPolicy.occlusionRunnerCode, + message: String( + format: + "scroll %@ refused: the keyboard leaves %.0fpt of visible surface above it, too little to swipe", + direction, + visibleHeight + ) + ) + ) } private func dragCommandName(message: String) -> String { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 2cb8b465b4..5e85e6ca09 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -909,18 +909,6 @@ extension RunnerTests { return CGRect(x: 0, y: 0, width: width, height: height) } - func synthesizedFrameAvoidingKeyboardWhenAllowed( - app: XCUIApplication, - context: SynthesizedCoordinateContext - ) -> CGRect { -#if os(iOS) - guard context.allowsKeyboardProbe else { return context.referenceFrame } - return frameAvoidingKeyboard(app: app, frame: context.referenceFrame) -#else - return context.referenceFrame -#endif - } - func keyboardAvoidingSynthesizedDragPoints( app: XCUIApplication, x: Double, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 60bb93d6d8..1aa612067e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -268,6 +268,10 @@ struct DataPayload: Codable { var gestureFallback: String? var gestureFallbackMessage: String? var gestureFallbackHint: String? + // Scroll keyboard avoidance evidence (#2500): the swipe was clipped to the band above an + // on-screen keyboard, and where that band ended. `referenceHeight` already names the clipped axis. + var keyboardAvoided: Bool? + var keyboardMinY: Double? var maestroNonHittableCoordinateFallbackUsed: Bool? var textEntryRoute: String? var runnerFatal: Bool? diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift index 90a7f4c6ed..c22c0cfb66 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift @@ -89,8 +89,14 @@ func synthesizedGesturePolicy(_ kind: SynthesizedGesturePolicyKind) -> Synthesiz fallbackPolicy: .xctestCoordinateAllowed ) case .scroll: + // Scroll places a viewport-center-symmetric swipe, so it cannot tell a keyboard-struck swipe + // from a scroll that reached the edge without reading the live keyboard frame (#2500). The + // probe is not free — `visibleKeyboardFrame` resolves `app.keyboards.firstMatch` with a live AX + // fetch — but skipping it on `.unknown` left the first scroll of a session swiping under the + // keys, which is the failure this command exists to avoid. `.unavailable` still skips it: there + // the fetch is known not to answer, and `ScrollViewportPolicy` fails open on a missing frame. return SynthesizedGesturePolicy( - keyboardPolicy: .whenAccessibilityHealthy, + keyboardPolicy: .requiredWhenAvailable, fallbackPolicy: .privateSynthesisRequired ) case .synthesizedDrag: @@ -182,7 +188,9 @@ extension RunnerTests { ) } - func testSynthesizedKeyboardPolicyKeepsUnknownDragProbeButNotUnknownScrollProbe() { + /// Keyboard-policy semantics only. Which command gets which policy is the table below; a probe + /// that is merely permitted still costs a live AX fetch, so the two questions stay separate. + func testSynthesizedKeyboardPolicyAllowsProbeOnlyWhenAccessibilityPermitsIt() { XCTAssertFalse( SynthesizedKeyboardPolicy.whenAccessibilityHealthy .allowsProbe(accessibilityHealth: .unknown) @@ -208,7 +216,7 @@ extension RunnerTests { XCTAssertEqual( synthesizedGesturePolicy(.scroll), SynthesizedGesturePolicy( - keyboardPolicy: .whenAccessibilityHealthy, + keyboardPolicy: .requiredWhenAvailable, fallbackPolicy: .privateSynthesisRequired ) ) From d562facdf7ad31a0057bce82925f49796bef1fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 14:10:14 +0200 Subject: [PATCH 2/4] chore(gates): run the scroll viewport policy tests on the iOS lane The parity table only detects drift if both halves run in CI. Two of these three were reachable by no lane, so the Swift half of the table was a local assertion. --- .github/workflows/ios.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index de94f9d663..6d037d7e75 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -165,6 +165,9 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEmptyReplacementWithoutResolvableTargetFailsClosed \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportKeyboardClipMatchesGoldenParityTable \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportPolicyUsesParityTableConstants \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedGesturePoliciesMatchCommandContracts \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFreshCoordinateTapContainsUnavailableTextInputProbe \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeIssueScopeIsThreadBound \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbePreservesEnclosingRunnerWait \ From 1b24fe0bdee81e56b26305d01e88479d9d5c7c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 18:45:04 +0200 Subject: [PATCH 3/4] fix(ios): keep the keyboard clip out of the scroll's rotation basis `resolvedScrollViewport` handed the command one frame for both jobs, and the coordinate rotation reads a frame's HEIGHT to map a `landscapeRight` native x. Clipping an 834pt landscape viewport to 576pt therefore moved the dispatched gesture 258pt sideways off the lane the plan had just been built for: the clip fixed the keyboard and broke the gesture. The resolved viewport now names both frames, and the gesture comes from one dispatch decision, so the band the plan is planned inside and the frame its coordinates rotate against cannot be swapped. The landscape case asserts through that decision and fails on the swap. --- .github/workflows/ios.yml | 1 + .../RunnerScrollViewportPolicy.swift | 140 ++++++++++++++++-- .../RunnerTests+CommandExecution.swift | 60 ++++---- .../RunnerTests+Interaction.swift | 2 +- 4 files changed, 159 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 6d037d7e75..089e2ea259 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -167,6 +167,7 @@ jobs: -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportKeyboardClipMatchesGoldenParityTable \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportPolicyUsesParityTableConstants \ + -only-testing:AgentDeviceRunnerUITests/RunnerTests/testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSynthesizedGesturePoliciesMatchCommandContracts \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFreshCoordinateTapContainsUnavailableTextInputProbe \ -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeIssueScopeIsThreadBound \ diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index 07b062c8d0..a174ec7b80 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -30,12 +30,71 @@ enum RunnerScrollKeyboardClip: Equatable { /** Where one directional scroll may place its swipe, once the keyboard has taken its share. */ enum RunnerScrollViewport { - /** The frame to plan inside, plus the keyboard top when the swipe was clipped for one. */ - case swipe(frame: CGRect, keyboardMinY: Double?) + /** + * The band to plan the swipe inside, the frame to rotate its coordinates against, and the keyboard + * top when the band was clipped for one. The two frames are separate on purpose: a clip shortens + * only the band, while `nativeSynthesizedPoint` derives a `landscapeRight` native x from the + * frame's HEIGHT, so rotating inside the band moves the dispatched path sideways off the planned + * one. + */ + case swipe(planFrame: CGRect, coordinateFrame: CGRect, keyboardMinY: Double?) /** Nothing to swipe. The caller answers `occlusionRunnerCode` and performs no gesture. */ case occluded(keyboardMinY: Double, visibleHeight: Double) } +/// The gesture one directional scroll dispatches, built from a resolved viewport in one place so the +/// band the plan was made inside and the frame its coordinates rotate against cannot be swapped. +struct ScrollGestureDispatch { + let plan: RunnerScrollGesturePlan + let planFrame: CGRect + let coordinateFrame: CGRect + let keyboardMinY: Double? +} + +/** What a resolved viewport turns into for the command: a gesture, or the reason there is none. */ +enum ScrollGestureOutcome { + case gesture(ScrollGestureDispatch) + case unusableFrame + case unusablePlan + case occluded(keyboardMinY: Double, visibleHeight: Double) +} + +extension RunnerScrollViewport { + /// Plans the swipe inside the band the keyboard left and keeps the viewport as the coordinate basis, + /// so a clip shortens the travel without moving the gesture's lane. + func gestureDispatch( + direction: RunnerScrollDirection, + amount: Double?, + pixels: Double? + ) -> ScrollGestureOutcome { + switch self { + case .occluded(let keyboardMinY, let visibleHeight): + return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) + case .swipe(let planFrame, let coordinateFrame, let keyboardMinY): + guard planFrame.width > 0, planFrame.height > 0 else { + return .unusableFrame + } + guard let plan = runnerScrollGesturePlan( + direction: direction, + amount: amount, + pixels: pixels, + referenceWidth: planFrame.width, + referenceHeight: planFrame.height + ) else { + return .unusablePlan + } + return .gesture( + ScrollGestureDispatch( + plan: plan, + planFrame: planFrame, + coordinateFrame: coordinateFrame, + keyboardMinY: keyboardMinY + ) + ) + } + } +} + enum ScrollViewportPolicy { /** Below this fraction of the viewport, the clipped band cannot hold a reliable swipe. */ static let minVisibleFraction: Double = 0.15 @@ -82,6 +141,20 @@ enum ScrollViewportPolicy { ) } + /// Splits a clip verdict into the two frames a dispatch needs. The gesture planner runs inside the + /// clipped band; the coordinate rotation keeps the frame the viewport was resolved against, because + /// the rotation basis is a property of the screen, not of what the keyboard left free. + static func frames(referenceFrame: CGRect, clip: RunnerScrollKeyboardClip) -> RunnerScrollViewport { + switch clip { + case .unobstructed: + return .swipe(planFrame: referenceFrame, coordinateFrame: referenceFrame, keyboardMinY: nil) + case .avoided(let frame, let keyboardMinY): + return .swipe(planFrame: frame, coordinateFrame: referenceFrame, keyboardMinY: keyboardMinY) + case .occluded(let keyboardMinY, let visibleHeight): + return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) + } + } + private static func isUsable(_ rect: CGRect) -> Bool { return [rect.minX, rect.minY, rect.width, rect.height].allSatisfy(\.isFinite) && rect.width > 0 && rect.height > 0 @@ -102,16 +175,16 @@ extension RunnerTests { // all: a policy that forbids the probe, and a probe that finds no keyboard. guard context.allowsKeyboardProbe else { logScrollViewport(decision: "probeSkipped", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return .swipe(frame: context.referenceFrame, keyboardMinY: nil) + return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed) } guard let keyboardFrame = visibleKeyboardFrame(app: app) else { logScrollViewport(decision: "noKeyboard", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return .swipe(frame: context.referenceFrame, keyboardMinY: nil) + return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: .unobstructed) } - switch ScrollViewportPolicy.clip(viewport: context.referenceFrame, keyboard: keyboardFrame) { + let clip = ScrollViewportPolicy.clip(viewport: context.referenceFrame, keyboard: keyboardFrame) + switch clip { case .unobstructed: logScrollViewport(decision: "unobstructed", keyboardMinY: nil, swipeHeight: context.referenceFrame.height, context: context) - return .swipe(frame: context.referenceFrame, keyboardMinY: nil) case .avoided(let frame, let keyboardMinY): logScrollViewport( decision: "avoided", @@ -119,7 +192,6 @@ extension RunnerTests { swipeHeight: frame.height, context: context ) - return .swipe(frame: frame, keyboardMinY: keyboardMinY) case .occluded(let keyboardMinY, let visibleHeight): logScrollViewport( decision: "occluded", @@ -127,10 +199,11 @@ extension RunnerTests { swipeHeight: visibleHeight, context: context ) - return .occluded(keyboardMinY: keyboardMinY, visibleHeight: visibleHeight) } + return ScrollViewportPolicy.frames(referenceFrame: context.referenceFrame, clip: clip) #else - return .swipe(frame: resolvedTouchReferenceFrame(app: app, appFrame: app.frame), keyboardMinY: nil) + let fallbackFrame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame) + return ScrollViewportPolicy.frames(referenceFrame: fallbackFrame, clip: .unobstructed) #endif } @@ -237,6 +310,55 @@ extension RunnerTests { XCTAssertEqual(constants.accessoryAllowance, ScrollViewportPolicy.accessoryAllowance) } + /// A clipped landscape band shortens the frame, and `nativeSynthesizedPoint` derives a + /// `landscapeRight` native x from the frame's HEIGHT. Rotating inside the band therefore moves the + /// dispatched path sideways by exactly what the keyboard took, off the lane the plan was built for, + /// so the plan band and the coordinate basis stay separate values through dispatch (#2500). + func testScrollViewportDispatchKeepsTheUnclippedFrameAsItsCoordinateRotationBasis() throws { + let viewport = CGRect(x: 0, y: 0, width: 1210, height: 834) + let keyboard = CGRect(x: 0, y: 588, width: 1210, height: 246) + let clip = ScrollViewportPolicy.clip(viewport: viewport, keyboard: keyboard) + guard case .avoided(let band, let keyboardMinY) = clip else { + return XCTFail("expected a landscape keyboard to be avoided, got \(clip)") + } + XCTAssertEqual(band.height, 576) + + guard case .gesture(let gesture) = ScrollViewportPolicy.frames( + referenceFrame: viewport, + clip: clip + ).gestureDispatch(direction: .up, amount: nil, pixels: nil) else { + return XCTFail("expected a gesture inside the clipped band") + } + XCTAssertEqual(gesture.planFrame, band) + XCTAssertEqual(gesture.keyboardMinY, keyboardMinY) + XCTAssertEqual(gesture.coordinateFrame, viewport, "the rotation basis must survive the clip") + XCTAssertLessThanOrEqual( + max(gesture.plan.y1, gesture.plan.y2), + keyboard.minY - ScrollViewportPolicy.accessoryAllowance, + "a landscape swipe must stay clear of the keys" + ) + + let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 + let dispatched = nativeSynthesizedPoint( + orientedX: gesture.planFrame.minX + gesture.plan.x1, + orientedY: orientedStartY, + in: gesture.coordinateFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + let clippedBasis = nativeSynthesizedPoint( + orientedX: gesture.planFrame.minX + gesture.plan.x1, + orientedY: orientedStartY, + in: gesture.planFrame, + interfaceOrientation: RunnerInterfaceOrientation.landscapeRight + ) + XCTAssertEqual( + dispatched.x - clippedBasis.x, + viewport.height - band.height, + accuracy: 0.001, + "rotating inside the clipped band would shift native x by what the keyboard took" + ) + } + private func loadScrollViewportPolicyFixture() throws -> ScrollViewportPolicyFixture { let fixtureURL = URL(fileURLWithPath: #filePath) .deletingLastPathComponent() // AgentDeviceRunnerUITests diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index ec7bbc3950..cd0724d8c0 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1954,33 +1954,24 @@ extension RunnerTests { ) } let viewport = resolvedScrollViewport(app: activeApp, context: scrollContext) - let frame: CGRect - let keyboardMinY: Double? - switch viewport { + let defaults = runnerDragCommandDefaults(command) + switch viewport.gestureDispatch( + direction: direction, + amount: defaults.scrollAmount, + pixels: command.pixels + ) { case .occluded(let occlusionKeyboardMinY, let visibleHeight): return scrollKeyboardOccludedResponse( direction: direction.rawValue, keyboardMinY: occlusionKeyboardMinY, visibleHeight: visibleHeight ) - case .swipe(let swipeFrame, let clippedAboveKeyboardMinY): - frame = swipeFrame - keyboardMinY = clippedAboveKeyboardMinY - } - guard frame.width > 0, frame.height > 0 else { + case .unusableFrame: return Response( ok: false, error: ErrorPayload(message: "scroll could not resolve a usable interaction frame") ) - } - let defaults = runnerDragCommandDefaults(command) - guard let plan = runnerScrollGesturePlan( - direction: direction, - amount: defaults.scrollAmount, - pixels: command.pixels, - referenceWidth: frame.width, - referenceHeight: frame.height - ) else { + case .unusablePlan: return Response( ok: false, error: ErrorPayload( @@ -1988,24 +1979,25 @@ extension RunnerTests { message: "scroll could not compute a gesture plan" ) ) + case .gesture(let gesture): + guard scrollDurationIsValid(command.durationMs) else { + return invalidScrollDurationResponse(commandName: "scroll") + } + return attachingScrollViewportEvidence( + executeScrollDragGesture( + activeApp: activeApp, + x: gesture.planFrame.minX + gesture.plan.x1, + y: gesture.planFrame.minY + gesture.plan.y1, + x2: gesture.planFrame.minX + gesture.plan.x2, + y2: gesture.planFrame.minY + gesture.plan.y2, + durationMs: defaults.durationMs, + message: "scrolled", + context: scrollContext.withReferenceFrame(gesture.coordinateFrame), + releaseBehavior: command.scrollReleaseBehavior + ), + keyboardMinY: gesture.keyboardMinY + ) } - guard scrollDurationIsValid(command.durationMs) else { - return invalidScrollDurationResponse(commandName: "scroll") - } - return attachingScrollViewportEvidence( - executeScrollDragGesture( - activeApp: activeApp, - x: frame.minX + plan.x1, - y: frame.minY + plan.y1, - x2: frame.minX + plan.x2, - y2: frame.minY + plan.y2, - durationMs: defaults.durationMs, - message: "scrolled", - context: scrollContext.withReferenceFrame(frame), - releaseBehavior: command.scrollReleaseBehavior - ), - keyboardMinY: keyboardMinY - ) case .desktopScroll: guard let rawDirection = command.direction, let direction = RunnerScrollDirection(rawValue: rawDirection) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 5e85e6ca09..6304bc43b5 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -11,7 +11,7 @@ private struct RunnerUnsupportedOperationError: LocalizedError { var errorDescription: String? { message } } -private enum RunnerInterfaceOrientation { +enum RunnerInterfaceOrientation { #if AGENT_DEVICE_RUNNER_UNIT_TESTS static let unknown = 0 #endif From b5a83ff682c92b58d789cfd7c09be71c7ebe8eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 20:51:25 +0200 Subject: [PATCH 4/4] fix(ios): report a scroll's clipped band in its response --- .../RunnerScrollViewportPolicy.swift | 35 +++++++++++++++++++ .../RunnerTests+CommandExecution.swift | 15 ++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift index a174ec7b80..1a69e33960 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift @@ -59,6 +59,25 @@ enum ScrollGestureOutcome { case occluded(keyboardMinY: Double, visibleHeight: Double) } +extension ScrollGestureDispatch { + /// Reports the gesture against the band its plan ran inside, beside the keyboard evidence. The + /// synthesis frame stays the full viewport so the coordinates rotate correctly, which leaves the + /// payload measured against an axis the caller never planned on: `pixels` are a fraction of the + /// band, so the band is what `referenceWidth` and `referenceHeight` have to name. + func attachingEvidence(to response: Response) -> Response { + guard response.ok else { return response } + var payload = response.data ?? DataPayload() + payload.referenceWidth = Double(planFrame.width) + payload.referenceHeight = Double(planFrame.height) + guard let keyboardMinY else { + return Response(ok: response.ok, data: payload, error: response.error) + } + payload.keyboardAvoided = true + payload.keyboardMinY = keyboardMinY + return Response(ok: response.ok, data: payload, error: response.error) + } +} + extension RunnerScrollViewport { /// Plans the swipe inside the band the keyboard left and keeps the viewport as the coordinate basis, /// so a clip shortens the travel without moving the gesture's lane. @@ -338,6 +357,22 @@ extension RunnerTests { "a landscape swipe must stay clear of the keys" ) + let reported = gesture.attachingEvidence( + to: Response( + ok: true, + data: DataPayload(referenceWidth: viewport.width, referenceHeight: viewport.height), + error: nil + ) + ) + XCTAssertEqual( + reported.data?.referenceHeight, + band.height, + "the payload names the band the plan ran inside, not the synthesis frame" + ) + XCTAssertEqual(reported.data?.referenceWidth, viewport.width) + XCTAssertEqual(reported.data?.keyboardMinY, keyboardMinY) + XCTAssertEqual(reported.data?.keyboardAvoided, true) + let orientedStartY = gesture.planFrame.minY + gesture.plan.y1 let dispatched = nativeSynthesizedPoint( orientedX: gesture.planFrame.minX + gesture.plan.x1, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index cd0724d8c0..a63598a582 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1983,8 +1983,8 @@ extension RunnerTests { guard scrollDurationIsValid(command.durationMs) else { return invalidScrollDurationResponse(commandName: "scroll") } - return attachingScrollViewportEvidence( - executeScrollDragGesture( + return gesture.attachingEvidence( + to: executeScrollDragGesture( activeApp: activeApp, x: gesture.planFrame.minX + gesture.plan.x1, y: gesture.planFrame.minY + gesture.plan.y1, @@ -1994,8 +1994,7 @@ extension RunnerTests { message: "scrolled", context: scrollContext.withReferenceFrame(gesture.coordinateFrame), releaseBehavior: command.scrollReleaseBehavior - ), - keyboardMinY: gesture.keyboardMinY + ) ) } case .desktopScroll: @@ -2575,14 +2574,6 @@ extension RunnerTests { /// Adds the #2500 avoidance evidence to a scroll response. Only the frame resolver knows whether /// it trimmed the swipe for a keyboard, and only `scroll` has this evidence to carry, so it is /// attached where the frame was resolved rather than threaded through every gesture response. - private func attachingScrollViewportEvidence(_ response: Response, keyboardMinY: Double?) -> Response { - guard response.ok, let keyboardMinY else { return response } - var payload = response.data ?? DataPayload() - payload.keyboardAvoided = true - payload.keyboardMinY = keyboardMinY - return Response(ok: response.ok, data: payload, error: response.error) - } - /// The refusal a keyboard forces. It performs no gesture: swiping into the keys would leave the /// surface where it was, which the daemon's no-progress fingerprint reads as a stuck container /// (#2499) and an agent reads as a broken scroll. The TS owner maps the code to the