From d4e7f4be12d318e9ad3b4c151fa9663b4d11b643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 14:10:45 +0200 Subject: [PATCH 1/2] fix(android): keep a scroll's swipe out of the IME window Android is the case the clip exists for beyond iOS: an `adjustPan` or `adjustNothing` activity keeps a window whose recorded bounds already run under the IME, so a plan built from them aims at the keys. The helper now reports the largest input method window beside the application window, in absolute screen pixels like the window next to it, and `scroll` clips its band with the shared rule or refuses when the keyboard owns the surface. An older helper reports no keyboard keys, and a provider-supplied viewport has no IME channel at all. Both read as "nothing to avoid", which is what the shared rule already does with a missing frame; neither turns into a refusal. `UiAutomation.getWindows()` answers with an empty list until the service asks for interactive windows, so the read applies the seam the tree capture already uses rather than depending on a snapshot capture having run first in the same instrumentation; the one-shot fallback below it has no such neighbour. Measured on a Pixel 7 emulator with an `adjust=pan` contact editor, the application window keeps its full 2400px height while the IME window reports `[0,1517][1080,2400]`, and `scroll down` answers with `referenceHeight: 1505`, `keyboardMinY: 1517`, `keyboardAvoided: true` and a swipe ending at 301 instead of starting at 1920 under the keys. --- .../snapshothelper/GestureViewportReader.java | 50 ++++++- .../snapshothelper/TouchCommandHandler.java | 18 ++- .../src/__tests__/input-actions.test.ts | 139 +++++++++++++++++- .../__tests__/touch-helper-session.test.ts | 21 +-- .../src/__tests__/touch-helper.test.ts | 76 +++++++++- .../platform-android/src/gesture-viewport.ts | 33 ++++- .../platform-android/src/input-actions.ts | 60 ++++++-- packages/platform-android/src/mechanics.ts | 2 +- .../platform-android/src/touch-executor.ts | 31 +++- packages/platform-android/src/touch-helper.ts | 33 ++++- website/docs/docs/commands.md | 2 +- 11 files changed, 415 insertions(+), 50 deletions(-) diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java index d54c3b6980..e2e2383928 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java @@ -7,12 +7,30 @@ import java.util.List; import java.util.concurrent.TimeoutException; -/** Resolves the active application window bounds used to validate planned gestures. */ +/** + * Resolves the active application window bounds used to validate planned gestures, plus the input + * method window's bounds when one is on screen. The keyboard half is what lets a scroll keep its + * swipe above the keys instead of flinging into them (#2500): the same {@code getWindows()} pass + * already lists {@code TYPE_INPUT_METHOD}, so reading it costs no extra automation round trip, and + * it is the live window list rather than a cached frame. + */ final class GestureViewportReader { private GestureViewportReader() {} + /** The application viewport a gesture may target, and the IME's share of the screen, if any. */ + static final class Reading { + final Rect application; + /** Null when no input method window is on screen; an unmeasurable keyboard is not occlusion. */ + final Rect inputMethod; + + Reading(Rect application, Rect inputMethod) { + this.application = application; + this.inputMethod = inputMethod; + } + } + @SuppressWarnings("deprecation") - static Rect read(UiAutomation automation) { + static Reading readReading(UiAutomation automation) { try { automation.waitForIdle(100, 2_000); } catch (TimeoutException ignored) { @@ -21,12 +39,29 @@ static Rect read(UiAutomation automation) { // UiAutomation.getWindows() transfers recyclable AccessibilityWindowInfo instances, and this // read runs repeatedly inside the persistent helper session: copy the bounds the precedence // below needs, then recycle every window before resolving. + // UiAutomation.getWindows() answers with an empty list until interactive retrieval is on, which + // is the same seam the tree capture already uses. Without it this read sees no windows at all and + // the keyboard below is invisible to it. + AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation); Rect activeBounds = null; Rect fallbackBounds = null; + Rect inputMethodBounds = null; List windows = automation.getWindows(); try { for (AccessibilityWindowInfo window : windows) { - if (window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION) continue; + int type = window.getType(); + if (type == AccessibilityWindowInfo.TYPE_INPUT_METHOD) { + // Keep the largest IME window: a composer bar and its key plane can be reported as + // separate windows, and the scroll only needs how far down the free surface reaches. + Rect bounds = new Rect(); + window.getBoundsInScreen(bounds); + if (!bounds.isEmpty() && (inputMethodBounds == null || bounds.height() * bounds.width() + > inputMethodBounds.height() * inputMethodBounds.width())) { + inputMethodBounds = bounds; + } + continue; + } + if (type != AccessibilityWindowInfo.TYPE_APPLICATION) continue; Rect bounds = new Rect(); window.getBoundsInScreen(bounds); if (activeBounds == null @@ -41,6 +76,15 @@ static Rect read(UiAutomation automation) { window.recycle(); } } + return new Reading(resolveApplication(automation, activeBounds, fallbackBounds), inputMethodBounds); + } + + static Rect read(UiAutomation automation) { + return readReading(automation).application; + } + + private static Rect resolveApplication( + UiAutomation automation, Rect activeBounds, Rect fallbackBounds) { if (activeBounds != null) return activeBounds; AccessibilityNodeInfo activeRoot = automation.getRootInActiveWindow(); if (activeRoot != null) { diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java index fd49995a35..93a6c82848 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java @@ -15,10 +15,11 @@ final class TouchCommandHandler { private TouchCommandHandler() {} static void populateViewport(Bundle result, UiAutomation automation) { - Rect viewport = GestureViewportReader.read(automation); + GestureViewportReader.Reading reading = GestureViewportReader.readReading(automation); result.putString("ok", "true"); result.putString("kind", "viewport"); - putViewportMetadata(result, viewport); + putViewportMetadata(result, reading.application); + putKeyboardMetadata(result, reading.inputMethod); } static void populateGesture(Bundle result, UiAutomation automation, String payloadBase64) @@ -69,4 +70,17 @@ private static void putViewportMetadata(Bundle result, Rect viewport) { result.putString("width", Integer.toString(viewport.width())); result.putString("height", Integer.toString(viewport.height())); } + + /** + * The input method window's screen bounds, reported only when one is on screen. An absent keyboard + * is reported by absence: a keyboard this helper cannot see is not evidence that a surface is + * blocked, so the caller must not read a zero frame as an occlusion. + */ + private static void putKeyboardMetadata(Bundle result, Rect inputMethod) { + if (inputMethod == null) return; + result.putString("keyboardX", Integer.toString(inputMethod.left)); + result.putString("keyboardY", Integer.toString(inputMethod.top)); + result.putString("keyboardWidth", Integer.toString(inputMethod.width())); + result.putString("keyboardHeight", Integer.toString(inputMethod.height())); + } } diff --git a/packages/platform-android/src/__tests__/input-actions.test.ts b/packages/platform-android/src/__tests__/input-actions.test.ts index 5b3c279059..f9615b83ed 100644 --- a/packages/platform-android/src/__tests__/input-actions.test.ts +++ b/packages/platform-android/src/__tests__/input-actions.test.ts @@ -1,4 +1,4 @@ -import { test, vi } from 'vitest'; +import { afterEach, beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { backAndroid, @@ -13,10 +13,147 @@ import { ANDROID_EMULATOR } from './test-utils/device-fixtures.ts'; import { withFakeAdb } from './test-utils/fake-adb.ts'; import { withAndroidAdbProvider } from '../adb-executor.ts'; import type { AndroidTouchInjector } from '../adb-executor.ts'; +import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; +import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts'; +import { + ANDROID_TOUCH_HELPER_MANIFEST as HELPER_MANIFEST, + androidTouchHelperResultRecord as helperRecord, +} from './touch-helper.fixtures.ts'; // The fake adb provider installs through the production withAndroidAdbProvider // scope, so `calls` records device-scoped args without a leading `-s `. +// The keyboard-aware viewport read goes through the snapshot helper rather than a touch provider, +// so the scroll tests below resolve the fixture APK instead of a bundled one. +vi.mock('../helper-package-install.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveAndroidHelperArtifact: async () => ({ + apkPath: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT.apkPath, + manifest: { + ...HELPER_MANIFEST, + sha256: ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT.manifest.sha256, + }, + }), + }; +}); + +beforeEach(async () => { + delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; + await resetAndroidSnapshotHelperSessions(); +}); + +afterEach(async () => { + delete process.env.AGENT_DEVICE_ANDROID_SNAPSHOT_HELPER_SESSION; + await resetAndroidSnapshotHelperSessions(); +}); + +/** + * Answers one helper session: the version probe says "current" so no install is faked, the viewport + * read reports `window` plus an optional IME window, and the gesture accepts whatever it is given. + */ +function helperRouteAdb(window: { + x: number; + y: number; + width: number; + height: number; + keyboard?: { x: number; y: number; width: number; height: number }; +}) { + return (args: string[]) => { + if (args.includes('--show-versioncode')) { + return { + stdout: `package:${HELPER_MANIFEST.packageName} versionCode:999999`, + stderr: '', + }; + } + if (args.includes('viewport')) { + const { keyboard, ...app } = window; + return { + stdout: [ + helperRecord({ + ok: 'true', + x: String(app.x), + y: String(app.y), + width: String(app.width), + height: String(app.height), + ...(keyboard + ? { + keyboardX: String(keyboard.x), + keyboardY: String(keyboard.y), + keyboardWidth: String(keyboard.width), + keyboardHeight: String(keyboard.height), + } + : {}), + }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + } + if (args[0] === 'shell' && args[1] === 'am') { + return { + stdout: [ + helperRecord({ ok: 'true', kind: 'pan', injectedEvents: '18', elapsedMs: '320' }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + }; + } + return undefined; + }; +} + +const PORTRAIT_WINDOW = { x: 0, y: 0, width: 1080, height: 2280 }; +const LOWER_HALF_KEYBOARD = { x: 0, y: 1600, width: 1080, height: 680 }; + +test('scrollAndroid keeps the swipe above the IME window and names the clipped band', async () => { + // The full window would place a center-symmetric swipe at y 1140..~1500 — on the keys. Clipping + // first means the injected path and the reported reference height both stop above the keyboard + // by the accessory allowance, so a focused field no longer swallows the gesture (#2500). + await withFakeAdb( + helperRouteAdb({ ...PORTRAIT_WINDOW, keyboard: LOWER_HALF_KEYBOARD }), + async ({ device }) => { + const result = await scrollAndroid(device, 'down', { pixels: 600 }); + const lowest = Math.max(Number(result.y1), Number(result.y2)); + assert.equal(result.keyboardAvoided, true); + assert.equal(result.keyboardMinY, 1600); + assert.equal(result.referenceHeight, 1588, 'clipped axis is the band above the allowance'); + assert.equal(result.pixels, 600, 'requested travel fits the clipped band'); + assert.ok(lowest <= 1588, `swipe endpoint ${lowest} landed under the keyboard`); + }, + ); +}); + +test('scrollAndroid swipes the whole window when no IME window is on screen', async () => { + await withFakeAdb(helperRouteAdb({ ...PORTRAIT_WINDOW }), async ({ device }) => { + const result = await scrollAndroid(device, 'down', { pixels: 600 }); + assert.equal('keyboardAvoided' in result, false); + assert.equal(result.referenceHeight, 2280); + }); +}); + +test('scrollAndroid refuses rather than flinging into a keyboard that owns the window', async () => { + // A landscape IME leaves 40px of a 900px window: a swipe there reads as a stuck surface, so the + // command refuses with its own typed reason instead of the generic no-progress stop. + await withFakeAdb( + helperRouteAdb({ + ...PORTRAIT_WINDOW, + keyboard: { x: 0, y: 120, width: 1080, height: 2160 }, + }), + async ({ device }) => { + await assert.rejects(scrollAndroid(device, 'down', { pixels: 600 }), (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal( + (error as { details?: { reason?: string } }).details?.reason, + 'scroll_keyboard_occludes_surface', + ); + return true; + }); + }, + ); +}); + test('scrollAndroid plans explicit pixel travel through semantic touch injection', async () => { const touchCalls: Parameters[0][] = []; const result = await withAndroidAdbProvider( diff --git a/packages/platform-android/src/__tests__/touch-helper-session.test.ts b/packages/platform-android/src/__tests__/touch-helper-session.test.ts index ac22694379..29b71f2485 100644 --- a/packages/platform-android/src/__tests__/touch-helper-session.test.ts +++ b/packages/platform-android/src/__tests__/touch-helper-session.test.ts @@ -22,7 +22,10 @@ import { captureAndroidSnapshotWithHelperSession } from '../snapshot-helper-sess import { resetAndroidSnapshotHelperSessions } from '../snapshot-helper-session-lifecycle.ts'; import { getAndroidSnapshotHelperSessionDeviceKey } from '../snapshot-helper-retirement.ts'; import { lowerAndroidTouchPlan } from '../touch-plan-lowering.ts'; -import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from '../touch-helper.ts'; +import { + executeAndroidTouchHelperPlan, + readAndroidTouchHelperViewportReading, +} from '../touch-helper.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts'; import { ANDROID_TOUCH_HELPER_MANIFEST as manifest, @@ -250,7 +253,7 @@ test('a daemon-session viewport read starts the session so the gesture reuses it }, { serial: device.id }, async () => { - const viewport = await readAndroidTouchHelperViewport(device, { + const viewport = await readAndroidTouchHelperViewportReading(device, { helperSessionScope: 'daemon-session', }); const gesture = await executeAndroidTouchHelperPlan( @@ -261,7 +264,7 @@ test('a daemon-session viewport read starts the session so the gesture reuses it }, ); - assert.deepEqual(result.viewport, { x: 0, y: 0, width: 400, height: 800 }); + assert.deepEqual(result.viewport, { viewport: { x: 0, y: 0, width: 400, height: 800 } }); assert.equal(result.gesture.helperTransport, 'persistent-session'); assert.equal(viewportCommands, 1); assert.equal(gestureCommands, 1); @@ -297,10 +300,10 @@ test('a command-scoped viewport read stays one-shot and starts no session', asyn }), }, { serial: device.id }, - async () => await readAndroidTouchHelperViewport(device), + async () => await readAndroidTouchHelperViewportReading(device), ); - assert.deepEqual(viewport, { x: 5, y: 6, width: 300, height: 400 }); + assert.deepEqual(viewport, { viewport: { x: 5, y: 6, width: 300, height: 400 } }); assert.ok(oneShotArgs?.includes('viewport')); assert.equal(sessionCommands, 0); }); @@ -673,10 +676,10 @@ test('viewport falls back to one-shot instrumentation after a session error', as }), }, { serial: device.id }, - async () => await readAndroidTouchHelperViewport(device), + async () => await readAndroidTouchHelperViewportReading(device), ); - assert.deepEqual(viewportResult, { x: 5, y: 6, width: 300, height: 400 }); + assert.deepEqual(viewportResult, { viewport: { x: 5, y: 6, width: 300, height: 400 } }); assert.ok(oneShotArgs?.includes('viewport')); assert.equal(await session.isSessionAlive(), false); }); @@ -719,9 +722,9 @@ test('a structured ok=false viewport response stops the session before the one-s }), }, { serial: device.id }, - async () => await readAndroidTouchHelperViewport(device), + async () => await readAndroidTouchHelperViewportReading(device), ); - assert.deepEqual(viewportResult, { x: 5, y: 6, width: 300, height: 400 }); + assert.deepEqual(viewportResult, { viewport: { x: 5, y: 6, width: 300, height: 400 } }); assert.ok(oneShotArgs?.includes('viewport')); }); diff --git a/packages/platform-android/src/__tests__/touch-helper.test.ts b/packages/platform-android/src/__tests__/touch-helper.test.ts index afebafee6f..036f2a8af3 100644 --- a/packages/platform-android/src/__tests__/touch-helper.test.ts +++ b/packages/platform-android/src/__tests__/touch-helper.test.ts @@ -12,7 +12,7 @@ import { executeAndroidTouchHelperPlan, normalizeAndroidTouchHelperGestureRequest, readAndroidTouchHelperFinalRecord, - readAndroidTouchHelperViewport, + readAndroidTouchHelperViewportReading, } from '../touch-helper.ts'; import { resolveAndroidHelperArtifact } from '../helper-package-install.ts'; import { ANDROID_SNAPSHOT_HELPER_FIXTURE_ARTIFACT } from './test-utils/android-snapshot-helper.ts'; @@ -394,7 +394,7 @@ test('one-shot viewport instruments the snapshot-helper runner and validates bou }), }, { serial: device.id }, - async () => await readAndroidTouchHelperViewport(device), + async () => await readAndroidTouchHelperViewportReading(device), ); assert.deepEqual(capturedArgs, [ @@ -407,7 +407,73 @@ test('one-shot viewport instruments the snapshot-helper runner and validates bou 'viewport', manifest.instrumentationRunner, ]); - assert.deepEqual(viewportResult, { x: 10, y: 20, width: 300, height: 500 }); + assert.deepEqual(viewportResult, { viewport: { x: 10, y: 20, width: 300, height: 500 } }); +}); + +test('one-shot viewport reports the input method window it read beside the app window', async () => { + // #2500: the scroll owner needs the live IME bounds, and the helper already walked the window list + // that contains them. They arrive in the same absolute screen space as the app window. + const device = makeIsolatedDevice(); + const reading = await withAndroidAdbProvider( + { + exec: currentVersionAdb(async () => ({ + exitCode: 0, + stdout: [ + resultRecord({ + ok: 'true', + x: '0', + y: '120', + width: '1080', + height: '2000', + keyboardX: '0', + keyboardY: '1600', + keyboardWidth: '1080', + keyboardHeight: '520', + }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + })), + }, + { serial: device.id }, + async () => await readAndroidTouchHelperViewportReading(device), + ); + + assert.deepEqual(reading, { + viewport: { x: 0, y: 120, width: 1080, height: 2000 }, + keyboard: { x: 0, y: 1600, width: 1080, height: 520 }, + }); +}); + +test('a helper that reports no input method window yields no keyboard rather than a zero frame', async () => { + // An installed helper older than the keyboard read omits the keys entirely; a helper that cannot + // see the IME must not become evidence that every scroll is blocked. + const device = makeIsolatedDevice(); + const reading = await withAndroidAdbProvider( + { + exec: currentVersionAdb(async () => ({ + exitCode: 0, + stdout: [ + resultRecord({ + ok: 'true', + x: '0', + y: '0', + width: '1080', + height: '2280', + keyboardX: '0', + keyboardY: '1600', + keyboardWidth: '1080', + }), + 'INSTRUMENTATION_CODE: 0', + ].join('\n'), + stderr: '', + })), + }, + { serial: device.id }, + async () => await readAndroidTouchHelperViewportReading(device), + ); + + assert.deepEqual(reading, { viewport: { x: 0, y: 0, width: 1080, height: 2280 } }); }); test('one-shot viewport rejects invalid bounds', async () => { @@ -425,7 +491,7 @@ test('one-shot viewport rejects invalid bounds', async () => { })), }, { serial: device.id }, - async () => await readAndroidTouchHelperViewport(device), + async () => await readAndroidTouchHelperViewportReading(device), ), { code: 'COMMAND_FAILED' }, ); @@ -450,7 +516,7 @@ test('one-shot viewport failure preserves its structured message and error type' })), }, { serial: device.id }, - async () => await readAndroidTouchHelperViewport(device), + async () => await readAndroidTouchHelperViewportReading(device), ), (error: unknown) => { assert.ok(error instanceof AppError); diff --git a/packages/platform-android/src/gesture-viewport.ts b/packages/platform-android/src/gesture-viewport.ts index 1131da493e..e79d0e2636 100644 --- a/packages/platform-android/src/gesture-viewport.ts +++ b/packages/platform-android/src/gesture-viewport.ts @@ -1,15 +1,32 @@ import { AppError } from '@agent-device/kernel/errors'; import type { Rect } from '@agent-device/kernel/snapshot'; +/** + * What the helper reports about the surface a gesture may target: the application window, and the + * input method window's share of the screen when a keyboard is on screen. + * + * The keyboard arrives as absolute screen pixels — the same space as the application window's + * `getBoundsInScreen()` — and is never converted into another platform's coordinate space. + */ +export type AndroidGestureViewportReading = Readonly<{ + viewport: Rect; + /** + * Absent when no input method window is on screen, or when the installed helper predates the + * keyboard read. Either way there is nothing to avoid; absence is not evidence of occlusion. + */ + keyboard?: Rect; +}>; + export function validateAndroidGestureViewport(viewport: Rect): Rect { - if ( - !Number.isFinite(viewport.x) || - !Number.isFinite(viewport.y) || - !Number.isFinite(viewport.width) || - !Number.isFinite(viewport.height) || - viewport.width <= 0 || - viewport.height <= 0 - ) + if (!isMeasurableRect(viewport)) throw new AppError('COMMAND_FAILED', 'Android helper returned an invalid gesture viewport'); return viewport; } + +export function isMeasurableRect(rect: Rect): boolean { + return ( + [rect.x, rect.y, rect.width, rect.height].every((value) => Number.isFinite(value)) && + rect.width > 0 && + rect.height > 0 + ); +} diff --git a/packages/platform-android/src/input-actions.ts b/packages/platform-android/src/input-actions.ts index 7648dec605..bafed2c593 100644 --- a/packages/platform-android/src/input-actions.ts +++ b/packages/platform-android/src/input-actions.ts @@ -12,13 +12,16 @@ import { import { type ScrollDirection, buildScrollGesturePlan, + clipScrollViewportAboveKeyboard, + scrollKeyboardOccludesSurfaceError, } from '@agent-device/contracts/scroll-gesture'; import { type TvRemoteButton, toAndroidTvRemoteKeyevent } from '@agent-device/contracts/tv-remote'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { AppError } from '@agent-device/kernel/errors'; +import type { Rect } from '@agent-device/kernel/snapshot'; import { sleep } from '@agent-device/host-kit/retry'; import { runAndroidAdb } from './adb.ts'; -import { executeAndroidTouchPlan, readAndroidGestureViewport } from './touch-executor.ts'; +import { executeAndroidTouchPlan, readAndroidGestureViewportReading } from './touch-executor.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; export async function pressAndroid(device: DeviceInfo, x: number, y: number): Promise { @@ -175,26 +178,27 @@ export async function scrollAndroid( ): Promise> { // The viewport read and the gesture are two helper calls one command apart: giving the read the // command's session scope keeps both on the same instrumentation. - const viewport = await readAndroidGestureViewport(device, { + const { viewport, keyboard } = await readAndroidGestureViewportReading(device, { helperSessionScope: options?.helperSessionScope, }); + const swipeSurface = resolveAndroidScrollSurface(direction, viewport, keyboard); const relativePlan = buildScrollGesturePlan({ direction, amount: options?.amount, pixels: options?.pixels, - referenceWidth: viewport.width, - referenceHeight: viewport.height, + referenceWidth: swipeSurface.viewport.width, + referenceHeight: swipeSurface.viewport.height, }); const scrollPlan = { ...relativePlan, // Injected coordinates are absolute, so their zero-origin reference frame // must include the viewport offset as well as its dimensions. - referenceWidth: viewport.x + viewport.width, - referenceHeight: viewport.y + viewport.height, - x1: viewport.x + relativePlan.x1, - y1: viewport.y + relativePlan.y1, - x2: viewport.x + relativePlan.x2, - y2: viewport.y + relativePlan.y2, + referenceWidth: swipeSurface.viewport.x + swipeSurface.viewport.width, + referenceHeight: swipeSurface.viewport.y + swipeSurface.viewport.height, + x1: swipeSurface.viewport.x + relativePlan.x1, + y1: swipeSurface.viewport.y + relativePlan.y1, + x2: swipeSurface.viewport.x + relativePlan.x2, + y2: swipeSurface.viewport.y + relativePlan.y2, }; const durationMs = Math.max( options?.durationMs ?? DEFAULT_MOBILE_SCROLL_DURATION_MS, @@ -211,7 +215,7 @@ export async function scrollAndroid( }, durationMs, }, - viewport, + swipeSurface.viewport, 'android', ), releaseBehavior: options?.releaseBehavior ?? 'controlled', @@ -219,11 +223,45 @@ export async function scrollAndroid( return { ...scrollPlan, + ...swipeSurface.evidence, ...(options?.durationMs !== undefined ? { durationMs } : {}), ...backend, }; } +/** + * The band one Android scroll may swipe, refusing when the keyboard owns the window (#2500). + * + * The IME bounds come from this command's own window read, in absolute screen pixels like the + * application window beside them; they are never shared with another platform's space. Android is + * the case the clip exists for beyond iOS: an `adjustPan` or `adjustNothing` activity keeps a window + * whose recorded bounds already run under the IME, so a plan built from them aims at keys. + */ +type AndroidScrollSurface = Readonly<{ + viewport: Rect; + evidence: Readonly<{ keyboardAvoided?: true; keyboardMinY?: number }>; +}>; + +function resolveAndroidScrollSurface( + direction: ScrollDirection, + viewport: Rect, + keyboard: Rect | undefined, +): AndroidScrollSurface { + if (keyboard === undefined) return { viewport, evidence: {} }; + const clip = clipScrollViewportAboveKeyboard(viewport, keyboard); + if (clip.kind === 'occluded') { + throw scrollKeyboardOccludesSurfaceError(direction, { + ...clip, + viewportHeight: viewport.height, + }); + } + if (clip.kind !== 'avoided') return { viewport, evidence: {} }; + return { + viewport: clip.viewport, + evidence: { keyboardAvoided: true, keyboardMinY: clip.keyboardMinY }, + }; +} + function resolveAndroidUserRotation(orientation: DeviceRotation): string { const index = DEVICE_ROTATION_SURFACE_INDEX[orientation]; if (index === undefined) { diff --git a/packages/platform-android/src/mechanics.ts b/packages/platform-android/src/mechanics.ts index 721349d43c..f1b89026aa 100644 --- a/packages/platform-android/src/mechanics.ts +++ b/packages/platform-android/src/mechanics.ts @@ -322,7 +322,7 @@ export { executeAndroidTouchHelperPlan, normalizeAndroidTouchHelperGestureRequest, readAndroidTouchHelperFinalRecord, - readAndroidTouchHelperViewport, + readAndroidTouchHelperViewportReading, } from './touch-helper.ts'; export { lowerAndroidTouchPlan, diff --git a/packages/platform-android/src/touch-executor.ts b/packages/platform-android/src/touch-executor.ts index 93bbe70e68..a88572a097 100644 --- a/packages/platform-android/src/touch-executor.ts +++ b/packages/platform-android/src/touch-executor.ts @@ -1,8 +1,14 @@ import type { DeviceInfo } from '@agent-device/kernel/device'; import type { Rect } from '@agent-device/kernel/snapshot'; import { resolveAndroidTouchProvider } from './adb-executor.ts'; -import { executeAndroidTouchHelperPlan, readAndroidTouchHelperViewport } from './touch-helper.ts'; -import { validateAndroidGestureViewport } from './gesture-viewport.ts'; +import { + executeAndroidTouchHelperPlan, + readAndroidTouchHelperViewportReading, +} from './touch-helper.ts'; +import { + validateAndroidGestureViewport, + type AndroidGestureViewportReading, +} from './gesture-viewport.ts'; import { lowerAndroidTouchPlan, type AndroidTouchPlan } from './touch-plan-lowering.ts'; import type { AndroidHelperSessionOptions } from './snapshot-helper-types.ts'; @@ -30,7 +36,24 @@ export async function readAndroidGestureViewport( device: DeviceInfo, helper: AndroidHelperSessionOptions = {}, ): Promise { + return (await readAndroidGestureViewportReading(device, helper)).viewport; +} + +/** + * The application viewport and the input method window's share of it, from one live window read. + * + * `scroll` needs both: a swipe planned against the unobstructed window lands on the keyboard when a + * field is focused, and a frame cached by an earlier command predates the keyboard. A + * provider-supplied viewport has no IME channel, so it reports no keyboard — which the clip rule + * reads as "nothing to avoid" rather than as an occlusion. + */ +export async function readAndroidGestureViewportReading( + device: DeviceInfo, + helper: AndroidHelperSessionOptions = {}, +): Promise { const provider = resolveAndroidTouchProvider(device); - if (provider) return validateAndroidGestureViewport(await provider.gestureViewport()); - return await readAndroidTouchHelperViewport(device, helper); + if (provider) { + return { viewport: validateAndroidGestureViewport(await provider.gestureViewport()) }; + } + return await readAndroidTouchHelperViewportReading(device, helper); } diff --git a/packages/platform-android/src/touch-helper.ts b/packages/platform-android/src/touch-helper.ts index c5a1c5619c..8b49554eab 100644 --- a/packages/platform-android/src/touch-helper.ts +++ b/packages/platform-android/src/touch-helper.ts @@ -14,7 +14,11 @@ import { parseInstrumentationRecords, readInstrumentationResultNumber, } from './instrumentation-helper.ts'; -import { validateAndroidGestureViewport } from './gesture-viewport.ts'; +import { + isMeasurableRect, + validateAndroidGestureViewport, + type AndroidGestureViewportReading, +} from './gesture-viewport.ts'; import type { AndroidLoweredTouchPlan } from './touch-plan-lowering.ts'; import { resolveAndroidHelperArtifact } from './helper-package-install.ts'; import { parseAndroidSnapshotHelperManifest } from './snapshot-helper-artifact.ts'; @@ -116,10 +120,10 @@ export async function executeAndroidTouchHelperPlan( }; } -export async function readAndroidTouchHelperViewport( +export async function readAndroidTouchHelperViewportReading( device: DeviceInfo, helper: AndroidHelperSessionOptions = {}, -): Promise { +): Promise { const prepared = await prepareAndroidTouchHelper(device); if (helper.helperSessionScope === 'daemon-session') { // Without a live session both this read and the gesture that follows would each start their @@ -329,7 +333,7 @@ function readGestureResult(record: Record): Record): Rect { +function readViewportResult(record: Record): AndroidGestureViewportReading { const x = readInstrumentationResultNumber(record.x); const y = readInstrumentationResultNumber(record.y); const width = readInstrumentationResultNumber(record.width); @@ -337,5 +341,24 @@ function readViewportResult(record: Record): Rect { if (x === undefined || y === undefined || width === undefined || height === undefined) { throw new AppError('COMMAND_FAILED', 'Android helper returned an invalid gesture viewport'); } - return validateAndroidGestureViewport({ x, y, width, height }); + const keyboard = readKeyboardResult(record); + return { + viewport: validateAndroidGestureViewport({ x, y, width, height }), + ...(keyboard ? { keyboard } : {}), + }; +} + +/** Absence of any keyboard key is the helper's way of saying no input method window is on screen. */ +function readKeyboardResult(record: Record): Rect | undefined { + const x = readInstrumentationResultNumber(record.keyboardX); + const y = readInstrumentationResultNumber(record.keyboardY); + const width = readInstrumentationResultNumber(record.keyboardWidth); + const height = readInstrumentationResultNumber(record.keyboardHeight); + if (x === undefined || y === undefined || width === undefined || height === undefined) { + return undefined; + } + // An IME window the helper cannot size is dropped, not refused: the clip rule already fails open + // on a missing frame, and a helper that cannot see it must not fail every scroll. + const keyboard = { x, y, width, height }; + return isMeasurableRect(keyboard) ? keyboard : undefined; } diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 4c389e95c9..c35c37f311 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -492,7 +492,7 @@ Target-authored drag is supported on Android touch devices and iOS/iPadOS. Backe On iOS simulators it uses private XCTest synthesis for a continuous two-finger pan/scale/rotation path, so verify app-level metrics instead of assuming the requested values map exactly to recognizer output. On Android, `gesture transform` injects a geometric two-finger path. App recognizers may report non-exact pan, scale, and rotation values, so verify qualitative state such as `pan changed yes`, `pinch changed yes`, and `rotate changed yes` unless the app explicitly promises exact centroid metrics. If exact app-state values matter, prefer isolated `gesture pan`, `gesture pinch`, or `gesture rotate` commands. `scroll` accepts either a relative amount (`0.5` means a finger path spanning half of the viewport on that axis) or `--pixels ` for a fixed-distance gesture. Directional scrolls decelerate through the drag on Android to reduce release momentum within the requested duration; `scroll top` and `scroll bottom` retain inertial release for edge traversal. Reduced momentum does not guarantee an exact content offset, especially for very short gestures: apps apply pan-recognition thresholds, collapsing headers, bounds, and their own scroll physics. Large distances are clamped to the usable drag band so the gesture stays reliable across Android, iOS, and macOS. -A directional scroll places its swipe across the middle of the viewport, so a focused field and its keyboard would put the swipe under the keys: the gesture would land on the keyboard, the surface would not move, and the scroll would read as stuck. On iOS the scroll instead keeps the whole swipe in the band above the keyboard, reporting `keyboardAvoided` and `keyboardMinY` alongside a `referenceHeight` and `pixels` measured against that shorter band. It never dismisses the keyboard, because dismissing drops focus and breaks a `fill`/`scroll`/`fill` loop; run `keyboard dismiss` yourself when you want that. When the keyboard leaves too little room to swipe, the command refuses with the `scroll_keyboard_occludes_surface` reason rather than swiping into the keys, so a scroll that cannot work says so instead of appearing stuck. +A directional scroll places its swipe across the middle of the viewport, so a focused field and its keyboard would put the swipe under the keys: the gesture would land on the keyboard, the surface would not move, and the scroll would read as stuck. On iOS and Android the scroll instead keeps the whole swipe in the band above the keyboard, reporting `keyboardAvoided` and `keyboardMinY` alongside a `referenceHeight` and `pixels` measured against that shorter band. It never dismisses the keyboard, because dismissing drops focus and breaks a `fill`/`scroll`/`fill` loop; run `keyboard dismiss` yourself when you want that. When the keyboard leaves too little room to swipe, the command refuses with the `scroll_keyboard_occludes_surface` reason rather than swiping into the keys, so a scroll that cannot work says so instead of appearing stuck. Default snapshot text output is visible-first, so off-screen interactive content is summarized instead of shown as tappable refs. When a target only appears in an off-screen summary, use `scroll --settle`: the response waits for the UI to go quiet and returns the diff against the tree you last observed, with fresh refs on the added lines, so no follow-up `snapshot -i` is needed. `back --settle` does the same for navigation. Both are best-effort and never fail the action. For repeated checks without settle, a small shell loop is enough: From 5d55192f3eb29613a81968de9420e4f77de798fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 12 Sep 2026 18:49:49 +0200 Subject: [PATCH 2/2] fix(android): clear the composer, not just the key plane, before a scroll swipes The helper kept the largest `TYPE_INPUT_METHOD` rectangle as the keyboard. A composer bar and its key plane can arrive as separate windows and the key plane is the larger one, so the earlier top edge was discarded and the clipped band still ended inside the composer: the swipe landed on keys the rule exists to keep it off. The read now copies every input method window and unions the ones the swipe's centre line crosses, which is the same line the shared clip rule tests. A candidate strip at the edge of the screen that the swipe can never reach no longer shortens the band either. The selection runs on plain window edges, because `Rect` is a device type whose constructors throw off-device, so the two-window case is a unit test rather than a simulator-only path. --- .../snapshothelper/GestureViewportReader.java | 75 ++++++++++++++-- .../GestureViewportReaderTest.java | 87 +++++++++++++++++++ .../SnapshotHelperTestSuite.java | 1 + 3 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/GestureViewportReaderTest.java diff --git a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java index e2e2383928..93cf14e16c 100644 --- a/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java +++ b/android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java @@ -4,6 +4,7 @@ import android.graphics.Rect; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeoutException; @@ -29,6 +30,33 @@ static final class Reading { } } + /** + * One reported window's edges in screen pixels. Plain fields because {@code Rect} is a device type + * whose constructors throw off-device, and which of several input method windows a swipe strikes is + * arithmetic that has to be testable without one. + */ + static final class WindowEdges { + final int left; + final int top; + final int right; + final int bottom; + + WindowEdges(int left, int top, int right, int bottom) { + this.left = left; + this.top = top; + this.right = right; + this.bottom = bottom; + } + + static WindowEdges of(Rect rect) { + return new WindowEdges(rect.left, rect.top, rect.right, rect.bottom); + } + + Rect toRect() { + return new Rect(left, top, right, bottom); + } + } + @SuppressWarnings("deprecation") static Reading readReading(UiAutomation automation) { try { @@ -45,20 +73,18 @@ static Reading readReading(UiAutomation automation) { AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation); Rect activeBounds = null; Rect fallbackBounds = null; - Rect inputMethodBounds = null; + List inputMethodWindows = new ArrayList<>(); List windows = automation.getWindows(); try { for (AccessibilityWindowInfo window : windows) { int type = window.getType(); if (type == AccessibilityWindowInfo.TYPE_INPUT_METHOD) { - // Keep the largest IME window: a composer bar and its key plane can be reported as - // separate windows, and the scroll only needs how far down the free surface reaches. + // Copy every input method window. Which of them a swipe has to clear depends on the + // application window, which this loop has not finished reading, so they are collected here + // and resolved once it has. Rect bounds = new Rect(); window.getBoundsInScreen(bounds); - if (!bounds.isEmpty() && (inputMethodBounds == null || bounds.height() * bounds.width() - > inputMethodBounds.height() * inputMethodBounds.width())) { - inputMethodBounds = bounds; - } + if (!bounds.isEmpty()) inputMethodWindows.add(WindowEdges.of(bounds)); continue; } if (type != AccessibilityWindowInfo.TYPE_APPLICATION) continue; @@ -76,7 +102,40 @@ static Reading readReading(UiAutomation automation) { window.recycle(); } } - return new Reading(resolveApplication(automation, activeBounds, fallbackBounds), inputMethodBounds); + Rect application = resolveApplication(automation, activeBounds, fallbackBounds); + WindowEdges struck = struckInputMethod( + inputMethodWindows, application == null ? null : WindowEdges.of(application)); + return new Reading(application, struck == null ? null : struck.toRect()); + } + + /** + * The input method share a swipe has to stay above, or null when none of it is in the way. + * + *

A composer bar and its key plane can arrive as separate windows, and the larger rectangle is + * usually the lower key plane: keeping only that leaves the swipe inside the composer reaching + * further up the screen. So this unions the windows the swipe's centre line crosses — the same line + * the shared clip rule tests — and ignores the ones beside it that the swipe cannot reach. + */ + static WindowEdges struckInputMethod(List inputMethodWindows, WindowEdges application) { + WindowEdges struck = null; + for (WindowEdges bounds : inputMethodWindows) { + if (application != null) { + double swipeCenterX = application.left + (application.right - application.left) / 2.0; + boolean strikesSwipePath = swipeCenterX >= bounds.left && swipeCenterX < bounds.right; + boolean overlapsWindow = bounds.bottom > application.top && bounds.top < application.bottom; + if (!strikesSwipePath || !overlapsWindow) continue; + } + if (struck == null) { + struck = bounds; + continue; + } + struck = new WindowEdges( + Math.min(struck.left, bounds.left), + Math.min(struck.top, bounds.top), + Math.max(struck.right, bounds.right), + Math.max(struck.bottom, bounds.bottom)); + } + return struck; } static Rect read(UiAutomation automation) { diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/GestureViewportReaderTest.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/GestureViewportReaderTest.java new file mode 100644 index 0000000000..cb12d6dcc2 --- /dev/null +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/GestureViewportReaderTest.java @@ -0,0 +1,87 @@ +package com.callstack.agentdevice.snapshothelper; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public final class GestureViewportReaderTest { + private GestureViewportReaderTest() {} + + private static final GestureViewportReader.WindowEdges APPLICATION = + new GestureViewportReader.WindowEdges(0, 0, 1080, 2400); + private static final GestureViewportReader.WindowEdges KEY_PLANE = + new GestureViewportReader.WindowEdges(0, 1517, 1080, 2400); + private static final GestureViewportReader.WindowEdges COMPOSER = + new GestureViewportReader.WindowEdges(0, 1400, 1080, 1517); + private static final GestureViewportReader.WindowEdges SIDE_STRIP = + new GestureViewportReader.WindowEdges(900, 1200, 1080, 2400); + + static void run() { + assertNoInputMethodOnScreen(); + assertComposerAboveItsKeyPlaneKeepsItsEarlierTopEdge(); + assertWindowBesideTheSwipePathIsIgnored(); + } + + private static void assertNoInputMethodOnScreen() { + assertEdges( + GestureViewportReader.struckInputMethod( + Collections.emptyList(), APPLICATION), + null, + "no input method window on screen"); + } + + private static void assertComposerAboveItsKeyPlaneKeepsItsEarlierTopEdge() { + List both = Arrays.asList(KEY_PLANE, COMPOSER); + // The key plane is the larger rectangle. Keeping only it would plan a swipe ending inside the + // composer, whose top edge reaches 117px further up the screen. + assertEdges( + GestureViewportReader.struckInputMethod(both, APPLICATION), + new GestureViewportReader.WindowEdges(0, 1400, 1080, 2400), + "composer above its key plane"); + assertEdges( + GestureViewportReader.struckInputMethod(Arrays.asList(COMPOSER, KEY_PLANE), APPLICATION), + new GestureViewportReader.WindowEdges(0, 1400, 1080, 2400), + "composer listed after its key plane"); + } + + private static void assertWindowBesideTheSwipePathIsIgnored() { + // A floating candidate strip at the right edge never crosses the centre line a vertical swipe + // travels, so its higher top edge must not shorten the band. + assertEdges( + GestureViewportReader.struckInputMethod(Arrays.asList(KEY_PLANE, SIDE_STRIP), APPLICATION), + KEY_PLANE, + "input method window beside the swipe path"); + assertEdges( + GestureViewportReader.struckInputMethod( + Collections.singletonList(SIDE_STRIP), APPLICATION), + null, + "only an unreachable input method window on screen"); + } + + private static void assertEdges( + GestureViewportReader.WindowEdges actual, + GestureViewportReader.WindowEdges expected, + String label) { + if (expected == null) { + if (actual != null) { + throw new AssertionError( + "Expected no input method rect for " + label + ", got " + describe(actual)); + } + return; + } + if (actual == null) { + throw new AssertionError("Expected " + describe(expected) + " for " + label + ", got none"); + } + if (actual.left != expected.left + || actual.top != expected.top + || actual.right != expected.right + || actual.bottom != expected.bottom) { + throw new AssertionError( + "Expected " + describe(expected) + " for " + label + ", got " + describe(actual)); + } + } + + private static String describe(GestureViewportReader.WindowEdges edges) { + return "[" + edges.left + "," + edges.top + "][" + edges.right + "," + edges.bottom + "]"; + } +} diff --git a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java index 22453fc334..868e1f0211 100644 --- a/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java +++ b/android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java @@ -7,5 +7,6 @@ public static void main(String[] args) throws Exception { PointerEventScheduleTest.run(); AccessibilityCaptureStabilizerTest.run(); BoundedUiAutomationConnectionTest.run(); + GestureViewportReaderTest.run(); } }