fix(android): keep a scroll's swipe out of the IME window

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/GestureViewportReader.java

@@ -4,15 +4,61 @@
 import android.graphics.Rect;
 import android.view.accessibility.AccessibilityNodeInfo;
 import android.view.accessibility.AccessibilityWindowInfo;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 
-/** Resolves the active application window bounds used to validate planned gestures. */
+/**
+ * Resolves the active application window bounds used to validate planned gestures, plus the input
+ * method window's bounds when one is on screen. The keyboard half is what lets a scroll keep its
+ * swipe above the keys instead of flinging into them (#2500): the same {@code getWindows()} pass
+ * already lists {@code TYPE_INPUT_METHOD}, so reading it costs no extra automation round trip, and
+ * it is the live window list rather than a cached frame.
+ */
 final class GestureViewportReader {
   private GestureViewportReader() {}
 
+  /** The application viewport a gesture may target, and the IME's share of the screen, if any. */
+  static final class Reading {
+    final Rect application;
+    /** Null when no input method window is on screen; an unmeasurable keyboard is not occlusion. */
+    final Rect inputMethod;
+
+    Reading(Rect application, Rect inputMethod) {
+      this.application = application;
+      this.inputMethod = inputMethod;
+    }
+  }
+
+  /**
+   * One reported window's edges in screen pixels. Plain fields because {@code Rect} is a device type
+   * whose constructors throw off-device, and which of several input method windows a swipe strikes is
+   * arithmetic that has to be testable without one.
+   */
+  static final class WindowEdges {
+    final int left;
+    final int top;
+    final int right;
+    final int bottom;
+
+    WindowEdges(int left, int top, int right, int bottom) {
+      this.left = left;
+      this.top = top;
+      this.right = right;
+      this.bottom = bottom;
+    }
+
+    static WindowEdges of(Rect rect) {
+      return new WindowEdges(rect.left, rect.top, rect.right, rect.bottom);
+    }
+
+    Rect toRect() {
+      return new Rect(left, top, right, bottom);
+    }
+  }
+
   @SuppressWarnings("deprecation")
-  static Rect read(UiAutomation automation) {
+  static Reading readReading(UiAutomation automation) {
     try {
       automation.waitForIdle(100, 2_000);
     } catch (TimeoutException ignored) {
@@ -21,12 +67,27 @@ static Rect read(UiAutomation automation) {
     // UiAutomation.getWindows() transfers recyclable AccessibilityWindowInfo instances, and this
     // read runs repeatedly inside the persistent helper session: copy the bounds the precedence
     // below needs, then recycle every window before resolving.
+    // UiAutomation.getWindows() answers with an empty list until interactive retrieval is on, which
+    // is the same seam the tree capture already uses. Without it this read sees no windows at all and
+    // the keyboard below is invisible to it.
+    AccessibilityTreeCapture.enableInteractiveWindowRetrieval(automation);
     Rect activeBounds = null;
     Rect fallbackBounds = null;
+    List<WindowEdges> inputMethodWindows = new ArrayList<>();
     List<AccessibilityWindowInfo> windows = automation.getWindows();
     try {
       for (AccessibilityWindowInfo window : windows) {
-        if (window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION) continue;
+        int type = window.getType();
+        if (type == AccessibilityWindowInfo.TYPE_INPUT_METHOD) {
+          // Copy every input method window. Which of them a swipe has to clear depends on the
+          // application window, which this loop has not finished reading, so they are collected here
+          // and resolved once it has.
+          Rect bounds = new Rect();
+          window.getBoundsInScreen(bounds);
+          if (!bounds.isEmpty()) inputMethodWindows.add(WindowEdges.of(bounds));
+          continue;
+        }
+        if (type != AccessibilityWindowInfo.TYPE_APPLICATION) continue;
         Rect bounds = new Rect();
         window.getBoundsInScreen(bounds);
         if (activeBounds == null
@@ -41,6 +102,48 @@ static Rect read(UiAutomation automation) {
         window.recycle();
       }
     }
+    Rect application = resolveApplication(automation, activeBounds, fallbackBounds);
+    WindowEdges struck = struckInputMethod(
+        inputMethodWindows, application == null ? null : WindowEdges.of(application));
+    return new Reading(application, struck == null ? null : struck.toRect());
+  }
+
+  /**
+   * The input method share a swipe has to stay above, or null when none of it is in the way.
+   *
+   * <p>A composer bar and its key plane can arrive as separate windows, and the larger rectangle is
+   * usually the lower key plane: keeping only that leaves the swipe inside the composer reaching
+   * further up the screen. So this unions the windows the swipe's centre line crosses — the same line
+   * the shared clip rule tests — and ignores the ones beside it that the swipe cannot reach.
+   */
+  static WindowEdges struckInputMethod(List<WindowEdges> inputMethodWindows, WindowEdges application) {
+    WindowEdges struck = null;
+    for (WindowEdges bounds : inputMethodWindows) {
+      if (application != null) {
+        double swipeCenterX = application.left + (application.right - application.left) / 2.0;
+        boolean strikesSwipePath = swipeCenterX >= bounds.left && swipeCenterX < bounds.right;
+        boolean overlapsWindow = bounds.bottom > application.top && bounds.top < application.bottom;
+        if (!strikesSwipePath || !overlapsWindow) continue;
+      }
+      if (struck == null) {
+        struck = bounds;
+        continue;
+      }
+      struck = new WindowEdges(
+          Math.min(struck.left, bounds.left),
+          Math.min(struck.top, bounds.top),
+          Math.max(struck.right, bounds.right),
+          Math.max(struck.bottom, bounds.bottom));
+    }
+    return struck;
+  }
+
+  static Rect read(UiAutomation automation) {
+    return readReading(automation).application;
+  }
+
+  private static Rect resolveApplication(
+      UiAutomation automation, Rect activeBounds, Rect fallbackBounds) {
     if (activeBounds != null) return activeBounds;
     AccessibilityNodeInfo activeRoot = automation.getRootInActiveWindow();
     if (activeRoot != null) {

android/snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/TouchCommandHandler.java

@@ -15,10 +15,11 @@ final class TouchCommandHandler {
   private TouchCommandHandler() {}
 
   static void populateViewport(Bundle result, UiAutomation automation) {
-    Rect viewport = GestureViewportReader.read(automation);
+    GestureViewportReader.Reading reading = GestureViewportReader.readReading(automation);
     result.putString("ok", "true");
     result.putString("kind", "viewport");
-    putViewportMetadata(result, viewport);
+    putViewportMetadata(result, reading.application);
+    putKeyboardMetadata(result, reading.inputMethod);
   }
 
   static void populateGesture(Bundle result, UiAutomation automation, String payloadBase64)
@@ -69,4 +70,17 @@ private static void putViewportMetadata(Bundle result, Rect viewport) {
     result.putString("width", Integer.toString(viewport.width()));
     result.putString("height", Integer.toString(viewport.height()));
   }
+
+  /**
+   * The input method window's screen bounds, reported only when one is on screen. An absent keyboard
+   * is reported by absence: a keyboard this helper cannot see is not evidence that a surface is
+   * blocked, so the caller must not read a zero frame as an occlusion.
+   */
+  private static void putKeyboardMetadata(Bundle result, Rect inputMethod) {
+    if (inputMethod == null) return;
+    result.putString("keyboardX", Integer.toString(inputMethod.left));
+    result.putString("keyboardY", Integer.toString(inputMethod.top));
+    result.putString("keyboardWidth", Integer.toString(inputMethod.width()));
+    result.putString("keyboardHeight", Integer.toString(inputMethod.height()));
+  }
 }

android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/GestureViewportReaderTest.java

@@ -0,0 +1,87 @@
+package com.callstack.agentdevice.snapshothelper;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+public final class GestureViewportReaderTest {
+  private GestureViewportReaderTest() {}
+
+  private static final GestureViewportReader.WindowEdges APPLICATION =
+      new GestureViewportReader.WindowEdges(0, 0, 1080, 2400);
+  private static final GestureViewportReader.WindowEdges KEY_PLANE =
+      new GestureViewportReader.WindowEdges(0, 1517, 1080, 2400);
+  private static final GestureViewportReader.WindowEdges COMPOSER =
+      new GestureViewportReader.WindowEdges(0, 1400, 1080, 1517);
+  private static final GestureViewportReader.WindowEdges SIDE_STRIP =
+      new GestureViewportReader.WindowEdges(900, 1200, 1080, 2400);
+
+  static void run() {
+    assertNoInputMethodOnScreen();
+    assertComposerAboveItsKeyPlaneKeepsItsEarlierTopEdge();
+    assertWindowBesideTheSwipePathIsIgnored();
+  }
+
+  private static void assertNoInputMethodOnScreen() {
+    assertEdges(
+        GestureViewportReader.struckInputMethod(
+            Collections.<GestureViewportReader.WindowEdges>emptyList(), APPLICATION),
+        null,
+        "no input method window on screen");
+  }
+
+  private static void assertComposerAboveItsKeyPlaneKeepsItsEarlierTopEdge() {
+    List<GestureViewportReader.WindowEdges> both = Arrays.asList(KEY_PLANE, COMPOSER);
+    // The key plane is the larger rectangle. Keeping only it would plan a swipe ending inside the
+    // composer, whose top edge reaches 117px further up the screen.
+    assertEdges(
+        GestureViewportReader.struckInputMethod(both, APPLICATION),
+        new GestureViewportReader.WindowEdges(0, 1400, 1080, 2400),
+        "composer above its key plane");
+    assertEdges(
+        GestureViewportReader.struckInputMethod(Arrays.asList(COMPOSER, KEY_PLANE), APPLICATION),
+        new GestureViewportReader.WindowEdges(0, 1400, 1080, 2400),
+        "composer listed after its key plane");
+  }
+
+  private static void assertWindowBesideTheSwipePathIsIgnored() {
+    // A floating candidate strip at the right edge never crosses the centre line a vertical swipe
+    // travels, so its higher top edge must not shorten the band.
+    assertEdges(
+        GestureViewportReader.struckInputMethod(Arrays.asList(KEY_PLANE, SIDE_STRIP), APPLICATION),
+        KEY_PLANE,
+        "input method window beside the swipe path");
+    assertEdges(
+        GestureViewportReader.struckInputMethod(
+            Collections.singletonList(SIDE_STRIP), APPLICATION),
+        null,
+        "only an unreachable input method window on screen");
+  }
+
+  private static void assertEdges(
+      GestureViewportReader.WindowEdges actual,
+      GestureViewportReader.WindowEdges expected,
+      String label) {
+    if (expected == null) {
+      if (actual != null) {
+        throw new AssertionError(
+            "Expected no input method rect for " + label + ", got " + describe(actual));
+      }
+      return;
+    }
+    if (actual == null) {
+      throw new AssertionError("Expected " + describe(expected) + " for " + label + ", got none");
+    }
+    if (actual.left != expected.left
+        || actual.top != expected.top
+        || actual.right != expected.right
+        || actual.bottom != expected.bottom) {
+      throw new AssertionError(
+          "Expected " + describe(expected) + " for " + label + ", got " + describe(actual));
+    }
+  }
+
+  private static String describe(GestureViewportReader.WindowEdges edges) {
+    return "[" + edges.left + "," + edges.top + "][" + edges.right + "," + edges.bottom + "]";
+  }
+}

android/snapshot-helper/src/test/java/com/callstack/agentdevice/snapshothelper/SnapshotHelperTestSuite.java

@@ -7,5 +7,6 @@ public static void main(String[] args) throws Exception {
     PointerEventScheduleTest.run();
     AccessibilityCaptureStabilizerTest.run();
     BoundedUiAutomationConnectionTest.run();
+    GestureViewportReaderTest.run();
   }
 }