Track motion events for reuse post gesture disambiguation (#19484)

This change makes it so that we track all the motion events encountered by `FlutterView` and all of its subviews in the `MotionEventTracker` class, indexed by a unique `MotionEventId`. This identifier is then passed to the Flutter framework as seen in flutter/flutter#60930. Once the gestures take part in gesture disambiguation and are sent back to the engine, we look-up the original motion event using the `MotionEventId` and dispatch it to the platform.

Bug: flutter/flutter#58837

Author: iskakaushik

ci/licenses_golden/licenses_flutter

@@ -697,6 +697,7 @@ FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/Flutt
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterSurfaceView.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterTextureView.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/FlutterView.java
+FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/RenderMode.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/SplashScreen.java
 FILE: ../../../flutter/shell/platform/android/io/flutter/embedding/android/SplashScreenProvider.java

lib/ui/hooks.dart

@@ -287,9 +287,9 @@ void _invoke3<A1, A2, A3>(void callback(A1 a1, A2 a2, A3 a3)?, Zone zone, A1 arg
 // If this value changes, update the encoding code in the following files:
 //
 //  * pointer_data.cc
-//  * pointers.dart
+//  * pointer.dart
 //  * AndroidTouchProcessor.java
-const int _kPointerDataFieldCount = 28;
+const int _kPointerDataFieldCount = 29;
 
 PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
   const int kStride = Int64List.bytesPerElement;
@@ -300,6 +300,7 @@ PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
   for (int i = 0; i < length; ++i) {
     int offset = i * _kPointerDataFieldCount;
     data.add(PointerData(
+      embedderId: packet.getInt64(kStride * offset++, _kFakeHostEndian),
       timeStamp: Duration(microseconds: packet.getInt64(kStride * offset++, _kFakeHostEndian)),
       change: PointerChange.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)],
       kind: PointerDeviceKind.values[packet.getInt64(kStride * offset++, _kFakeHostEndian)],

lib/ui/pointer.dart

@@ -72,6 +72,7 @@ enum PointerSignalKind {
 class PointerData {
   /// Creates an object that represents the state of a pointer.
   const PointerData({
+    this.embedderId = 0,
     this.timeStamp = Duration.zero,
     this.change = PointerChange.cancel,
     this.kind = PointerDeviceKind.touch,
@@ -102,6 +103,13 @@ class PointerData {
     this.scrollDeltaY = 0.0,
   });
 
+  /// Unique identifier that ties the [PointerEvent] to embedder event created it.
+  ///
+  /// No two pointer events can have the same [embedderId]. This is different from
+  /// [pointerIdentifier] - used for hit-testing, whereas [embedderId] is used to
+  /// identify the platform event.
+  final int embedderId;
+
   /// Time of event dispatch, relative to an arbitrary timeline.
   final Duration timeStamp;
 
@@ -263,6 +271,7 @@ class PointerData {
   /// Returns a complete textual description of the information in this object.
   String toStringFull() {
     return '$runtimeType('
+             'embedderId: $embedderId, '
              'timeStamp: $timeStamp, '
              'change: $change, '
              'kind: $kind, '

lib/ui/window/pointer_data.h

@@ -10,7 +10,7 @@
 namespace flutter {
 
 // If this value changes, update the pointer data unpacking code in hooks.dart.
-static constexpr int kPointerDataFieldCount = 28;
+static constexpr int kPointerDataFieldCount = 29;
 static constexpr int kBytesPerField = sizeof(int64_t);
 // Must match the button constants in events.dart.
 enum PointerButtonMouse : int64_t {
@@ -58,6 +58,7 @@ struct alignas(8) PointerData {
     kScroll,
   };
 
+  int64_t embedder_id;
   int64_t time_stamp;
   Change change;
   DeviceKind kind;

lib/web_ui/lib/src/ui/pointer.dart

@@ -71,6 +71,7 @@ enum PointerSignalKind {
 class PointerData {
   /// Creates an object that represents the state of a pointer.
   const PointerData({
+    this.embedderId = 0,
     this.timeStamp = Duration.zero,
     this.change = PointerChange.cancel,
     this.kind = PointerDeviceKind.touch,
@@ -101,6 +102,13 @@ class PointerData {
     this.scrollDeltaY = 0.0,
   });
 
+  /// Unique identifier that ties the [PointerEvent] to embedder event created it.
+  ///
+  /// No two pointer events can have the same [embedderId]. This is different from
+  /// [pointerIdentifier] - used for hit-testing, whereas [embedderId] is used to
+  /// identify the platform event.
+  final int embedderId;
+
   /// Time of event dispatch, relative to an arbitrary timeline.
   final Duration timeStamp;
 
@@ -257,46 +265,47 @@ class PointerData {
   final double scrollDeltaY;
 
   @override
-  String toString() => '$runtimeType(x: $physicalX, y: $physicalY)';
+  String toString() => 'PointerData(x: $physicalX, y: $physicalY)';
 
   /// Returns a complete textual description of the information in this object.
   String toStringFull() {
     return '$runtimeType('
-        'timeStamp: $timeStamp, '
-        'change: $change, '
-        'kind: $kind, '
-        'signalKind: $signalKind, '
-        'device: $device, '
-        'pointerIdentifier: $pointerIdentifier, '
-        'physicalX: $physicalX, '
-        'physicalY: $physicalY, '
-        'physicalDeltaX: $physicalDeltaX, '
-        'physicalDeltaY: $physicalDeltaY, '
-        'buttons: $buttons, '
-        'synthesized: $synthesized, '
-        'pressure: $pressure, '
-        'pressureMin: $pressureMin, '
-        'pressureMax: $pressureMax, '
-        'distance: $distance, '
-        'distanceMax: $distanceMax, '
-        'size: $size, '
-        'radiusMajor: $radiusMajor, '
-        'radiusMinor: $radiusMinor, '
-        'radiusMin: $radiusMin, '
-        'radiusMax: $radiusMax, '
-        'orientation: $orientation, '
-        'tilt: $tilt, '
-        'platformData: $platformData, '
-        'scrollDeltaX: $scrollDeltaX, '
-        'scrollDeltaY: $scrollDeltaY'
-        ')';
+             'embedderId: $embedderId, '
+             'timeStamp: $timeStamp, '
+             'change: $change, '
+             'kind: $kind, '
+             'signalKind: $signalKind, '
+             'device: $device, '
+             'pointerIdentifier: $pointerIdentifier, '
+             'physicalX: $physicalX, '
+             'physicalY: $physicalY, '
+             'physicalDeltaX: $physicalDeltaX, '
+             'physicalDeltaY: $physicalDeltaY, '
+             'buttons: $buttons, '
+             'synthesized: $synthesized, '
+             'pressure: $pressure, '
+             'pressureMin: $pressureMin, '
+             'pressureMax: $pressureMax, '
+             'distance: $distance, '
+             'distanceMax: $distanceMax, '
+             'size: $size, '
+             'radiusMajor: $radiusMajor, '
+             'radiusMinor: $radiusMinor, '
+             'radiusMin: $radiusMin, '
+             'radiusMax: $radiusMax, '
+             'orientation: $orientation, '
+             'tilt: $tilt, '
+             'platformData: $platformData, '
+             'scrollDeltaX: $scrollDeltaX, '
+             'scrollDeltaY: $scrollDeltaY'
+           ')';
   }
 }
 
 /// A sequence of reports about the state of pointers.
 class PointerDataPacket {
   /// Creates a packet of pointer data reports.
-  const PointerDataPacket({this.data = const <PointerData>[]}) : assert(data != null); // ignore: unnecessary_null_comparison
+  const PointerDataPacket({ this.data = const <PointerData>[] }) : assert(data != null); // ignore: unnecessary_null_comparison
 
   /// Data about the individual pointers in this packet.
   ///

shell/platform/android/BUILD.gn

@@ -140,6 +140,7 @@ android_java_sources = [
   "io/flutter/embedding/android/FlutterSurfaceView.java",
   "io/flutter/embedding/android/FlutterTextureView.java",
   "io/flutter/embedding/android/FlutterView.java",
+  "io/flutter/embedding/android/MotionEventTracker.java",
   "io/flutter/embedding/android/RenderMode.java",
   "io/flutter/embedding/android/SplashScreen.java",
   "io/flutter/embedding/android/SplashScreenProvider.java",

shell/platform/android/io/flutter/embedding/android/AndroidTouchProcessor.java

@@ -57,14 +57,15 @@ public class AndroidTouchProcessor {
   }
 
   // Must match the unpacking code in hooks.dart.
-  private static final int POINTER_DATA_FIELD_COUNT = 28;
+  private static final int POINTER_DATA_FIELD_COUNT = 29;
   private static final int BYTES_PER_FIELD = 8;
 
   // This value must match the value in framework's platform_view.dart.
   // This flag indicates whether the original Android pointer events were batched together.
   private static final int POINTER_DATA_FLAG_BATCHED = 1;
 
   @NonNull private final FlutterRenderer renderer;
+  @NonNull private final MotionEventTracker motionEventTracker;
 
   private static final int _POINTER_BUTTON_PRIMARY = 1;
 
@@ -76,6 +77,7 @@ public class AndroidTouchProcessor {
   // FlutterRenderer
   public AndroidTouchProcessor(@NonNull FlutterRenderer renderer) {
     this.renderer = renderer;
+    this.motionEventTracker = MotionEventTracker.getInstance();
   }
 
   /** Sends the given {@link MotionEvent} data to Flutter in a format that Flutter understands. */
@@ -174,6 +176,8 @@ private void addPointerForIndex(
       return;
     }
 
+    MotionEventTracker.MotionEventId motionEventId = motionEventTracker.track(event);
+
     int pointerKind = getPointerDeviceTypeForToolType(event.getToolType(pointerIndex));
 
     int signalKind =
@@ -183,6 +187,7 @@ private void addPointerForIndex(
 
     long timeStamp = event.getEventTime() * 1000; // Convert from milliseconds to microseconds.
 
+    packet.putLong(motionEventId.getId());
     packet.putLong(timeStamp); // time_stamp
     packet.putLong(pointerChange); // change
     packet.putLong(pointerKind); // kind

shell/platform/android/io/flutter/embedding/android/MotionEventTracker.java

@@ -0,0 +1,79 @@
+package io.flutter.embedding.android;
+
+import android.util.LongSparseArray;
+import android.view.MotionEvent;
+import androidx.annotation.Nullable;
+import java.util.PriorityQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Tracks the motion events received by the FlutterView. */
+public final class MotionEventTracker {
+
+  /** Represents a unique identifier corresponding to a motion event. */
+  public static class MotionEventId {
+    private static final AtomicLong ID_COUNTER = new AtomicLong(0);
+    private final long id;
+
+    private MotionEventId(long id) {
+      this.id = id;
+    }
+
+    public static MotionEventId from(long id) {
+      return new MotionEventId(id);
+    }
+
+    public static MotionEventId createUnique() {
+      return MotionEventId.from(ID_COUNTER.incrementAndGet());
+    }
+
+    public long getId() {
+      return id;
+    }
+  }
+
+  private final LongSparseArray<MotionEvent> eventById;
+  private final PriorityQueue<Long> unusedEvents;
+  private static MotionEventTracker INSTANCE;
+
+  public static MotionEventTracker getInstance() {
+    if (INSTANCE == null) {
+      INSTANCE = new MotionEventTracker();
+    }
+    return INSTANCE;
+  }
+
+  private MotionEventTracker() {
+    eventById = new LongSparseArray<>();
+    unusedEvents = new PriorityQueue<>();
+  }
+
+  /** Tracks the event and returns a unique MotionEventId identifying the event. */
+  public MotionEventId track(MotionEvent event) {
+    MotionEventId eventId = MotionEventId.createUnique();
+    eventById.put(eventId.id, event);
+    unusedEvents.add(eventId.id);
+    return eventId;
+  }
+
+  /**
+   * Returns the MotionEvent corresponding to the eventId while discarding all the motion events
+   * that occured prior to the event represented by the eventId. Returns null if this event was
+   * popped or discarded.
+   */
+  @Nullable
+  public MotionEvent pop(MotionEventId eventId) {
+    // remove all the older events.
+    while (!unusedEvents.isEmpty() && unusedEvents.peek() < eventId.id) {
+      eventById.remove(unusedEvents.poll());
+    }
+
+    // remove the current event from the heap if it exists.
+    if (!unusedEvents.isEmpty() && unusedEvents.peek() == eventId.id) {
+      unusedEvents.poll();
+    }
+
+    MotionEvent event = eventById.get(eventId.id);
+    eventById.remove(eventId.id);
+    return event;
+  }
+}

shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java

@@ -166,7 +166,8 @@ private void touch(@NonNull MethodCall call, @NonNull MethodChannel.Result resul
                   (int) args.get(11),
                   (int) args.get(12),
                   (int) args.get(13),
-                  (int) args.get(14));
+                  (int) args.get(14),
+                  ((Number) args.get(15)).longValue());
 
           try {
             handler.onTouch(touch);
@@ -380,6 +381,8 @@ public static class PlatformViewTouch {
     public final int source;
     /** TODO(mattcarroll): javadoc */
     public final int flags;
+    /** TODO(iskakaushik): javadoc */
+    public final long motionEventId;
 
     PlatformViewTouch(
         int viewId,
@@ -396,7 +399,8 @@ public static class PlatformViewTouch {
         int deviceId,
         int edgeFlags,
         int source,
-        int flags) {
+        int flags,
+        long motionEventId) {
       this.viewId = viewId;
       this.downTime = downTime;
       this.eventTime = eventTime;
@@ -412,6 +416,7 @@ public static class PlatformViewTouch {
       this.edgeFlags = edgeFlags;
       this.source = source;
       this.flags = flags;
+      this.motionEventId = motionEventId;
     }
   }
 }

shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java

@@ -21,6 +21,7 @@
 import androidx.annotation.VisibleForTesting;
 import io.flutter.embedding.android.FlutterImageView;
 import io.flutter.embedding.android.FlutterView;
+import io.flutter.embedding.android.MotionEventTracker;
 import io.flutter.embedding.engine.FlutterOverlaySurface;
 import io.flutter.embedding.engine.dart.DartExecutor;
 import io.flutter.embedding.engine.mutatorsstack.*;
@@ -93,6 +94,9 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
   // Platform view IDs that were displayed since the start of the current frame.
   private HashSet<Integer> currentFrameUsedPlatformViewIds;
 
+  // Used to acquire the original motion events using the motionEventIds.
+  private final MotionEventTracker motionEventTracker;
+
   private final PlatformViewsChannel.PlatformViewsHandler channelHandler =
       new PlatformViewsChannel.PlatformViewsHandler() {
 
@@ -301,8 +305,16 @@ private void ensureValidAndroidVersion(int minSdkVersion) {
         }
       };
 
-  private static MotionEvent toMotionEvent(
-      float density, PlatformViewsChannel.PlatformViewTouch touch) {
+  private MotionEvent toMotionEvent(float density, PlatformViewsChannel.PlatformViewTouch touch) {
+    MotionEventTracker.MotionEventId motionEventId =
+        MotionEventTracker.MotionEventId.from(touch.motionEventId);
+    MotionEvent trackedEvent = motionEventTracker.pop(motionEventId);
+    if (trackedEvent != null) {
+      return trackedEvent;
+    }
+
+    // TODO (kaushikiska) : warn that we are potentially using an untracked
+    // event in the platform views.
     PointerProperties[] pointerProperties =
         parsePointerPropertiesList(touch.rawPointerPropertiesList)
             .toArray(new PointerProperties[touch.pointerCount]);
@@ -339,6 +351,8 @@ public PlatformViewsController() {
     platformViewRequests = new SparseArray<>();
     platformViews = new SparseArray<>();
     mutatorViews = new SparseArray<>();
+
+    motionEventTracker = MotionEventTracker.getInstance();
   }
 
   /**

shell/platform/embedder/embedder.cc

@@ -1225,6 +1225,8 @@ FlutterEngineResult FlutterEngineSendPointerEvent(
   for (size_t i = 0; i < events_count; ++i) {
     flutter::PointerData pointer_data;
     pointer_data.Clear();
+    // this is currely in use only on android embedding.
+    pointer_data.embedder_id = 0;
     pointer_data.time_stamp = SAFE_ACCESS(current, timestamp, 0);
     pointer_data.change = ToPointerDataChange(
         SAFE_ACCESS(current, phase, FlutterPointerPhase::kCancel));