Switch to FlutterSurfaceView if no Android view is in the frame (#19487)

Author: Emmanuel Garcia

shell/common/rasterizer.cc

@@ -377,12 +377,6 @@ RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
   TRACE_EVENT0("flutter", "Rasterizer::DrawToSurface");
   FML_DCHECK(surface_);
 
-  auto frame = surface_->AcquireFrame(layer_tree.frame_size());
-
-  if (frame == nullptr) {
-    return RasterStatus::kFailed;
-  }
-
   // There is no way for the compositor to know how long the layer tree
   // construction took. Fortunately, the layer tree does. Grab that time
   // for instrumentation.
@@ -398,6 +392,16 @@ RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
     embedder_root_canvas = external_view_embedder->GetRootCanvas();
   }
 
+  // On Android, the external view embedder deletes surfaces in `BeginFrame`.
+  //
+  // Deleting a surface also clears the GL context. Therefore, acquire the
+  // frame after calling `BeginFrame` as this operation resets the GL context.
+  auto frame = surface_->AcquireFrame(layer_tree.frame_size());
+
+  if (frame == nullptr) {
+    return RasterStatus::kFailed;
+  }
+
   // If the external view embedder has specified an optional root surface, the
   // root surface transformation is set by the embedder instead of
   // having to apply it here.

shell/platform/android/external_view_embedder/external_view_embedder.cc

@@ -85,18 +85,20 @@ bool AndroidExternalViewEmbedder::SubmitFrame(
   std::unordered_map<int64_t, std::list<SkRect>> overlay_layers;
   std::unordered_map<int64_t, sk_sp<SkPicture>> pictures;
   SkCanvas* background_canvas = frame->SkiaCanvas();
+  auto current_frame_view_count = composition_order_.size();
 
   // Restore the clip context after exiting this method since it's changed
   // below.
   SkAutoCanvasRestore save(background_canvas, /*doSave=*/true);
 
-  for (size_t i = 0; i < composition_order_.size(); i++) {
+  for (size_t i = 0; i < current_frame_view_count; i++) {
     int64_t view_id = composition_order_[i];
 
     sk_sp<SkPicture> picture =
         picture_recorders_.at(view_id)->finishRecordingAsPicture();
     FML_CHECK(picture);
     pictures.insert({view_id, picture});
+
     overlay_layers.insert({view_id, {}});
 
     sk_sp<RTree> rtree = view_rtrees_.at(view_id);
@@ -142,8 +144,14 @@ bool AndroidExternalViewEmbedder::SubmitFrame(
     background_canvas->drawPicture(pictures.at(view_id));
   }
   // Submit the background canvas frame before switching the GL context to
-  // the surfaces above.
-  frame->Submit();
+  // the overlay surfaces.
+  //
+  // Skip a frame if the embedding is switching surfaces.
+  auto should_submit_current_frame =
+      previous_frame_view_count_ > 0 || current_frame_view_count == 0;
+  if (should_submit_current_frame) {
+    frame->Submit();
+  }
 
   for (int64_t view_id : composition_order_) {
     SkRect view_rect = GetViewRect(view_id);
@@ -161,12 +169,15 @@ bool AndroidExternalViewEmbedder::SubmitFrame(
         params.mutatorsStack()  //
     );
     for (const SkRect& overlay_rect : overlay_layers.at(view_id)) {
-      CreateSurfaceIfNeeded(context,               //
-                            view_id,               //
-                            pictures.at(view_id),  //
-                            overlay_rect           //
-                            )
-          ->Submit();
+      std::unique_ptr<SurfaceFrame> frame =
+          CreateSurfaceIfNeeded(context,               //
+                                view_id,               //
+                                pictures.at(view_id),  //
+                                overlay_rect           //
+          );
+      if (should_submit_current_frame) {
+        frame->Submit();
+      }
     }
   }
   return true;
@@ -233,6 +244,8 @@ SkCanvas* AndroidExternalViewEmbedder::GetRootCanvas() {
 }
 
 void AndroidExternalViewEmbedder::Reset() {
+  previous_frame_view_count_ = composition_order_.size();
+
   composition_order_.clear();
   picture_recorders_.clear();
 }
@@ -244,6 +257,9 @@ void AndroidExternalViewEmbedder::BeginFrame(
     double device_pixel_ratio,
     fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger) {
   Reset();
+
+  // The surface size changed. Therefore, destroy existing surfaces as
+  // the existing surfaces in the pool can't be recycled.
   if (frame_size_ != frame_size) {
     surface_pool_->DestroyLayers(jni_facade_);
   }

shell/platform/android/external_view_embedder/external_view_embedder.h

@@ -124,6 +124,9 @@ class AndroidExternalViewEmbedder final : public ExternalViewEmbedder {
   // The r-tree that captures the operations for the picture recorders.
   std::unordered_map<int64_t, sk_sp<RTree>> view_rtrees_;
 
+  // The number of platform views in the previous frame.
+  int64_t previous_frame_view_count_;
+
   // Resets the state.
   void Reset();
 

shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc

@@ -250,7 +250,7 @@ TEST(AndroidExternalViewEmbedder, PlatformViewRect__ChangedParams) {
   ASSERT_EQ(SkRect::MakeXYWH(75, 90, 105, 120), embedder->GetViewRect(view_id));
 }
 
-TEST(AndroidExternalViewEmbedder, SubmitFrame__RecycleSurfaces) {
+TEST(AndroidExternalViewEmbedder, SubmitFrame) {
   auto jni_mock = std::make_shared<JNIMock>();
   auto android_context =
       std::make_shared<AndroidContext>(AndroidRenderingAPI::kSoftware);
@@ -294,6 +294,27 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame__RecycleSurfaces) {
   auto raster_thread_merger = GetThreadMergerFromPlatformThread();
 
   // ------------------ First frame ------------------ //
+  {
+    auto did_submit_frame = false;
+    auto surface_frame = std::make_unique<SurfaceFrame>(
+        SkSurface::MakeNull(1000, 1000), false,
+        [&did_submit_frame](const SurfaceFrame& surface_frame,
+                            SkCanvas* canvas) mutable {
+          if (canvas != nullptr) {
+            did_submit_frame = true;
+          }
+          return true;
+        });
+
+    embedder->SubmitFrame(gr_context.get(), std::move(surface_frame));
+    // Submits frame if no Android view in the current frame.
+    EXPECT_TRUE(did_submit_frame);
+
+    EXPECT_CALL(*jni_mock, FlutterViewEndFrame());
+    embedder->EndFrame(/*should_resubmit_frame=*/false, raster_thread_merger);
+  }
+
+  // ------------------ Second frame ------------------ //
   {
     EXPECT_CALL(*jni_mock, FlutterViewBeginFrame());
     embedder->BeginFrame(frame_size, nullptr, 1.5, raster_thread_merger);
@@ -338,18 +359,26 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame__RecycleSurfaces) {
     EXPECT_CALL(*jni_mock,
                 FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200));
 
-    auto surface_frame =
-        std::make_unique<SurfaceFrame>(SkSurface::MakeNull(1000, 1000), false,
-                                       [](const SurfaceFrame& surface_frame,
-                                          SkCanvas* canvas) { return true; });
+    auto did_submit_frame = false;
+    auto surface_frame = std::make_unique<SurfaceFrame>(
+        SkSurface::MakeNull(1000, 1000), false,
+        [&did_submit_frame](const SurfaceFrame& surface_frame,
+                            SkCanvas* canvas) mutable {
+          if (canvas != nullptr) {
+            did_submit_frame = true;
+          }
+          return true;
+        });
 
     embedder->SubmitFrame(gr_context.get(), std::move(surface_frame));
+    // Doesn't submit frame if there aren't Android views in the previous frame.
+    EXPECT_FALSE(did_submit_frame);
 
     EXPECT_CALL(*jni_mock, FlutterViewEndFrame());
     embedder->EndFrame(/*should_resubmit_frame=*/false, raster_thread_merger);
   }
 
-  // ------------------ Second frame ------------------ //
+  // ------------------ Third frame ------------------ //
   {
     EXPECT_CALL(*jni_mock, FlutterViewBeginFrame());
     embedder->BeginFrame(frame_size, nullptr, 1.5, raster_thread_merger);
@@ -392,11 +421,19 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame__RecycleSurfaces) {
     EXPECT_CALL(*jni_mock,
                 FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200));
 
-    auto surface_frame =
-        std::make_unique<SurfaceFrame>(SkSurface::MakeNull(1000, 1000), false,
-                                       [](const SurfaceFrame& surface_frame,
-                                          SkCanvas* canvas) { return true; });
+    auto did_submit_frame = false;
+    auto surface_frame = std::make_unique<SurfaceFrame>(
+        SkSurface::MakeNull(1000, 1000), false,
+        [&did_submit_frame](const SurfaceFrame& surface_frame,
+                            SkCanvas* canvas) mutable {
+          if (canvas != nullptr) {
+            did_submit_frame = true;
+          }
+          return true;
+        });
     embedder->SubmitFrame(gr_context.get(), std::move(surface_frame));
+    // Submits frame if there are Android views in the previous frame.
+    EXPECT_TRUE(did_submit_frame);
 
     EXPECT_CALL(*jni_mock, FlutterViewEndFrame());
     embedder->EndFrame(/*should_resubmit_frame=*/false, raster_thread_merger);
@@ -423,15 +460,83 @@ TEST(AndroidExternalViewEmbedder, DoesNotCallJNIPlatformThreadOnlyMethods) {
 
 TEST(AndroidExternalViewEmbedder, DestroyOverlayLayersOnSizeChange) {
   auto jni_mock = std::make_shared<JNIMock>();
-  auto embedder =
-      std::make_unique<AndroidExternalViewEmbedder>(nullptr, jni_mock, nullptr);
+  auto android_context =
+      std::make_shared<AndroidContext>(AndroidRenderingAPI::kSoftware);
 
+  auto window = fml::MakeRefCounted<AndroidNativeWindow>(nullptr);
+  auto gr_context = GrContext::MakeMock(nullptr);
+  auto frame_size = SkISize::Make(1000, 1000);
+  auto surface_factory =
+      [gr_context, window, frame_size](
+          std::shared_ptr<AndroidContext> android_context,
+          std::shared_ptr<PlatformViewAndroidJNI> jni_facade) {
+        auto surface_frame_1 = std::make_unique<SurfaceFrame>(
+            SkSurface::MakeNull(1000, 1000), false,
+            [](const SurfaceFrame& surface_frame, SkCanvas* canvas) {
+              return true;
+            });
+
+        auto surface_mock = std::make_unique<SurfaceMock>();
+        EXPECT_CALL(*surface_mock, AcquireFrame(frame_size))
+            .WillOnce(Return(ByMove(std::move(surface_frame_1))));
+
+        auto android_surface_mock = std::make_unique<AndroidSurfaceMock>();
+        EXPECT_CALL(*android_surface_mock, IsValid()).WillOnce(Return(true));
+
+        EXPECT_CALL(*android_surface_mock, CreateGPUSurface(gr_context.get()))
+            .WillOnce(Return(ByMove(std::move(surface_mock))));
+
+        EXPECT_CALL(*android_surface_mock, SetNativeWindow(window));
+
+        return android_surface_mock;
+      };
+
+  auto embedder = std::make_unique<AndroidExternalViewEmbedder>(
+      android_context, jni_mock, surface_factory);
   auto raster_thread_merger = GetThreadMergerFromPlatformThread();
-  ASSERT_FALSE(raster_thread_merger->IsMerged());
 
-  embedder->BeginFrame(SkISize::Make(10, 20), nullptr, 1.0,
-                       raster_thread_merger);
+  // ------------------ First frame ------------------ //
+  {
+    EXPECT_CALL(*jni_mock, FlutterViewBeginFrame());
+    embedder->BeginFrame(frame_size, nullptr, 1.5, raster_thread_merger);
+
+    // Add an Android view.
+    MutatorsStack stack1;
+    // TODO(egarciad): Investigate why Flow applies the device pixel ratio to
+    // the offsetPixels, but not the sizePoints.
+    auto view_params_1 = std::make_unique<EmbeddedViewParams>(
+        SkMatrix(), SkSize::Make(200, 200), stack1);
+
+    embedder->PrerollCompositeEmbeddedView(0, std::move(view_params_1));
+
+    // This simulates Flutter UI that intersects with the Android view.
+    embedder->CompositeEmbeddedView(0)->drawRect(
+        SkRect::MakeXYWH(50, 50, 200, 200), SkPaint());
+
+    // Create a new overlay surface.
+    EXPECT_CALL(*jni_mock, FlutterViewCreateOverlaySurface())
+        .WillOnce(Return(
+            ByMove(std::make_unique<PlatformViewAndroidJNI::OverlayMetadata>(
+                0, window))));
+    // The JNI call to display the Android view.
+    EXPECT_CALL(*jni_mock, FlutterViewOnDisplayPlatformView(0, 0, 0, 200, 200,
+                                                            300, 300, stack1));
+    EXPECT_CALL(*jni_mock,
+                FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200));
+
+    auto surface_frame =
+        std::make_unique<SurfaceFrame>(SkSurface::MakeNull(1000, 1000), false,
+                                       [](const SurfaceFrame& surface_frame,
+                                          SkCanvas* canvas) { return true; });
+    embedder->SubmitFrame(gr_context.get(), std::move(surface_frame));
+
+    EXPECT_CALL(*jni_mock, FlutterViewEndFrame());
+    embedder->EndFrame(/*should_resubmit_frame=*/false, raster_thread_merger);
+  }
+
   EXPECT_CALL(*jni_mock, FlutterViewDestroyOverlaySurfaces());
+  EXPECT_CALL(*jni_mock, FlutterViewBeginFrame());
+  // Change the frame size.
   embedder->BeginFrame(SkISize::Make(30, 40), nullptr, 1.0,
                        raster_thread_merger);
 }

shell/platform/android/external_view_embedder/surface_pool.cc

@@ -72,9 +72,11 @@ void SurfacePool::RecycleLayers() {
 
 void SurfacePool::DestroyLayers(
     std::shared_ptr<PlatformViewAndroidJNI> jni_facade) {
+  if (layers_.size() > 0) {
+    jni_facade->FlutterViewDestroyOverlaySurfaces();
+  }
   layers_.clear();
   available_layer_index_ = 0;
-  jni_facade->FlutterViewDestroyOverlaySurfaces();
 }
 
 std::vector<std::shared_ptr<OverlayLayer>> SurfacePool::GetUnusedLayers() {

shell/platform/android/external_view_embedder/surface_pool_unittests.cc

@@ -164,5 +164,41 @@ TEST(SurfacePool, GetLayer__AllocateTwoLayers) {
   ASSERT_EQ(1, layer_2->id);
 }
 
+TEST(SurfacePool, DestroyLayers) {
+  auto pool = std::make_unique<SurfacePool>();
+  auto jni_mock = std::make_shared<JNIMock>();
+
+  EXPECT_CALL(*jni_mock, FlutterViewDestroyOverlaySurfaces()).Times(0);
+  pool->DestroyLayers(jni_mock);
+
+  auto gr_context = GrContext::MakeMock(nullptr);
+  auto android_context =
+      std::make_shared<AndroidContext>(AndroidRenderingAPI::kSoftware);
+
+  auto window = fml::MakeRefCounted<AndroidNativeWindow>(nullptr);
+  EXPECT_CALL(*jni_mock, FlutterViewCreateOverlaySurface())
+      .Times(1)
+      .WillOnce(Return(
+          ByMove(std::make_unique<PlatformViewAndroidJNI::OverlayMetadata>(
+              0, window))));
+
+  auto surface_factory =
+      [gr_context, window](std::shared_ptr<AndroidContext> android_context,
+                           std::shared_ptr<PlatformViewAndroidJNI> jni_facade) {
+        auto android_surface_mock = std::make_unique<AndroidSurfaceMock>();
+        EXPECT_CALL(*android_surface_mock, CreateGPUSurface(gr_context.get()));
+        EXPECT_CALL(*android_surface_mock, SetNativeWindow(window));
+        EXPECT_CALL(*android_surface_mock, IsValid()).WillOnce(Return(true));
+        return android_surface_mock;
+      };
+  pool->GetLayer(gr_context.get(), android_context, jni_mock, surface_factory);
+
+  EXPECT_CALL(*jni_mock, FlutterViewDestroyOverlaySurfaces());
+  pool->DestroyLayers(jni_mock);
+
+  pool->RecycleLayers();
+  ASSERT_TRUE(pool->GetUnusedLayers().empty());
+}
+
 }  // namespace testing
 }  // namespace flutter