[canvaskit] read pixels back in Picture.toImage

Author: yjbanov

lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart

@@ -92,6 +92,11 @@ extension CanvasKitExtension on CanvasKit {
     double width,
     double height,
   );
+  external SkSurface MakeRasterDirectSurface(
+    SkImageInfo imageInfo,
+    MallocObj pixels,
+    double bytesPerRow,
+  );
   external Uint8List getDataBytes(
     SkData skData,
   );
@@ -1277,6 +1282,9 @@ external _NativeType get _nativeFloat32ArrayType;
 @JS('Uint32Array')
 external _NativeType get _nativeUint32ArrayType;
 
+@JS('Uint8Array')
+external _NativeType get _nativeUint8ArrayType;
+
 @JS('window.flutterCanvasKit.Malloc')
 external Object _malloc(_NativeType nativeType, double length);
 
@@ -1296,6 +1304,10 @@ SkUint32List mallocUint32List(int length) {
   return _malloc(_nativeUint32ArrayType, length.toDouble()) as SkUint32List;
 }
 
+SkUint8List mallocUint8List(int length) {
+  return _malloc(_nativeUint8ArrayType, length.toDouble()) as SkUint8List;
+}
+
 /// Frees the WASM memory occupied by a [SkFloat32List] or [SkUint32List].
 ///
 /// The [list] is no longer usable after calling this function.
@@ -1354,6 +1366,20 @@ extension SkUint32ListExtension on SkUint32List {
   external Uint32List toTypedArray();
 }
 
+/// Wraps a [Uint8List] backed by WASM memory.
+///
+/// This wrapper is necessary because the raw [Uint8List] will get detached
+/// when WASM grows its memory. Call [toTypedArray] to get a new instance
+/// that's attached to the current WASM memory block.
+@JS()
+@staticInterop
+class SkUint8List extends MallocObj {}
+
+extension SkUint8ListExtension on SkUint8List {
+  external double length;
+  external Uint8List toTypedArray();
+}
+
 /// Writes [color] information into the given [skColor] buffer.
 Float32List _populateSkColor(SkFloat32List skColor, ui.Color color) {
   final Float32List array = skColor.toTypedArray();

lib/web_ui/lib/src/engine/canvaskit/picture.dart

@@ -100,28 +100,42 @@ class CkPicture extends ManagedSkiaObject<SkPicture> implements ui.Picture {
   }
 
   @override
-  ui.Image toImageSync(int width, int height) {
-    SurfaceFactory.instance.baseSurface.ensureSurface();
-    if (SurfaceFactory.instance.baseSurface.usingSoftwareBackend) {
+  CkImage toImageSync(int width, int height) {
+    // TODO(yjbanov): toImageSyncGPU regressed Image.toByteData. See:
+    //    https://github.com/flutter/flutter/issues/121758
+    //    https://github.com/flutter/flutter/issues/117786#issuecomment-1456975886
+    // SurfaceFactory.instance.baseSurface.ensureSurface();
+    // if (SurfaceFactory.instance.baseSurface.usingSoftwareBackend) {
       return toImageSyncSoftware(width, height);
-    }
-    return toImageSyncGPU(width, height);
+    // }
+    // return toImageSyncGPU(width, height);
   }
 
-  ui.Image toImageSyncGPU(int width, int height) {
+  CkImage toImageSyncGPU(int width, int height) {
     assert(debugCheckNotDisposed('Cannot convert picture to image.'));
 
-    final CkSurface ckSurface = SurfaceFactory.instance.baseSurface
-      .createRenderTargetSurface(ui.Size(width.toDouble(), height.toDouble()));
-    final CkCanvas ckCanvas = ckSurface.getCanvas();
-    ckCanvas.clear(const ui.Color(0x00000000));
-    ckCanvas.drawPicture(this);
-    final SkImage skImage = ckSurface.surface.makeImageSnapshot();
-    ckSurface.dispose();
+    final SkUint8List memory = mallocUint8List(4 * width * height);
+    final SkSurface skSurface = canvasKit.MakeRasterDirectSurface(
+      SkImageInfo(
+        width: width.toDouble(),
+        height: height.toDouble(),
+        colorType: canvasKit.ColorType.RGBA_8888,
+        alphaType: canvasKit.AlphaType.Premul,
+        colorSpace: SkColorSpaceSRGB,
+      ),
+      memory,
+      (4 * width).toDouble(),
+    );
+    final SkCanvas skCanvas = skSurface.getCanvas();
+    skCanvas.clear(toSharedSkColor1(const ui.Color(0x00000000)));
+    skCanvas.drawPicture(skiaObject);
+    final SkImage skImage = skSurface.makeImageSnapshot();
+    skSurface.dispose();
+    free(memory);
     return CkImage(skImage);
   }
 
-  ui.Image toImageSyncSoftware(int width, int height) {
+  CkImage toImageSyncSoftware(int width, int height) {
     assert(debugCheckNotDisposed('Cannot convert picture to image.'));
 
     final Surface surface = SurfaceFactory.instance.pictureToImageSurface;

lib/web_ui/test/canvaskit/canvas_golden_test.dart

@@ -828,6 +828,64 @@ void testMain() {
       await matchGoldenFile('canvaskit_empty_scene.png',
           region: const ui.Rect.fromLTRB(0, 0, 100, 100));
     });
+
+    // Regression test for https://github.com/flutter/flutter/issues/121758
+    test('resources used in temporary surfaces for Image.toByteData can cross to rendering overlays', () async {
+      final Rasterizer rasterizer = CanvasKitRenderer.instance.rasterizer;
+      SurfaceFactory.instance.debugClear();
+
+      ui.platformViewRegistry.registerViewFactory(
+        'test-platform-view',
+        (int viewId) => createDomHTMLDivElement()..id = 'view-0',
+      );
+      await createPlatformView(0, 'test-platform-view');
+
+      CkPicture makeTextPicture(String text, ui.Offset offset) {
+        final CkPictureRecorder recorder = CkPictureRecorder();
+        final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest);
+        final CkParagraphBuilder builder = CkParagraphBuilder(CkParagraphStyle());
+        builder.addText(text);
+        final CkParagraph paragraph = builder.build();
+        paragraph.layout(const ui.ParagraphConstraints(width: 100));
+        canvas.drawRect(
+          ui.Rect.fromLTWH(offset.dx, offset.dy, paragraph.width, paragraph.height).inflate(10),
+          CkPaint()..color = const ui.Color(0xFF00FF00)
+        );
+        canvas.drawParagraph(paragraph, offset);
+        return recorder.endRecording();
+      }
+
+      CkPicture imageToPicture(CkImage image, ui.Offset offset) {
+        final CkPictureRecorder recorder = CkPictureRecorder();
+        final CkCanvas canvas = recorder.beginRecording(ui.Rect.largest);
+        canvas.drawImage(image, offset, CkPaint());
+        return recorder.endRecording();
+      }
+
+      final CkPicture helloPicture = makeTextPicture('Hello', ui.Offset.zero);
+
+      final CkImage helloImage = helloPicture.toImageSync(100, 100);
+
+      // Calling toByteData is essential to hit the bug.
+      await helloImage.toByteData(format: ui.ImageByteFormat.png);
+
+      final LayerSceneBuilder sb = LayerSceneBuilder();
+      sb.pushOffset(0, 0);
+      sb.addPicture(ui.Offset.zero, helloPicture);
+      sb.addPlatformView(0, width: 10, height: 10);
+
+      // The image is rendered after the platform view so that it's rendered into
+      // a separate surface, which is what triggers the bug. If the bug is present
+      // the image will not appear on the UI.
+      sb.addPicture(const ui.Offset(0, 50), imageToPicture(helloImage, ui.Offset.zero));
+      sb.pop();
+
+      // The below line should not throw an error.
+      rasterizer.draw(sb.build().layerTree);
+
+      await matchGoldenFile('cross_overlay_resources.png', region: const ui.Rect.fromLTRB(0, 0, 100, 100));
+    });
+
     // TODO(hterkelsen): https://github.com/flutter/flutter/issues/71520
   }, skip: isSafari || isFirefox);
 }