[camera] Implemented ImageStream ImageFormat setting for Dart and Android (flutter#3359)

* Implemented ImageStream ImageFormat setting for Dart and Android

* Fixed formatting and toString test

* Apply suggestions from code review

* Removed imageStreamImageFormat from CameraValue

* Removed imageStreamImageFormat from CameraValue

* Removed imageStreamImageFormat from CameraValue

* fixed formatting

* fixed formatting

* fixed formatting

* WIP: iOS implementation

* Imaplemented suggested changes, added tests.

* iOS switch case videoFormat

* Added imageFormatGroup to initialize

* Apply suggestions from code review

Co-authored-by: Maurits van Beusekom <[email protected]>

* Added period to sentence

* Moved ImageFormatGroup to platform_interface; Added extension to convert ImageFormatGroup to name; Changed int to ImageFormatGroup for initializeCamera

* Fixed test

* Separated Android and iOS in name extension

* Clarified returns on name extension

* updated Android implementation to support String output

* removed getOrDefault

* Updated camera implementation to use ImageFormatGroupName; Updated to Dart 2.7.0 to support extensions;

* removed unused import

* Export image_format_group.dart in types.dart

* Changed enum values to lowercase

* Added ImageFormatGroup test

* Fixed formatting

* made enum strings lowercase

* Removed target platform switch.

* Fixed formatting

* Updated Android implementation

* Updated iOS implementation

* updated log message for unsupported ImageFormatGroup

* Updated Android implementation

* fixed period in docs

* Switch change to if-statement

* Moved switching videoFormat to method in iOS

* Implemented feedback

* fixed formatting

* fixed mistakingly removed bracket

* fixed formatting

* Updated version

* Updated version

* fixed formatting

* Define TAG correctly

Co-authored-by: anniek <[email protected]>
Co-authored-by: Maurits van Beusekom <[email protected]>
Co-authored-by: Maurits van Beusekom <[email protected]>

Author: 3 people

packages/camera/camera/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.5
+
+* Adds ImageFormat selection for ImageStream and Video(iOS only).
 ## 0.6.4+5
 
 * Update the example app: remove the deprecated `RaisedButton` and `FlatButton` widgets.
@@ -16,7 +19,7 @@
 
 ## 0.6.4+1
 
-* Added closeCaptureSession() to stopVideoRecording in Camera.java to fix an Android 6 crash
+* Added closeCaptureSession() to stopVideoRecording in Camera.java to fix an Android 6 crash.
 
 ## 0.6.4
 

packages/camera/camera/android/src/main/java/io/flutter/plugins/camera/Camera.java

@@ -67,6 +67,8 @@ interface ErrorCallback {
 }
 
 public class Camera {
+  private static final String TAG = "Camera";
+
   private final SurfaceTextureEntry flutterTexture;
   private final CameraManager cameraManager;
   private final OrientationEventListener orientationEventListener;
@@ -99,6 +101,14 @@ public class Camera {
   private boolean useAutoFocus = true;
   private Range<Integer> fpsRange;
 
+  private static final HashMap<String, Integer> supportedImageFormats;
+  // Current supported outputs
+  static {
+    supportedImageFormats = new HashMap<>();
+    supportedImageFormats.put("yuv420", 35);
+    supportedImageFormats.put("jpeg", 256);
+  }
+
   public Camera(
       final Activity activity,
       final SurfaceTextureEntry flutterTexture,
@@ -183,15 +193,20 @@ private void prepareMediaRecorder(String outputFilePath) throws IOException {
   }
 
   @SuppressLint("MissingPermission")
-  public void open() throws CameraAccessException {
+  public void open(String imageFormatGroup) throws CameraAccessException {
     pictureImageReader =
         ImageReader.newInstance(
             captureSize.getWidth(), captureSize.getHeight(), ImageFormat.JPEG, 2);
 
+    Integer imageFormat = supportedImageFormats.get(imageFormatGroup);
+    if (imageFormat == null) {
+      Log.w(TAG, "The selected imageFormatGroup is not supported by Android. Defaulting to yuv420");
+      imageFormat = ImageFormat.YUV_420_888;
+    }
+
     // Used to steam image byte data to dart side.
     imageStreamReader =
-        ImageReader.newInstance(
-            previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2);
+        ImageReader.newInstance(previewSize.getWidth(), previewSize.getHeight(), imageFormat, 2);
 
     cameraManager.openCamera(
         cameraName,

packages/camera/camera/android/src/main/java/io/flutter/plugins/camera/MethodCallHandlerImpl.java

@@ -80,7 +80,7 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull final Result result)
         {
           if (camera != null) {
             try {
-              camera.open();
+              camera.open(call.argument("imageFormatGroup"));
               result.success(null);
             } catch (Exception e) {
               handleException(e, result);

packages/camera/camera/example/lib/main.dart

@@ -486,6 +486,7 @@ class _CameraExampleHomeState extends State<CameraExampleHome>
       cameraDescription,
       ResolutionPreset.medium,
       enableAudio: enableAudio,
+      imageFormatGroup: ImageFormatGroup.jpeg,
     );
 
     // If the controller is updated then update the UI.

packages/camera/camera/example/pubspec.yaml

@@ -22,5 +22,5 @@ flutter:
   uses-material-design: true
 
 environment:
-  sdk: ">=2.0.0-dev.28.0 <3.0.0"
+  sdk: ">=2.7.0 <3.0.0"
   flutter: ">=1.9.1+hotfix.4 <2.0.0"

packages/camera/camera/ios/Classes/CameraPlugin.m

@@ -162,6 +162,17 @@ static FlashMode getFlashModeForString(NSString *mode) {
   }
 }
 
+static OSType getVideoFormatFromString(NSString *videoFormatString) {
+  if ([videoFormatString isEqualToString:@"bgra8888"]) {
+    return kCVPixelFormatType_32BGRA;
+  } else if ([videoFormatString isEqualToString:@"yuv420"]) {
+    return kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+  } else {
+    NSLog(@"The selected imageFormatGroup is not supported by iOS. Defaulting to brga8888");
+    return kCVPixelFormatType_32BGRA;
+  }
+}
+
 static AVCaptureFlashMode getAVCaptureFlashModeForFlashMode(FlashMode mode) {
   switch (mode) {
     case FlashModeOff:
@@ -296,7 +307,7 @@ @implementation FLTCam {
   dispatch_queue_t _dispatchQueue;
 }
 // Format used for video and image streaming.
-FourCharCode const videoFormat = kCVPixelFormatType_32BGRA;
+FourCharCode videoFormat = kCVPixelFormatType_32BGRA;
 NSString *const errorMethod = @"error";
 
 - (instancetype)initWithCameraName:(NSString *)cameraName
@@ -1147,6 +1158,9 @@ - (void)handleMethodCallAsync:(FlutterMethodCall *)call result:(FlutterResult)re
     NSDictionary *argsMap = call.arguments;
     NSUInteger cameraId = ((NSNumber *)argsMap[@"cameraId"]).unsignedIntegerValue;
     if ([@"initialize" isEqualToString:call.method]) {
+      NSString *videoFormatValue = ((NSString *)argsMap[@"imageFormatGroup"]);
+      videoFormat = getVideoFormatFromString(videoFormatValue);
+
       __weak CameraPlugin *weakSelf = self;
       _camera.onFrameAvailable = ^{
         [weakSelf.registry textureFrameAvailable:cameraId];

packages/camera/camera/lib/camera.dart

@@ -14,4 +14,5 @@ export 'package:camera_platform_interface/camera_platform_interface.dart'
         FlashMode,
         ExposureMode,
         ResolutionPreset,
-        XFile;
+        XFile,
+        ImageFormatGroup;

packages/camera/camera/lib/src/camera_controller.dart

@@ -160,6 +160,7 @@ class CameraController extends ValueNotifier<CameraValue> {
     this.description,
     this.resolutionPreset, {
     this.enableAudio = true,
+    this.imageFormatGroup,
   }) : super(const CameraValue.uninitialized());
 
   /// The properties of the camera device controlled by this controller.
@@ -176,6 +177,11 @@ class CameraController extends ValueNotifier<CameraValue> {
   /// Whether to include audio when recording a video.
   final bool enableAudio;
 
+  /// The [ImageFormatGroup] describes the output of the raw image format.
+  ///
+  /// When null the imageFormat will fallback to the platforms default.
+  final ImageFormatGroup imageFormatGroup;
+
   int _cameraId;
   bool _isDisposed = false;
   StreamSubscription<dynamic> _imageStreamSubscription;
@@ -217,7 +223,10 @@ class CameraController extends ValueNotifier<CameraValue> {
         _initializeCompleter.complete(event);
       }));
 
-      await CameraPlatform.instance.initializeCamera(_cameraId);
+      await CameraPlatform.instance.initializeCamera(
+        _cameraId,
+        imageFormatGroup: imageFormatGroup,
+      );
 
       value = value.copyWith(
         isInitialized: true,

packages/camera/camera/lib/src/camera_image.dart

@@ -6,6 +6,7 @@ import 'dart:typed_data';
 
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
+import 'package:camera_platform_interface/camera_platform_interface.dart';
 
 /// A single color plane of image data.
 ///
@@ -41,32 +42,6 @@ class Plane {
   final int width;
 }
 
-// TODO:(bmparr) Turn [ImageFormatGroup] to a class with int values.
-/// Group of image formats that are comparable across Android and iOS platforms.
-enum ImageFormatGroup {
-  /// The image format does not fit into any specific group.
-  unknown,
-
-  /// Multi-plane YUV 420 format.
-  ///
-  /// This format is a generic YCbCr format, capable of describing any 4:2:0
-  /// chroma-subsampled planar or semiplanar buffer (but not fully interleaved),
-  /// with 8 bits per color sample.
-  ///
-  /// On Android, this is `android.graphics.ImageFormat.YUV_420_888`. See
-  /// https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
-  ///
-  /// On iOS, this is `kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange`. See
-  /// https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers/kcvpixelformattype_420ypcbcr8biplanarvideorange?language=objc
-  yuv420,
-
-  /// 32-bit BGRA.
-  ///
-  /// On iOS, this is `kCVPixelFormatType_32BGRA`. See
-  /// https://developer.apple.com/documentation/corevideo/1563591-pixel_format_identifiers/kcvpixelformattype_32bgra?language=objc
-  bgra8888,
-}
-
 /// Describes how pixels are represented in an image.
 class ImageFormat {
   ImageFormat._fromPlatformData(this.raw) : group = _asImageFormatGroup(raw);
@@ -86,9 +61,13 @@ class ImageFormat {
 
 ImageFormatGroup _asImageFormatGroup(dynamic rawFormat) {
   if (defaultTargetPlatform == TargetPlatform.android) {
-    // android.graphics.ImageFormat.YUV_420_888
-    if (rawFormat == 35) {
-      return ImageFormatGroup.yuv420;
+    switch (rawFormat) {
+      // android.graphics.ImageFormat.YUV_420_888
+      case 35:
+        return ImageFormatGroup.yuv420;
+      // android.graphics.ImageFormat.JPEG
+      case 256:
+        return ImageFormatGroup.jpeg;
     }
   }
 

packages/camera/camera/pubspec.yaml

@@ -2,13 +2,13 @@ name: camera
 description: A Flutter plugin for getting information about and controlling the
   camera on Android and iOS. Supports previewing the camera feed, capturing images, capturing video,
   and streaming image buffers to dart.
-version: 0.6.4+5
+version: 0.6.5
 homepage: https://github.com/flutter/plugins/tree/master/packages/camera/camera
 
 dependencies:
   flutter:
     sdk: flutter
-  camera_platform_interface: ^1.2.0
+  camera_platform_interface: ^1.3.0
   pedantic: ^1.8.0
 
 dev_dependencies:
@@ -31,5 +31,5 @@ flutter:
         pluginClass: CameraPlugin
 
 environment:
-  sdk: ">=2.1.0 <3.0.0"
+  sdk: ">=2.7.0 <3.0.0"
   flutter: ">=1.12.13+hotfix.5"

packages/camera/camera/test/camera_image_test.dart

@@ -5,6 +5,7 @@
 import 'dart:typed_data';
 
 import 'package:camera/camera.dart';
+import 'package:camera_platform_interface/camera_platform_interface.dart';
 import 'package:flutter/cupertino.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter_test/flutter_test.dart';

packages/camera/camera/test/camera_test.dart

@@ -8,6 +8,7 @@ import 'dart:ui';
 
 import 'package:camera/camera.dart';
 import 'package:camera_platform_interface/camera_platform_interface.dart';
+import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';
@@ -164,6 +165,22 @@ void main() {
       mockPlatformException = false;
     });
 
+    test('initialize() sets imageFormat', () async {
+      debugDefaultTargetPlatformOverride = TargetPlatform.android;
+      CameraController cameraController = CameraController(
+        CameraDescription(
+            name: 'cam',
+            lensDirection: CameraLensDirection.back,
+            sensorOrientation: 90),
+        ResolutionPreset.max,
+        imageFormatGroup: ImageFormatGroup.yuv420,
+      );
+      await cameraController.initialize();
+      verify(CameraPlatform.instance
+              .initializeCamera(13, imageFormatGroup: ImageFormatGroup.yuv420))
+          .called(1);
+    });
+
     test('prepareForVideoRecording() calls $CameraPlatform ', () async {
       CameraController cameraController = CameraController(
           CameraDescription(
@@ -1004,6 +1021,10 @@ class MockCameraPlatform extends Mock
     with MockPlatformInterfaceMixin
     implements CameraPlatform {
   @override
+  Future<void> initializeCamera(int cameraId,
+      {ImageFormatGroup imageFormatGroup});
+
+  @override
   Future<List<CameraDescription>> availableCameras() =>
       Future.value(mockAvailableCameras);