[camera] Sample buffer handling on session queue (flutter#4709)

Author: hellohuanlin

packages/camera/camera/CHANGELOG.md

@@ -1,5 +1,6 @@
-## NEXT
+## 0.9.4+9
 
+* iOS performance improvement by moving sample buffer handling from the main queue to a background session queue. 
 * Minor iOS internal code cleanup related to camera class and its delegate. 
 * Minor iOS internal code cleanup related to resolution preset, video format, focus mode, exposure mode and device orientation.
 * Minor iOS internal code cleanup related to flash mode.

packages/camera/camera/example/ios/Runner.xcodeproj/project.pbxproj

@@ -26,6 +26,7 @@
 		E0C6E2012770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0C6E1FE2770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m */; };
 		E0C6E2022770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0C6E1FF2770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m */; };
 		E0F95E3D27A32AB900699390 /* CameraPropertiesTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0F95E3C27A32AB900699390 /* CameraPropertiesTests.m */; };
+		E0F95E4427A36B9200699390 /* SampleBufferQueueTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E0F95E4327A36B9200699390 /* SampleBufferQueueTests.m */; };
 		E487C86026D686A10034AC92 /* CameraPreviewPauseTests.m in Sources */ = {isa = PBXBuildFile; fileRef = E487C85F26D686A10034AC92 /* CameraPreviewPauseTests.m */; };
 		F6EE622F2710A6FC00905E4A /* MockFLTThreadSafeFlutterResult.m in Sources */ = {isa = PBXBuildFile; fileRef = F6EE622E2710A6FC00905E4A /* MockFLTThreadSafeFlutterResult.m */; };
 /* End PBXBuildFile section */
@@ -86,6 +87,7 @@
 		E0C6E1FE2770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ThreadSafeTextureRegistryTests.m; sourceTree = "<group>"; };
 		E0C6E1FF2770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ThreadSafeEventChannelTests.m; sourceTree = "<group>"; };
 		E0F95E3C27A32AB900699390 /* CameraPropertiesTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CameraPropertiesTests.m; sourceTree = "<group>"; };
+		E0F95E4327A36B9200699390 /* SampleBufferQueueTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = SampleBufferQueueTests.m; sourceTree = "<group>"; };
 		E487C85F26D686A10034AC92 /* CameraPreviewPauseTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CameraPreviewPauseTests.m; sourceTree = "<group>"; };
 		F63F9EED27143B19002479BF /* MockFLTThreadSafeFlutterResult.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MockFLTThreadSafeFlutterResult.h; sourceTree = "<group>"; };
 		F6EE622E2710A6FC00905E4A /* MockFLTThreadSafeFlutterResult.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MockFLTThreadSafeFlutterResult.m; sourceTree = "<group>"; };
@@ -122,6 +124,7 @@
 				E0C6E1FF2770F01A00EA6AA3 /* ThreadSafeEventChannelTests.m */,
 				E0C6E1FD2770F01A00EA6AA3 /* ThreadSafeMethodChannelTests.m */,
 				E0C6E1FE2770F01A00EA6AA3 /* ThreadSafeTextureRegistryTests.m */,
+				E0F95E4327A36B9200699390 /* SampleBufferQueueTests.m */,
 				E01EE4A72799F3A5008C1950 /* QueueHelperTests.m */,
 				E487C85F26D686A10034AC92 /* CameraPreviewPauseTests.m */,
 				F6EE622E2710A6FC00905E4A /* MockFLTThreadSafeFlutterResult.m */,
@@ -390,6 +393,7 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				E0F95E4427A36B9200699390 /* SampleBufferQueueTests.m in Sources */,
 				03F6F8B226CBB4670024B8D3 /* ThreadSafeFlutterResultTests.m in Sources */,
 				033B94BE269C40A200B4DF97 /* CameraMethodChannelTests.m in Sources */,
 				E0F95E3D27A32AB900699390 /* CameraPropertiesTests.m in Sources */,

packages/camera/camera/example/ios/RunnerTests/SampleBufferQueueTests.m

@@ -0,0 +1,37 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+@import camera;
+@import camera.Test;
+@import AVFoundation;
+@import XCTest;
+#import <OCMock/OCMock.h>
+
+@interface SampleBufferQueueTests : XCTestCase
+
+@end
+
+@implementation SampleBufferQueueTests
+
+- (void)testSampleBufferCallbackQueueMustBeCaptureSessionQueue {
+  id inputMock = OCMClassMock([AVCaptureDeviceInput class]);
+  OCMStub([inputMock deviceInputWithDevice:[OCMArg any] error:[OCMArg setTo:nil]])
+      .andReturn(inputMock);
+
+  id sessionMock = OCMClassMock([AVCaptureSession class]);
+  OCMStub([sessionMock alloc]).andReturn(sessionMock);
+  OCMStub([sessionMock addInputWithNoConnections:[OCMArg any]]);  // no-op
+  OCMStub([sessionMock canSetSessionPreset:[OCMArg any]]).andReturn(YES);
+
+  dispatch_queue_t captureSessionQueue = dispatch_queue_create("testing", NULL);
+  FLTCam *cam = [[FLTCam alloc] initWithCameraName:@"camera"
+                                  resolutionPreset:@"medium"
+                                       enableAudio:true
+                                       orientation:UIDeviceOrientationPortrait
+                               captureSessionQueue:captureSessionQueue
+                                             error:nil];
+  XCTAssertEqual(captureSessionQueue, cam.captureVideoOutput.sampleBufferCallbackQueue);
+}
+
+@end

packages/camera/camera/ios/Classes/CameraPlugin.modulemap

@@ -8,5 +8,6 @@ framework module camera {
     header "CameraPlugin_Test.h"
     header "CameraProperties.h"
     header "FLTCam.h"
+    header "FLTCam_Test.h"
   }
 }

packages/camera/camera/ios/Classes/FLTCam.m

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #import "FLTCam.h"
+#import "FLTCam_Test.h"
 
 @import CoreMotion;
 #import <libkern/OSAtomic.h>
@@ -114,7 +115,6 @@ @interface FLTCam () <AVCaptureVideoDataOutputSampleBufferDelegate,
 @property(readonly, nonatomic) AVCaptureSession *captureSession;
 
 @property(readonly, nonatomic) AVCapturePhotoOutput *capturePhotoOutput API_AVAILABLE(ios(10));
-@property(readonly, nonatomic) AVCaptureVideoDataOutput *captureVideoOutput;
 @property(readonly, nonatomic) AVCaptureInput *captureVideoInput;
 @property(readonly) CVPixelBufferRef volatile latestPixelBuffer;
 @property(readonly, nonatomic) CGSize captureSize;
@@ -184,7 +184,7 @@ - (instancetype)initWithCameraName:(NSString *)cameraName
   _captureVideoOutput.videoSettings =
       @{(NSString *)kCVPixelBufferPixelFormatTypeKey : @(_videoFormat)};
   [_captureVideoOutput setAlwaysDiscardsLateVideoFrames:YES];
-  [_captureVideoOutput setSampleBufferDelegate:self queue:dispatch_get_main_queue()];
+  [_captureVideoOutput setSampleBufferDelegate:self queue:captureSessionQueue];
 
   AVCaptureConnection *connection =
       [AVCaptureConnection connectionWithInputPorts:_captureVideoInput.ports

packages/camera/camera/ios/Classes/FLTCam_Test.h

@@ -0,0 +1,12 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "FLTCam.h"
+
+// APIs exposed for unit testing.
+@interface FLTCam ()
+
+@property(readonly, nonatomic) AVCaptureVideoDataOutput *captureVideoOutput;
+
+@end

packages/camera/camera/pubspec.yaml

@@ -4,7 +4,7 @@ description: A Flutter plugin for controlling the camera. Supports previewing
   Dart.
 repository: https://github.com/flutter/plugins/tree/main/packages/camera/camera
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.4+8
+version: 0.9.4+9
 
 environment:
   sdk: ">=2.14.0 <3.0.0"