[video_player_avfoundation] Applies the standardized transform for videos with different orientations (#5069)

Author: AlexV525

packages/video_player/video_player_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.3.2
+
+* Applies the standardized transform for videos with different orientations.
+
 ## 2.3.1
 
 * Renames internal method channels to avoid potential confusion with the

packages/video_player/video_player_avfoundation/example/ios/RunnerTests/VideoPlayerTests.m

@@ -7,6 +7,7 @@
 @import XCTest;
 
 #import <OCMock/OCMock.h>
+#import <video_player_avfoundation/AVAssetTrackUtils.h>
 
 @interface FLTVideoPlayer : NSObject <FlutterStreamHandler>
 @property(readonly, nonatomic) AVPlayer *player;
@@ -17,6 +18,44 @@ @interface FLTVideoPlayerPlugin (Test) <FLTAVFoundationVideoPlayerApi>
     NSMutableDictionary<NSNumber *, FLTVideoPlayer *> *playersByTextureId;
 @end
 
+@interface FakeAVAssetTrack : AVAssetTrack
+@property(readonly, nonatomic) CGAffineTransform preferredTransform;
+@property(readonly, nonatomic) CGSize naturalSize;
+@property(readonly, nonatomic) UIImageOrientation orientation;
+- (instancetype)initWithOrientation:(UIImageOrientation)orientation;
+@end
+
+@implementation FakeAVAssetTrack
+
+- (instancetype)initWithOrientation:(UIImageOrientation)orientation {
+  _orientation = orientation;
+  _naturalSize = CGSizeMake(800, 600);
+  return self;
+}
+
+- (CGAffineTransform)preferredTransform {
+  switch (_orientation) {
+    case UIImageOrientationUp:
+      return CGAffineTransformMake(1, 0, 0, 1, 0, 0);
+    case UIImageOrientationDown:
+      return CGAffineTransformMake(-1, 0, 0, -1, 0, 0);
+    case UIImageOrientationLeft:
+      return CGAffineTransformMake(0, -1, 1, 0, 0, 0);
+    case UIImageOrientationRight:
+      return CGAffineTransformMake(0, 1, -1, 0, 0, 0);
+    case UIImageOrientationUpMirrored:
+      return CGAffineTransformMake(-1, 0, 0, 1, 0, 0);
+    case UIImageOrientationDownMirrored:
+      return CGAffineTransformMake(1, 0, 0, -1, 0, 0);
+    case UIImageOrientationLeftMirrored:
+      return CGAffineTransformMake(0, -1, -1, 0, 0, 0);
+    case UIImageOrientationRightMirrored:
+      return CGAffineTransformMake(0, 1, 1, 0, 0, 0);
+  }
+}
+
+@end
+
 @interface VideoPlayerTests : XCTestCase
 @end
 
@@ -121,6 +160,17 @@ - (void)testHLSControls {
   XCTAssertEqualWithAccuracy([videoInitialization[@"duration"] intValue], 4000, 200);
 }
 
+- (void)testTransformFix {
+  [self validateTransformFixForOrientation:UIImageOrientationUp];
+  [self validateTransformFixForOrientation:UIImageOrientationDown];
+  [self validateTransformFixForOrientation:UIImageOrientationLeft];
+  [self validateTransformFixForOrientation:UIImageOrientationRight];
+  [self validateTransformFixForOrientation:UIImageOrientationUpMirrored];
+  [self validateTransformFixForOrientation:UIImageOrientationDownMirrored];
+  [self validateTransformFixForOrientation:UIImageOrientationLeftMirrored];
+  [self validateTransformFixForOrientation:UIImageOrientationRightMirrored];
+}
+
 - (NSDictionary<NSString *, id> *)testPlugin:(FLTVideoPlayerPlugin *)videoPlayerPlugin
                                          uri:(NSString *)uri {
   FlutterError *error;
@@ -175,4 +225,47 @@ - (void)testHLSControls {
   return initializationEvent;
 }
 
+- (void)validateTransformFixForOrientation:(UIImageOrientation)orientation {
+  AVAssetTrack *track = [[FakeAVAssetTrack alloc] initWithOrientation:orientation];
+  CGAffineTransform t = FLTGetStandardizedTransformForTrack(track);
+  CGSize size = track.naturalSize;
+  CGFloat expectX, expectY;
+  switch (orientation) {
+    case UIImageOrientationUp:
+      expectX = 0;
+      expectY = 0;
+      break;
+    case UIImageOrientationDown:
+      expectX = size.width;
+      expectY = size.height;
+      break;
+    case UIImageOrientationLeft:
+      expectX = 0;
+      expectY = size.width;
+      break;
+    case UIImageOrientationRight:
+      expectX = size.height;
+      expectY = 0;
+      break;
+    case UIImageOrientationUpMirrored:
+      expectX = size.width;
+      expectY = 0;
+      break;
+    case UIImageOrientationDownMirrored:
+      expectX = 0;
+      expectY = size.height;
+      break;
+    case UIImageOrientationLeftMirrored:
+      expectX = size.height;
+      expectY = size.width;
+      break;
+    case UIImageOrientationRightMirrored:
+      expectX = 0;
+      expectY = 0;
+      break;
+  }
+  XCTAssertEqual(t.tx, expectX);
+  XCTAssertEqual(t.ty, expectY);
+}
+
 @end

packages/video_player/video_player_avfoundation/ios/Classes/AVAssetTrackUtils.h

@@ -0,0 +1,14 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import <AVFoundation/AVFoundation.h>
+
+/**
+ * Returns a standardized transform
+ * according to the orientation of the track.
+ *
+ * Note: https://stackoverflow.com/questions/64161544
+ * `AVAssetTrack.preferredTransform` can have wrong `tx` and `ty`.
+ */
+CGAffineTransform FLTGetStandardizedTransformForTrack(AVAssetTrack* track);

packages/video_player/video_player_avfoundation/ios/Classes/AVAssetTrackUtils.m

@@ -0,0 +1,46 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import <AVFoundation/AVFoundation.h>
+
+CGAffineTransform FLTGetStandardizedTransformForTrack(AVAssetTrack *track) {
+  CGAffineTransform t = track.preferredTransform;
+  CGSize size = track.naturalSize;
+  // Each case of control flows corresponds to a specific
+  // `UIImageOrientation`, with 8 cases in total.
+  if (t.a == 1 && t.b == 0 && t.c == 0 && t.d == 1) {
+    // UIImageOrientationUp
+    t.tx = 0;
+    t.ty = 0;
+  } else if (t.a == -1 && t.b == 0 && t.c == 0 && t.d == -1) {
+    // UIImageOrientationDown
+    t.tx = size.width;
+    t.ty = size.height;
+  } else if (t.a == 0 && t.b == -1 && t.c == 1 && t.d == 0) {
+    // UIImageOrientationLeft
+    t.tx = 0;
+    t.ty = size.width;
+  } else if (t.a == 0 && t.b == 1 && t.c == -1 && t.d == 0) {
+    // UIImageOrientationRight
+    t.tx = size.height;
+    t.ty = 0;
+  } else if (t.a == -1 && t.b == 0 && t.c == 0 && t.d == 1) {
+    // UIImageOrientationUpMirrored
+    t.tx = size.width;
+    t.ty = 0;
+  } else if (t.a == 1 && t.b == 0 && t.c == 0 && t.d == -1) {
+    // UIImageOrientationDownMirrored
+    t.tx = 0;
+    t.ty = size.height;
+  } else if (t.a == 0 && t.b == -1 && t.c == -1 && t.d == 0) {
+    // UIImageOrientationLeftMirrored
+    t.tx = size.height;
+    t.ty = size.width;
+  } else if (t.a == 0 && t.b == 1 && t.c == 1 && t.d == 0) {
+    // UIImageOrientationRightMirrored
+    t.tx = 0;
+    t.ty = 0;
+  }
+  return t;
+}

packages/video_player/video_player_avfoundation/ios/Classes/FLTVideoPlayerPlugin.m

@@ -3,8 +3,11 @@
 // found in the LICENSE file.
 
 #import "FLTVideoPlayerPlugin.h"
+
 #import <AVFoundation/AVFoundation.h>
 #import <GLKit/GLKit.h>
+
+#import "AVAssetTrackUtils.h"
 #import "messages.g.h"
 
 #if !__has_feature(objc_arc)
@@ -187,29 +190,6 @@ - (instancetype)initWithURL:(NSURL *)url
   return [self initWithPlayerItem:item frameUpdater:frameUpdater];
 }
 
-- (CGAffineTransform)fixTransform:(AVAssetTrack *)videoTrack {
-  CGAffineTransform transform = videoTrack.preferredTransform;
-  // TODO(@recastrodiaz): why do we need to do this? Why is the preferredTransform incorrect?
-  // At least 2 user videos show a black screen when in portrait mode if we directly use the
-  // videoTrack.preferredTransform Setting tx to the height of the video instead of 0, properly
-  // displays the video https://github.com/flutter/flutter/issues/17606#issuecomment-413473181
-  if (transform.tx == 0 && transform.ty == 0) {
-    NSInteger rotationDegrees = (NSInteger)round(radiansToDegrees(atan2(transform.b, transform.a)));
-    NSLog(@"TX and TY are 0. Rotation: %ld. Natural width,height: %f, %f", (long)rotationDegrees,
-          videoTrack.naturalSize.width, videoTrack.naturalSize.height);
-    if (rotationDegrees == 90) {
-      NSLog(@"Setting transform tx");
-      transform.tx = videoTrack.naturalSize.height;
-      transform.ty = 0;
-    } else if (rotationDegrees == 270) {
-      NSLog(@"Setting transform ty");
-      transform.tx = 0;
-      transform.ty = videoTrack.naturalSize.width;
-    }
-  }
-  return transform;
-}
-
 - (instancetype)initWithPlayerItem:(AVPlayerItem *)item
                       frameUpdater:(FLTFrameUpdater *)frameUpdater {
   self = [super init];
@@ -226,7 +206,7 @@ - (instancetype)initWithPlayerItem:(AVPlayerItem *)item
           if ([videoTrack statusOfValueForKey:@"preferredTransform"
                                         error:nil] == AVKeyValueStatusLoaded) {
             // Rotate the video by using a videoComposition and the preferredTransform
-            self->_preferredTransform = [self fixTransform:videoTrack];
+            self->_preferredTransform = FLTGetStandardizedTransformForTrack(videoTrack);
             // Note:
             // https://developer.apple.com/documentation/avfoundation/avplayeritem/1388818-videocomposition
             // Video composition can only be used with file-based media and is not supported for

packages/video_player/video_player_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: video_player_avfoundation
 description: iOS implementation of the video_player plugin.
 repository: https://github.com/flutter/plugins/tree/master/packages/video_player/video_player_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+video_player%22
-version: 2.3.1
+version: 2.3.2
 
 environment:
   sdk: ">=2.14.0 <3.0.0"