Apply the correct transform for videos with different orientations

Author: AlexV525

packages/video_player/video_player_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2.3.2
+
+* Apply the correct transform for videos with different orientations.
+
 ## 2.3.1
 
 * Renames internal method channels to avoid potential confusion with the

packages/video_player/video_player_avfoundation/example/ios/RunnerTests/VideoPlayerTests.m

@@ -7,19 +7,59 @@
 @import XCTest;
 
 #import <OCMock/OCMock.h>
+#import <video_player_avfoundation/AVAssetTrack+Utils.h>
 
 @interface FLTVideoPlayer : NSObject <FlutterStreamHandler>
 @property(readonly, nonatomic) AVPlayer *player;
+@property(readonly, nonatomic) CGAffineTransform preferredTransform;
 @end
 
 @interface FLTVideoPlayerPlugin (Test) <FLTAVFoundationVideoPlayerApi>
 @property(readonly, strong, nonatomic)
     NSMutableDictionary<NSNumber *, FLTVideoPlayer *> *playersByTextureId;
 @end
 
+@interface FakeAVAssetTrack : AVAssetTrack
+@property(readonly, nonatomic) CGAffineTransform preferredTransform;
+@property(readonly, nonatomic) CGSize naturalSize;
+@property(readonly, nonatomic) UIImageOrientation orientation;
+- (instancetype)initWithOrientation:(UIImageOrientation)orientation;
+@end
+
 @interface VideoPlayerTests : XCTestCase
 @end
 
+@implementation FakeAVAssetTrack
+
+- (instancetype)initWithOrientation:(UIImageOrientation)orientation {
+    _orientation = orientation;
+    _naturalSize = CGSizeMake(800, 600);
+    return self;
+}
+
+- (CGAffineTransform) preferredTransform {
+  switch (_orientation) {
+    case UIImageOrientationUp:
+      return CGAffineTransformMake(1, 0, 0, 1, 0, 0);
+    case UIImageOrientationUpMirrored:
+      return CGAffineTransformMake(-1, 0, 0, 1, 0, 0);
+    case UIImageOrientationDown:
+      return CGAffineTransformMake(-1, 0, 0, -1, 0, 0);
+    case UIImageOrientationDownMirrored:
+      return CGAffineTransformMake(1, 0, 0, -1, 0, 0);
+    case UIImageOrientationLeft:
+      return CGAffineTransformMake(0, -1, 1, 0, 0, 0);
+    case UIImageOrientationLeftMirrored:
+      return CGAffineTransformMake(0, -1, -1, 0, 0, 0);
+    case UIImageOrientationRight:
+      return CGAffineTransformMake(0, 1, -1, 0, 0, 0);
+    case UIImageOrientationRightMirrored:
+      return CGAffineTransformMake(0, 1, 1, 0, 0, 0);
+  }
+}
+
+@end
+
 @implementation VideoPlayerTests
 
 - (void)testSeekToInvokesTextureFrameAvailableOnTextureRegistry {
@@ -121,6 +161,17 @@ - (void)testHLSControls {
   XCTAssertEqualWithAccuracy([videoInitialization[@"duration"] intValue], 4000, 200);
 }
 
+- (void)testTransformFix {
+  [self testTransformFixByOrientation:UIImageOrientationUp];
+  [self testTransformFixByOrientation:UIImageOrientationDown];
+  [self testTransformFixByOrientation:UIImageOrientationLeft];
+  [self testTransformFixByOrientation:UIImageOrientationRight];
+  [self testTransformFixByOrientation:UIImageOrientationUpMirrored];
+  [self testTransformFixByOrientation:UIImageOrientationDownMirrored];
+  [self testTransformFixByOrientation:UIImageOrientationLeftMirrored];
+  [self testTransformFixByOrientation:UIImageOrientationRightMirrored];
+}
+
 - (NSDictionary<NSString *, id> *)testPlugin:(FLTVideoPlayerPlugin *)videoPlayerPlugin
                                          uri:(NSString *)uri {
   FlutterError *error;
@@ -175,4 +226,47 @@ - (void)testHLSControls {
   return initializationEvent;
 }
 
+- (void)testTransformFixByOrientation:(UIImageOrientation)orientation {
+  AVAssetTrack *track = [[FakeAVAssetTrack alloc] initWithOrientation:orientation];
+  CGAffineTransform t = [track fixTransform];
+  CGSize size = track.naturalSize;
+  CGFloat expectX, expectY;
+  switch (orientation) {
+  case UIImageOrientationUp:
+    expectX = 0;
+    expectY = 0;
+    break;
+  case UIImageOrientationDown:
+    expectX = size.width;
+    expectY = size.height;
+    break;
+  case UIImageOrientationLeft:
+    expectX = 0;
+    expectY = size.width;
+    break;
+  case UIImageOrientationRight:
+    expectX = size.height;
+    expectY = 0;
+    break;
+  case UIImageOrientationUpMirrored:
+    expectX = size.width;
+    expectY = 0;
+    break;
+  case UIImageOrientationDownMirrored:
+    expectX = 0;
+    expectY = size.height;
+    break;
+  case UIImageOrientationLeftMirrored:
+    expectX = size.height;
+    expectY = size.width;
+    break;
+  case UIImageOrientationRightMirrored:
+    expectX = 0;
+    expectY = 0;
+    break;
+  }
+  XCTAssertEqual(t.tx, expectX);
+  XCTAssertEqual(t.ty, expectY);
+}
+
 @end

packages/video_player/video_player_avfoundation/ios/Classes/AVAssetTrack+Utils.h

@@ -0,0 +1,15 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import <AVFoundation/AVFoundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+// Fix the transform for the track.
+// Each fix case corresponding to `UIImage.Orientation`, with 8 cases in total.
+@interface AVAssetTrack (Utils)
+- (CGAffineTransform)fixTransform;
+@end
+
+NS_ASSUME_NONNULL_END

packages/video_player/video_player_avfoundation/ios/Classes/AVAssetTrack+Utils.m

@@ -0,0 +1,48 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "AVAssetTrack+Utils.h"
+
+@implementation AVAssetTrack (Utils)
+
+- (CGAffineTransform)fixTransform {
+  CGAffineTransform t = self.preferredTransform;
+  CGSize size = self.naturalSize;
+  if (t.a == 1 && t.b == 0 && t.c == 0 && t.d == 1) {
+    // UIImageOrientationUp
+    t.tx = 0;
+    t.ty = 0;
+  } else if (t.a == -1 && t.b == 0 && t.c == 0 && t.d == -1) {
+    // UIImageOrientationDown
+    t.tx = size.width;
+    t.ty = size.height;
+  } else if (t.a == 0 && t.b == -1 && t.c == 1 && t.d == 0) {
+    // UIImageOrientationLeft
+    t.tx = 0;
+    t.ty = size.width;
+  } else if (t.a == 0 && t.b == 1 && t.c == -1 && t.d == 0) {
+    // UIImageOrientationRight
+    t.tx = size.height;
+    t.ty = 0;
+  } else if (t.a == -1 && t.b == 0 && t.c == 0 && t.d == 1) {
+    // UIImageOrientationUpMirrored
+    t.tx = size.width;
+    t.ty = 0;
+  } else if (t.a == 1 && t.b == 0 && t.c == 0 && t.d == -1) {
+    // UIImageOrientationDownMirrored
+    t.tx = 0;
+    t.ty = size.height;
+  } else if (t.a == 0 && t.b == -1 && t.c == -1 && t.d == 0) {
+    // UIImageOrientationLeftMirrored
+    t.tx = size.height;
+    t.ty = size.width;
+  } else if (t.a == 0 && t.b == 1 && t.c == 1 && t.d == 0) {
+    // UIImageOrientationRightMirrored
+    t.tx = 0;
+    t.ty = 0;
+  }
+  return t;
+}
+
+@end

packages/video_player/video_player_avfoundation/ios/Classes/FLTVideoPlayerPlugin.m

@@ -2,9 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#import "FLTVideoPlayerPlugin.h"
 #import <AVFoundation/AVFoundation.h>
 #import <GLKit/GLKit.h>
+#import "AVAssetTrack+Utils.h"
+#import "FLTVideoPlayerPlugin.h"
 #import "messages.g.h"
 
 #if !__has_feature(objc_arc)
@@ -187,29 +188,6 @@ - (instancetype)initWithURL:(NSURL *)url
   return [self initWithPlayerItem:item frameUpdater:frameUpdater];
 }
 
-- (CGAffineTransform)fixTransform:(AVAssetTrack *)videoTrack {
-  CGAffineTransform transform = videoTrack.preferredTransform;
-  // TODO(@recastrodiaz): why do we need to do this? Why is the preferredTransform incorrect?
-  // At least 2 user videos show a black screen when in portrait mode if we directly use the
-  // videoTrack.preferredTransform Setting tx to the height of the video instead of 0, properly
-  // displays the video https://github.com/flutter/flutter/issues/17606#issuecomment-413473181
-  if (transform.tx == 0 && transform.ty == 0) {
-    NSInteger rotationDegrees = (NSInteger)round(radiansToDegrees(atan2(transform.b, transform.a)));
-    NSLog(@"TX and TY are 0. Rotation: %ld. Natural width,height: %f, %f", (long)rotationDegrees,
-          videoTrack.naturalSize.width, videoTrack.naturalSize.height);
-    if (rotationDegrees == 90) {
-      NSLog(@"Setting transform tx");
-      transform.tx = videoTrack.naturalSize.height;
-      transform.ty = 0;
-    } else if (rotationDegrees == 270) {
-      NSLog(@"Setting transform ty");
-      transform.tx = 0;
-      transform.ty = videoTrack.naturalSize.width;
-    }
-  }
-  return transform;
-}
-
 - (instancetype)initWithPlayerItem:(AVPlayerItem *)item
                       frameUpdater:(FLTFrameUpdater *)frameUpdater {
   self = [super init];
@@ -226,7 +204,7 @@ - (instancetype)initWithPlayerItem:(AVPlayerItem *)item
           if ([videoTrack statusOfValueForKey:@"preferredTransform"
                                         error:nil] == AVKeyValueStatusLoaded) {
             // Rotate the video by using a videoComposition and the preferredTransform
-            self->_preferredTransform = [self fixTransform:videoTrack];
+            self->_preferredTransform = [videoTrack fixTransform];
             // Note:
             // https://developer.apple.com/documentation/avfoundation/avplayeritem/1388818-videocomposition
             // Video composition can only be used with file-based media and is not supported for

packages/video_player/video_player_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: video_player_avfoundation
 description: iOS implementation of the video_player plugin.
 repository: https://github.com/flutter/plugins/tree/master/packages/video_player/video_player_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+video_player%22
-version: 2.3.1
+version: 2.3.2
 
 environment:
   sdk: ">=2.14.0 <3.0.0"