Reduce flicker when entering non-native full screen

In macvim-dev#1547, flicker during font size change and showing tab/scrollbar were
reduced. Here, we do something similar for the flicker that happens when
entering non-native full screen, where the temporarily moved text view
shows a brief temporary draw before the updated Vim redraws over it,
leading to a flicker. Use the same mechanism here to block rendering and
offset the text view draw to preserve visual stability as much as
possible. No need for native full screen as the smooth but slow
transition means there isn't a sharp flicker anyway.

Also, reduce the "fill right" behavior of Core Text renderer. It's
designed to make smooth resizing more seamless but it fills all the way
to the right which makes situations like this or maximizing the window
jarrying as Vim looks stretched horizontally but not vertically. Just
put a sane cap of 4 cell sizes. This way if Vim is a little slow smooth
resize still looks good, but it won't stretch across the whole screen.

Author: ychin

src/MacVim/MMCoreTextView.m

@@ -1499,6 +1499,8 @@ - (NSRect)rectForRow:(int)row column:(int)col numRows:(int)nr
     if (col + nc == grid.cols) {
         const NSInteger insetRight = [[NSUserDefaults standardUserDefaults] integerForKey:MMTextInsetRightKey];
         CGFloat extraWidth = frame.size.width - insetRight - (rect.size.width + rect.origin.x);
+        if (extraWidth > cellSize.width * 4) // just a sane cap so Vim doesn't look really stretched when resized before Vim could catch up
+            extraWidth = cellSize.width * 4;
         rect.size.width += extraWidth;
     }
 

src/MacVim/MMWindow.m

@@ -213,11 +213,12 @@ - (IBAction)zoom:(id)sender
 
 - (IBAction)toggleFullScreen:(id)sender
 {
-    // HACK! This is an NSWindow method used to enter full-screen on OS X 10.7.
-    // We override it so that we can interrupt and pass this on to Vim first.
-    // An alternative hack would be to reroute the action message sent by the
-    // full-screen button in the top right corner of a window, but there could
-    // be other places where this action message is sent from.
+    // This is an NSWindow method used to enter full-screen since OS X 10.7
+    // Lion. We override it so that we can interrupt and pass this on to Vim
+    // first, as it is full-screen aware (":set fullscreen") and it's better to
+    // only have one path to enter full screen. For non-native full screen this
+    // does mean this button will now enter non-native full screen instead of
+    // native one.
     // To get to the original method (and enter Lion full-screen) we need to
     // call realToggleFullScreen: defined below.
 
@@ -228,9 +229,8 @@ - (IBAction)toggleFullScreen:(id)sender
 - (IBAction)realToggleFullScreen:(id)sender
 {
 #if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_7)
-    // HACK! See toggleFullScreen: comment above.
-    if ([NSWindow instancesRespondToSelector:@selector(toggleFullScreen:)])
-        [super toggleFullScreen:sender];
+    // See toggleFullScreen: comment above.
+    [super toggleFullScreen:sender];
 #endif
 }
 

src/MacVim/MMWindowController.h

@@ -31,7 +31,9 @@
 
     BOOL                shouldResizeVimView; ///< Indicates there is a pending command to resize the Vim view
     BOOL                shouldKeepGUISize; ///< If on, the Vim view resize will try to fit in the existing window. If off, the window resizes to fit Vim view.
+
     BOOL                blockRenderUntilResize; ///< Indicates that there should be no text rendering until a Vim view resize is completed to avoid flicker.
+    NSRect              blockedRenderTextViewFrame; ///< The old screen-based coords for the text view when render was blocked.
 
     BOOL                shouldRestoreUserTopLeft;
     int                 updateToolbarFlag;

src/MacVim/MMWindowController.m

@@ -420,6 +420,7 @@ - (void)setTextDimensionsWithRows:(int)rows columns:(int)cols isLive:(BOOL)live
     vimView.pendingLiveResize = NO;
     if (blockRenderUntilResize) {
         blockRenderUntilResize = NO;
+        blockedRenderTextViewFrame = NSZeroRect;
         [vimView.textView setDrawRectOffset:NSZeroSize];
     }
     if (vimView.pendingLiveResizeQueued) {
@@ -506,6 +507,9 @@ - (void)resizeVimViewBlockRender
         shouldResizeVimView = YES;
         shouldKeepGUISize = YES;
         blockRenderUntilResize = YES;
+        blockedRenderTextViewFrame = [self.window convertRectToScreen:
+                                      [vimView convertRect:vimView.textView.frame
+                                                    toView:nil]];
         if (!vimController.isHandlingInputQueue)
             [self processInputQueueDidFinish];
     }
@@ -884,7 +888,6 @@ - (void)processInputQueueDidFinish
 
     const int oldTextViewRows = vimView.textView.pendingMaxRows;
     const int oldTextViewCols = vimView.textView.pendingMaxColumns;
-    const NSRect oldTextViewFrame = vimView.textView.frame;
     BOOL vimViewSizeChanged = NO;
 
     // NOTE: If the window has not been presented then we must avoid resizing
@@ -899,8 +902,6 @@ - (void)processInputQueueDidFinish
         // Setting 'guioptions+=k' will make shouldKeepGUISize true, which
         // means avoid resizing the window. Instead, resize the view instead
         // to keep the GUI window's size consistent.
-        // Note: Vim should always have requested shouldKeepGUISize to be true
-        //       when in full screen, but we check for it anyway for safety.
         bool avoidWindowResize = shouldKeepGUISize || fullScreenEnabled;
 
         if (!avoidWindowResize) {
@@ -939,7 +940,7 @@ - (void)processInputQueueDidFinish
 
     if (blockRenderUntilResize) {
         if (vimViewSizeChanged) {
-            const NSRect newTextViewFrame = vimView.textView.frame;
+            const NSRect newTextViewFrame = [self.window convertRectToScreen:[vimView convertRect:vimView.textView.frame toView:nil]];
 
             // We are currently blocking all rendering to prevent flicker. If
             // the view frame moved (this happens if the tab or left scroll bar
@@ -948,7 +949,9 @@ - (void)processInputQueueDidFinish
             // to match the new size. To alleviate this, we temporarily apply
             // a drawing offset in the text view to counter the offset. To the
             // user it would appear that the text view hasn't moved at all.
-            [vimView.textView setDrawRectOffset:NSMakeSize(NSMinX(oldTextViewFrame) - NSMinX(newTextViewFrame), NSMaxY(oldTextViewFrame) - NSMaxY(newTextViewFrame))];
+            [vimView.textView setDrawRectOffset:
+             NSMakeSize(NSMinX(blockedRenderTextViewFrame) - NSMinX(newTextViewFrame),
+                        NSMaxY(blockedRenderTextViewFrame) - NSMaxY(newTextViewFrame))];
         } else {
             // We were blocking all rendering until Vim has been resized. However
             // in situations where we turned out to not need to resize Vim to
@@ -959,6 +962,7 @@ - (void)processInputQueueDidFinish
             // we need to resize) but turned out we set it to the same font so
             // the grid size is the same and no need to resize.
             blockRenderUntilResize = NO;
+            blockedRenderTextViewFrame = NSZeroRect;
             [vimView.textView setDrawRectOffset:NSZeroSize];
 
             [vimController sendMessage:RedrawMsgID data:nil];
@@ -1139,6 +1143,22 @@ - (void)enterFullScreen:(int)fuoptions backgroundColor:(NSColor *)back
         // custom full-screen can appear on any screen, as opposed to native
         // full-screen which always uses the main screen.)
         if (windowPresented) {
+            const BOOL shouldPreventFlicker = (fuoptions & FUOPT_MAXVERT) && (fuoptions & FUOPT_MAXHORZ);
+            if (shouldPreventFlicker) {
+                // Prevent visual flickering by temporarily blocking new render
+                // until Vim has updated/resized itself.
+                // We don't do the same when exiting full screen because when
+                // going in this direction the flickering is less noticeable
+                // and it looks odd when the user sees a clamped view.
+                // Also, don't do this if maxvert/maxhorz not set because it
+                // looks quite off in that situation as Vim is supposed to move
+                // visually.
+                blockRenderUntilResize = YES;
+                blockedRenderTextViewFrame = [decoratedWindow convertRectToScreen:
+                                              [vimView convertRect:vimView.textView.frame
+                                                            toView:nil]];
+            }
+
             [fullScreenWindow enterFullScreen];
             fullScreenEnabled = YES;
 
@@ -1149,8 +1169,6 @@ - (void)enterFullScreen:(int)fuoptions backgroundColor:(NSColor *)back
             if (blurRadius != 0)
                 [MMWindow setBlurRadius:blurRadius onWindow:fullScreenWindow];
 
-            // The resize handle disappears so the vim view needs to update the
-            // scrollbars.
             shouldResizeVimView = YES;
         }
     }

src/MacVim/MacVimTests/MacVimTests.m

@@ -17,6 +17,7 @@
 #import "MMApplication.h"
 #import "MMFullScreenWindow.h"
 #import "MMWindow.h"
+#import "MMTabline.h"
 #import "MMTextView.h"
 #import "MMWindowController.h"
 #import "MMVimController.h"
@@ -887,6 +888,8 @@ - (void) testResizeVimView {
     XCTAssertLessThan(textView.pendingMaxRows, 30); // confirms that we have an outstanding resize request to make it smaller
     XCTAssertLessThan(textView.pendingMaxColumns, 80);
     XCTAssertTrue(win.isRenderBlocked);
+    XCTAssertEqual(textView.drawRectOffset.width, 0);
+    XCTAssertEqual(textView.drawRectOffset.height, 0);
     // Vim has responded to the size change. We should now have unblocked rendering.
     [self waitForVimMessage:SetTextDimensionsNoResizeWindowMsgID blockFutureMessages:YES];
     XCTAssertLessThan(textView.maxRows, 30);
@@ -910,7 +913,7 @@ - (void) testResizeVimView {
     [self waitForVimMessage:ShowTabBarMsgID blockFutureMessages:YES];
     XCTAssertEqual(textView.maxRows, 30);
     XCTAssertLessThan(textView.pendingMaxRows, 30);
-    XCTAssertGreaterThan(textView.drawRectOffset.height, 0);
+    XCTAssertEqual(textView.drawRectOffset.height, MMTablineHeight);
     XCTAssertTrue(win.isRenderBlocked);
     [self waitForVimMessage:SetTextDimensionsNoResizeWindowMsgID blockFutureMessages:YES];
     XCTAssertLessThan(textView.maxRows, 30);
@@ -923,7 +926,11 @@ - (void) testResizeVimView {
     // was not explicitly set.
     [self setDefault:MMNativeFullScreenKey toValue:@NO]; // non-native is faster so use that
     [self sendStringToVim:@":set guioptions-=k fullscreen\n" withMods:0];
+    [self waitForVimMessage:EnterFullScreenMsgID blockFutureMessages:YES];
+    XCTAssertTrue(win.isRenderBlocked);
+    [self blockVimProcessInput:NO];
     [self waitForFullscreenTransitionIsEnter:YES isNative:NO];
+    XCTAssertFalse(win.isRenderBlocked);
     int fuRows = textView.maxRows;
     int fuCols = textView.maxColumns;
     [self sendStringToVim:@":set guifont=Menlo:h13\n" withMods:0];