Fix 10.12 transparent title bar, and 10.14 'transparency' issues

src/MacVim/MMBackend.m

@@ -1187,7 +1187,7 @@ - (void)enterFullScreen:(int)fuoptions background:(int)bg
 {
     NSMutableData *data = [NSMutableData data];
     [data appendBytes:&fuoptions length:sizeof(int)];
-    bg = MM_COLOR(bg);
+    bg = MM_COLOR_WITH_TRANSP(bg,p_transp);
     [data appendBytes:&bg length:sizeof(int)];
     [self queueMessage:EnterFullScreenMsgID data:data];
 }

src/MacVim/MMCoreTextView.m

@@ -452,7 +452,7 @@ - (BOOL)_wantsKeyDownForEvent:(id)event
 
 - (void)setFrameSize:(NSSize)newSize {
     if (!NSEqualSizes(newSize, self.bounds.size)) {
-        if (!drawPending && !cgBufferDrawEnabled) {
+        if (!drawPending && !cgBufferDrawEnabled && drawData.count == 0) {
             // When resizing a window, it will invalidate the buffer and cause
             // MacVim to draw black until we get the draw commands from Vim and
             // we draw them out in drawRect. Use beginGrouping to stop the

src/MacVim/MMVimView.m

@@ -497,14 +497,12 @@ - (void)setDefaultColorsBackground:(NSColor *)back foreground:(NSColor *)fore
 
     CALayer *backedLayer = [self layer];
     if (backedLayer) {
-        // This would only trigger in 10.14 where all views are layer-backed.
-        //
-        // Note: This doesn't do much now. Should fix this class to use
-        // updateLayer: instead of drawRect: at a later time, which would draw
-        // the background color automatically. When we do that we can remove the
-        // hack at drawKnobSlotInRect: since it would overlay properly without
-        // needing to manually draw the background color itself.
-        [backedLayer setBackgroundColor:[back CGColor]];
+        // This only happens in 10.14+, where everything is layer-backed by
+        // default. Since textView draws itself as a separate layer, we don't
+        // want this layer to draw anything. This is especially important with
+        // 'transparency' where there's alpha blending and we don't want this
+        // layer to be in the way and double-blending things.
+        [backedLayer setBackgroundColor:CGColorGetConstantColor(kCGColorClear)];
     }
 
     for (NSUInteger i = 0, count = [scrollbars count]; i < count; ++i) {
@@ -1005,10 +1003,10 @@ - (void)drawKnobSlotInRect:(NSRect)slotRect highlight:(BOOL)flag
     // show through rendering artifacts (e.g. if guioption 'k' is on, and you
     // turn off the bar bar, the artiacts will show through in the overlay).
     //
-    // Note: This should ideally be done on MMVimView itself by setting a background
-    // color. This would be fixed at a later time by telling the view to just
-    // use the background color form the backed CALayer (mandated since Mojave
-    // 10.14).
+    // Note: Another way to fix this is to make sure to draw the underlying
+    // MMVimView or the window with the proper color so the scrollbar would just
+    // draw on top, but this doesn't work properly right now, and it's difficult
+    // to get that to work with the 'transparency' setting as well.
     MMVimView *vimView = [self target];
     NSColor *defaultBackgroundColor = [[vimView textView] defaultBackgroundColor];
     [defaultBackgroundColor setFill];

src/MacVim/MMWindowController.m

@@ -543,7 +543,39 @@ - (void)setDefaultColorsBackground:(NSColor *)back foreground:(NSColor *)fore
     [decoratedWindow setOpaque:isOpaque];
     if (fullScreenWindow)
         [fullScreenWindow setOpaque:isOpaque];
-    [decoratedWindow setBackgroundColor:back];
+
+#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_14
+    if (@available(macos 10.14, *)) {
+        // We usually don't really need to change the background color of the
+        // window, but in 10.14+ we switched to using layer-backed drawing.
+        // That's fine except when we set 'transparency' to non-zero. The alpha
+        // is set on the text view, but it won't work if drawn on top of a solid
+        // window, so we need to set a transparency color here to make the
+        // transparency show through.
+        if ([back alphaComponent] == 1) {
+            // Here, any solid color would do, but setting it with "back" has an
+            // interesting effect where the title bar gets subtly tinted by it
+            // as well, so do that. (Note that this won't play well in <=10.12
+            // since we are using the deprecated
+            // NSWindowStyleMaskTexturedBackground which makes the titlebars
+            // transparent in those. Consider not using textured background.)
+            [decoratedWindow setBackgroundColor:back];
+            if (fullScreenWindow) {
+                [fullScreenWindow setBackgroundColor:back];
+            }
+        } else {
+            // HACK! We really want a transparent background color to avoid
+            // double blending the transparency, but setting alpha=0 leads to
+            // the window border disappearing and also drag-to-resize becomes a
+            // lot slower. So hack around it by making it virtually transparent.
+            NSColor *clearColor = [back colorWithAlphaComponent:0.001];
+            [decoratedWindow setBackgroundColor:clearColor];
+            if (fullScreenWindow) {
+                [fullScreenWindow setBackgroundColor:clearColor];
+            }
+        }
+    }
+#endif
 
     [vimView setDefaultColorsBackground:back foreground:fore];
 }
@@ -758,8 +790,16 @@ - (void)enterFullScreen:(int)fuoptions backgroundColor:(NSColor *)back
         // times during startup.
         [fullScreenWindow release];
 
+        NSColor *fullscreenBg = back;
+
+        // See setDefaultColorsBackground: for why set a transparent
+        // background color, and why 0.001 instead of 0.
+        if ([fullscreenBg alphaComponent] != 1) {
+            fullscreenBg = [fullscreenBg colorWithAlphaComponent:0.001];
+        }
+
         fullScreenWindow = [[MMFullScreenWindow alloc]
-            initWithWindow:decoratedWindow view:vimView backgroundColor:back];
+            initWithWindow:decoratedWindow view:vimView backgroundColor:fullscreenBg];
         [fullScreenWindow setOptions:fuoptions];
         [fullScreenWindow setRepresentedFilename:
             [decoratedWindow representedFilename]];
@@ -1544,7 +1584,7 @@ - (void)updateTablineSeparator
     BOOL windowTextured = ([decoratedWindow styleMask] &
                             NSWindowStyleMaskTexturedBackground) != 0;
     BOOL hideSeparator  = NO;
-    
+
     if (floor(NSAppKitVersionNumber) >= NSAppKitVersionNumber10_10) {
         // The tabline separator is mostly an old feature and not necessary
         // modern macOS versions.