macOS: add methods to setup transparent titlebar with custom height (#100)

* macOS: add methods to setup transparent titlebar with custom height * make windowTransparentTitleBarHeight CPlatfromWindow property * add windowTransparentTitleBarHeight test * Prevent mouseUp events on the transparent header on macOS when the window is being dragged Co-authored-by: Manuel Unterhofer <manuel.unterhofer@jetbrains.com> Custom macOS window decorations via JBR API JBR-4460 Fix window drag with custom decorations on macOS JBR-4553 Add logging to setUpTransparentTitleBar and resetTitleBar (cherry picked from commit f5552eed4d)
2026-01-08 01:21:42 +01:00 · 2022-01-26 22:23:22 +03:00
parent b340188215
commit 7d27fc695d
5 changed files with 516 additions and 1 deletions
--- a/src/java.desktop/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java
+++ b/src/java.desktop/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java
@@ -67,12 +67,15 @@ import sun.awt.AWTAccessor.WindowAccessor;
 import sun.awt.AWTThreading;
 import sun.java2d.SurfaceData;
 import sun.lwawt.LWKeyboardFocusManagerPeer;
+import sun.lwawt.LWComponentPeer;
 import sun.lwawt.LWLightweightFramePeer;
 import sun.lwawt.LWToolkit;
 import sun.lwawt.LWWindowPeer;
 import sun.lwawt.LWWindowPeer.PeerType;
 import sun.lwawt.PlatformWindow;
 import sun.util.logging.PlatformLogger;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;

 public class CPlatformWindow extends CFRetainedResource implements PlatformWindow {
    private native long nativeCreateNSWindow(long nsViewPtr,long ownerPtr, long styleBits, double x, double y, double w, double h);
@@ -102,6 +105,7 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
    static native CPlatformWindow nativeGetTopmostPlatformWindowUnderMouse();
    private static native void nativeRaiseLevel(long nsWindowPtr, boolean popup, boolean onlyIfParentIsActive);
    private static native boolean nativeDelayShowing(long nsWindowPtr);
+    private static native void nativeSetTransparentTitleBarHeight(long nsWindowPtr, float height);

    // Logger to report issues happened during execution but that do not affect functionality
    private static final PlatformLogger logger = PlatformLogger.getLogger("sun.lwawt.macosx.CPlatformWindow");
@@ -134,6 +138,7 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
    public static final String WINDOW_TRANSPARENT_TITLE_BAR = "apple.awt.transparentTitleBar";
    public static final String WINDOW_TITLE_VISIBLE = "apple.awt.windowTitleVisible";
    public static final String WINDOW_APPEARANCE = "apple.awt.windowAppearance";
+    public static final String WINDOW_TRANSPARENT_TITLE_BAR_HEIGHT = "apple.awt.windowTransparentTitleBarHeight";

    // This system property is named as jdk.* because it is not specific to AWT
    // and it is also used in JavaFX
@@ -281,6 +286,13 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
                    c.execute(ptr -> nativeSetNSWindowAppearance(ptr, (String) value));
                }
            }
+        },
+        new Property<CPlatformWindow>(WINDOW_TRANSPARENT_TITLE_BAR_HEIGHT) {
+            public void applyProperty(final CPlatformWindow c, final Object value) {
+                if (value != null && (value instanceof Float)) {
+                    c.execute(ptr -> AWTThreading.executeWaitToolkit(wait -> nativeSetTransparentTitleBarHeight(ptr, (float) value)));
+                }
+            }
        }
    }) {
        public CPlatformWindow convertJComponentToTarget(final JRootPane p) {
@@ -1477,4 +1489,21 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
        isInFullScreen = false;
        isFullScreenAnimationOn = false;
    }
+
+    // JBR API internals
+    private static void setCustomDecorationTitleBarHeight(Window target, ComponentPeer peer, float height) {
+        if (peer instanceof LWComponentPeer) {
+            PlatformWindow platformWindow = ((LWComponentPeer<?, ?>) peer).getPlatformWindow();
+            if (platformWindow instanceof CPlatformWindow) {
+                ((CPlatformWindow) platformWindow).execute(ptr -> {
+                    AWTThreading.executeWaitToolkit(wait -> nativeSetTransparentTitleBarHeight(ptr, height));
+                });
+                if (target instanceof javax.swing.RootPaneContainer) {
+                    final javax.swing.JRootPane rootpane = ((javax.swing.RootPaneContainer)target).getRootPane();
+                    if (rootpane != null) rootpane.putClientProperty(WINDOW_TRANSPARENT_TITLE_BAR_HEIGHT, height);
+                }
+            }
+        }
+    }
+
 }
--- a/src/java.desktop/macosx/native/libawt_lwawt/awt/AWTWindow.h
+++ b/src/java.desktop/macosx/native/libawt_lwawt/awt/AWTWindow.h
@@ -49,6 +49,13 @@
    BOOL isJustCreated;
    NSWindowTabbingMode javaWindowTabbingMode;
    BOOL isEnterFullScreen;
+    CGFloat _transparentTitleBarHeight;
+    id<NSObject> _windowWillEnterFullScreenNotification;
+    id<NSObject> _windowWillExitFullScreenNotification;
+    id<NSObject> _windowDidExitFullScreenNotification;
+    NSMutableArray* _transparentTitleBarConstraints;
+    NSLayoutConstraint *_transparentTitleBarHeightConstraint;
+    NSMutableArray *_transparentTitleBarButtonCenterXConstraints;
 }

 // An instance of either AWTWindow_Normal or AWTWindow_Panel
@@ -104,4 +111,9 @@
            contentView:(NSView *)view;
@end

+@interface AWTWindowDragView : NSView
+@property (nonatomic) jobject javaPlatformWindow;
+- (id) initWithPlatformWindow:(jobject)javaPlatformWindow;
+@end
+
 #endif _AWTWINDOW_H
--- a/src/java.desktop/macosx/native/libawt_lwawt/awt/AWTWindow.m
+++ b/src/java.desktop/macosx/native/libawt_lwawt/awt/AWTWindow.m
@@ -26,6 +26,7 @@
 #include <objc/objc-runtime.h>
 #import <Cocoa/Cocoa.h>

+#include <java_awt_Window_CustomWindowDecoration.h>
 #import "sun_lwawt_macosx_CPlatformWindow.h"
 #import "com_apple_eawt_event_GestureHandler.h"
 #import "com_apple_eawt_FullScreenHandler.h"
@@ -1264,8 +1265,303 @@ AWT_ASSERT_APPKIT_THREAD;
    return lastKeyWindow;
 }

+static const CGFloat DefaultHorizontalTitleBarButtonOffset = 20.0;
+
+- (CGFloat) getTransparentTitleBarButtonShrinkingFactor
+{
+    CGFloat minimumHeightWithoutShrinking = 28.0; // This is the smallest macOS title bar availabe with public APIs as of Monterey
+    CGFloat shrinkingFactor = fmin(_transparentTitleBarHeight / minimumHeightWithoutShrinking, 1.0);
+    return shrinkingFactor;
+}
+
+- (void) setUpTransparentTitleBar
+{
+
+    /**
+     * The view hierarchy normally looks as follows:
+     * NSThemeFrame
+     * ├─NSView (content view)
+     * └─NSTitlebarContainerView
+     *   ├─_NSTitlebarDecorationView (only on Mojave 10.14 and newer)
+     *   └─NSTitlebarView
+     *     ├─NSVisualEffectView (only on Big Sur 11 and newer)
+     *     ├─NSView (only on Big Sur and newer)
+     *     ├─_NSThemeCloseWidget - Close
+     *     ├─_NSThemeZoomWidget - Full Screen
+     *     ├─_NSThemeWidget - Minimize (note the different order compared to their layout)
+     *     └─AWTWindowDragView (we will create this)
+     *
+     * But the order and presence of decorations and effects has been unstable across different macOS versions,
+     * even patch upgrades, which is why the code below uses scans instead of indexed access
+     */
+    NSView* closeButtonView = [self.nsWindow standardWindowButton:NSWindowCloseButton];
+    NSView* zoomButtonView = [self.nsWindow standardWindowButton:NSWindowZoomButton];
+    NSView* miniaturizeButtonView = [self.nsWindow standardWindowButton:NSWindowMiniaturizeButton];
+    if (!closeButtonView || !zoomButtonView || !miniaturizeButtonView) {
+        NSLog(@"WARNING: setUpTransparentTitleBar closeButtonView=%@, zoomButtonView=%@, miniaturizeButtonView=%@",
+              closeButtonView, zoomButtonView, miniaturizeButtonView);
+        return;
+    }
+    NSView* titlebar = closeButtonView.superview;
+    NSView* titlebarContainer = titlebar.superview;
+    NSView* themeFrame = titlebarContainer.superview;
+    if (!themeFrame) {
+        NSLog(@"WARNING: setUpTransparentTitleBar titlebar=%@, titlebarContainer=%@, themeFrame=%@",
+              titlebar, titlebarContainer, themeFrame);
+        return;
+    }
+
+    _transparentTitleBarConstraints = [[NSMutableArray alloc] init];
+    titlebarContainer.translatesAutoresizingMaskIntoConstraints = NO;
+    _transparentTitleBarHeightConstraint = [titlebarContainer.heightAnchor constraintEqualToConstant:_transparentTitleBarHeight];
+    [_transparentTitleBarConstraints addObjectsFromArray:@[
+        [titlebarContainer.leftAnchor constraintEqualToAnchor:themeFrame.leftAnchor],
+        [titlebarContainer.widthAnchor constraintEqualToAnchor:themeFrame.widthAnchor],
+        [titlebarContainer.topAnchor constraintEqualToAnchor:themeFrame.topAnchor],
+        _transparentTitleBarHeightConstraint,
+    ]];
+
+    AWTWindowDragView* windowDragView = [[AWTWindowDragView alloc] initWithPlatformWindow:self.javaPlatformWindow];
+    [titlebar addSubview:windowDragView positioned:NSWindowBelow relativeTo:closeButtonView];
+
+    for (NSView* view in @[titlebar, windowDragView])
+    {
+        view.translatesAutoresizingMaskIntoConstraints = NO;
+        [_transparentTitleBarConstraints addObjectsFromArray:@[
+            [view.leftAnchor constraintEqualToAnchor:titlebarContainer.leftAnchor],
+            [view.rightAnchor constraintEqualToAnchor:titlebarContainer.rightAnchor],
+            [view.topAnchor constraintEqualToAnchor:titlebarContainer.topAnchor],
+            [view.bottomAnchor constraintEqualToAnchor:titlebarContainer.bottomAnchor],
+        ]];
+    }
+
+    CGFloat shrinkingFactor = [self getTransparentTitleBarButtonShrinkingFactor];
+    CGFloat horizontalButtonOffset = shrinkingFactor * DefaultHorizontalTitleBarButtonOffset;
+    _transparentTitleBarButtonCenterXConstraints = [[NSMutableArray alloc] initWithCapacity:3];
+    [@[closeButtonView, miniaturizeButtonView, zoomButtonView] enumerateObjectsUsingBlock:^(NSView* button, NSUInteger index, BOOL* stop)
+    {
+        button.translatesAutoresizingMaskIntoConstraints = NO;
+        NSLayoutConstraint* buttonCenterXConstraint = [button.centerXAnchor constraintEqualToAnchor:titlebarContainer.leftAnchor constant:(_transparentTitleBarHeight/2.0 + (index * horizontalButtonOffset))];
+        [_transparentTitleBarButtonCenterXConstraints addObject:buttonCenterXConstraint];
+        [_transparentTitleBarConstraints addObjectsFromArray:@[
+            [button.widthAnchor constraintLessThanOrEqualToAnchor:titlebarContainer.heightAnchor multiplier:0.5],
+            // Those corrections are required to keep the icons perfectly round because macOS adds a constant 2 px in resulting height to their frame
+            [button.heightAnchor constraintEqualToAnchor: button.widthAnchor multiplier:14.0/12.0 constant:-2.0],
+            [button.centerYAnchor constraintEqualToAnchor:titlebarContainer.centerYAnchor],
+            buttonCenterXConstraint,
+        ]];
+    }];
+
+    [NSLayoutConstraint activateConstraints:_transparentTitleBarConstraints];
+}
+
+- (void) updateTransparentTitleBarConstraints
+{
+    dispatch_sync(dispatch_get_main_queue(), ^{
+        _transparentTitleBarHeightConstraint.constant = _transparentTitleBarHeight;
+        CGFloat shrinkingFactor = [self getTransparentTitleBarButtonShrinkingFactor];
+        CGFloat horizontalButtonOffset = shrinkingFactor * DefaultHorizontalTitleBarButtonOffset;
+        [_transparentTitleBarButtonCenterXConstraints enumerateObjectsUsingBlock:^(NSLayoutConstraint* buttonConstraint, NSUInteger index, BOOL *stop)
+        {
+            buttonConstraint.constant = (_transparentTitleBarHeight/2.0 + (index * horizontalButtonOffset));
+        }];
+    });
+}
+
+- (void) resetTitleBar
+{
+    // See [setUpTransparentTitleBar] for the view hierarchy we're working with
+    NSView* closeButtonView = [self.nsWindow standardWindowButton:NSWindowCloseButton];
+    NSView* zoomButtonView = [self.nsWindow standardWindowButton:NSWindowZoomButton];
+    NSView* miniaturizeButtonView = [self.nsWindow standardWindowButton:NSWindowMiniaturizeButton];
+    NSView* titlebar = closeButtonView.superview;
+    NSView* titlebarContainer = titlebar.superview;
+    if (!titlebarContainer) {
+        NSLog(@"WARNING: resetTitleBar closeButtonView=%@, titlebar=%@, titlebarContainer=%@",
+              closeButtonView, titlebar, titlebarContainer);
+        return;
+    }
+
+    [NSLayoutConstraint deactivateConstraints:_transparentTitleBarConstraints];
+
+    AWTWindowDragView* windowDragView;
+    for (NSView* view in titlebar.subviews) {
+        if ([view isMemberOfClass:[AWTWindowDragView class]]) {
+            windowDragView = view;
+        }
+        if (view.translatesAutoresizingMaskIntoConstraints == NO) {
+            view.translatesAutoresizingMaskIntoConstraints = YES;
+        }
+    }
+    [windowDragView removeFromSuperview];
+    titlebarContainer.translatesAutoresizingMaskIntoConstraints = YES;
+    titlebar.translatesAutoresizingMaskIntoConstraints = YES;
+
+    _transparentTitleBarConstraints = nil;
+    _transparentTitleBarHeightConstraint = nil;
+    _transparentTitleBarButtonCenterXConstraints = nil;
+}
+
+- (void) setWindowControlsHidden: (BOOL) hidden
+{
+    [self.nsWindow standardWindowButton:NSWindowCloseButton].superview.hidden = hidden;
+}
+
+- (BOOL) isFullScreen
+{
+    NSUInteger masks = [self.nsWindow styleMask];
+    return (masks & NSWindowStyleMaskFullScreen) != 0;
+}
+
+- (void) configureWindowAndListenersForTransparentTitleBar
+{
+    dispatch_sync(dispatch_get_main_queue(), ^{
+        [self.nsWindow setTitlebarAppearsTransparent:YES];
+        [self.nsWindow setTitleVisibility:NSWindowTitleHidden];
+        [self.nsWindow setStyleMask:[self.nsWindow styleMask]|NSWindowStyleMaskFullSizeContentView];
+
+        if (!self.isFullScreen) {
+            [self setUpTransparentTitleBar];
+        }
+    });
+    NSNotificationCenter* defaultCenter = [NSNotificationCenter defaultCenter];
+    NSOperationQueue* mainQueue = [NSOperationQueue mainQueue];
+    _windowWillEnterFullScreenNotification = [defaultCenter addObserverForName:NSWindowWillEnterFullScreenNotification object:self.nsWindow queue:mainQueue usingBlock:^(NSNotification* notification) {
+        [self resetTitleBar];
+    }];
+    _windowWillExitFullScreenNotification = [defaultCenter addObserverForName:NSWindowWillExitFullScreenNotification object:self.nsWindow queue:mainQueue usingBlock:^(NSNotification* notification) {
+        [self setWindowControlsHidden:YES];
+    }];
+    _windowDidExitFullScreenNotification = [defaultCenter addObserverForName:NSWindowDidExitFullScreenNotification object:self.nsWindow queue:mainQueue usingBlock:^(NSNotification* notification) {
+        [self setUpTransparentTitleBar];
+        [self setWindowControlsHidden:NO];
+    }];
+}
+
+- (void) configureWindowAndListenersForDefaultTitleBar
+{
+    dispatch_sync(dispatch_get_main_queue(), ^{
+        [self.nsWindow setTitlebarAppearsTransparent:NO];
+        [self.nsWindow setTitleVisibility:NSWindowTitleVisible];
+        [self.nsWindow setStyleMask:[self.nsWindow styleMask]&(~NSWindowStyleMaskFullSizeContentView)];
+
+        if (!self.isFullScreen) {
+            [self resetTitleBar];
+        }
+    });
+    NSNotificationCenter* defaultCenter = [NSNotificationCenter defaultCenter];
+    [defaultCenter removeObserver:_windowWillEnterFullScreenNotification];
+    [defaultCenter removeObserver:_windowWillExitFullScreenNotification];
+    [defaultCenter removeObserver:_windowDidExitFullScreenNotification];
+    _windowWillEnterFullScreenNotification = _windowWillExitFullScreenNotification = _windowDidExitFullScreenNotification = nil;
+}
+
+- (void) setTransparentTitleBarHeight: (CGFloat) transparentTitleBarHeight
+{
+    if (_transparentTitleBarHeight == transparentTitleBarHeight) return;
+    if (_transparentTitleBarHeight != 0.0f) {
+        _transparentTitleBarHeight = transparentTitleBarHeight;
+        if (transparentTitleBarHeight == 0.0f) {
+            [self configureWindowAndListenersForDefaultTitleBar];
+        } else if (_transparentTitleBarHeightConstraint != nil || _transparentTitleBarButtonCenterXConstraints != nil) {
+            [self updateTransparentTitleBarConstraints];
+        }
+    } else {
+        _transparentTitleBarHeight = transparentTitleBarHeight;
+        [self configureWindowAndListenersForTransparentTitleBar];
+    }
+}
+
@end // AWTWindow

+@implementation AWTWindowDragView {
+    CGFloat _accumulatedDragDelta;
+    enum WindowDragState {
+        NO_DRAG,   // Mouse not dragging
+        SKIP_DRAG, // Mouse dragging in non-draggable area
+        DRAG,      // Mouse is dragging window
+    } _draggingWindow;
+}
+
+- (id) initWithPlatformWindow:(jobject)javaPlatformWindow {
+    self = [super init];
+    if (self == nil) return nil; // no hope
+
+    self.javaPlatformWindow = javaPlatformWindow;
+    return self;
+}
+
+- (BOOL)mouseDownCanMoveWindow
+{
+    return NO;
+}
+
+- (jint)hitTestCustomDecoration:(NSPoint)point
+{
+    jint returnValue = java_awt_Window_CustomWindowDecoration_NO_HIT_SPOT;
+    JNIEnv *env = [ThreadUtilities getJNIEnvUncached];
+    jobject platformWindow = (*env)->NewLocalRef(env, self.javaPlatformWindow);
+    if (platformWindow != NULL) {
+        GET_CPLATFORM_WINDOW_CLASS_RETURN(YES);
+        DECLARE_FIELD_RETURN(jf_target, jc_CPlatformWindow, "target", "Ljava/awt/Window;", YES);
+        DECLARE_CLASS_RETURN(jc_Window, "java/awt/Window", YES);
+        DECLARE_METHOD_RETURN(jm_hitTestCustomDecoration, jc_Window, "hitTestCustomDecoration", "(II)I", YES);
+        jobject awtWindow = (*env)->GetObjectField(env, platformWindow, jf_target);
+        if (awtWindow != NULL) {
+            NSRect frame = [self.window frame];
+            float windowHeight = frame.size.height;
+            returnValue = (*env)->CallIntMethod(env, awtWindow, jm_hitTestCustomDecoration, (jint) point.x,  (jint) (windowHeight - point.y));
+            CHECK_EXCEPTION();
+            (*env)->DeleteLocalRef(env, awtWindow);
+        }
+        (*env)->DeleteLocalRef(env, platformWindow);
+    }
+    return returnValue;
+}
+
+- (void)mouseDown:(NSEvent *)event
+{
+    _draggingWindow = NO_DRAG;
+    _accumulatedDragDelta = 0.0;
+    // We don't follow the regular responder chain here since the native window swallows events in some cases
+    [[self.window contentView] deliverJavaMouseEvent:event];
+}
+
+- (void)mouseDragged:(NSEvent *)event
+{
+    if (_draggingWindow == NO_DRAG) {
+        jint hitSpot = [self hitTestCustomDecoration:event.locationInWindow];
+        switch (hitSpot) {
+            case java_awt_Window_CustomWindowDecoration_DRAGGABLE_AREA:
+                // Start drag only after 4px threshold inside DRAGGABLE_AREA
+                if ((_accumulatedDragDelta += fabs(event.deltaX) + fabs(event.deltaY)) <= 4.0) break;
+            case java_awt_Window_CustomWindowDecoration_NO_HIT_SPOT:
+                [self.window performWindowDragWithEvent:event];
+                _draggingWindow = DRAG;
+                break;
+            default:
+                _draggingWindow = SKIP_DRAG;
+        }
+    }
+}
+
+- (void)mouseUp:(NSEvent *)event
+{
+    if (_draggingWindow == DRAG) {
+        _draggingWindow = NO_DRAG;
+    } else {
+        jint hitSpot = [self hitTestCustomDecoration:event.locationInWindow];
+        if (event.clickCount == 2 && hitSpot == java_awt_Window_CustomWindowDecoration_NO_HIT_SPOT) {
+            [self.window performZoom:nil];
+        }
+
+        // We don't follow the regular responder chain here since the native window swallows events in some cases
+        [[self.window contentView] deliverJavaMouseEvent:event];
+    }
+}
+
+@end
+
 /*
 * Class:     sun_lwawt_macosx_CPlatformWindow
 * Method:    nativeSetAllAllowAutomaticTabbingProperty
@@ -2059,3 +2355,16 @@ JNIEXPORT jboolean JNICALL Java_sun_lwawt_macosx_CPlatformWindow_nativeDelayShow

    return result;
 }
+
+
+JNIEXPORT void JNICALL Java_sun_lwawt_macosx_CPlatformWindow_nativeSetTransparentTitleBarHeight
+(JNIEnv *env, jclass clazz, jlong windowPtr, jfloat transparentTitleBarHeight)
+{
+    JNI_COCOA_ENTER(env);
+
+    NSWindow *nsWindow = (NSWindow *)jlong_to_ptr(windowPtr);
+    AWTWindow *window = (AWTWindow*)[nsWindow delegate];
+    [window setTransparentTitleBarHeight:((CGFloat) transparentTitleBarHeight)];
+
+    JNI_COCOA_EXIT(env);
+}
--- a/src/java.desktop/share/classes/java/awt/Window.java
+++ b/src/java.desktop/share/classes/java/awt/Window.java
@@ -3931,6 +3931,8 @@ public class Window extends Container implements Accessible {
            window.hasCustomDecoration = enabled;
            if (Win.INSTANCE != null) {
                Win.INSTANCE.updateCustomDecoration(window.peer);
+            } else if (MacOS.INSTANCE != null && window.customDecorTitleBarHeight > 0f) {
+                MacOS.INSTANCE.setTitleBarHeight(window, window.peer, enabled ? window.customDecorTitleBarHeight : 0);
            }
        }
        boolean isCustomDecorationEnabled(Window window) {
@@ -3945,7 +3947,12 @@ public class Window extends Container implements Accessible {
        }

        void setCustomDecorationTitleBarHeight(Window window, int height) {
-            if (height >= 0) window.customDecorTitleBarHeight = height;
+            if (height >= 0) {
+                window.customDecorTitleBarHeight = height;
+                if (MacOS.INSTANCE != null && window.hasCustomDecoration) {
+                    MacOS.INSTANCE.setTitleBarHeight(window, window.peer, height);
+                }
+            }
        }
        int getCustomDecorationTitleBarHeight(Window window) {
            return window.customDecorTitleBarHeight;
@@ -3956,6 +3963,12 @@ public class Window extends Container implements Accessible {
                    .withStatic("updateCustomDecoration", "sun.awt.windows.WFramePeer").build();
            void updateCustomDecoration(ComponentPeer peer);
        }
+
+        private interface MacOS {
+            MacOS INSTANCE = (MacOS) JBRApi.internalServiceBuilder(MethodHandles.lookup(), null)
+                    .withStatic("setTitleBarHeight", "sun.lwawt.macosx.CPlatformWindow", "setCustomDecorationTitleBarHeight").build();
+            void setTitleBarHeight(Window target, ComponentPeer peer, float height);
+        }
    }

    @Deprecated
--- a/test/jdk/jb/java/awt/Window/MacNativeTransparentTitlebarWithCustomHeight.java
+++ b/test/jdk/jb/java/awt/Window/MacNativeTransparentTitlebarWithCustomHeight.java
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, JetBrains s.r.o.. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @key headful
+ * @summary [macosx] transparent titlebar with custom height test
+ * @author Grigorii Kargin
+ * @run main MacNativeTransparentTitleBarWithCustomHeight
+ * @requires (os.family == "mac")
+ */
+
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.lang.reflect.InvocationTargetException;
+import javax.swing.*;
+
+public class MacNativeTransparentTitleBarWithCustomHeight
+{
+    private static final int TD = 10;
+    private static final Color darkSystemGray4 = new Color(58, 58, 60);
+    private static final Color lightSystemGray6 = new Color(242, 242, 247);
+    static MacNativeTransparentTitleBarWithCustomHeight theTest;
+    private Robot robot;
+    private JFrame frame;
+    private JRootPane rootPane;
+
+    private int DELAY = 1000;
+
+    public MacNativeTransparentTitleBarWithCustomHeight() {
+        try {
+            robot = new Robot();
+        } catch (AWTException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void performTest() {
+
+        runSwing(() -> {
+            frame = new JFrame("");
+            frame.setBounds(100, 100, 300, 150);
+            rootPane = frame.getRootPane();
+            JComponent contentPane = (JComponent) frame.getContentPane();
+            JPanel comp = new JPanel();
+            contentPane.add(comp);
+            comp.setBackground(Color.RED);
+            frame.setVisible(true);
+        });
+
+        // check that titlebar is not of background color
+        for (int px = 140; px < 160; px++) {
+            for (int py = 5; py < 20; py++) {
+                Color c = getTestPixel(px, py);
+                if (validateColor(c, Color.RED)) {
+                    throw new RuntimeException("Test failed. Incorrect color " + c +
+                            "at (" + px + "," + py + ")");
+                }
+            }
+        }
+
+        robot.delay(DELAY);
+        runSwing(() -> rootPane.putClientProperty("apple.awt.windowTransparentTitleBarHeight", 42f));
+        robot.delay(DELAY);
+
+        // check that titlebar is of background color
+        for (int px = 140; px < 160; px++) {
+            for (int py = 5; py < 20; py++) {
+                Color c = getTestPixel(px, py);
+                if (!validateColor(c, Color.RED)) {
+                    throw new RuntimeException("Test failed. Incorrect color " + c +
+                            "at (" + px + "," + py + ")");
+                }
+            }
+        }
+
+        runSwing(() -> frame.dispose());
+
+        frame = null;
+        rootPane = null;
+    }
+
+    private Color getTestPixel(int x, int y) {
+        Rectangle bounds = frame.getBounds();
+        BufferedImage screenImage = robot.createScreenCapture(bounds);
+        int rgb = screenImage.getRGB(x, y);
+        int red = (rgb >> 16) & 0xFF;
+        int green = (rgb >> 8) & 0xFF;
+        int blue = rgb & 0xFF;
+        Color c = new Color(red, green, blue);
+        return c;
+    }
+
+    private boolean validateColor(Color c, Color expected) {
+        return Math.abs(c.getRed() - expected.getRed()) <= TD &&
+            Math.abs(c.getGreen() - expected.getGreen()) <= TD &&
+            Math.abs(c.getBlue() - expected.getBlue()) <= TD;
+    }
+
+    public void dispose() {
+        if (frame != null) {
+            frame.dispose();
+            frame = null;
+        }
+    }
+
+    private static void runSwing(Runnable r) {
+        try {
+            SwingUtilities.invokeAndWait(r);
+        } catch (InterruptedException e) {
+        } catch (InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static void main(String[] args) {
+        if (!System.getProperty("os.name").contains("OS X")) {
+            System.out.println("This test is for MacOS only. Automatically passed on other platforms.");
+            return;
+        }
+
+        try {
+            runSwing(() -> theTest = new MacNativeTransparentTitleBarWithCustomHeight());
+            theTest.performTest();
+        } finally {
+            if (theTest != null) {
+                runSwing(() -> theTest.dispose());
+            }
+        }
+    }
+}