JBR-4364 Port macOS native file dialog patches

IDEA-146669 Enable Mac native file dialogs
IDEA-159507 Mac native dialogs: multiple open dialogs are possible
JBR-1784 File dialogs aren't themed on macOS
JBR-1752 Floating windows overlap modal dialogs
JBR-2005: don't set appearance of file chooser if OSX version < 10.14
JBR-4546 Focus is not returned back to IDE after closing "Open" dialog

src/java.desktop/macosx/classes/sun/lwawt/macosx/CFileDialog.java

@@ -57,6 +57,7 @@ import java.util.List;
 import com.jetbrains.desktop.JBRFileDialog;
 import sun.awt.AWTAccessor;
 import sun.java2d.pipe.Region;
+import sun.lwawt.LWWindowPeer;
 import sun.security.action.GetBooleanAction;
 import sun.util.logging.PlatformLogger;
 
@@ -93,7 +94,16 @@ class CFileDialog implements FileDialogPeer {
                     title = " ";
                 }
 
-                String[] userFileNames = nativeRunFileDialog(title,
+                Window owner = target.getOwner();
+
+                long ownerPtr = owner == null ?
+                        0L :
+                        ((CPlatformWindow) ((LWWindowPeer) AWTAccessor.getComponentAccessor().getPeer(owner))
+                                .getPlatformWindow()).executeGet(ptr -> ptr);
+
+                String[] userFileNames = nativeRunFileDialog(
+                        ownerPtr,
+                        title,
                         dialogMode,
                         target.isMultipleMode(),
                         navigateApps,
@@ -108,7 +118,7 @@ class CFileDialog implements FileDialogPeer {
                 String file = null;
                 File[] files = null;
 
-                if (userFileNames != null) {
+                if (userFileNames != null && userFileNames.length > 0) {
                     // the dialog wasn't cancelled
                     int filesNumber = userFileNames.length;
                     files = new File[filesNumber];
@@ -191,7 +201,7 @@ class CFileDialog implements FileDialogPeer {
         return ret;
     }
 
-    private native String[] nativeRunFileDialog(String title, int mode,
+    private native String[] nativeRunFileDialog(long ownerPtr, String title, int mode,
             boolean multipleMode, boolean shouldNavigateApps,
             boolean canChooseDirectories, boolean canChooseFiles,
             boolean canCreateDirectories, boolean hasFilenameFilter,

src/java.desktop/macosx/native/libawt_lwawt/awt/CFileDialog.h

@@ -26,6 +26,8 @@
 #import <Cocoa/Cocoa.h>
 
 @interface CFileDialog : NSObject <NSOpenSavePanelDelegate> {
+    NSWindow* fOwner;
+
     // Should we query back to Java for a file filter?
     jboolean fHasFileFilter;
 
@@ -65,7 +67,8 @@
 }
 
 // Allocator
-- (id) initWithFilter:(jboolean)inHasFilter
+- (id) initWithOwner:(NSWindow*) owner
+           filter:(jboolean)inHasFilter
            fileDialog:(jobject)inDialog
                 title:(NSString *)inTitle
             directory:(NSString *)inPath

src/java.desktop/macosx/native/libawt_lwawt/awt/CFileDialog.m

@@ -29,6 +29,7 @@
 #import "ThreadUtilities.h"
 #import "JNIUtilities.h"
 #import "CFileDialog.h"
+#import "AWTWindow.h"
 #import "CMenuBar.h"
 #import "ApplicationDelegate.h"
 
@@ -37,7 +38,8 @@
 
 @implementation CFileDialog
 
-- (id)initWithFilter:(jboolean)inHasFilter
+- (id)initWithOwner:(NSWindow*)owner
+              filter:(jboolean)inHasFilter
           fileDialog:(jobject)inDialog
                title:(NSString *)inTitle
            directory:(NSString *)inPath
@@ -50,7 +52,9 @@ canChooseDirectories:(BOOL)inChooseDirectories
 canCreateDirectories:(BOOL)inCreateDirectories
              withEnv:(JNIEnv*)env;
 {
-  if (self = [super init]) {
+    if (self = [super init]) {
+        fOwner = owner;
+        [fOwner retain];
         fHasFileFilter = inHasFilter;
         fFileDialog = (*env)->NewGlobalRef(env, inDialog);
         fDirectory = inPath;
@@ -92,6 +96,9 @@ canCreateDirectories:(BOOL)inCreateDirectories
     [fURLs release];
     fURLs = nil;
 
+    [fOwner release];
+    fOwner = nil;
+
     [super dealloc];
 }
 
@@ -149,10 +156,55 @@ canCreateDirectories:(BOOL)inCreateDirectories
             [editMenuItem release];
         }
 
-        fPanelResult = [thePanel runModalForDirectory:fDirectory file:fFile];
-        [thePanel setDelegate:nil];
-        CMenuBar *menuBar = [[ApplicationDelegate sharedDelegate] defaultMenuBar];
-        [CMenuBar activate:menuBar modallyDisabled:NO];
+
+        if (fOwner != nil) {
+            if (fDirectory != nil) {
+                 [thePanel setDirectoryURL:[NSURL fileURLWithPath:[fDirectory stringByExpandingTildeInPath]]];
+            }
+
+            if (fFile != nil) {
+                 [thePanel setNameFieldStringValue:fFile];
+            }
+
+            CMenuBar *menuBar = nil;
+            if (fOwner != nil) {
+
+                // Finds appropriate menubar in our hierarchy,
+                AWTWindow *awtWindow = (AWTWindow *)fOwner.delegate;
+                while (awtWindow.ownerWindow != nil) {
+                    awtWindow = awtWindow.ownerWindow;
+                }
+
+                BOOL isDisabled = NO;
+                if ([awtWindow.nsWindow isVisible]){
+                    menuBar = awtWindow.javaMenuBar;
+                    isDisabled = !awtWindow.isEnabled;
+                }
+
+                if (menuBar == nil) {
+                    menuBar = [[ApplicationDelegate sharedDelegate] defaultMenuBar];
+                    isDisabled = NO;
+                }
+
+                [CMenuBar activate:menuBar modallyDisabled:isDisabled];
+            }
+
+            if (@available(macOS 10.14, *)) {
+                [thePanel setAppearance:fOwner.appearance];
+            }
+
+            fPanelResult = [thePanel runModal];
+
+            if (menuBar != nil) {
+                [CMenuBar activate:menuBar modallyDisabled:NO];
+            }
+        }
+        else
+        {
+            fPanelResult = [thePanel runModalForDirectory:fDirectory file:fFile];
+            CMenuBar *menuBar = [[ApplicationDelegate sharedDelegate] defaultMenuBar];
+            [CMenuBar activate:menuBar modallyDisabled:NO];
+        }
 
         if (editMenuItem != nil) {
             [menu removeItem:editMenuItem];
@@ -167,6 +219,8 @@ canCreateDirectories:(BOOL)inCreateDirectories
             }
             [fURLs retain];
         }
+
+        [thePanel setDelegate:nil];
     }
 
     [self disposer];
@@ -234,7 +288,7 @@ canCreateDirectories:(BOOL)inCreateDirectories
 }
 
 - (BOOL) userClickedOK {
-    return fPanelResult == NSOKButton;
+    return fPanelResult == NSFileHandlingPanelOKButton;
 }
 
 - (NSArray *)URLs {
@@ -248,7 +302,7 @@ canCreateDirectories:(BOOL)inCreateDirectories
  */
 JNIEXPORT jobjectArray JNICALL
 Java_sun_lwawt_macosx_CFileDialog_nativeRunFileDialog
-(JNIEnv *env, jobject peer, jstring title, jint mode, jboolean multipleMode,
+(JNIEnv *env, jobject peer, jlong ownerPtr, jstring title, jint mode, jboolean multipleMode,
  jboolean navigateApps, jboolean chooseDirectories, jboolean chooseFiles, jboolean createDirectories,
  jboolean hasFilter, jstring directory, jstring file)
 {
@@ -260,7 +314,8 @@ JNI_COCOA_ENTER(env);
         dialogTitle = @" ";
     }
 
-    CFileDialog *dialogDelegate = [[CFileDialog alloc] initWithFilter:hasFilter
+    CFileDialog *dialogDelegate = [[CFileDialog alloc] initWithOwner:(NSWindow *)jlong_to_ptr(ownerPtr)
+                                                               filter:hasFilter
                                                            fileDialog:peer
                                                                 title:dialogTitle
                                                             directory:JavaStringToNSString(env, directory)

test/jdk/jb/java/awt/Focus/FileDialogClosing.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2022 JetBrains s.r.o.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.InputEvent;
+import java.awt.event.KeyEvent;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * @test
+ * @summary Regression test for JBR-4546 Focus is not returned back to IDE after closing "Open" dialog
+ * @key headful
+ */
+
+public class FileDialogClosing {
+    private static final AtomicInteger pressedCount = new AtomicInteger();
+
+    private static Robot robot;
+    private static Frame frame;
+    private static Button button;
+
+    public static void main(String[] args) throws Exception {
+        robot = new Robot();
+        try {
+            SwingUtilities.invokeAndWait(FileDialogClosing::initUI);
+            robot.delay(1000);
+            clickOn(button);
+            robot.delay(1000);
+            pressAndRelease(KeyEvent.VK_ESCAPE);
+            robot.delay(1000);
+            pressAndRelease(KeyEvent.VK_SPACE);
+            robot.delay(1000);
+            if (pressedCount.get() != 2) {
+                throw new RuntimeException("Unexpected pressed count: " + pressedCount);
+            }
+        } finally {
+            SwingUtilities.invokeAndWait(FileDialogClosing::disposeUI);
+        }
+    }
+
+    private static void initUI() {
+        frame = new Frame("FileDialogClosing");
+        button = new Button("Open dialog");
+        button.addActionListener(e -> {
+            pressedCount.incrementAndGet();
+            new FileDialog(frame).setVisible(true);
+        });
+        frame.add(button);
+        frame.setBounds(200, 200, 200, 200);
+        frame.setVisible(true);
+    }
+
+    private static void disposeUI() {
+        if (frame != null) frame.dispose();
+    }
+
+    private static void clickAt(int x, int y) {
+        robot.mouseMove(x, y);
+        robot.mousePress(InputEvent.BUTTON1_DOWN_MASK);
+        robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK);
+    }
+
+    private static void clickOn(Component component) {
+        Point location = component.getLocationOnScreen();
+        clickAt(location.x + component.getWidth() / 2, location.y + component.getHeight() / 2);
+    }
+
+    private static void pressAndRelease(int keyCode) {
+        robot.keyPress(keyCode);
+        robot.keyRelease(keyCode);
+    }
+}