JBR-3640 java/awt/Modal/ModalFocusTransferTests/FocusTransferDWFAppModalTest.java: window Open button lost focus when it should not

(cherry picked from commit a7d486ca3d)

src/java.desktop/macosx/classes/sun/lwawt/LWComponentPeer.java

@@ -1342,7 +1342,7 @@ public abstract class LWComponentPeer<T extends Component, D extends JComponent>
 
     public Point windowToLocal(Point p, LWWindowPeer wp) {
         LWComponentPeer<?, ?> cp = this;
-        while (cp != wp) {
+        while (cp != wp && cp != null) {
             Rectangle cpb = cp.getBounds();
             p.x -= cpb.x;
             p.y -= cpb.y;

src/java.desktop/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -1138,7 +1138,6 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
             }
             nativeSetEnabled(ptr, !blocked);
         });
-        checkBlockingAndOrder();
     }
 
     public final void invalidateShadow() {
@@ -1316,14 +1315,16 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
                 isFocusable ? focusableStyleBits : 0); // set both bits at once
     }
 
-    private boolean checkBlockingAndOrder() {
+    private void checkBlockingAndOrder() {
         LWWindowPeer blocker = (peer == null)? null : peer.getBlocker();
         if (blocker == null) {
-            return false;
+            // If it's not blocked, make sure it's above its siblings
+            orderAboveSiblings();
+            return;
         }
 
         if (blocker instanceof CPrinterDialogPeer) {
-            return true;
+            return;
         }
 
         CPlatformWindow pWindow = (CPlatformWindow)blocker.getPlatformWindow();
@@ -1340,7 +1341,6 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
                 CWrapper.NSWindow.makeMainWindow(ptr);
             });
         }
-        return true;
     }
 
     private boolean isIconified() {
@@ -1402,7 +1402,7 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
             if (p instanceof LWWindowPeer) {
                 CPlatformWindow pw = (CPlatformWindow)((LWWindowPeer)p).getPlatformWindow();
                 iconified = isIconified();
-                if (pw != null && pw.isVisible() && !iconified) {
+                if (pw != null && pw.isVisible() && !iconified && !pw.delayShowing()) {
                     // If the window is one of ancestors of 'main window' or is going to become main by itself,
                     // the window should be ordered above its siblings; otherwise the window is just ordered
                     // above its nearest parent.
@@ -1474,9 +1474,7 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
 
     private void windowDidBecomeMain() {
         lastBecomeMainTime = System.currentTimeMillis();
-        if (checkBlockingAndOrder()) return;
-        // If it's not blocked, make sure it's above its siblings
-        orderAboveSiblings();
+        checkBlockingAndOrder();
     }
 
     private void windowWillEnterFullScreen() {

src/java.desktop/macosx/native/libawt_lwawt/awt/AWTWindow.m

@@ -612,8 +612,8 @@ AWT_ASSERT_APPKIT_THREAD;
     jobject platformWindow = [self.javaPlatformWindow jObjectWithEnv:env];
     if (platformWindow != NULL) {
         static JNF_MEMBER_CACHE(jm_checkBlockingAndOrder, jc_CPlatformWindow,
-                                "checkBlockingAndOrder", "()Z");
-        JNFCallBooleanMethod(env, platformWindow, jm_checkBlockingAndOrder);
+                                "checkBlockingAndOrder", "()V");
+        JNFCallVoidMethod(env, platformWindow, jm_checkBlockingAndOrder);
         (*env)->DeleteLocalRef(env, platformWindow);
     }
 }
@@ -666,7 +666,7 @@ AWT_ASSERT_APPKIT_THREAD;
                         // back to normal window level
                         [window setLevel:NSNormalWindowLevel];
                     }
-                    if (window.onActiveSpace) {
+                    if (window.onActiveSpace && owner.nsWindow.onActiveSpace) {
                         // The childWindow should be displayed in front of
                         // its nearest parentWindow
                         [window orderWindow:NSWindowAbove relativeTo:[owner.nsWindow windowNumber]];

src/java.desktop/share/classes/java/awt/Window.java

@@ -1071,7 +1071,13 @@ public class Window extends Container implements Accessible {
             } else {
                 // fix for 6532736: after this window is shown, its blocker
                 // should be raised to front
-                modalBlocker.toFront_NoClientCode();
+                boolean storedValue = modalBlocker.isAutoRequestFocus();
+                modalBlocker.setAutoRequestFocus(false);
+                try {
+                    modalBlocker.toFront_NoClientCode();
+                } finally {
+                    modalBlocker.setAutoRequestFocus(storedValue);
+                }
             }
             if (this instanceof Frame || this instanceof Dialog) {
                 updateChildFocusableWindowState(this);

test/jdk/jb/java/awt/Window/FullScreenInactiveDialog.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright 2021 JetBrains s.r.o.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import com.apple.eawt.Application;
+import com.apple.eawt.FullScreenAdapter;
+import com.apple.eawt.FullScreenUtilities;
+import com.apple.eawt.event.FullScreenEvent;
+
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * @test
+ * @summary Regression test for JBR-3633 Modal dialog is shown not at the same space as its parent
+ * @key headful
+ * @requires (os.family == "mac")
+ * @modules java.desktop/com.apple.eawt
+ *          java.desktop/com.apple.eawt.event
+ */
+
+public class FullScreenInactiveDialog {
+    private static final CompletableFuture<Boolean> shownAtFullScreen = new CompletableFuture<>();
+
+    private static Robot robot;
+    private static JFrame frame;
+    private static JDialog dialog;
+
+    public static void main(String[] args) throws Exception {
+        robot = new Robot();
+        try {
+            SwingUtilities.invokeAndWait(FullScreenInactiveDialog::initUI);
+            shownAtFullScreen.get(5, TimeUnit.SECONDS);
+            switchToPreviousSpace();
+            SwingUtilities.invokeLater(FullScreenInactiveDialog::openDialog);
+            robot.delay(1000);
+            if (isWindowReallyShowing(dialog)) {
+                throw new RuntimeException("Dialog is showing earlier than expected");
+            }
+            activateApp(); // simulates clicking on app icon in dock or switch using Cmd+Tab
+            if (!isWindowReallyShowing(dialog)) {
+                throw new RuntimeException("Dialog isn't showing when expected");
+            }
+            if (!isWindowReallyShowing(frame)) {
+                throw new RuntimeException("Frame isn't showing when expected");
+            }
+        } finally {
+            SwingUtilities.invokeAndWait(FullScreenInactiveDialog::disposeUI);
+        }
+    }
+
+    private static void initUI() {
+        frame = new JFrame("FullScreenInactiveDialog");
+        frame.setSize(300, 100);
+        frame.setVisible(true);
+
+
+        FullScreenUtilities.addFullScreenListenerTo(frame, new FullScreenAdapter() {
+            @Override
+            public void windowEnteredFullScreen(FullScreenEvent e) {
+                shownAtFullScreen.complete(true);
+            }
+        });
+        Application.getApplication().requestToggleFullScreen(frame);
+    }
+
+    private static void openDialog() {
+        dialog = new JDialog(frame);
+        dialog.setSize(100, 100);
+        dialog.setVisible(true);
+    }
+
+    private static void disposeUI() {
+        if (frame != null) frame.dispose();
+    }
+
+    private static void switchToPreviousSpace() {
+        robot.keyPress(KeyEvent.VK_CONTROL);
+        robot.keyPress(KeyEvent.VK_LEFT);
+        robot.keyRelease(KeyEvent.VK_LEFT);
+        robot.keyRelease(KeyEvent.VK_CONTROL);
+        robot.delay(1000); // wait for animation to finish
+    }
+
+    private static void activateApp() {
+        Desktop.getDesktop().requestForeground(false);
+        robot.delay(1000); // wait for animation to finish
+    }
+
+    private static boolean isWindowReallyShowing(Window window) throws Exception {
+        Point[] location = new Point[1];
+        AtomicBoolean movementDetected = new AtomicBoolean();
+        SwingUtilities.invokeAndWait(() -> {
+            if (window.isVisible()) {
+                Rectangle bounds = window.getBounds();
+                Insets insets = window.getInsets();
+                bounds.x += insets.left;
+                bounds.y += insets.top;
+                bounds.width -= insets.left + insets.right;
+                bounds.height -= insets.top + insets.bottom;
+                if (!bounds.isEmpty()) {
+                    location[0] = new Point(bounds.x + bounds.width / 2, bounds.y + bounds.height / 2);
+                    window.addMouseMotionListener(new MouseMotionAdapter() {
+                        @Override
+                        public void mouseMoved(MouseEvent e) {
+                            movementDetected.set(true);
+                        }
+                    });
+                }
+            }
+        });
+        Point target = location[0];
+        if (target == null) {
+            return false;
+        }
+        robot.mouseMove(target.x, target.y);
+        robot.delay(100);
+        robot.mouseMove(target.x + 1, target.y + 1);
+        robot.delay(1000);
+        return movementDetected.get();
+    }
+}