JRE-373 [macos] nativeCreateNSWindow deadlocks with a11y

(cherry picked from commit 72c77a992bbf1b95b82ffc08cb2f4f3bc36b3657)

(cherry picked from commit aa09fa2c85)

src/java.desktop/macosx/classes/sun/lwawt/macosx/CPlatformWindow.java

@@ -52,6 +52,7 @@ import java.util.Comparator;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.Callable;
 
 import javax.swing.JRootPane;
 import javax.swing.RootPaneContainer;
@@ -343,26 +344,29 @@ public class CPlatformWindow extends CFRetainedResource implements PlatformWindo
         } else {
             bounds = _peer.constrainBounds(_target.getBounds());
         }
-        AtomicLong ref = new AtomicLong();
-        contentView.execute(viewPtr -> {
-            boolean hasOwnerPtr = false;
+        long nativeWindowPtr = LWCToolkit.SelectorPerformer.perform(() -> {
+            AtomicLong ref = new AtomicLong();
+            contentView.execute(viewPtr -> {
+                boolean hasOwnerPtr = false;
 
-            if (owner != null) {
-                hasOwnerPtr = 0L != owner.executeGet(ownerPtr -> {
-                    ref.set(nativeCreateNSWindow(viewPtr, ownerPtr, styleBits,
-                                                    bounds.x, bounds.y,
-                                                    bounds.width, bounds.height));
-                    return 1;
-                });
-            }
+                if (owner != null) {
+                    hasOwnerPtr = 0L != owner.executeGet(ownerPtr -> {
+                        ref.set(nativeCreateNSWindow(viewPtr, ownerPtr, styleBits,
+                                bounds.x, bounds.y,
+                                bounds.width, bounds.height));
+                        return 1;
+                    });
+                }
 
-            if (!hasOwnerPtr) {
-                ref.set(nativeCreateNSWindow(viewPtr, 0,
-                                             styleBits, bounds.x, bounds.y,
-                                             bounds.width, bounds.height));
-            }
+                if (!hasOwnerPtr) {
+                    ref.set(nativeCreateNSWindow(viewPtr, 0,
+                            styleBits, bounds.x, bounds.y,
+                            bounds.width, bounds.height));
+                }
+            });
+            return ref.get();
         });
-        setPtr(ref.get());
+        setPtr(nativeWindowPtr);
 
         if (target instanceof javax.swing.RootPaneContainer) {
             final javax.swing.JRootPane rootpane = ((javax.swing.RootPaneContainer)target).getRootPane();

src/java.desktop/macosx/classes/sun/lwawt/macosx/LWCToolkit.java

@@ -87,16 +87,13 @@ import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.MissingResourceException;
-import java.util.Objects;
-import java.util.ResourceBundle;
-import java.util.concurrent.Callable;
-
+import java.security.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.net.MalformedURLException;
 import javax.swing.UIManager;
 
 import com.apple.laf.AquaMenuBarUI;
@@ -703,6 +700,139 @@ public final class LWCToolkit extends LWToolkit {
         }
     }
 
+    /**
+     * Performs a wrapped native selector on the main thread, waiting on EDT, preventing the threads from a deadlock.
+     */
+    public static class SelectorPerformer {
+        private ExecutorService executor;
+        // every perform() pushes a queue of invocations
+        private Stack<LinkedBlockingQueue<InvocationEvent>> invocations = new Stack<>();
+
+        // invocations should be dispatched on proper EDT (per AppContext)
+        private static final Map<Thread, SelectorPerformer> edt2performer = new ConcurrentHashMap<>();
+
+        private static final int WAIT_LIMIT_SECONDS = 5;
+
+        private SelectorPerformer() {}
+
+        /**
+         * Performs the selector wrapped in the callable. The selector should be executed via [JNFRunLoop performOnMainThreadWaiting:YES ...]
+         * on the native side so that the native doAWTRunLoop, which is run in [JNFRunLoop javaRunLoopMode], accepts it.
+         * The callable wrapper should not call any Java code which would normally be called on EDT.
+         * <p>
+         * If the main thread posts invocation events caused by the selector, those events are intercepted and dispatched on EDT out of order.
+         * <p>
+         * When called on non-EDT, the method performs the selector in place. The method is reentrant.
+         *
+         * @param selector the native selector wrapper
+         * @param <T> the selector return type
+         * @return the selector result
+         */
+        public static <T> T perform(Callable<T> selector) {
+            if (selector == null) return null;
+
+            if (EventQueue.isDispatchThread()) {
+                SelectorPerformer performer = getInstance(Thread.currentThread());
+                if (performer != null) {
+                    return performer.performImpl(selector);
+                }
+            }
+            // fallback to default
+            try {
+                return selector.call();
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            return null;
+        }
+
+        private <T> T performImpl(Callable<T> selector) {
+            assert EventQueue.isDispatchThread();
+            if (executor == null) {
+                // init on EDT
+                executor = new ThreadPoolExecutor(1, Integer.MAX_VALUE,
+                        60L, TimeUnit.SECONDS,
+                        new SynchronousQueue<>(),
+                        Executors.privilegedThreadFactory());
+            }
+            LinkedBlockingQueue<InvocationEvent> currentQueue;
+            synchronized (invocations) {
+                invocations.push(currentQueue = new LinkedBlockingQueue<>());
+            }
+
+            FutureTask<T> task = new FutureTask(selector) {
+                @Override
+                protected void done() {
+                    synchronized (invocations) {
+                        // Done with the current queue, wake it up.
+                        invocations.pop().add(new InvocationEvent(executor, () -> {}));
+                    }
+                }
+            };
+            executor.execute(task);
+
+            try {
+                while (!task.isDone() || !currentQueue.isEmpty()) {
+                    InvocationEvent event = currentQueue.poll(WAIT_LIMIT_SECONDS, TimeUnit.SECONDS);
+                    if (event == null) {
+                        new RuntimeException("Waiting for the invocation event timed out").printStackTrace();
+                        break;
+                    }
+                    event.dispatch();
+                }
+                return task.isDone() ? task.get() : null;
+
+            } catch (InterruptedException | ExecutionException e) {
+                e.printStackTrace();
+            }
+            return null;
+        }
+
+        /**
+         * Checks if there's an active SelectorPerformer corresponding to the invocation's AppContext,
+         * adds the invocation to the SelectorPerformer's queue and returns true.
+         * Otherwise does nothing and returns false.
+         */
+        public static boolean offer(InvocationEvent invocation) {
+            Object source = invocation.getSource();
+
+            SelectorPerformer performer = (source instanceof Component) ?
+                    getInstance((Component)source) :
+                    getInstance(Toolkit.getDefaultToolkit().getSystemEventQueue());
+
+            if (performer == null) return false;
+
+            synchronized (performer.invocations) {
+                if (!performer.invocations.isEmpty()) {
+                    performer.invocations.peek().add(invocation);
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        private static SelectorPerformer getInstance(Component comp) {
+            if (comp == null) return null;
+
+            AppContext appContext = SunToolkit.targetToAppContext(comp);
+            if (appContext == null) return null;
+
+            return getInstance((EventQueue)appContext.get(AppContext.EVENT_QUEUE_KEY));
+        }
+
+        private static SelectorPerformer getInstance(EventQueue eq) {
+            if (eq == null) return null;
+
+            return getInstance(AWTAccessor.getEventQueueAccessor().getDispatchThread(eq));
+        }
+
+        private static SelectorPerformer getInstance(Thread edt) {
+            if (edt == null) return null;
+
+            return edt2performer.computeIfAbsent(edt, key -> new SelectorPerformer());
+        }
+    }
+
     /**
      * Kicks an event over to the appropriate event queue and waits for it to
      * finish To avoid deadlocking, we manually run the NSRunLoop while waiting
@@ -726,12 +856,19 @@ public final class LWCToolkit extends LWToolkit {
                         },
                         true);
 
-        AppContext appContext = SunToolkit.targetToAppContext(component);
-        SunToolkit.postEvent(appContext, invocationEvent);
-        // 3746956 - flush events from PostEventQueue to prevent them from getting stuck and causing a deadlock
-        SunToolkit.flushPendingEvents(appContext);
-        doAWTRunLoop(mediator, false);
+        if (!SelectorPerformer.offer(invocationEvent)) {
+            if (component != null) {
+                AppContext appContext = SunToolkit.targetToAppContext(component);
+                SunToolkit.postEvent(appContext, invocationEvent);
 
+                // 3746956 - flush events from PostEventQueue to prevent them from getting stuck and causing a deadlock
+                SunToolkit.flushPendingEvents(appContext);
+            } else {
+                // This should be the equivalent to EventQueue.invokeAndWait
+                ((LWCToolkit) Toolkit.getDefaultToolkit()).getSystemEventQueueForInvokeAndWait().postEvent(invocationEvent);
+            }
+        }
+        doAWTRunLoop(mediator, false);
         checkException(invocationEvent);
     }