JBR-7237 Fix cyclic dependency of Wayland and Vulkan initialization (#396)

(cherry picked from commit cde482c1fd)

src/java.desktop/share/classes/sun/java2d/vulkan/VKInstance.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, JetBrains s.r.o.. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.java2d.vulkan;
+
+import sun.util.logging.PlatformLogger;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+public class VKInstance {
+
+    private static final PlatformLogger log = PlatformLogger.getLogger("sun.java2d.vulkan.VKInstance");
+    private static Boolean initialized;
+
+    private static native boolean initNative(long nativePtr, boolean verbose, int deviceNumber);
+
+    @SuppressWarnings({"removal", "restricted"})
+    public static void init(long nativePtr) {
+        String vulkanOption = AccessController.doPrivileged(
+                        (PrivilegedAction<String>) () -> System.getProperty("sun.java2d.vulkan", ""));
+        if ("true".equalsIgnoreCase(vulkanOption)) {
+            String deviceNumberOption = AccessController.doPrivileged(
+                    (PrivilegedAction<String>) () -> System.getProperty("sun.java2d.vulkan.deviceNumber", "0"));
+            int parsedDeviceNumber = 0;
+            try {
+                parsedDeviceNumber = Integer.parseInt(deviceNumberOption);
+            } catch (NumberFormatException e) {
+                log.warning("Invalid Vulkan device number:" + deviceNumberOption);
+            }
+            final int deviceNumber = parsedDeviceNumber;
+            final boolean verbose = "True".equals(vulkanOption);
+            System.loadLibrary("awt");
+            initialized = initNative(nativePtr, verbose, deviceNumber);
+        } else initialized = false;
+        if (log.isLoggable(PlatformLogger.Level.FINE)) {
+            log.fine("Vulkan rendering enabled: " + (initialized ? "YES" : "NO"));
+        }
+    }
+
+    public static boolean isVulkanEnabled() {
+        if (initialized == null) throw new RuntimeException("Vulkan not initialized");
+        return initialized;
+    }
+}

src/java.desktop/share/native/common/java2d/vulkan/VKBase.c

@@ -43,15 +43,11 @@ static const uint32_t REQUIRED_VULKAN_VERSION = VK_MAKE_API_VERSION(0, 1, 2, 0);
 #define MAX_ENABLED_EXTENSIONS 5
 #define VALIDATION_LAYER_NAME "VK_LAYER_KHRONOS_validation"
 #define COUNT_OF(x) (sizeof(x)/sizeof(x[0]))
-#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
-extern struct wl_display *wl_display;
-#endif
 
 static jboolean verbose;
 static jint requestedDeviceNumber = -1;
 static VKGraphicsEnvironment* geInstance = NULL;
 static void* pVulkanLib = NULL;
-#define DEBUG
 #define INCLUDE_BYTECODE
 #define SHADER_ENTRY(NAME, TYPE) static uint32_t NAME ## _ ## TYPE ## _data[] = {
 #define BYTECODE_END };
@@ -151,12 +147,21 @@ void* vulkanLibProc(VkInstance vkInstance, char* procName) {
     return vkProc;
 }
 
-
-jboolean VK_Init(jboolean verb, jint requestedDevice) {
+/*
+ * Class:     sun_java2d_vulkan_VKInstance
+ * Method:    init
+ * Signature: (JZI)Z
+ */
+JNIEXPORT jboolean JNICALL
+Java_sun_java2d_vulkan_VKInstance_initNative(JNIEnv *env, jclass wlge, jlong nativePtr, jboolean verb, jint requestedDevice) {
     verbose = verb;
-    if (VKGE_graphics_environment() == NULL) {
+    VKGraphicsEnvironment* geInstance = VKGE_graphics_environment();
+    if (geInstance == NULL) {
         return JNI_FALSE;
     }
+#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
+    geInstance->waylandDisplay = (struct wl_display*) jlong_to_ptr(nativePtr);
+#endif
     if (!VK_FindDevices()) {
         return JNI_FALSE;
     }
@@ -547,7 +552,7 @@ jboolean VK_FindDevices() {
 #if defined(VK_USE_PLATFORM_WAYLAND_KHR)
             VkBool32 presentationSupported =
                 geInstance->vkGetPhysicalDeviceWaylandPresentationSupportKHR(
-                    geInstance->physicalDevices[i], j, wl_display);
+                    geInstance->physicalDevices[i], j, geInstance->waylandDisplay);
 #endif
             char logFlags[5] = {
                     queueFamilies[j].queueFlags & VK_QUEUE_GRAPHICS_BIT ? 'G' : '-',

src/java.desktop/share/native/common/java2d/vulkan/VKBase.h

@@ -71,6 +71,10 @@ typedef struct {
     VkExtensionProperties*  extensions;
     VkLayerProperties*      layers;
 
+#if defined(VK_USE_PLATFORM_WAYLAND_KHR)
+    struct wl_display* waylandDisplay;
+#endif
+
     PFN_vkEnumeratePhysicalDevices vkEnumeratePhysicalDevices;
     PFN_vkGetPhysicalDeviceFeatures2 vkGetPhysicalDeviceFeatures2;
     PFN_vkGetPhysicalDeviceProperties2 vkGetPhysicalDeviceProperties2;

src/java.desktop/unix/classes/sun/awt/wl/WLGraphicsDevice.java

@@ -27,6 +27,7 @@
 package sun.awt.wl;
 
 import sun.awt.AWTAccessor;
+import sun.java2d.vulkan.VKInstance;
 import sun.java2d.vulkan.WLVKGraphicsConfig;
 
 import java.awt.GraphicsConfiguration;
@@ -99,7 +100,7 @@ public class WLGraphicsDevice extends GraphicsDevice {
             WLGraphicsConfig newDefaultConfig;
             // It is necessary to create a new object whenever config changes as its
             // identity is used to detect changes in scale, among other things.
-            if (WLGraphicsEnvironment.isVulkanEnabled()) {
+            if (VKInstance.isVulkanEnabled()) {
                 newDefaultConfig = WLVKGraphicsConfig.getConfig(this, width, height, scale);
                 newConfigs = new GraphicsConfiguration[1];
                 newConfigs[0] = newDefaultConfig;

src/java.desktop/unix/classes/sun/awt/wl/WLGraphicsEnvironment.java

@@ -30,8 +30,6 @@ import java.awt.Dimension;
 import java.awt.GraphicsConfiguration;
 import java.awt.GraphicsDevice;
 import java.awt.Rectangle;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -45,46 +43,17 @@ import sun.util.logging.PlatformLogger.Level;
 public class WLGraphicsEnvironment extends SunGraphicsEnvironment {
     private static final PlatformLogger log = PlatformLogger.getLogger("sun.awt.wl.WLGraphicsEnvironment");
 
-    private static boolean vulkanEnabled = false;
-    private static boolean verboseVulkanStatus = false;
-    private static boolean vulkanRequested = false;
-    private static int vulkanRequestedDeviceNumber = -1;
     private static final boolean debugScaleEnabled;
-    @SuppressWarnings("removal")
-    private static String vulkanOption =
-            AccessController.doPrivileged(
-                    (PrivilegedAction<String>) () -> System.getProperty("sun.java2d.vulkan", ""));
-
-    @SuppressWarnings("removal")
-    private static String vulkanOptionDeviceNumber =
-            AccessController.doPrivileged(
-                    (PrivilegedAction<String>) () -> System.getProperty("sun.java2d.vulkan.deviceNumber", "0"));
+    private final Dimension totalDisplayBounds = new Dimension();
 
     @SuppressWarnings("restricted")
     private static void loadAwt() {
         System.loadLibrary("awt");
     }
 
-    private final Dimension totalDisplayBounds = new Dimension();
-
     static {
-        vulkanRequested = "true".equalsIgnoreCase(vulkanOption);
-        try {
-            vulkanRequestedDeviceNumber = Integer.parseInt(vulkanOptionDeviceNumber);
-        } catch (NumberFormatException e) {
-            log.warning("Invalid Vulkan device number:" + vulkanOptionDeviceNumber);
-        }
-        verboseVulkanStatus = "True".equals(vulkanOption);
-
         loadAwt();
-
         SurfaceManagerFactory.setInstance(new UnixSurfaceManagerFactory());
-        if (vulkanRequested) {
-            vulkanEnabled = initVKWL(verboseVulkanStatus, vulkanRequestedDeviceNumber);
-        }
-        if (log.isLoggable(Level.FINE)) {
-            log.fine("Vulkan rendering enabled: " + (vulkanEnabled?"YES":"NO"));
-        }
 
         debugScaleEnabled = SunGraphicsEnvironment.isUIScaleEnabled() && SunGraphicsEnvironment.getDebugScale() >= 1;
 
@@ -92,15 +61,9 @@ public class WLGraphicsEnvironment extends SunGraphicsEnvironment {
         WLToolkit.isInitialized();
     }
 
-    private static native boolean initVKWL(boolean verbose, int deviceNumber);
-
     private WLGraphicsEnvironment() {
     }
 
-    public static boolean isVulkanEnabled() {
-        return vulkanEnabled;
-    }
-
     private static class Holder {
         static final WLGraphicsEnvironment INSTANCE = new WLGraphicsEnvironment();
     }

src/java.desktop/unix/classes/sun/awt/wl/WLToolkit.java

@@ -35,7 +35,7 @@ import sun.awt.PeerEvent;
 import sun.awt.SunToolkit;
 import sun.awt.UNIXToolkit;
 import sun.awt.datatransfer.DataTransferer;
-import sun.awt.wl.WLDisplay;
+import sun.java2d.vulkan.VKInstance;
 import sun.util.logging.PlatformLogger;
 
 import java.awt.*;
@@ -142,7 +142,9 @@ public class WLToolkit extends UNIXToolkit implements Runnable {
     static {
         if (!GraphicsEnvironment.isHeadless()) {
             keyboard = new WLKeyboard();
-            initIDs(WLDisplay.getInstance().getDisplayPtr());
+            long display = WLDisplay.getInstance().getDisplayPtr();
+            VKInstance.init(display);
+            initIDs(display);
         }
         initialized = true;
     }

src/java.desktop/unix/native/common/java2d/vulkan/WLVKSurfaceData.c

@@ -104,7 +104,7 @@ Java_sun_java2d_vulkan_WLVKSurfaceData_assignSurface(JNIEnv *env, jobject wsd, j
     VKSD_InitWindowSurface(vkwinsdo);
     J2dRlsTraceLn(J2D_TRACE_INFO, "WLVKSurfaceData_assignSurface: Created WaylandSurfaceKHR");
 
-    J2dTraceLn2(J2D_TRACE_INFO, "WLVKSurfaceData_assignSurface wl_surface(%p) wl_display(%p)", wlSurface, wl_display);
+    J2dTraceLn2(J2D_TRACE_INFO, "WLVKSurfaceData_assignSurface wl_surface(%p) wl_display(%p)", wlvksdo->wl_surface, wl_display);
 
 #endif /* !HEADLESS */
 }

src/java.desktop/unix/native/libawt_wlawt/WLGraphicsEnvironment.c

@@ -31,7 +31,6 @@
 
 #include "JNIUtilities.h"
 #include "WLToolkit.h"
-#include "VKInit.h"
 
 typedef struct WLOutput {
     struct WLOutput *  next;
@@ -280,27 +279,4 @@ WLOutputByID(uint32_t id)
 
     return NULL;
 }
-
-/*
- * Class:     sun_awt_wl_WLGraphicsEnvironment
- * Method:    initVKWL
- * Signature: ()Z
- */
-JNIEXPORT jboolean JNICALL
-Java_sun_awt_wl_WLGraphicsEnvironment_initVKWL(JNIEnv *env, jclass wlge, jboolean verbose, jint requestedDevice)
-{
-    jboolean vkwlAvailable = JNI_FALSE;
-    /* TODO: The following sequence lead to uninitialized awt lock
-       BufferedImage.getGraphics()
-       BufferedImage.createGraphics()
-       GraphicsEnvironment.getLocalGraphicsEnvironment()
-       GraphicsEnvironment$LocalGE.createGE()
-       PlatformGraphicsInfo.createGE()
-       WLGraphicsEnvironment.initVKWL()
-    */
-    //AWT_LOCK();
-    vkwlAvailable = VK_Init(verbose, requestedDevice);
-    //AWT_UNLOCK();
-    return vkwlAvailable;
-}
 #endif // #ifndef HEADLESS