JBR-4544 Enable OpenGL pipeline by default for Wayland sessions

The OpenGL pipeline is enabled only if all of the following is true:
- WAYLAND is detected,
- VMWare virtualization is detected,
- rendering pipeline is not a software one (llvmpipe).
As a side effect, a system property 'jbr.virtualization.information'
is set to the value of detected virtualization type. The value is the
same as provided by JFR.

src/hotspot/share/runtime/arguments.cpp

@@ -36,6 +36,7 @@
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #include "jvm.h"
+#include "jfr/periodic/jfrOSInterface.hpp"
 #include "logging/log.hpp"
 #include "logging/logConfiguration.hpp"
 #include "logging/logStream.hpp"
@@ -4335,5 +4336,11 @@ void Arguments::parse_malloc_limits(size_t* total_limit, size_t limits[mt_number
   } while (p != nullptr);
 
   os::free(copy);
-
+}
+
+void Arguments::add_virtualization_information_property()
+{
+  const char *virt_name = "undetected";
+  JFR_ONLY(virt_name = JfrOSInterface::virtualization_name();)
+  PropertyList_add(&_system_properties, new SystemProperty("jbr.virtualization.information", virt_name, false));
 }

src/hotspot/share/runtime/arguments.hpp

@@ -578,6 +578,8 @@ class Arguments : AllStatic {
     _vm_info->set_value(vm_info);
   }
 
+  static void add_virtualization_information_property();
+
   // Property List manipulation
   static void PropertyList_add(SystemProperty *element);
   static void PropertyList_add(SystemProperty** plist, SystemProperty *element);

src/hotspot/share/runtime/threads.cpp

@@ -574,6 +574,8 @@ jint Threads::create_vm(JavaVMInitArgs* args, bool* canTryAgain) {
 
   JFR_ONLY(Jfr::on_create_vm_1();)
 
+  Arguments::add_virtualization_information_property();
+
   // Should be done after the heap is fully created
   main_thread->cache_global_variables();
 

src/java.desktop/unix/classes/sun/awt/X11GraphicsEnvironment.java

@@ -76,6 +76,7 @@ public final class X11GraphicsEnvironment extends SunGraphicsEnvironment {
                 if (!isHeadless()) {
                     // first check the OGL system property
                     boolean glxRequested = false;
+                    boolean glxRecommended = false;
                     String prop = System.getProperty("sun.java2d.opengl");
                     if (prop != null) {
                         if (prop.equals("true") || prop.equals("t")) {
@@ -84,6 +85,9 @@ public final class X11GraphicsEnvironment extends SunGraphicsEnvironment {
                             glxRequested = true;
                             glxVerbose = true;
                         }
+                    } else if (openGLRecommended()) {
+                        glxRequested = true;
+                        glxRecommended = true;
                     }
 
                     // Now check for XRender system property
@@ -108,7 +112,7 @@ public final class X11GraphicsEnvironment extends SunGraphicsEnvironment {
 
                     // only attempt to initialize GLX if it was requested
                     if (glxRequested) {
-                        glxAvailable = initGLX();
+                        glxAvailable = initGLX(glxRecommended);
                         if (glxVerbose && !glxAvailable) {
                             System.out.println(
                                 "Could not enable OpenGL " +
@@ -139,11 +143,21 @@ public final class X11GraphicsEnvironment extends SunGraphicsEnvironment {
 
     }
 
+    private static boolean isVMWare() {
+        final String virtName = System.getProperty("jbr.virtualization.information");
+        return virtName != null && virtName.equals("VMWare virtualization");
+    }
+
+    private static boolean openGLRecommended() {
+        final String sessionType = System.getenv("XDG_SESSION_TYPE");
+        return (sessionType != null && sessionType.equals("wayland") && isVMWare());
+    }
+
 
     private static boolean glxAvailable;
     private static boolean glxVerbose;
 
-    private static native boolean initGLX();
+    private static native boolean initGLX(boolean glxRecommended);
 
     public static boolean isGLXAvailable() {
         return glxAvailable;

src/java.desktop/unix/native/common/java2d/opengl/GLXGraphicsConfig.c

@@ -47,6 +47,64 @@ extern Bool usingXinerama;
  */
 static GLXContext sharedContext = 0;
 
+static jboolean
+isSoftwareRenderer(void)
+{
+    jboolean isLLVMPipeline = JNI_TRUE; // Assume the worst
+    const int attributes[] = {
+      GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
+      GLX_RENDER_TYPE,   GLX_RGBA_BIT,
+      GLX_RED_SIZE,      1,   /* Request a single buffered color buffer */
+      GLX_GREEN_SIZE,    1,   /* with the maximum number of color bits  */
+      GLX_BLUE_SIZE,     1,   /* for each component                     */
+      None
+    };
+    const int scrnum = DefaultScreen(awt_display);
+    int nconfs;
+    GLXFBConfig *fbConfigs = j2d_glXChooseFBConfig(awt_display, scrnum,
+                                                   attributes, &nconfs);
+    if ((fbConfigs == NULL) || (nconfs <= 0)) {
+        return isLLVMPipeline;
+    }
+
+    XVisualInfo *visinfo = j2d_glXGetVisualFromFBConfig(awt_display, fbConfigs[0]);
+    if (visinfo == NULL) {
+        XFree(fbConfigs);
+        return isLLVMPipeline;
+    }
+
+    GLXContext ctx = j2d_glXCreateNewContext(awt_display, fbConfigs[0],
+                                             GLX_RGBA_TYPE, 0, GL_TRUE);
+    if (ctx == 0) {
+        XFree(visinfo);
+        XFree(fbConfigs);
+        return isLLVMPipeline;
+    }
+
+    const Window root = RootWindow(awt_display, scrnum);
+    XSetWindowAttributes attr;
+    attr.background_pixel = 0;
+    attr.border_pixel = 0;
+    attr.colormap = XCreateColormap(awt_display, root, visinfo->visual, AllocNone);
+    attr.event_mask = StructureNotifyMask | ExposureMask;
+    const unsigned long mask = CWBackPixel | CWBorderPixel | CWColormap | CWEventMask;
+    const Window win = XCreateWindow(awt_display, root, 0, 0, 1, 1,
+                                     0, visinfo->depth, InputOutput,
+                                     visinfo->visual, mask, &attr);
+
+    if (j2d_glXMakeContextCurrent(awt_display, win, win, ctx)) {
+        const char * renderer = j2d_glXQueryCurrentRendererStringMESA(GLX_RENDERER_DEVICE_ID_MESA);
+        isLLVMPipeline = (strstr(renderer, "llvmpipe") != NULL);
+    }
+
+    XDestroyWindow(awt_display, win);
+    j2d_glXDestroyContext(awt_display, ctx);
+    XFree(visinfo);
+    XFree(fbConfigs);
+
+    return isLLVMPipeline;
+}
+
 /**
  * Attempts to initialize GLX and the core OpenGL library.  For this method
  * to return JNI_TRUE, the following must be true:
@@ -59,7 +117,7 @@ static GLXContext sharedContext = 0;
  * GraphicsConfig in the environment.
  */
 static jboolean
-GLXGC_InitGLX()
+GLXGC_InitGLX(jboolean glxRecommended)
 {
     int errorbase, eventbase;
     const char *version;
@@ -105,6 +163,14 @@ GLXGC_InitGLX()
         return JNI_FALSE;
     }
 
+    if (glxRecommended) {
+        if (isSoftwareRenderer()) {
+          // There are severe glitches when using software renderer, so
+          // if the OpenGL pipeline is merely recommended and not forced,
+          // report that it is not useable.
+          return JNI_FALSE;
+        }
+    }
     return JNI_TRUE;
 }
 
@@ -115,7 +181,7 @@ GLXGC_InitGLX()
  * calling this method.
  */
 jboolean
-GLXGC_IsGLXAvailable()
+GLXGC_IsGLXAvailable(jboolean glxRecommended)
 {
     static jboolean glxAvailable = JNI_FALSE;
     static jboolean firstTime = JNI_TRUE;
@@ -123,7 +189,7 @@ GLXGC_IsGLXAvailable()
     J2dTraceLn(J2D_TRACE_INFO, "GLXGC_IsGLXAvailable");
 
     if (firstTime) {
-        glxAvailable = GLXGC_InitGLX();
+        glxAvailable = GLXGC_InitGLX(glxRecommended);
         firstTime = JNI_FALSE;
     }
 
@@ -339,7 +405,7 @@ GLXGC_FindBestVisual(JNIEnv *env, jint screen)
 
     J2dRlsTraceLn1(J2D_TRACE_INFO, "GLXGC_FindBestVisual: scn=%d", screen);
 
-    if (!GLXGC_IsGLXAvailable()) {
+    if (!GLXGC_IsGLXAvailable(JNI_FALSE)) {
         J2dRlsTraceLn(J2D_TRACE_ERROR,
             "GLXGC_FindBestVisual: could not initialize GLX");
         return 0;

src/java.desktop/unix/native/common/java2d/opengl/GLXGraphicsConfig.h

@@ -86,7 +86,7 @@ typedef struct _GLXCtxInfo {
     GLXPbuffer  scratchSurface;
 } GLXCtxInfo;
 
-jboolean GLXGC_IsGLXAvailable();
+jboolean GLXGC_IsGLXAvailable(jboolean glxRecommended);
 VisualID GLXGC_FindBestVisual(JNIEnv *env, jint screen);
 
 #endif /* HEADLESS */

src/java.desktop/unix/native/common/java2d/opengl/OGLFuncs_md.h

@@ -68,6 +68,11 @@ typedef int (GLAPIENTRY *glXQueryContextType)(Display *dpy, GLXContext ctx, int
 typedef void (GLAPIENTRY *glXSelectEventType)(Display *dpy, GLXDrawable draw, unsigned long event_mask);
 typedef void (GLAPIENTRY *glXGetSelectedEventType)(Display *dpy, GLXDrawable draw, unsigned long *event_mask);
 
+/**
+ * These are required for just one purpose of checking the available renderer in isSoftwareRenderer()
+ */
+typedef const char* (GLAPIENTRY *glXQueryCurrentRendererStringMESAType) (int attribute);
+
 /**
  * GLX extension functions
  */
@@ -165,7 +170,8 @@ do { \
     OGL_##action##_FUNC(glXMakeContextCurrent); \
     OGL_##action##_FUNC(glXGetCurrentReadDrawable); \
     OGL_##action##_FUNC(glXQueryContext); \
-    OGL_##action##_FUNC(glXSelectEvent); \
+    OGL_##action##_FUNC(glXSelectEvent);   \
+    OGL_##action##_FUNC(glXQueryCurrentRendererStringMESA);   \
     OGL_##action##_FUNC(glXGetSelectedEvent);
 
 #define OGL_EXPRESS_PLATFORM_EXT_FUNCS(action)

src/java.desktop/unix/native/libawt_xawt/awt/awt_GraphicsEnv.c

@@ -818,12 +818,12 @@ Java_sun_awt_X11GraphicsEnvironment_initDisplay(JNIEnv *env, jobject this,
  * Signature: ()Z
  */
 JNIEXPORT jboolean JNICALL
-Java_sun_awt_X11GraphicsEnvironment_initGLX(JNIEnv *env, jclass x11ge)
+Java_sun_awt_X11GraphicsEnvironment_initGLX(JNIEnv *env, jclass x11ge, jboolean glxRecommended)
 {
     jboolean glxAvailable;
 
     AWT_LOCK();
-    glxAvailable = GLXGC_IsGLXAvailable();
+    glxAvailable = GLXGC_IsGLXAvailable(glxRecommended);
     AWT_UNLOCK();
 
     return glxAvailable;