8266269: Lookup::accessClass fails with IAE when accessing an arrayClass with a protected inner class as component class

Reviewed-by: chegar, alanb
(cherry picked from commit 6eb734a60f)

src/java.base/share/classes/java/lang/invoke/MethodHandles.java

@@ -2768,6 +2768,7 @@ assertEquals("[x, y, z]", pb.command().toString());
          * @throws ClassNotFoundException if the class cannot be loaded by the lookup class' loader.
          * @throws IllegalAccessException if the class is not accessible, using the allowed access
          * modes.
+         * @throws NullPointerException if {@code targetName} is null
          * @since 9
          * @jvms 5.4.3.1 Class and Interface Resolution
          */
@@ -2837,8 +2838,12 @@ assertEquals("[x, y, z]", pb.command().toString());
         /**
          * Determines if a class can be accessed from the lookup context defined by
          * this {@code Lookup} object. The static initializer of the class is not run.
+         * If {@code targetClass} is an array class, {@code targetClass} is accessible
+         * if the element type of the array class is accessible.  Otherwise,
+         * {@code targetClass} is determined as accessible as follows.
+         *
          * <p>
-         * If the {@code targetClass} is in the same module as the lookup class,
+         * If {@code targetClass} is in the same module as the lookup class,
          * the lookup class is {@code LC} in module {@code M1} and
          * the previous lookup class is in module {@code M0} or
          * {@code null} if not present,
@@ -2861,7 +2866,7 @@ assertEquals("[x, y, z]", pb.command().toString());
          * can access public types in all modules when the type is in a package
          * that is exported unconditionally.
          * <p>
-         * Otherwise, the target class is in a different module from {@code lookupClass},
+         * Otherwise, {@code targetClass} is in a different module from {@code lookupClass},
          * and if this lookup does not have {@code PUBLIC} access, {@code lookupClass}
          * is inaccessible.
          * <p>
@@ -2897,13 +2902,14 @@ assertEquals("[x, y, z]", pb.command().toString());
          * @return the class that has been access-checked
          * @throws IllegalAccessException if the class is not accessible from the lookup class
          * and previous lookup class, if present, using the allowed access modes.
-         * @throws    SecurityException if a security manager is present and it
-         *                              <a href="MethodHandles.Lookup.html#secmgr">refuses access</a>
+         * @throws SecurityException if a security manager is present and it
+         *                           <a href="MethodHandles.Lookup.html#secmgr">refuses access</a>
+         * @throws NullPointerException if {@code targetClass} is {@code null}
          * @since 9
          * @see <a href="#cross-module-lookup">Cross-module lookups</a>
          */
         public Class<?> accessClass(Class<?> targetClass) throws IllegalAccessException {
-            if (!VerifyAccess.isClassAccessible(targetClass, lookupClass, prevLookupClass, allowedModes)) {
+            if (!isClassAccessible(targetClass)) {
                 throw makeAccessException(targetClass);
             }
             checkSecurityManager(targetClass);
@@ -3684,7 +3690,11 @@ return mh1;
         boolean isClassAccessible(Class<?> refc) {
             Objects.requireNonNull(refc);
             Class<?> caller = lookupClassOrNull();
-            return caller == null || VerifyAccess.isClassAccessible(refc, caller, prevLookupClass, allowedModes);
+            Class<?> type = refc;
+            while (type.isArray()) {
+                type = type.getComponentType();
+            }
+            return caller == null || VerifyAccess.isClassAccessible(type, caller, prevLookupClass, allowedModes);
         }
 
         /** Check name for an illegal leading "&lt;" character. */

test/jdk/java/lang/invoke/t8150782/TestAccessClass.java

@@ -24,13 +24,17 @@
  */
 
 /* @test
- * @bug 8150782 8207027
- * @compile TestAccessClass.java TestCls.java
+ * @bug 8150782 8207027 8266269
+ * @compile TestAccessClass.java TestCls.java p/Foo.java q/Bar.java
  * @run testng/othervm -ea -esa test.java.lang.invoke.t8150782.TestAccessClass
  */
 package test.java.lang.invoke.t8150782;
 
 import java.lang.invoke.*;
+import java.lang.reflect.Modifier;
+
+import p.Foo;
+import q.Bar;
 
 import static java.lang.invoke.MethodHandles.*;
 
@@ -55,13 +59,13 @@ public class TestAccessClass {
     }
 
     @Test
-    public void returnsSameClassInSamePackage() throws IllegalAccessException, ClassNotFoundException {
+    public void returnsSameClassInSamePackage() throws IllegalAccessException {
         Class<?> aClass = lookup().accessClass(Class1.class);
         assertEquals(Class1.class, aClass);
     }
 
     @Test
-    public void returnsSameArrayClassInSamePackage() throws IllegalAccessException, ClassNotFoundException {
+    public void returnsSameArrayClassInSamePackage() throws IllegalAccessException {
         Class<?> aClass = lookup().accessClass(Class1[].class);
         assertEquals(Class1[].class, aClass);
     }
@@ -75,7 +79,7 @@ public class TestAccessClass {
     }
 
     @Test(dataProvider = "illegalAccessAccess", expectedExceptions = {IllegalAccessException.class})
-    public void illegalAccessExceptionTest(Lookup lookup, Class<?> klass) throws IllegalAccessException, ClassNotFoundException {
+    public void illegalAccessExceptionTest(Lookup lookup, Class<?> klass) throws IllegalAccessException {
         lookup.accessClass(klass);
     }
 
@@ -84,4 +88,20 @@ public class TestAccessClass {
         lookup().accessClass(TestCls.getPrivateSIC());
     }
 
+    /**
+     * Verify that a protected Q is as accessible as a public Q during linkage
+     * (see JLS 15.12.4.3).
+     */
+    @Test
+    public void protectedInnerClass() throws Throwable {
+        lookup().accessClass(Bar.T_CLS);
+        lookup().accessClass(Bar.T_ARRAY_CLS);
+        MethodHandle mh = lookup().findStatic(Bar.class, "meth", MethodType.methodType(void.class, Bar.T_ARRAY_CLS));
+        mh.invoke(null);
+    }
+
+    @Test(expectedExceptions = {NullPointerException.class})
+    public void illegalArgument() throws IllegalAccessException {
+        lookup().accessClass(null);
+    }
 }

test/jdk/java/lang/invoke/t8150782/TestFindClass.java

@@ -24,13 +24,15 @@
  */
 
 /* @test
- * @bug 8150782 8207027
- * @compile TestFindClass.java TestCls.java
+ * @bug 8150782 8207027 8266269
+ * @compile TestFindClass.java TestCls.java p/Foo.java q/Bar.java
  * @run testng/othervm -ea -esa test.java.lang.invoke.t8150782.TestFindClass
  */
 package test.java.lang.invoke.t8150782;
 
 import java.lang.invoke.*;
+import p.Foo;
+import q.Bar;
 
 import static java.lang.invoke.MethodHandles.*;
 
@@ -94,4 +96,18 @@ public class TestFindClass {
         lookup().findClass(PACKAGE_PREFIX + "TestCls$PrivateSIC");
     }
 
+    /**
+     * Verify that a protected Q is as accessible as a public Q during linkage
+     * (see JLS 15.12.4.3).
+     */
+    @Test
+    public void protectedInnerClass() throws IllegalAccessException, ClassNotFoundException {
+        lookup().findClass("p.Foo$T");
+        lookup().findClass("[Lp.Foo$T;");
+    }
+
+    @Test(expectedExceptions = {NullPointerException.class})
+    public void illegalArgument() throws IllegalAccessException, ClassNotFoundException {
+        lookup().findClass(null);
+    }
 }

test/jdk/java/lang/invoke/t8150782/p/Foo.java

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package p;
+
+public class Foo {
+     protected enum T { ONE }
+}
+

test/jdk/java/lang/invoke/t8150782/q/Bar.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package q;
+
+import p.Foo;
+
+// access protected inner class Foo.T
+public class Bar extends Foo {
+    public static final Class<?> T_CLS = T.class;
+    public static final Class<?> T_ARRAY_CLS = T[].class;
+
+    public static void meth(T[] arr) {
+        System.out.println("called method");
+    }
+}