8272613: CharsetDecoder.decode(ByteBuffer) throws IllegalArgumentException

Reviewed-by: alanb, bpb

src/java.base/share/classes/java/nio/charset/Charset-X-Coder.java.template

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,6 +35,7 @@ import java.nio.BufferUnderflowException;
 import java.lang.ref.WeakReference;
 import java.nio.charset.CoderMalfunctionError;                  // javadoc
 import java.util.Arrays;
+import jdk.internal.util.ArraysSupport;
 
 
 /**
@@ -791,11 +792,16 @@ public abstract class Charset$Coder$ {
      *          position cannot be mapped to an equivalent $otype$ sequence and
      *          the current unmappable-character action is {@link
      *          CodingErrorAction#REPORT}
+     *
+     * @throws  OutOfMemoryError
+     *          If the output $otype$ buffer for the requested size of the input
+     *          $itype$ buffer cannot be allocated
      */
     public final $Otype$Buffer $code$($Itype$Buffer in)
         throws CharacterCodingException
     {
-        int n = (int)(in.remaining() * average$ItypesPerOtype$());
+        int n = Math.min((int)(in.remaining() * average$ItypesPerOtype$()),
+                    ArraysSupport.SOFT_MAX_ARRAY_LENGTH);
         $Otype$Buffer out = $Otype$Buffer.allocate(n);
 
         if ((n == 0) && (in.remaining() == 0))
@@ -810,7 +816,8 @@ public abstract class Charset$Coder$ {
             if (cr.isUnderflow())
                 break;
             if (cr.isOverflow()) {
-                n = 2*n + 1;    // Ensure progress; n might be 0!
+                // Ensure progress; n might be 0!
+                n = ArraysSupport.newLength(n, Math.min(n + 1, 1_024), n + 1);
                 $Otype$Buffer o = $Otype$Buffer.allocate(n);
                 out.flip();
                 o.put(out);

test/jdk/java/nio/charset/CharsetDecoder/XcodeOverflow.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8272613
+ * @summary Make sure IAE is not thrown on `int` overflow, turning negative
+ *          size. The test should either not throw any Throwable, or an OOME
+ *          with real Java heap space error (not "exceeds VM limit").
+ * @requires sun.arch.data.model == "64"
+ * @run junit/othervm XcodeOverflow
+ */
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.StandardCharsets;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.Arguments;
+
+public class XcodeOverflow {
+    private static Stream<Arguments> sizes() {
+        return Stream.of(
+            // SOFT_MAX_ARRAY_LENGTH: copied from ArraysSupport. No overflow; no OOME.
+            Arguments.of(Integer.MAX_VALUE - 8),
+
+            // overflow case: OOME w/ "Java heap space" is thrown on decoding
+            Arguments.of(Integer.MAX_VALUE - 1000000)
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("sizes")
+    public void testEncodeOverflow(int size) throws CharacterCodingException {
+        try {
+            StandardCharsets.UTF_8
+                    .newEncoder()
+                    .encode(CharBuffer.wrap(new char[size], 0, size));
+            System.out.println("Encoded without error");
+        } catch (OutOfMemoryError oome) {
+            if (oome.getMessage().equals("Java heap space")) {
+                System.out.println("OOME for \"Java heap space\" is thrown correctly during encoding");
+            } else {
+                throw new RuntimeException("Unexpected OOME", oome);
+            }
+        }
+    }
+
+    @ParameterizedTest
+    @MethodSource("sizes")
+    public void testDecodeOverflow(int size) throws CharacterCodingException {
+        try {
+            StandardCharsets.UTF_8
+                .newDecoder()
+                .decode(ByteBuffer.wrap(new byte[size], 0, size));
+            System.out.println("Decoded without error");
+        } catch (OutOfMemoryError oome) {
+            if (oome.getMessage().equals("Java heap space")) {
+                System.out.println("OOME for \"Java heap space\" is thrown correctly during decoding");
+            } else {
+                throw new RuntimeException("Unexpected OOME", oome);
+            }
+        }
+    }
+}