JBR-2442 fix memory leak of fileBuffer

fix was suggested by Nikita Gubarkov

jb/project/docker/Dockerfile

@@ -9,5 +9,5 @@ RUN wget https://cdn.azul.com/zulu/bin/zulu16.28.11-ca-jdk16.0.0-linux_x64.tar.g
 RUN mv /zulu16.28.11-ca-jdk16.0.0-linux_x64 /jdk16.0.0
 ENV PATH /opt/rh/devtoolset-8/root/usr/bin:$PATH
 RUN mkdir .git
-RUN git config user.email "builduser@jetbrains.com"
+RUN git config user.email "teamcity@jetbrains.com"
 RUN git config user.name "builduser"

make/autoconf/toolchain.m4

@@ -221,6 +221,12 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETERMINE_TOOLCHAIN_TYPE],
   AC_ARG_WITH(toolchain-type, [AS_HELP_STRING([--with-toolchain-type],
       [the toolchain type (or family) to use, use '--help' to show possible values @<:@platform dependent@:>@])])
 
+  # Linux x86_64 needs higher binutils after 8265783
+  # (this really is a dependency on as version, but we take ld as a check for a general binutils version)
+  if test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then
+    TOOLCHAIN_MINIMUM_LD_VERSION_gcc="2.25"
+  fi
+
   # Use indirect variable referencing
   toolchain_var_name=VALID_TOOLCHAINS_$OPENJDK_BUILD_OS
   VALID_TOOLCHAINS=${!toolchain_var_name}
@@ -677,9 +683,10 @@ AC_DEFUN_ONCE([TOOLCHAIN_DETECT_TOOLCHAIN_CORE],
   TOOLCHAIN_PREPARE_FOR_LD_VERSION_COMPARISONS
 
   if test "x$TOOLCHAIN_MINIMUM_LD_VERSION" != x; then
+    AC_MSG_NOTICE([comparing linker version to minimum version $TOOLCHAIN_MINIMUM_LD_VERSION])
     TOOLCHAIN_CHECK_LINKER_VERSION(VERSION: $TOOLCHAIN_MINIMUM_LD_VERSION,
         IF_OLDER_THAN: [
-          AC_MSG_WARN([You are using a linker older than $TOOLCHAIN_MINIMUM_LD_VERSION. This is not a supported configuration.])
+          AC_MSG_ERROR([You are using a linker older than $TOOLCHAIN_MINIMUM_LD_VERSION. This is not a supported configuration.])
         ]
     )
   fi

make/test/JtregNativeJdk.gmk

@@ -53,6 +53,8 @@ BUILD_JDK_JTREG_EXECUTABLES_CFLAGS_exeJliLaunchTest := \
     -I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS_TYPE)/native/libjli \
     -I$(TOPDIR)/src/java.base/$(OPENJDK_TARGET_OS)/native/libjli
 
+BUILD_JDK_JTREG_LIBRARIES_LDFLAGS_libAsyncStackWalk := $(LIBCXX)
+
 # Platform specific setup
 ifeq ($(call isTargetOs, windows), true)
   BUILD_JDK_JTREG_EXCLUDE += libDirectIO.c libInheritedChannel.c exelauncher.c
@@ -63,6 +65,7 @@ ifeq ($(call isTargetOs, windows), true)
   BUILD_JDK_JTREG_EXECUTABLES_LIBS_exeJliLaunchTest := $(WIN_LIB_JLI)
   BUILD_JDK_JTREG_EXECUTABLES_LIBS_exeCallerAccessTest := jvm.lib
   BUILD_JDK_JTREG_EXECUTABLES_LIBS_exerevokeall := advapi32.lib
+  BUILD_JDK_JTREG_LIBRARIES_CFLAGS_libAsyncStackWalk := /EHsc
 else
   BUILD_JDK_JTREG_LIBRARIES_LIBS_libstringPlatformChars := -ljava
   BUILD_JDK_JTREG_LIBRARIES_LIBS_libDirectIO := -ljava

src/demo/share/java2d/RenderPerfTest/Makefile

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#   - Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+#
+#   - Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+#   - Neither the name of Oracle nor the names of its
+#     contributors may be used to endorse or promote products derived
+#     from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+SOURCEPATH=src
+CLASSES=build
+DIST=dist
+RESOURCES=resources
+
+RENDERPERF_CLASSES = $(CLASSES)/renderperf/RenderPerfTest.class
+RENDERPERF_SOURCES = $(SOURCEPATH)/renderperf/RenderPerfTest.java
+
+RENDERPERF_RESOURCES = $(CLASSES)/renderperf/images/duke.png 
+
+all: mkdirs $(DIST)/RenderPerfTest.jar
+
+run: mkdirs $(DIST)/RenderPerfTest.jar
+	java -jar $(DIST)/RenderPerfTest.jar
+
+$(DIST)/RenderPerfTest.jar: \
+	$(RENDERPERF_CLASSES) $(RENDERPERF_RESOURCES) \
+	$(CLASSES)/renderperf.manifest
+	jar cvmf $(CLASSES)/renderperf.manifest $(DIST)/RenderPerfTest.jar -C $(CLASSES) .
+
+
+$(CLASSES)/renderperf/images/%: $(RESOURCES)/images/%
+	cp -r $< $@
+
+
+$(CLASSES)/renderperf.manifest:
+	echo "Main-Class: renderperf.RenderPerfTest" > $@
+
+$(DIST):
+	mkdir  $(DIST)
+
+$(CLASSES):
+	mkdir $(CLASSES)
+	mkdir -p $(CLASSES)/renderperf/images
+
+mkdirs: $(DIST) $(CLASSES)
+
+$(RENDERPERF_CLASSES): $(RENDERPERF_SOURCES)
+	javac -g:none -d $(CLASSES) -sourcepath $(SOURCEPATH) $<
+
+clean:
+	rm -rf $(CLASSES)
+	rm -rf $(DIST)

src/demo/share/java2d/RenderPerfTest/README

@@ -0,0 +1,34 @@
+-----------------------------------------------------------------------
+Introduction
+-----------------------------------------------------------------------
+
+RenderPerfTest is a set of on-screen rendering microbenchmarks  to
+analyze the  performance of Java2D graphical primitives rendering
+
+-----------------------------------------------------------------------
+How To Compile
+-----------------------------------------------------------------------
+
+#> cd RenderPerfTest
+
+The benchmark can be compiled by using either ant:
+
+#> ant
+
+or gnumake (assuming there's 'javac' in the path):
+
+#> gnumake
+
+The jar files will be generated into RenderPerfTest/dist directory.
+
+-----------------------------------------------------------------------
+How To Run RenderPerfTest
+-----------------------------------------------------------------------
+Run all tests
+#> ant run
+  or
+#> java -jar dist/RenderPerfTest.jar
+
+Run particular test cases
+
+#> java -jar dist/RenderPerfTest.jar testWhiteTextBubblesGray ...

src/demo/share/java2d/RenderPerfTest/build.xml

@@ -0,0 +1,94 @@
+<!--
+ Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+   - Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in the
+     documentation and/or other materials provided with the distribution.
+
+   - Neither the name of Oracle nor the names of its
+     contributors may be used to endorse or promote products derived
+     from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+-->
+
+<project name="RenderPerfTest" default="dist" basedir=".">
+    <description>
+        simple example build file
+    </description>
+  <!-- set global properties for this build -->
+  <property name="src" location="src"/>
+  <property name="build" location="build"/>
+  <property name="dist"  location="dist"/>
+  <property name="resources"  location="resources"/>
+
+  <target name="init">
+    <!-- Create the time stamp -->
+    <tstamp/>
+    <!-- Create the build directory structure used by compile -->
+    <mkdir dir="${build}"/>
+  </target>
+
+  <target name="compile" depends="init"
+        description="compile the source " >
+    <!-- Compile the java code from ${src} into ${build} -->
+    <javac includeantruntime="false" debug="off" srcdir="${src}" destdir="${build}"/>
+  </target>
+
+  <target name="run" depends="dist"
+    description="run RenderPerfTest" >
+    <java jar="${dist}/RenderPerfTest.jar"
+       fork="true"
+    >
+    </java>
+  </target>
+
+  <target name="resources" depends="init"
+        description="copy resources into build dir" >
+    <!-- Copy the resource files from ${resources} into ${build}/ -->
+    <mkdir dir="${dist}"/>
+    <mkdir dir="${dist}/renderperf"/>
+    <mkdir dir="${build}/renderperf/images"/>
+    <copy todir="${build}/renderperf/images">
+      <fileset dir="resources/renderperf/images" />
+    </copy>
+  </target>
+
+  <target name="dist" depends="compile, resources"
+        description="generate the distribution" >
+    <!-- Create the distribution directory -->
+    <mkdir dir="${dist}"/>
+
+    <!-- Put everything in ${build} into the J2DBench.jar file -->
+    <jar jarfile="${dist}/RenderPerfTest.jar" basedir="${build}">
+      <manifest>
+        <attribute name="Built-By" value="${user.name}"/>
+	<attribute name="Main-Class" value="renderperf.RenderPerfTest"/>
+      </manifest>
+    </jar>
+  </target>
+
+  <target name="clean"
+        description="clean up" >
+    <!-- Delete the ${build} and ${dist} directory trees -->
+    <delete dir="${build}"/>
+    <delete dir="${dist}"/>
+  </target>
+</project>

src/demo/share/java2d/RenderPerfTest/src/renderperf/RenderPerfTest.java

@@ -0,0 +1,739 @@
+/*
+ * Copyright (c) 2019, 2020 Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package renderperf;
+
+
+import javax.imageio.ImageIO;
+import javax.swing.*;
+import java.awt.*;
+import java.awt.event.WindowAdapter;
+import java.awt.event.WindowEvent;
+import java.awt.geom.AffineTransform;
+import java.awt.geom.Point2D;
+import java.awt.geom.QuadCurve2D;
+import java.awt.image.BufferedImage;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+
+public class RenderPerfTest {
+    private static HashSet<String> ignoredTests = new HashSet<>();
+
+    private final static int N = 1000;
+    private final static float WIDTH = 800;
+    private final static float HEIGHT = 800;
+    private final static float R = 25;
+    private final static int BW = 50;
+    private final static int BH = 50;
+    private final static int COUNT = 300;
+    private final static int DELAY = 10;
+    private final static int RESOLUTION = 5;
+    private final static int COLOR_TOLERANCE = 10;
+    private final static int MAX_MEASURE_TIME = 5000;
+
+
+    interface Configurable {
+        void configure(Graphics2D g2d);
+    }
+
+    interface Renderable {
+        void setup(Graphics2D g2d);
+        void render(Graphics2D g2d);
+        void update();
+    }
+
+    static class Particles {
+        private float[] bx;
+        private float[] by;
+        private float[] vx;
+        private float[] vy;
+        private float r;
+        private int n;
+
+        private float x0;
+        private float y0;
+        private float width;
+        private float height;
+
+        Particles(int n, float r, float x0, float y0, float width, float height) {
+            bx = new float[n];
+            by = new float[n];
+            vx = new float[n];
+            vy = new float[n];
+            this.n = n;
+            this.r = r;
+            this.x0 = x0;
+            this.y0 = y0;
+            this.width = width;
+            this.height = height;
+            for (int i = 0; i < n; i++) {
+                bx[i] = (float) (x0 + r + 0.1 + Math.random() * (width - 2 * r - 0.2 - x0));
+                by[i] = (float) (y0 + r + 0.1 + Math.random() * (height - 2 * r - 0.2 - y0));
+                vx[i] = 0.1f * (float) (Math.random() * 2 * r - r);
+                vy[i] = 0.1f * (float) (Math.random() * 2 * r - r);
+            }
+
+        }
+
+        void render(Graphics2D g2d, ParticleRenderer renderer) {
+            for (int i = 0; i < n; i++) {
+                renderer.render(g2d, i, bx, by, vx, vy);
+            }
+        }
+
+        void update() {
+            for (int i = 0; i < n; i++) {
+                bx[i] += vx[i];
+                if (bx[i] + r > width || bx[i] - r < x0) vx[i] = -vx[i];
+                by[i] += vy[i];
+                if (by[i] + r > height || by[i] - r < y0) vy[i] = -vy[i];
+            }
+
+        }
+
+    }
+
+    ParticleRenderable createPR(ParticleRenderer renderer) {
+        return new ParticleRenderable(renderer);
+    }
+
+    static class ParticleRenderable implements Renderable {
+        ParticleRenderer renderer;
+        Configurable configure;
+
+        ParticleRenderable(ParticleRenderer renderer, Configurable configure) {
+            this.renderer = renderer;
+            this.configure = configure;
+        }
+
+        ParticleRenderable(ParticleRenderer renderer) {
+            this(renderer, null);
+        }
+
+        @Override
+        public void setup(Graphics2D g2d) {
+            if (configure != null) configure.configure(g2d);
+        }
+
+        @Override
+        public void render(Graphics2D g2d) {
+            balls.render(g2d, renderer);
+        }
+
+        @Override
+        public void update() {
+            balls.update();
+        }
+
+        public ParticleRenderable configure(Configurable configure) {
+            this.configure = configure;
+            return this;
+        }
+    }
+
+    interface ParticleRenderer {
+        void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy);
+
+    }
+
+    static class FlatParticleRenderer implements ParticleRenderer {
+        Color[] colors;
+        float r;
+
+        FlatParticleRenderer(int n, float r) {
+            colors = new Color[n];
+            this.r = r;
+            for (int i = 0; i < n; i++) {
+                colors[i] = new Color((float) Math.random(),
+                        (float) Math.random(), (float) Math.random());
+            }
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            g2d.setColor(colors[id % colors.length]);
+            g2d.fillOval((int)(x[id] - r), (int)(y[id] - r), (int)(2*r), (int)(2*r));
+        }
+
+    }
+
+    static class WhiteTextParticleRenderer implements ParticleRenderer {
+        float r;
+
+        WhiteTextParticleRenderer(float r) {
+            this.r = r;
+        }
+
+        void setPaint(Graphics2D g2d, int id) {
+            g2d.setColor(Color.WHITE);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            setPaint(g2d, id);
+            g2d.drawString("The quick brown fox jumps over the lazy dog",
+                    (int)(x[id] - r), (int)(y[id] - r));
+            g2d.drawString("The quick brown fox jumps over the lazy dog",
+                    (int)(x[id] - r), (int)y[id]);
+            g2d.drawString("The quick brown fox jumps over the lazy dog",
+                    (int)(x[id] - r), (int)(y[id] + r));
+        }
+    }
+
+    static class TextParticleRenderer extends WhiteTextParticleRenderer {
+        Color[] colors;
+
+        float r;
+
+        TextParticleRenderer(int n, float r) {
+            super(r);
+            colors = new Color[n];
+            this.r = r;
+            for (int i = 0; i < n; i++) {
+                colors[i] = new Color((float) Math.random(),
+                        (float) Math.random(), (float) Math.random());
+            }
+        }
+
+        void setPaint(Graphics2D g2d, int id) {
+            g2d.setColor(colors[id % colors.length]);
+        }
+    }
+
+    static class LargeTextParticleRenderer extends TextParticleRenderer {
+
+        LargeTextParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            setPaint(g2d, id);
+            Font font = new Font("LucidaGrande", Font.PLAIN, 32);
+            g2d.setFont(font);
+            g2d.drawString("The quick brown fox jumps over the lazy dog",
+                    (int)(x[id] - r), (int)(y[id] - r));
+            g2d.drawString("The quick brown fox jumps over the lazy dog",
+                    (int)(x[id] - r), (int)y[id]);
+            g2d.drawString("The quick brown fox jumps over the lazy dog",
+                    (int)(x[id] - r), (int)(y[id] + r));
+        }
+    }
+
+    static class FlatOvalRotParticleRenderer extends FlatParticleRenderer {
+
+
+        FlatOvalRotParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        void setPaint(Graphics2D g2d, int id) {
+            g2d.setColor(colors[id % colors.length]);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            setPaint(g2d, id);
+            if (Math.abs(vx[id] + vy[id]) > 0.001) {
+                AffineTransform t = (AffineTransform) g2d.getTransform().clone();
+                double l = vx[id] / Math.sqrt(vx[id] * vx[id] + vy[id] * vy[id]);
+                if (vy[id] < 0) {
+                    l = -l;
+                }
+                g2d.translate(x[id], y[id]);
+                g2d.rotate(Math.acos(l));
+                g2d.fillOval(-(int)r, (int)(-0.5*r), (int) (2 * r), (int)r);
+                g2d.setTransform(t);
+            } else {
+                g2d.fillOval((int)(x[id] - r), (int)(y[id] - 0.5*r),
+                        (int) (2 * r), (int) r);
+            }
+        }
+    }
+
+    static class LinGradOvalRotParticleRenderer extends FlatOvalRotParticleRenderer {
+
+
+        LinGradOvalRotParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        @Override
+        void setPaint(Graphics2D g2d, int id) {
+            Point2D start = new Point2D.Double(- r,  - 0.5*r);
+            Point2D end = new Point2D.Double( 2 * r, r);
+            float[] dist = {0.0f, 1.0f};
+            Color[] cls = {colors[id %colors.length], colors[(colors.length - id) %colors.length]};
+            LinearGradientPaint p =
+                    new LinearGradientPaint(start, end, dist, cls);
+            g2d.setPaint(p);
+        }
+    }
+
+    static class FlatBoxParticleRenderer extends FlatParticleRenderer {
+
+
+        FlatBoxParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            g2d.setColor(colors[id % colors.length]);
+            g2d.fillRect((int)(x[id] - r), (int)(y[id] - r), (int)(2*r), (int)(2*r));
+
+        }
+
+    }
+
+    static class ImgParticleRenderer extends FlatParticleRenderer {
+        BufferedImage dukeImg;
+
+        ImgParticleRenderer(int n, float r) {
+            super(n, r);
+            try {
+                dukeImg = ImageIO.read(
+                        Objects.requireNonNull(
+                                RenderPerfTest.class.getClassLoader().getResourceAsStream(
+                                        "renderperf/images/duke.png")));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            g2d.setColor(colors[id % colors.length]);
+            g2d.drawImage(dukeImg, (int)(x[id] - r), (int)(y[id] - r), (int)(2*r), (int)(2*r), null);
+        }
+
+    }
+
+    static class FlatBoxRotParticleRenderer extends FlatParticleRenderer {
+
+
+        FlatBoxRotParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            g2d.setColor(colors[id % colors.length]);
+            if (Math.abs(vx[id] + vy[id]) > 0.001) {
+                AffineTransform t = (AffineTransform) g2d.getTransform().clone();
+                double l = vx[id] / Math.sqrt(vx[id] * vx[id] + vy[id] * vy[id]);
+                if (vy[id] < 0) {
+                    l = -l;
+                }
+                g2d.translate(x[id], y[id]);
+                g2d.rotate(Math.acos(l));
+                g2d.fillRect(-(int)r, -(int)r, (int) (2 * r), (int) (2 * r));
+                g2d.setTransform(t);
+            } else {
+                g2d.fillRect((int)(x[id] - r), (int)(y[id] - r),
+                        (int) (2 * r), (int) (2 * r));
+            }
+        }
+    }
+
+    static class WiredParticleRenderer extends FlatParticleRenderer {
+
+
+        WiredParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            g2d.setColor(colors[id % colors.length]);
+            g2d.drawOval((int)(x[id] - r), (int)(y[id] - r), (int)(2*r), (int)(2*r));
+        }
+
+    }
+    static class WiredBoxParticleRenderer extends FlatParticleRenderer {
+
+        WiredBoxParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            g2d.setColor(colors[id % colors.length]);
+            g2d.drawRect((int)(x[id] - r), (int)(y[id] - r), (int)(2*r), (int)(2*r));
+        }
+
+    }
+    static class SegParticleRenderer extends FlatParticleRenderer {
+
+        SegParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            double v = Math.sqrt(vx[id]*vx[id]+vy[id]*vy[id]);
+            float nvx = (float) (vx[id]/v);
+            float nvy = (float) (vy[id]/v);
+            g2d.setColor(colors[id % colors.length]);
+            g2d.drawLine((int)(x[id] - r*nvx), (int)(y[id] - r*nvy),
+                    (int)(x[id] + 2*r*nvx), (int)(y[id] + 2*r*nvy));
+        }
+
+    }
+
+
+    static class WiredQuadParticleRenderer extends FlatParticleRenderer {
+
+        WiredQuadParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            if (id > 2 && (id % 3) == 0) {
+                g2d.setColor(colors[id % colors.length]);
+                g2d.draw(new QuadCurve2D.Float(x[id-3], y[id-3], x[id-2], y[id-2], x[id-1], y[id-1]));
+            }
+
+        }
+    }
+
+    static class FlatQuadParticleRenderer extends FlatParticleRenderer {
+
+        FlatQuadParticleRenderer(int n, float r) {
+            super(n, r);
+        }
+
+        @Override
+        public void render(Graphics2D g2d, int id, float[] x, float[] y, float[] vx, float[] vy) {
+            if (id > 2 && (id % 3) == 0) {
+                g2d.setColor(colors[id % colors.length]);
+                g2d.fill(new QuadCurve2D.Float(x[id-3], y[id-3], x[id-2], y[id-2], x[id-1], y[id-1]));
+            }
+
+        }
+    }
+
+    static class PerfMeter {
+        private String name;
+        private int frame = 0;
+
+        private JPanel panel;
+
+        private long time;
+        private double execTime = 0;
+        private Color expColor = Color.RED;
+        AtomicBoolean waiting = new AtomicBoolean(false);
+        private double fps;
+
+        PerfMeter(String name) {
+            this.name = name;
+        }
+
+        PerfMeter exec(final Renderable renderable) throws Exception {
+            final CountDownLatch latch = new CountDownLatch(COUNT);
+            final CountDownLatch latchFrame = new CountDownLatch(1);
+            final long endTime = System.currentTimeMillis() + MAX_MEASURE_TIME;
+
+            final JFrame f = new JFrame();
+            f.addWindowListener(new WindowAdapter() {
+                @Override
+                public void windowClosed(WindowEvent e) {
+                    latchFrame.countDown();
+                }
+            });
+
+            SwingUtilities.invokeAndWait(new Runnable() {
+                @Override
+                public void run() {
+
+                    panel = new JPanel()
+                    {
+                        @Override
+                        protected void paintComponent(Graphics g) {
+
+                            super.paintComponent(g);
+                            time = System.nanoTime();
+                            Graphics2D g2d = (Graphics2D) g.create();
+                            renderable.setup(g2d);
+                            renderable.render(g2d);
+                            g2d.setColor(expColor);
+                            g.fillRect(0, 0, BW, BH);
+                        }
+                    };
+
+                    panel.setPreferredSize(new Dimension((int)(WIDTH + BW), (int)(HEIGHT + BH)));
+                    panel.setBackground(Color.BLACK);
+                    f.add(panel);
+                    f.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
+                    f.pack();
+                    f.setVisible(true);
+                }
+            });
+            Robot robot = new Robot();
+
+            Timer timer = new Timer(DELAY, e -> {
+
+                if (waiting.compareAndSet(false, true)) {
+                    Color c = robot.getPixelColor(
+                            panel.getTopLevelAncestor().getX() + panel.getTopLevelAncestor().getInsets().left + BW / 2,
+                            panel.getTopLevelAncestor().getY() + panel.getTopLevelAncestor().getInsets().top + BW / 2);
+                    if (isAlmostEqual(c, Color.BLUE)) {
+                        expColor = Color.RED;
+                    } else {
+                        expColor = Color.BLUE;
+                    }
+                    renderable.update();
+                    panel.getParent().repaint();
+
+                } else {
+                    while (!isAlmostEqual(
+                            robot.getPixelColor(
+                                    panel.getTopLevelAncestor().getX() + panel.getTopLevelAncestor().getInsets().left + BW/2,
+                                    panel.getTopLevelAncestor().getY() + panel.getTopLevelAncestor().getInsets().top + BH/2),
+                            expColor))
+                    {
+                        try {
+                            Thread.sleep(RESOLUTION);
+                        } catch (InterruptedException ex) {
+                            ex.printStackTrace();
+                        }
+                    }
+                    time = System.nanoTime() - time;
+                    execTime += time;
+                    frame++;
+                    waiting.set(false);
+                }
+
+                if (System.currentTimeMillis() < endTime) {
+                    latch.countDown();
+                } else {
+                    while(latch.getCount() > 0) latch.countDown();
+                }
+            });
+            timer.start();
+            latch.await();
+            SwingUtilities.invokeAndWait(() -> {
+                timer.stop();
+                f.setVisible(false);
+                f.dispose();
+            });
+
+            latchFrame.await();
+            if (execTime != 0 && frame != 0) {
+                fps = 1e9 / (execTime / frame);
+            } else {
+                fps = 0;
+            }
+
+            return this;
+        }
+
+        private void report() {
+            System.err.println(name + " : " + String.format("%.2f FPS", fps));
+        }
+
+        private boolean isAlmostEqual(Color c1, Color c2) {
+            return Math.abs(c1.getRed() - c2.getRed()) < COLOR_TOLERANCE ||
+                    Math.abs(c1.getGreen() - c2.getGreen()) < COLOR_TOLERANCE ||
+                    Math.abs(c1.getBlue() - c2.getBlue()) < COLOR_TOLERANCE;
+
+        }
+    }
+
+    private static final Particles balls = new Particles(N, R, BW, BH, WIDTH, HEIGHT);
+    private static final ParticleRenderer flatRenderer = new FlatParticleRenderer(N, R);
+    private static final ParticleRenderer flatOvalRotRenderer = new FlatOvalRotParticleRenderer(N, R);
+    private static final ParticleRenderer flatBoxRenderer = new FlatBoxParticleRenderer(N, R);
+    private static final ParticleRenderer flatBoxRotRenderer = new FlatBoxRotParticleRenderer(N, R);
+    private static final ParticleRenderer linGradOvalRotRenderer = new LinGradOvalRotParticleRenderer(N, R);
+    private static final ParticleRenderer wiredRenderer = new WiredParticleRenderer(N, R);
+    private static final ParticleRenderer wiredBoxRenderer = new WiredBoxParticleRenderer(N, R);
+    private static final ParticleRenderer segRenderer = new SegParticleRenderer(N, R);
+    private static final ParticleRenderer flatQuadRenderer = new FlatQuadParticleRenderer(N, R);
+    private static final ParticleRenderer wiredQuadRenderer = new WiredQuadParticleRenderer(N, R);
+    private static final ParticleRenderer imgRenderer = new ImgParticleRenderer(N, R);
+    private static final ParticleRenderer textRenderer = new TextParticleRenderer(N, R);
+    private static final ParticleRenderer largeTextRenderer = new LargeTextParticleRenderer(N, R);
+    private static final ParticleRenderer whiteTextRenderer = new WhiteTextParticleRenderer(R);
+
+    private static final Configurable AA = (Graphics2D g2d) ->
+        g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING,
+            RenderingHints.VALUE_ANTIALIAS_ON);
+
+    private static final Configurable TextLCD = (Graphics2D g2d) ->
+        g2d.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING,
+                RenderingHints.VALUE_TEXT_ANTIALIAS_LCD_HRGB);
+
+    private static final Configurable TextAA = (Graphics2D g2d) ->
+        g2d.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING,
+                RenderingHints.VALUE_TEXT_ANTIALIAS_ON);
+
+    public void testFlatBubbles() throws Exception {
+        (new PerfMeter("FlatOval")).exec(createPR(flatRenderer)).report();
+    }
+
+    public void testFlatBubblesAA() throws Exception {
+        (new PerfMeter("FlatOvalAA")).exec(createPR(flatRenderer).configure(AA)).report();
+    }
+
+    public void testFlatBoxBubbles() throws Exception {
+        (new PerfMeter("FlatBox")).exec(createPR(flatBoxRenderer)).report();
+    }
+
+    public void testFlatBoxBubblesAA() throws Exception {
+        (new PerfMeter("FlatBoxAA")).exec(createPR(flatBoxRenderer).configure(AA)).report();
+    }
+
+    public void testImgBubbles() throws Exception {
+        (new PerfMeter("Image")).exec(createPR(imgRenderer)).report();
+    }
+
+    public void testImgBubblesAA() throws Exception {
+        (new PerfMeter("ImageAA")).exec(createPR(imgRenderer).configure(AA)).report();
+    }
+
+    public void testFlatBoxRotBubbles() throws Exception {
+        (new PerfMeter("RotatedBox")).exec(createPR(flatBoxRotRenderer)).report();
+    }
+
+    public void testFlatBoxRotBubblesAA() throws Exception {
+        (new PerfMeter("RotatedBoxAA")).exec(createPR(flatBoxRotRenderer).configure(AA)).report();
+    }
+
+    public void testFlatOvalRotBubbles() throws Exception {
+        (new PerfMeter("RotatedOval")).exec(createPR(flatOvalRotRenderer)).report();
+    }
+
+    public void testFlatOvalRotBubblesAA() throws Exception {
+        (new PerfMeter("RotatedOvalAA")).exec(createPR(flatOvalRotRenderer).configure(AA)).report();
+    }
+
+    public void testLinGradOvalRotBubbles() throws Exception {
+        (new PerfMeter("LinGradRotatedOval")).exec(createPR(linGradOvalRotRenderer)).report();
+    }
+
+    public void testLinGradOvalRotBubblesAA() throws Exception {
+        (new PerfMeter("LinGradRotatedOvalAA")).exec(createPR(linGradOvalRotRenderer).configure(AA)).report();
+    }
+
+    public void testWiredBubbles() throws Exception {
+        (new PerfMeter("WiredBubbles")).exec(createPR(wiredRenderer)).report();
+    }
+
+    public void testWiredBubblesAA() throws Exception {
+        (new PerfMeter("WiredBubblesAA")).exec(createPR(wiredRenderer).configure(AA)).report();
+    }
+
+    public void testWiredBoxBubbles() throws Exception {
+        (new PerfMeter("WiredBox")).exec(createPR(wiredBoxRenderer)).report();
+    }
+
+    public void testWiredBoxBubblesAA() throws Exception {
+        (new PerfMeter("WiredBoxAA")).exec(createPR(wiredBoxRenderer).configure(AA)).report();
+    }
+
+    public void testLines() throws Exception {
+        (new PerfMeter("Lines")).exec(createPR(segRenderer)).report();
+    }
+
+    public void testLinesAA() throws Exception {
+        (new PerfMeter("LinesAA")).exec(createPR(segRenderer).configure(AA)).report();
+    }
+
+    public void testFlatQuad() throws Exception {
+        (new PerfMeter("FlatQuad")).exec(createPR(flatQuadRenderer)).report();
+    }
+
+    public void testFlatQuadAA() throws Exception {
+        (new PerfMeter("FlatQuadAA")).exec(createPR(flatQuadRenderer).configure(AA)).report();
+    }
+
+    public void testWiredQuad() throws Exception {
+        (new PerfMeter("WiredQuad")).exec(createPR(wiredQuadRenderer)).report();
+    }
+
+    public void testWiredQuadAA() throws Exception {
+        (new PerfMeter("WiredQuadAA")).exec(createPR(wiredQuadRenderer).configure(AA)).report();
+    }
+
+    public void testTextBubblesNoAA() throws Exception {
+        (new PerfMeter("TextNoAA")).exec(createPR(textRenderer)).report();
+    }
+
+    public void testTextBubblesLCD() throws Exception {
+        (new PerfMeter("TextLCD")).exec(createPR(textRenderer).configure(TextLCD)).report();
+    }
+
+    public void testTextBubblesGray() throws Exception {
+        (new PerfMeter("TextGray")).exec(createPR(textRenderer).configure(TextAA)).report();
+    }
+
+    public void testLargeTextBubblesNoAA() throws Exception {
+        (new PerfMeter("LargeTextNoAA")).exec(createPR(largeTextRenderer)).report();
+    }
+
+    public void testLargeTextBubblesLCD() throws Exception {
+        (new PerfMeter("LargeTextLCD")).exec(createPR(largeTextRenderer).configure(TextLCD)).report();
+    }
+
+    public void testLargeTextBubblesGray() throws Exception {
+        (new PerfMeter("LargeTextGray")).exec(createPR(largeTextRenderer).configure(TextAA)).report();
+    }
+    public void testWhiteTextBubblesNoAA() throws Exception {
+        (new PerfMeter("WhiteTextNoAA")).exec(createPR(whiteTextRenderer)).report();
+    }
+
+    public void testWhiteTextBubblesLCD() throws Exception {
+        (new PerfMeter("WhiteTextLCD")).exec(createPR(whiteTextRenderer).configure(TextLCD)).report();
+    }
+
+    public void testWhiteTextBubblesGray() throws Exception {
+        (new PerfMeter("WhiteTextGray")).exec(createPR(whiteTextRenderer).configure(TextAA)).report();
+    }
+
+    public static void main(String[] args)
+            throws InvocationTargetException, IllegalAccessException, NoSuchMethodException
+    {
+        RenderPerfTest test = new RenderPerfTest();
+
+        if (args.length > 0) {
+            for (String testCase : args) {
+                Method m = RenderPerfTest.class.getDeclaredMethod(testCase);
+                m.invoke(test);
+            }
+        } else {
+            Method[] methods = RenderPerfTest.class.getDeclaredMethods();
+            for (Method m : methods) {
+                if (m.getName().startsWith("test") && !ignoredTests.contains(m.getName())) {
+                    m.invoke(test);
+                }
+            }
+        }
+    }
+}

src/hotspot/cpu/aarch64/aarch64.ad

@@ -2438,10 +2438,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 // Is this branch offset short enough that a short branch can be used?
 //
 // NOTE: If the platform does not provide any short branch variants, then
@@ -2554,6 +2550,31 @@ bool Matcher::is_spillable_arg(int reg)
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  // JDK-8183543: When taking the number of available registers as int
+  // register pressure threshold, the jtreg test:
+  // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
+  // failed due to C2 compilation failure with
+  // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
+  //
+  // A derived pointer is live at CallNode and then is flagged by RA
+  // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
+  // derived pointers and lastly fail to spill after reaching maximum
+  // number of iterations. Lowering the default pressure threshold to
+  // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
+  // a high register pressure area of the code so that split_DEF can
+  // generate DefinitionSpillCopy for the derived pointer.
+  uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
+  return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
+  return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
   return false;
 }

src/hotspot/cpu/aarch64/c1_CodeStubs_aarch64.cpp

@@ -361,8 +361,10 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->add_call_info_here(info());
 
 #ifndef PRODUCT
-  __ lea(rscratch2, ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
-  __ incrementw(Address(rscratch2));
+  if (PrintC1Statistics) {
+    __ lea(rscratch2, ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
+    __ incrementw(Address(rscratch2));
+  }
 #endif
 
   __ b(_continuation);

src/hotspot/cpu/aarch64/c2_globals_aarch64.hpp

@@ -44,10 +44,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         3);
-define_pd_global(intx, FLOATPRESSURE,                32);
 define_pd_global(intx, FreqInlineSize,               325);
 define_pd_global(intx, MinJumpTableSize,             10);
-define_pd_global(intx, INTPRESSURE,                  24);
 define_pd_global(intx, InteriorEntryAlignment,       16);
 define_pd_global(intx, NewSizeThreadIncrease, ScaleForWordSize(4*K));
 define_pd_global(intx, LoopUnrollLimit,              60);

src/hotspot/cpu/aarch64/frame_aarch64.cpp

@@ -367,6 +367,11 @@ JavaFrameAnchor* OptimizedEntryBlob::jfa_for_frame(const frame& frame) const {
   return nullptr;
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_optimized_entry_frame(RegisterMap* map) const {
   ShouldNotCallThis();
   return {};

src/hotspot/cpu/arm/arm.ad

@@ -1001,10 +1001,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 // Vector width in bytes
 const int Matcher::vector_width_in_bytes(BasicType bt) {
   return MaxVectorSize;
@@ -1100,6 +1096,16 @@ bool Matcher::is_spillable_arg( int reg ) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? 12 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 30 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
   return false;
 }

src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp

@@ -234,8 +234,9 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj,
   // -2- test (hdr - SP) if the low two bits are 0
   sub(tmp2, hdr, SP, eq);
   movs(tmp2, AsmOperand(tmp2, lsr, exact_log2(os::vm_page_size())), eq);
-  // If 'eq' then OK for recursive fast locking: store 0 into a lock record.
-  str(tmp2, Address(disp_hdr, mark_offset), eq);
+  // If still 'eq' then recursive locking OK
+  // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8267042)
+  str(tmp2, Address(disp_hdr, mark_offset));
   b(fast_lock_done, eq);
   // else need slow case
   b(slow_case);

src/hotspot/cpu/arm/c2_globals_arm.hpp

@@ -45,9 +45,7 @@ define_pd_global(intx, CompileThreshold,             10000);
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         4);
 // C2 gets to use all the float/double registers
-define_pd_global(intx, FLOATPRESSURE,                30);
 define_pd_global(intx, FreqInlineSize,               175);
-define_pd_global(intx, INTPRESSURE,                  12);
 define_pd_global(intx, InteriorEntryAlignment,       16);  // = CodeEntryAlignment
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 // The default setting 16/16 seems to work best.

src/hotspot/cpu/arm/frame_arm.cpp

@@ -313,6 +313,11 @@ frame frame::sender_for_entry_frame(RegisterMap* map) const {
   return fr;
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 //------------------------------------------------------------------------------
 // frame::verify_deopt_original_pc
 //

src/hotspot/cpu/arm/sharedRuntime_arm.cpp

@@ -1180,8 +1180,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     // -2- test (hdr - SP) if the low two bits are 0
     __ sub(Rtemp, mark, SP, eq);
     __ movs(Rtemp, AsmOperand(Rtemp, lsr, exact_log2(os::vm_page_size())), eq);
-    // If still 'eq' then recursive locking OK: set displaced header to 0
-    __ str(Rtemp, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()), eq);
+    // If still 'eq' then recursive locking OK
+    // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8267042)
+    __ str(Rtemp, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
     __ b(lock_done, eq);
     __ b(slow_lock);
 

src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp

@@ -491,12 +491,14 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->verify_oop_map(info());
 
 #ifndef PRODUCT
-  const address counter = (address)&Runtime1::_arraycopy_slowcase_cnt;
-  const Register tmp = R3, tmp2 = R4;
-  int simm16_offs = __ load_const_optimized(tmp, counter, tmp2, true);
-  __ lwz(tmp2, simm16_offs, tmp);
-  __ addi(tmp2, tmp2, 1);
-  __ stw(tmp2, simm16_offs, tmp);
+  if (PrintC1Statistics) {
+    const address counter = (address)&Runtime1::_arraycopy_slowcase_cnt;
+    const Register tmp = R3, tmp2 = R4;
+    int simm16_offs = __ load_const_optimized(tmp, counter, tmp2, true);
+    __ lwz(tmp2, simm16_offs, tmp);
+    __ addi(tmp2, tmp2, 1);
+    __ stw(tmp2, simm16_offs, tmp);
+  }
 #endif
 
   __ b(_continuation);

src/hotspot/cpu/ppc/c2_globals_ppc.hpp

@@ -44,10 +44,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         3);
-define_pd_global(intx, FLOATPRESSURE,                28);
 define_pd_global(intx, FreqInlineSize,               175);
 define_pd_global(intx, MinJumpTableSize,             10);
-define_pd_global(intx, INTPRESSURE,                  26);
 define_pd_global(intx, InteriorEntryAlignment,       16);
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 define_pd_global(intx, RegisterCostAreaRatio,        16000);

src/hotspot/cpu/ppc/frame_ppc.cpp

@@ -197,6 +197,11 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   return fr;
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_interpreter_frame(RegisterMap *map) const {
   // Pass callers initial_caller_sp as unextended_sp.
   return frame(sender_sp(), sender_pc(), (intptr_t*)get_ijava_state()->sender_sp);

src/hotspot/cpu/ppc/ppc.ad

@@ -2193,10 +2193,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 // Vector width in bytes.
 const int Matcher::vector_width_in_bytes(BasicType bt) {
   if (SuperwordUseVSX) {
@@ -2363,6 +2359,16 @@ bool Matcher::is_spillable_arg(int reg) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? 26 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 28 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
   return false;
 }

src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp

@@ -454,8 +454,10 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->verify_oop_map(info());
 
 #ifndef PRODUCT
-  __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_slowcase_cnt);
-  __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
+  if (PrintC1Statistics) {
+    __ load_const_optimized(Z_R1_scratch, (address)&Runtime1::_arraycopy_slowcase_cnt);
+    __ add2mem_32(Address(Z_R1_scratch), 1, Z_R0_scratch);
+  }
 #endif
 
   __ branch_optimized(Assembler::bcondAlways, _continuation);

src/hotspot/cpu/s390/c2_globals_s390.hpp

@@ -44,10 +44,8 @@ define_pd_global(intx, CompileThreshold,             10000);
 
 define_pd_global(intx, OnStackReplacePercentage,     140);
 define_pd_global(intx, ConditionalMoveLimit,         4);
-define_pd_global(intx, FLOATPRESSURE,                15);
 define_pd_global(intx, FreqInlineSize,               175);
 // 10 prevents spill-split-recycle sanity check in JVM2008.xml.transform.
-define_pd_global(intx, INTPRESSURE,                  10); // Medium size register set, 6 special purpose regs, 3 SOE regs.
 define_pd_global(intx, InteriorEntryAlignment,       2);
 define_pd_global(size_t, NewSizeThreadIncrease,      ScaleForWordSize(4*K));
 define_pd_global(intx, RegisterCostAreaRatio,        12000);

src/hotspot/cpu/s390/frame_s390.cpp

@@ -208,6 +208,11 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   return fr;
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_interpreter_frame(RegisterMap *map) const {
   // Pass callers sender_sp as unextended_sp.
   return frame(sender_sp(), sender_pc(), (intptr_t*)(ijava_state()->sender_sp));

src/hotspot/cpu/s390/s390.ad

@@ -1554,10 +1554,6 @@ OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
   return OptoRegPair(0, 0);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  return default_pressure_threshold;
-}
-
 //----------SUPERWORD HELPERS----------------------------------------
 
 // Vector width in bytes.
@@ -1665,6 +1661,17 @@ bool Matcher::is_spillable_arg(int reg) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  // Medium size register set, 6 special purpose regs, 3 SOE regs.
+  return (INTPRESSURE == -1) ? 10 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 15 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
   return false;
 }

src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp

@@ -543,7 +543,9 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   ce->add_call_info_here(info());
 
 #ifndef PRODUCT
-  __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
+  if (PrintC1Statistics) {
+    __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
+  }
 #endif
 
   __ jmp(_continuation);

src/hotspot/cpu/x86/c2_globals_x86.hpp

@@ -46,8 +46,6 @@ define_pd_global(intx, FreqInlineSize,               325);
 define_pd_global(intx, MinJumpTableSize,             10);
 define_pd_global(intx, LoopPercentProfileLimit,      30);
 #ifdef AMD64
-define_pd_global(intx,  INTPRESSURE,                 13);
-define_pd_global(intx,  FLOATPRESSURE,               14);
 define_pd_global(intx,  InteriorEntryAlignment,      16);
 define_pd_global(size_t, NewSizeThreadIncrease,     ScaleForWordSize(4*K));
 define_pd_global(intx,  LoopUnrollLimit,             60);
@@ -58,8 +56,6 @@ define_pd_global(uintx, CodeCacheExpansionSize,      64*K);
 // Ergonomics related flags
 define_pd_global(uint64_t, MaxRAM,                   128ULL*G);
 #else
-define_pd_global(intx,  INTPRESSURE,                 6);
-define_pd_global(intx,  FLOATPRESSURE,               6);
 define_pd_global(intx,  InteriorEntryAlignment,      4);
 define_pd_global(size_t, NewSizeThreadIncrease,      4*K);
 define_pd_global(intx,  LoopUnrollLimit,             50);     // Design center runs on 1.3.1

src/hotspot/cpu/x86/frame_x86.cpp

@@ -358,12 +358,20 @@ JavaFrameAnchor* OptimizedEntryBlob::jfa_for_frame(const frame& frame) const {
   return reinterpret_cast<JavaFrameAnchor*>(reinterpret_cast<char*>(frame.unextended_sp()) + in_bytes(jfa_sp_offset()));
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  assert(is_optimized_entry_frame(), "must be optimzed entry frame");
+  OptimizedEntryBlob* blob = _cb->as_optimized_entry_blob();
+  JavaFrameAnchor* jfa = blob->jfa_for_frame(*this);
+  return jfa->last_Java_sp() == NULL;
+}
+
 frame frame::sender_for_optimized_entry_frame(RegisterMap* map) const {
   assert(map != NULL, "map must be set");
   OptimizedEntryBlob* blob = _cb->as_optimized_entry_blob();
   // Java frame called from C; skip all C frames and return top C
   // frame of that chunk as the sender
   JavaFrameAnchor* jfa = blob->jfa_for_frame(*this);
+  assert(!optimized_entry_frame_is_first(), "must have a frame anchor to go back to");
   assert(jfa->last_Java_sp() > sp(), "must be above this frame on stack");
   // Since we are walking the stack now this nested anchor is obviously walkable
   // even if it wasn't when it was stacked.

src/hotspot/cpu/x86/x86.ad

@@ -1898,18 +1898,6 @@ const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
   return new TypeVectMask(TypeInt::BOOL, length);
 }
 
-const int Matcher::float_pressure(int default_pressure_threshold) {
-  int float_pressure_threshold = default_pressure_threshold;
-#ifdef _LP64
-  if (UseAVX > 2) {
-    // Increase pressure threshold on machines with AVX3 which have
-    // 2x more XMM registers.
-    float_pressure_threshold = default_pressure_threshold * 2;
-  }
-#endif
-  return float_pressure_threshold;
-}
-
 // Max vector size in bytes. 0 if not supported.
 const int Matcher::vector_width_in_bytes(BasicType bt) {
   assert(is_java_primitive(bt), "only primitive type vectors");
@@ -4844,7 +4832,8 @@ instruct vaddB_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddB_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVB src (LoadVector mem)));
   format %{ "vpaddb  $dst,$src,$mem\t! add packedB" %}
   ins_encode %{
@@ -4877,7 +4866,8 @@ instruct vaddS_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddS_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVS src (LoadVector mem)));
   format %{ "vpaddw  $dst,$src,$mem\t! add packedS" %}
   ins_encode %{
@@ -4911,7 +4901,8 @@ instruct vaddI_reg(vec dst, vec src1, vec src2) %{
 
 
 instruct vaddI_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVI src (LoadVector mem)));
   format %{ "vpaddd  $dst,$src,$mem\t! add packedI" %}
   ins_encode %{
@@ -4944,7 +4935,8 @@ instruct vaddL_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddL_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVL src (LoadVector mem)));
   format %{ "vpaddq  $dst,$src,$mem\t! add packedL" %}
   ins_encode %{
@@ -4977,7 +4969,8 @@ instruct vaddF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVF src (LoadVector mem)));
   format %{ "vaddps  $dst,$src,$mem\t! add packedF" %}
   ins_encode %{
@@ -5010,7 +5003,8 @@ instruct vaddD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vaddD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AddVD src (LoadVector mem)));
   format %{ "vaddpd  $dst,$src,$mem\t! add packedD" %}
   ins_encode %{
@@ -5045,7 +5039,8 @@ instruct vsubB_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubB_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVB src (LoadVector mem)));
   format %{ "vpsubb  $dst,$src,$mem\t! sub packedB" %}
   ins_encode %{
@@ -5079,7 +5074,8 @@ instruct vsubS_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubS_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVS src (LoadVector mem)));
   format %{ "vpsubw  $dst,$src,$mem\t! sub packedS" %}
   ins_encode %{
@@ -5112,7 +5108,8 @@ instruct vsubI_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubI_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVI src (LoadVector mem)));
   format %{ "vpsubd  $dst,$src,$mem\t! sub packedI" %}
   ins_encode %{
@@ -5146,7 +5143,8 @@ instruct vsubL_reg(vec dst, vec src1, vec src2) %{
 
 
 instruct vsubL_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVL src (LoadVector mem)));
   format %{ "vpsubq  $dst,$src,$mem\t! sub packedL" %}
   ins_encode %{
@@ -5179,7 +5177,8 @@ instruct vsubF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVF src (LoadVector mem)));
   format %{ "vsubps  $dst,$src,$mem\t! sub packedF" %}
   ins_encode %{
@@ -5212,7 +5211,8 @@ instruct vsubD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vsubD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (SubVD src (LoadVector mem)));
   format %{ "vsubpd  $dst,$src,$mem\t! sub packedD" %}
   ins_encode %{
@@ -5360,7 +5360,8 @@ instruct vmulS_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulS_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVS src (LoadVector mem)));
   format %{ "vpmullw $dst,$src,$mem\t! mul packedS" %}
   ins_encode %{
@@ -5394,7 +5395,8 @@ instruct vmulI_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulI_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVI src (LoadVector mem)));
   format %{ "vpmulld $dst,$src,$mem\t! mul packedI" %}
   ins_encode %{
@@ -5418,7 +5420,8 @@ instruct vmulL_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulL_mem(vec dst, vec src, memory mem) %{
-  predicate(VM_Version::supports_avx512dq());
+  predicate(VM_Version::supports_avx512dq() &&
+              (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVL src (LoadVector mem)));
   format %{ "vpmullq $dst,$src,$mem\t! mul packedL" %}
   ins_encode %{
@@ -5503,7 +5506,8 @@ instruct vmulF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVF src (LoadVector mem)));
   format %{ "vmulps  $dst,$src,$mem\t! mul packedF" %}
   ins_encode %{
@@ -5536,7 +5540,8 @@ instruct vmulD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vmulD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (MulVD src (LoadVector mem)));
   format %{ "vmulpd  $dst,$src,$mem\t! mul packedD" %}
   ins_encode %{
@@ -5607,7 +5612,8 @@ instruct vdivF_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vdivF_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (DivVF src (LoadVector mem)));
   format %{ "vdivps  $dst,$src,$mem\t! div packedF" %}
   ins_encode %{
@@ -5640,7 +5646,8 @@ instruct vdivD_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vdivD_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (DivVD src (LoadVector mem)));
   format %{ "vdivpd  $dst,$src,$mem\t! div packedD" %}
   ins_encode %{
@@ -5824,6 +5831,7 @@ instruct vsqrtF_reg(vec dst, vec src) %{
 %}
 
 instruct vsqrtF_mem(vec dst, memory mem) %{
+  predicate(vector_length_in_bytes(n->in(1)) > 8);
   match(Set dst (SqrtVF (LoadVector mem)));
   format %{ "vsqrtps  $dst,$mem\t! sqrt packedF" %}
   ins_encode %{
@@ -5847,6 +5855,7 @@ instruct vsqrtD_reg(vec dst, vec src) %{
 %}
 
 instruct vsqrtD_mem(vec dst, memory mem) %{
+  predicate(vector_length_in_bytes(n->in(1)) > 8);
   match(Set dst (SqrtVD (LoadVector mem)));
   format %{ "vsqrtpd  $dst,$mem\t! sqrt packedD" %}
   ins_encode %{
@@ -6459,7 +6468,8 @@ instruct vand_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vand_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (AndV src (LoadVector mem)));
   format %{ "vpand   $dst,$src,$mem\t! and vectors" %}
   ins_encode %{
@@ -6493,7 +6503,8 @@ instruct vor_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vor_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (OrV src (LoadVector mem)));
   format %{ "vpor    $dst,$src,$mem\t! or vectors" %}
   ins_encode %{
@@ -6527,7 +6538,8 @@ instruct vxor_reg(vec dst, vec src1, vec src2) %{
 %}
 
 instruct vxor_mem(vec dst, vec src, memory mem) %{
-  predicate(UseAVX > 0);
+  predicate((UseAVX > 0) &&
+            (vector_length_in_bytes(n->in(1)) > 8));
   match(Set dst (XorV src (LoadVector mem)));
   format %{ "vpxor   $dst,$src,$mem\t! xor vectors" %}
   ins_encode %{
@@ -7947,6 +7959,7 @@ instruct vfmaF_reg(vec a, vec b, vec c) %{
 %}
 
 instruct vfmaF_mem(vec a, memory b, vec c) %{
+  predicate(vector_length_in_bytes(n->in(1)) > 8);
   match(Set c (FmaVF  c (Binary a (LoadVector b))));
   format %{ "fmaps $a,$b,$c\t# $c = $a * $b + $c fma packedF" %}
   ins_cost(150);
@@ -7971,6 +7984,7 @@ instruct vfmaD_reg(vec a, vec b, vec c) %{
 %}
 
 instruct vfmaD_mem(vec a, memory b, vec c) %{
+  predicate(vector_length_in_bytes(n->in(1)) > 8);
   match(Set c (FmaVD  c (Binary a (LoadVector b))));
   format %{ "fmapd $a,$b,$c\t# $c = $a * $b + $c fma packedD" %}
   ins_cost(150);
@@ -8048,6 +8062,7 @@ instruct vpternlog(vec dst, vec src2, vec src3, immU8 func) %{
 %}
 
 instruct vpternlog_mem(vec dst, vec src2, memory src3, immU8 func) %{
+  predicate(vector_length_in_bytes(n->in(1)) > 8);
   match(Set dst (MacroLogicV (Binary dst src2) (Binary (LoadVector src3) func)));
   effect(TEMP dst);
   format %{ "vpternlogd $dst,$src2,$src3,$func\t! vector ternary logic" %}

src/hotspot/cpu/x86/x86_32.ad

@@ -1440,6 +1440,16 @@ bool Matcher::is_spillable_arg( int reg ) {
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? 6 : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  return (FLOATPRESSURE == -1) ? 6 : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
   // Use hardware integer DIV instruction when
   // it is faster than a code which use multiply.

src/hotspot/cpu/x86/x86_64.ad

@@ -322,6 +322,7 @@ extern RegMask _LONG_NO_RCX_REG_mask;
 extern RegMask _INT_REG_mask;
 extern RegMask _INT_NO_RAX_RDX_REG_mask;
 extern RegMask _INT_NO_RCX_REG_mask;
+extern RegMask _FLOAT_REG_mask;
 
 extern RegMask _STACK_OR_PTR_REG_mask;
 extern RegMask _STACK_OR_LONG_REG_mask;
@@ -350,6 +351,7 @@ RegMask _LONG_NO_RCX_REG_mask;
 RegMask _INT_REG_mask;
 RegMask _INT_NO_RAX_RDX_REG_mask;
 RegMask _INT_NO_RCX_REG_mask;
+RegMask _FLOAT_REG_mask;
 RegMask _STACK_OR_PTR_REG_mask;
 RegMask _STACK_OR_LONG_REG_mask;
 RegMask _STACK_OR_INT_REG_mask;
@@ -425,6 +427,10 @@ void reg_mask_init() {
   _INT_NO_RCX_REG_mask = _INT_REG_mask;
   _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 
+  // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
+  // from the float_reg_legacy/float_reg_evex register class.
+  _FLOAT_REG_mask = VM_Version::supports_evex() ? _FLOAT_REG_EVEX_mask : _FLOAT_REG_LEGACY_mask;
+
   if (Matcher::has_predicated_vectors()) {
     // Post-loop multi-versioning expects mask to be present in K1 register, till the time
     // its fixed, RA should not be allocting K1 register, this shall prevent any accidental
@@ -1758,6 +1764,20 @@ bool Matcher::is_spillable_arg(int reg)
   return can_be_java_arg(reg);
 }
 
+uint Matcher::int_pressure_limit()
+{
+  return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
+}
+
+uint Matcher::float_pressure_limit()
+{
+  // After experiment around with different values, the following default threshold
+  // works best for LCM's register pressure scheduling on x64.
+  uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
+  uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
+  return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
+}
+
 bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
   // In 64 bit mode a code which use multiply when
   // devisor is constant is faster than hardware

src/hotspot/cpu/zero/frame_zero.cpp

@@ -61,6 +61,11 @@ frame frame::sender_for_entry_frame(RegisterMap *map) const {
   return frame(zeroframe()->next(), sender_sp());
 }
 
+bool frame::optimized_entry_frame_is_first() const {
+  ShouldNotCallThis();
+  return false;
+}
+
 frame frame::sender_for_nonentry_frame(RegisterMap *map) const {
   assert(zeroframe()->is_interpreter_frame() ||
          zeroframe()->is_fake_stub_frame(), "wrong type of frame");
@@ -95,7 +100,7 @@ void frame::patch_pc(Thread* thread, address pc) {
     // We borrow this call to set the thread pointer in the interpreter
     // state; the hook to set up deoptimized frames isn't supplied it.
     assert(pc == NULL, "should be");
-    get_interpreterState()->set_thread(thread->as_Java_thread());
+    get_interpreterState()->set_thread(JavaThread::cast(thread));
   }
 }
 

src/hotspot/os/windows/os_windows.cpp

@@ -2500,7 +2500,7 @@ LONG Handle_Exception(struct _EXCEPTION_POINTERS* exceptionInfo,
 
   // Save pc in thread
   if (thread != nullptr && thread->is_Java_thread()) {
-    thread->as_Java_thread()->set_saved_exception_pc((address)(DWORD_PTR)exceptionInfo->ContextRecord->PC_NAME);
+    JavaThread::cast(thread)->set_saved_exception_pc((address)(DWORD_PTR)exceptionInfo->ContextRecord->PC_NAME);
   }
 
   // Set pc to handler
@@ -2794,7 +2794,7 @@ LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
 #endif
 
   if (t != NULL && t->is_Java_thread()) {
-    JavaThread* thread = t->as_Java_thread();
+    JavaThread* thread = JavaThread::cast(t);
     bool in_java = thread->thread_state() == _thread_in_Java;
     bool in_native = thread->thread_state() == _thread_in_native;
     bool in_vm = thread->thread_state() == _thread_in_vm;

src/hotspot/share/c1/c1_Runtime1.cpp

@@ -348,8 +348,11 @@ const char* Runtime1::name_for_address(address entry) {
 
 
 JRT_ENTRY(void, Runtime1::new_instance(JavaThread* current, Klass* klass))
-  NOT_PRODUCT(_new_instance_slowcase_cnt++;)
-
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_instance_slowcase_cnt++;
+  }
+#endif
   assert(klass->is_klass(), "not a class");
   Handle holder(current, klass->klass_holder()); // keep the klass alive
   InstanceKlass* h = InstanceKlass::cast(klass);
@@ -363,7 +366,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::new_type_array(JavaThread* current, Klass* klass, jint length))
-  NOT_PRODUCT(_new_type_array_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_type_array_slowcase_cnt++;
+  }
+#endif
   // Note: no handle for klass needed since they are not used
   //       anymore after new_typeArray() and no GC can happen before.
   //       (This may have to change if this code changes!)
@@ -381,8 +388,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::new_object_array(JavaThread* current, Klass* array_klass, jint length))
-  NOT_PRODUCT(_new_object_array_slowcase_cnt++;)
-
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_object_array_slowcase_cnt++;
+  }
+#endif
   // Note: no handle for klass needed since they are not used
   //       anymore after new_objArray() and no GC can happen before.
   //       (This may have to change if this code changes!)
@@ -400,8 +410,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::new_multi_array(JavaThread* current, Klass* klass, int rank, jint* dims))
-  NOT_PRODUCT(_new_multi_array_slowcase_cnt++;)
-
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _new_multi_array_slowcase_cnt++;
+  }
+#endif
   assert(klass->is_klass(), "not a class");
   assert(rank >= 1, "rank must be nonzero");
   Handle holder(current, klass->klass_holder()); // keep the klass alive
@@ -653,7 +666,11 @@ address Runtime1::exception_handler_for_pc(JavaThread* current) {
 
 
 JRT_ENTRY(void, Runtime1::throw_range_check_exception(JavaThread* current, int index, arrayOopDesc* a))
-  NOT_PRODUCT(_throw_range_check_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_range_check_exception_count++;
+  }
+#endif
   const int len = 35;
   assert(len < strlen("Index %d out of bounds for length %d"), "Must allocate more space for message.");
   char message[2 * jintAsStringSize + len];
@@ -663,7 +680,11 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_index_exception(JavaThread* current, int index))
-  NOT_PRODUCT(_throw_index_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_index_exception_count++;
+  }
+#endif
   char message[16];
   sprintf(message, "%d", index);
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_IndexOutOfBoundsException(), message);
@@ -671,19 +692,31 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_div0_exception(JavaThread* current))
-  NOT_PRODUCT(_throw_div0_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_div0_exception_count++;
+  }
+#endif
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_ArithmeticException(), "/ by zero");
 JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_null_pointer_exception(JavaThread* current))
-  NOT_PRODUCT(_throw_null_pointer_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_null_pointer_exception_count++;
+  }
+#endif
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException());
 JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_class_cast_exception(JavaThread* current, oopDesc* object))
-  NOT_PRODUCT(_throw_class_cast_exception_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_class_cast_exception_count++;
+  }
+#endif
   ResourceMark rm(current);
   char* message = SharedRuntime::generate_class_cast_message(current, object->klass());
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_ClassCastException(), message);
@@ -691,14 +724,22 @@ JRT_END
 
 
 JRT_ENTRY(void, Runtime1::throw_incompatible_class_change_error(JavaThread* current))
-  NOT_PRODUCT(_throw_incompatible_class_change_error_count++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _throw_incompatible_class_change_error_count++;
+  }
+#endif
   ResourceMark rm(current);
   SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_IncompatibleClassChangeError());
 JRT_END
 
 
 JRT_BLOCK_ENTRY(void, Runtime1::monitorenter(JavaThread* current, oopDesc* obj, BasicObjectLock* lock))
-  NOT_PRODUCT(_monitorenter_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _monitorenter_slowcase_cnt++;
+  }
+#endif
   if (!UseFastLocking) {
     lock->set_obj(obj);
   }
@@ -708,7 +749,11 @@ JRT_END
 
 
 JRT_LEAF(void, Runtime1::monitorexit(JavaThread* current, BasicObjectLock* lock))
-  NOT_PRODUCT(_monitorexit_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _monitorexit_slowcase_cnt++;
+  }
+#endif
   assert(current->last_Java_sp(), "last_Java_sp must be set");
   oop obj = lock->obj();
   assert(oopDesc::is_oop(obj), "must be NULL or an object");
@@ -860,7 +905,11 @@ static Klass* resolve_field_return_klass(const methodHandle& caller, int bci, TR
 // patch only naturally aligned words, as single, full-word writes.
 
 JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, Runtime1::StubID stub_id ))
-  NOT_PRODUCT(_patch_code_slowcase_cnt++;)
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _patch_code_slowcase_cnt++;
+  }
+#endif
 
   ResourceMark rm(current);
   RegisterMap reg_map(current, false);
@@ -1255,7 +1304,11 @@ JRT_END
 #else // DEOPTIMIZE_WHEN_PATCHING
 
 void Runtime1::patch_code(JavaThread* current, Runtime1::StubID stub_id) {
-  NOT_PRODUCT(_patch_code_slowcase_cnt++);
+#ifndef PRODUCT
+  if (PrintC1Statistics) {
+    _patch_code_slowcase_cnt++;
+  }
+#endif
 
   // Enable WXWrite: the function is called by c1 stub as a runtime function
   // (see another implementation above).

src/hotspot/share/code/codeCache.cpp

@@ -1077,12 +1077,6 @@ void CodeCache::old_nmethods_do(MetadataClosure* f) {
   log_debug(redefine, class, nmethod)("Walked %d nmethods for mark_on_stack", length);
 }
 
-// Just marks the methods in this class as needing deoptimization
-void CodeCache::mark_for_evol_deoptimization(InstanceKlass* dependee) {
-  assert(SafepointSynchronize::is_at_safepoint(), "Can only do this at a safepoint!");
-}
-
-
 // Walk compiled methods and mark dependent methods for deoptimization.
 int CodeCache::mark_dependents_for_evol_deoptimization() {
   assert(SafepointSynchronize::is_at_safepoint(), "Can only do this at a safepoint!");

src/hotspot/share/code/codeCache.hpp

@@ -271,7 +271,6 @@ class CodeCache : AllStatic {
 
   // RedefineClasses support
   // Flushing and deoptimization in case of evolution
-  static void mark_for_evol_deoptimization(InstanceKlass* dependee);
   static int  mark_dependents_for_evol_deoptimization();
   static void mark_all_nmethods_for_evol_deoptimization();
   static void flush_evol_dependents();

src/hotspot/share/code/dependencies.cpp

@@ -51,7 +51,7 @@
 static bool must_be_in_vm() {
   Thread* thread = Thread::current();
   if (thread->is_Java_thread()) {
-    return thread->as_Java_thread()->thread_state() == _thread_in_vm;
+    return JavaThread::cast(thread)->thread_state() == _thread_in_vm;
   } else {
     return true;  // Could be VMThread or GC thread
   }

src/hotspot/share/code/nmethod.cpp

@@ -1596,8 +1596,18 @@ void nmethod::post_compiled_method_load_event(JvmtiThreadState* state) {
 
   // Don't post this nmethod load event if it is already dying
   // because the sweeper might already be deleting this nmethod.
-  if (is_not_entrant() && can_convert_to_zombie()) {
-    return;
+  {
+    MutexLocker ml(CompiledMethod_lock, Mutex::_no_safepoint_check_flag);
+    // When the nmethod is acquired from the CodeCache iterator, it can racingly become zombie
+    // before this code is called. Filter them out here under the CompiledMethod_lock.
+    if (!is_alive()) {
+      return;
+    }
+    // As for is_alive() nmethods, we also don't want them to racingly become zombie once we
+    // release this lock, so we check that this is not going to be the case.
+    if (is_not_entrant() && can_convert_to_zombie()) {
+      return;
+    }
   }
 
   // This is a bad time for a safepoint.  We don't want

src/hotspot/share/compiler/compileBroker.cpp

@@ -1276,7 +1276,7 @@ void CompileBroker::compile_method_base(const methodHandle& method,
 
       if (!UseJVMCINativeLibrary) {
         // Don't allow blocking compiles if inside a class initializer or while performing class loading
-        vframeStream vfst(thread->as_Java_thread());
+        vframeStream vfst(JavaThread::cast(thread));
         for (; !vfst.at_end(); vfst.next()) {
           if (vfst.method()->is_static_initializer() ||
               (vfst.method()->method_holder()->is_subclass_of(vmClasses::ClassLoader_klass()) &&

src/hotspot/share/gc/g1/g1Arguments.cpp

@@ -25,6 +25,8 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1Arguments.hpp"
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1HeapVerifier.hpp"
 #include "gc/g1/heapRegion.hpp"
@@ -50,16 +52,17 @@ void G1Arguments::initialize_alignments() {
   // around this we use the unaligned values for the heap.
   HeapRegion::setup_heap_region_size(MaxHeapSize);
 
-  // The remembered set needs the heap regions set up.
-  HeapRegionRemSet::setup_remset_size();
+  SpaceAlignment = HeapRegion::GrainBytes;
+  HeapAlignment = calculate_heap_alignment(SpaceAlignment);
+
+  // We need to initialize card set configuration as soon as heap region size is
+  // known as it depends on it and is used really early.
+  initialize_card_set_configuration();
   // Needs remembered set initialization as the ergonomics are based
   // on it.
   if (FLAG_IS_DEFAULT(G1EagerReclaimRemSetThreshold)) {
-    FLAG_SET_ERGO(G1EagerReclaimRemSetThreshold, G1RSetSparseRegionEntries);
+    FLAG_SET_ERGO(G1EagerReclaimRemSetThreshold, G1RemSetArrayOfCardsEntries);
   }
-
-  SpaceAlignment = HeapRegion::GrainBytes;
-  HeapAlignment = calculate_heap_alignment(SpaceAlignment);
 }
 
 size_t G1Arguments::conservative_max_heap_alignment() {
@@ -90,6 +93,8 @@ void G1Arguments::parse_verification_type(const char* type) {
     G1HeapVerifier::enable_verification_type(G1HeapVerifier::G1VerifyConcurrentStart);
   } else if (strcmp(type, "mixed") == 0) {
     G1HeapVerifier::enable_verification_type(G1HeapVerifier::G1VerifyMixed);
+  } else if (strcmp(type, "young-evac-fail") == 0) {
+    G1HeapVerifier::enable_verification_type(G1HeapVerifier::G1VerifyYoungEvacFail);
   } else if (strcmp(type, "remark") == 0) {
     G1HeapVerifier::enable_verification_type(G1HeapVerifier::G1VerifyRemark);
   } else if (strcmp(type, "cleanup") == 0) {
@@ -98,7 +103,7 @@ void G1Arguments::parse_verification_type(const char* type) {
     G1HeapVerifier::enable_verification_type(G1HeapVerifier::G1VerifyFull);
   } else {
     log_warning(gc, verify)("VerifyGCType: '%s' is unknown. Available types are: "
-                            "young-normal, concurrent-start, mixed, remark, cleanup and full", type);
+                            "young-normal, young-evac-fail, concurrent-start, mixed, remark, cleanup and full", type);
   }
 }
 
@@ -119,6 +124,40 @@ void G1Arguments::initialize_mark_stack_size() {
   log_trace(gc)("MarkStackSize: %uk  MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
 }
 
+
+void G1Arguments::initialize_card_set_configuration() {
+  assert(HeapRegion::LogOfHRGrainBytes != 0, "not initialized");
+  // Array of Cards card set container globals.
+  const int LOG_M = 20;
+  uint region_size_log_mb = (uint)MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0);
+
+  if (FLAG_IS_DEFAULT(G1RemSetArrayOfCardsEntries)) {
+    uint num_cards_in_inline_ptr = G1CardSetConfiguration::num_cards_in_inline_ptr(HeapRegion::LogOfHRGrainBytes - CardTable::card_shift);
+    FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries, MAX2(num_cards_in_inline_ptr * 2,
+                                                    G1RemSetArrayOfCardsEntriesBase * (1u << (region_size_log_mb + 1))));
+  }
+
+  // Round to next 8 byte boundary for array to maximize space usage.
+  size_t const cur_size = G1CardSetArray::size_in_bytes(G1RemSetArrayOfCardsEntries);
+  FLAG_SET_ERGO(G1RemSetArrayOfCardsEntries,
+                G1RemSetArrayOfCardsEntries + (uint)(align_up(cur_size, G1CardSetAllocOptions::BufferAlignment) - cur_size) / sizeof(G1CardSetArray::EntryDataType));
+
+  // Howl card set container globals.
+  if (FLAG_IS_DEFAULT(G1RemSetHowlNumBuckets)) {
+    FLAG_SET_ERGO(G1RemSetHowlNumBuckets, G1CardSetHowl::num_buckets(HeapRegion::CardsPerRegion,
+                                                                     G1RemSetArrayOfCardsEntries,
+                                                                     G1RemSetHowlMaxNumBuckets));
+  }
+
+  if (FLAG_IS_DEFAULT(G1RemSetHowlMaxNumBuckets)) {
+    FLAG_SET_ERGO(G1RemSetHowlMaxNumBuckets, MAX2(G1RemSetHowlMaxNumBuckets, G1RemSetHowlNumBuckets));
+  } else if (G1RemSetHowlMaxNumBuckets < G1RemSetHowlNumBuckets) {
+    FormatBuffer<> buf("Maximum Howl card set container bucket size %u smaller than requested bucket size %u",
+                       G1RemSetHowlMaxNumBuckets, G1RemSetHowlNumBuckets);
+    vm_exit_during_initialization(buf);
+  }
+}
+
 void G1Arguments::initialize() {
   GCArguments::initialize();
   assert(UseG1GC, "Error");
@@ -196,6 +235,14 @@ void G1Arguments::initialize() {
 
   initialize_mark_stack_size();
   initialize_verification_types();
+
+  // Verify that the maximum parallelism isn't too high to eventually overflow
+  // the refcount in G1CardSetContainer.
+  uint max_parallel_refinement_threads = G1ConcRefinementThreads + G1DirtyCardQueueSet::num_par_ids();
+  uint const divisor = 3;  // Safe divisor; we increment by 2 for each claim, but there is a small initial value.
+  if (max_parallel_refinement_threads > UINTPTR_MAX / divisor) {
+    vm_exit_during_initialization("Too large parallelism for remembered sets.");
+  }
 }
 
 void G1Arguments::initialize_heap_flags_and_sizes() {

src/hotspot/share/gc/g1/g1Arguments.hpp

@@ -34,6 +34,7 @@ class G1Arguments : public GCArguments {
   friend class G1HeapVerifierTest;
 
   static void initialize_mark_stack_size();
+  static void initialize_card_set_configuration();
   static void initialize_verification_types();
   static void parse_verification_type(const char* type);
 

src/hotspot/share/gc/g1/g1CardSet.cpp

@@ -0,0 +1,887 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CardSet.inline.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "gc/g1/g1CardSetMemory.inline.hpp"
+#include "gc/g1/g1FromCardCache.hpp"
+#include "gc/g1/heapRegion.inline.hpp"
+#include "memory/allocation.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/globals_extension.hpp"
+#include "runtime/mutex.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/concurrentHashTable.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/lockFreeStack.hpp"
+#include "utilities/spinYield.hpp"
+
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
+#include "runtime/java.hpp"
+
+G1CardSet::CardSetPtr G1CardSet::FullCardSet = (G1CardSet::CardSetPtr)-1;
+
+G1CardSetConfiguration::G1CardSetConfiguration() :
+  _inline_ptr_bits_per_card(HeapRegion::LogOfHRGrainBytes - CardTable::card_shift) {
+
+  // Array of Cards card set container size calculation
+  _num_cards_in_array = G1RemSetArrayOfCardsEntries;
+
+  // Full card set container size calculation
+  _max_cards_in_card_set = (uint)HeapRegion::CardsPerRegion;
+  assert(is_power_of_2(_max_cards_in_card_set),
+        "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set);
+  _cards_in_howl_threshold = _max_cards_in_card_set * (double)G1RemSetCoarsenHowlToFullPercent / 100;
+
+  // Howl card set container size calculation.
+  _num_buckets_in_howl = G1RemSetHowlNumBuckets;
+
+  // Howl Bitmap card set container size calculation.
+  _num_cards_in_howl_bitmap = G1CardSetHowl::bitmap_size(_max_cards_in_card_set, _num_buckets_in_howl);
+  _log2_num_cards_in_howl_bitmap = log2i_exact(_num_cards_in_howl_bitmap);
+  _cards_in_howl_bitmap_threshold = _num_cards_in_howl_bitmap * (double)G1RemSetCoarsenHowlBitmapToHowlFullPercent / 100;
+  _bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);
+
+  log_configuration();
+}
+
+G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
+                                               uint num_cards_in_array,
+                                               double cards_in_bitmap_threshold,
+                                               uint max_buckets_in_howl,
+                                               double cards_in_howl_threshold,
+                                               uint max_cards_in_cardset) :
+  _inline_ptr_bits_per_card(inline_ptr_bits_per_card),
+  _num_cards_in_array(num_cards_in_array),
+  _max_cards_in_card_set(max_cards_in_cardset),
+  _cards_in_howl_threshold(max_cards_in_cardset * cards_in_howl_threshold) {
+
+  assert(is_power_of_2(_max_cards_in_card_set),
+        "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set);
+
+  _num_buckets_in_howl = G1CardSetHowl::num_buckets(_max_cards_in_card_set, _num_cards_in_array, max_buckets_in_howl);
+
+  _num_cards_in_howl_bitmap = G1CardSetHowl::bitmap_size(_max_cards_in_card_set, _num_buckets_in_howl);
+  _cards_in_howl_bitmap_threshold = _num_cards_in_howl_bitmap * cards_in_bitmap_threshold;
+  _log2_num_cards_in_howl_bitmap = log2i_exact(_num_cards_in_howl_bitmap);
+  _bitmap_hash_mask = ~(~(0) << _log2_num_cards_in_howl_bitmap);
+
+  log_configuration();
+}
+
+void G1CardSetConfiguration::log_configuration() {
+  log_debug_p(gc, remset)("Card Set container configuration: "
+                          "InlinePtr #elems %u size %zu "
+                          "Array Of Cards #elems %u size %zu "
+                          "Howl #buckets %u coarsen threshold %u "
+                          "Howl Bitmap #elems %u size %zu coarsen threshold %u",
+                          num_cards_in_inline_ptr(), sizeof(void*),
+                          num_cards_in_array(), G1CardSetArray::size_in_bytes(num_cards_in_array()),
+                          num_buckets_in_howl(), cards_in_howl_threshold(),
+                          num_cards_in_howl_bitmap(), G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), cards_in_howl_bitmap_threshold());
+}
+
+uint G1CardSetConfiguration::num_cards_in_inline_ptr() const {
+  return num_cards_in_inline_ptr(_inline_ptr_bits_per_card);
+}
+
+uint G1CardSetConfiguration::num_cards_in_inline_ptr(uint bits_per_card) {
+  return G1CardSetInlinePtr::max_cards_in_inline_ptr(bits_per_card);
+}
+
+G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options() {
+  G1CardSetAllocOptions* result = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC);
+
+  result[0] = { (uint)CardSetHash::get_node_size() };
+  result[1] = { (uint)G1CardSetArray::size_in_bytes(num_cards_in_array()), 2, 256 };
+  result[2] = { (uint)G1CardSetBitMap::size_in_bytes(num_cards_in_howl_bitmap()), 2, 256 };
+  result[3] = { (uint)G1CardSetHowl::size_in_bytes(num_buckets_in_howl()), 2, 256 };
+
+  return result;
+}
+
+const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) {
+  const char* names[] = { "Node", "Array", "Bitmap", "Howl" };
+  return names[index];
+}
+
+void G1CardSetCoarsenStats::reset() {
+  STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
+  for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
+    _coarsen_from[i] = 0;
+    _coarsen_collision[i] = 0;
+  }
+}
+
+void G1CardSetCoarsenStats::subtract_from(G1CardSetCoarsenStats& other) {
+  STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision));
+  for (uint i = 0; i < ARRAY_SIZE(_coarsen_from); i++) {
+    _coarsen_from[i] = other._coarsen_from[i] - _coarsen_from[i];
+    _coarsen_collision[i] = other._coarsen_collision[i] - _coarsen_collision[i];
+  }
+}
+
+void G1CardSetCoarsenStats::record_coarsening(uint tag, bool collision) {
+  assert(tag < ARRAY_SIZE(_coarsen_from), "tag %u out of bounds", tag);
+  Atomic::inc(&_coarsen_from[tag], memory_order_relaxed);
+  if (collision) {
+    Atomic::inc(&_coarsen_collision[tag], memory_order_relaxed);
+  }
+}
+
+void G1CardSetCoarsenStats::print_on(outputStream* out) {
+  out->print_cr("Inline->AoC %zu (%zu) "
+                "AoC->Howl %zu (%zu) "
+                "Howl->Full %zu (%zu) "
+                "Inline->AoC %zu (%zu) "
+                "AoC->BitMap %zu (%zu) "
+                "BitMap->Full %zu (%zu) ",
+                _coarsen_from[0], _coarsen_collision[0],
+                _coarsen_from[1], _coarsen_collision[1],
+                // There is no BitMap at the first level so we can't .
+                _coarsen_from[3], _coarsen_collision[3],
+                _coarsen_from[4], _coarsen_collision[4],
+                _coarsen_from[5], _coarsen_collision[5],
+                _coarsen_from[6], _coarsen_collision[6]
+               );
+}
+
+class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  // Did we insert at least one element in the table?
+  bool volatile _inserted_elem;
+
+  G1CardSetMemoryManager* _mm;
+  CardSetHash _table;
+
+  class G1CardSetHashTableLookUp : public StackObj {
+    uint _region_idx;
+  public:
+    explicit G1CardSetHashTableLookUp(uint region_idx) : _region_idx(region_idx) { }
+
+    uintx get_hash() const { return _region_idx; }
+
+    bool equals(G1CardSetHashTableValue* value, bool* is_dead) {
+      *is_dead = false;
+      return value->_region_idx == _region_idx;
+    }
+  };
+
+  class G1CardSetHashTableFound : public StackObj {
+    G1CardSetHashTableValue* _value;
+  public:
+    void operator()(G1CardSetHashTableValue* value) {
+      _value = value;
+    }
+
+    G1CardSetHashTableValue* value() const { return _value; }
+  };
+
+  class G1CardSetHashTableScan : public StackObj {
+    G1CardSet::G1CardSetPtrIterator* _scan_f;
+  public:
+    explicit G1CardSetHashTableScan(G1CardSet::G1CardSetPtrIterator* f) : _scan_f(f) { }
+
+    bool operator()(G1CardSetHashTableValue* value) {
+      _scan_f->do_cardsetptr(value->_region_idx, value->_num_occupied, value->_card_set);
+      return true;
+    }
+  };
+
+
+public:
+  static const size_t InitialLogTableSize = 2;
+
+  G1CardSetHashTable(G1CardSetMemoryManager* mm,
+                     size_t initial_log_table_size = InitialLogTableSize) :
+    _inserted_elem(false),
+    _mm(mm),
+    _table(mm, initial_log_table_size) {
+  }
+
+  ~G1CardSetHashTable() {
+    reset();
+  }
+
+  G1CardSetHashTableValue* get_or_add(uint region_idx, bool* should_grow) {
+    G1CardSetHashTableLookUp lookup(region_idx);
+    G1CardSetHashTableFound found;
+
+    if (_table.get(Thread::current(), lookup, found)) {
+      return found.value();
+    }
+
+    G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr());
+    bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow);
+
+    if (!_inserted_elem && inserted) {
+      // It does not matter to us who is setting the flag so a regular atomic store
+      // is sufficient.
+      Atomic::store(&_inserted_elem, true);
+    }
+
+    return found.value();
+  }
+
+  CardSetPtr get(uint region_idx) {
+    G1CardSetHashTableLookUp lookup(region_idx);
+    G1CardSetHashTableFound found;
+
+    if (_table.get(Thread::current(), lookup, found)) {
+      return found.value()->_card_set;
+    }
+    return nullptr;
+  }
+
+  void iterate_safepoint(G1CardSet::G1CardSetPtrIterator* cl2) {
+    G1CardSetHashTableScan cl(cl2);
+    _table.do_safepoint_scan(cl);
+  }
+
+  void iterate(G1CardSet::G1CardSetPtrIterator* cl2) {
+    G1CardSetHashTableScan cl(cl2);
+    _table.do_scan(Thread::current(), cl);
+  }
+
+  void reset() {
+    if (Atomic::load(&_inserted_elem)) {
+       _table.unsafe_reset(InitialLogTableSize);
+      Atomic::store(&_inserted_elem, false);
+    }
+  }
+
+  void print(outputStream* os) {
+    os->print("TBL " PTR_FORMAT " size %zu mem %zu ", p2i(&_table), _table.get_size_log2(Thread::current()), _table.get_mem_size(Thread::current()));
+  }
+
+  void grow() {
+    size_t new_limit = _table.get_size_log2(Thread::current()) + 1;
+    _table.grow(Thread::current(), new_limit);
+  }
+
+  size_t mem_size() {
+    return sizeof(*this) +
+      _table.get_mem_size(Thread::current()) - sizeof(_table);
+  }
+
+  size_t log_table_size() { return _table.get_size_log2(Thread::current()); }
+};
+
+void* G1CardSetHashTableConfig::allocate_node(void* context, size_t size, Value const& value) {
+  G1CardSetMemoryManager* mm = (G1CardSetMemoryManager*)context;
+  return mm->allocate_node();
+}
+
+void G1CardSetHashTableConfig::free_node(void* context, void* memory, Value const& value) {
+  G1CardSetMemoryManager* mm = (G1CardSetMemoryManager*)context;
+  mm->free_node(memory);
+}
+
+G1CardSetCoarsenStats G1CardSet::_coarsen_stats;
+G1CardSetCoarsenStats G1CardSet::_last_coarsen_stats;
+
+G1CardSet::G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm) :
+  _mm(mm),
+  _config(config),
+  _table(new G1CardSetHashTable(mm)),
+  _num_occupied(0) {
+}
+
+G1CardSet::~G1CardSet() {
+  delete _table;
+  _mm->flush();
+}
+
+uint G1CardSet::card_set_type_to_mem_object_type(uintptr_t type) const {
+  assert(type == G1CardSet::CardSetArrayOfCards ||
+         type == G1CardSet::CardSetBitMap ||
+         type == G1CardSet::CardSetHowl, "should not allocate card set type %zu", type);
+
+  return (uint)type;
+}
+
+uint8_t* G1CardSet::allocate_mem_object(uintptr_t type) {
+  return _mm->allocate(card_set_type_to_mem_object_type(type));
+}
+
+void G1CardSet::free_mem_object(CardSetPtr card_set) {
+  assert(card_set != G1CardSet::FreeCardSet, "should not free Free card set");
+  assert(card_set != G1CardSet::FullCardSet, "should not free Full card set");
+
+  uintptr_t type = card_set_type(card_set);
+  void* value = strip_card_set_type(card_set);
+
+  assert(type == G1CardSet::CardSetArrayOfCards ||
+         type == G1CardSet::CardSetBitMap ||
+         type == G1CardSet::CardSetHowl, "should not free card set type %zu", type);
+
+#ifdef ASSERT
+  if (type == G1CardSet::CardSetArrayOfCards ||
+      type == G1CardSet::CardSetBitMap ||
+      type == G1CardSet::CardSetHowl) {
+    G1CardSetContainer* card_set = (G1CardSetContainer*)value;
+    assert((card_set->refcount() == 1), "must be");
+  }
+#endif
+
+  _mm->free(card_set_type_to_mem_object_type(type), value);
+}
+
+G1CardSet::CardSetPtr G1CardSet::acquire_card_set(CardSetPtr volatile* card_set_addr) {
+  // Update reference counts under RCU critical section to avoid a
+  // use-after-cleapup bug where we increment a reference count for
+  // an object whose memory has already been cleaned up and reused.
+  GlobalCounter::CriticalSection cs(Thread::current());
+  while (true) {
+    // Get cardsetptr and increment refcount atomically wrt to memory reuse.
+    CardSetPtr card_set = Atomic::load_acquire(card_set_addr);
+    uint cs_type = card_set_type(card_set);
+    if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
+      return card_set;
+    }
+
+    G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
+
+    if (card_set_on_heap->try_increment_refcount()) {
+      assert(card_set_on_heap->refcount() >= 3, "Smallest value is 3");
+      return card_set;
+    }
+  }
+}
+
+bool G1CardSet::release_card_set(CardSetPtr card_set) {
+  uint cs_type = card_set_type(card_set);
+  if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
+    return false;
+  }
+
+  G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
+  return card_set_on_heap->decrement_refcount() == 1;
+}
+
+void G1CardSet::release_and_maybe_free_card_set(CardSetPtr card_set) {
+  if (release_card_set(card_set)) {
+    free_mem_object(card_set);
+  }
+}
+
+void G1CardSet::release_and_must_free_card_set(CardSetPtr card_set) {
+  bool should_free = release_card_set(card_set);
+  assert(should_free, "should have been the only one having a reference");
+  free_mem_object(card_set);
+}
+
+class G1ReleaseCardsets : public StackObj {
+  G1CardSet* _card_set;
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  void coarsen_to_full(CardSetPtr* card_set_addr) {
+    while (true) {
+      CardSetPtr cur_card_set = Atomic::load_acquire(card_set_addr);
+      uint cs_type = G1CardSet::card_set_type(cur_card_set);
+      if (cur_card_set == G1CardSet::FullCardSet) {
+        return;
+      }
+
+      CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, G1CardSet::FullCardSet);
+
+      if (old_value == cur_card_set) {
+        _card_set->release_and_maybe_free_card_set(cur_card_set);
+        return;
+      }
+    }
+  }
+
+public:
+  explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { }
+
+  void operator ()(CardSetPtr* card_set_addr) {
+    coarsen_to_full(card_set_addr);
+  }
+};
+
+G1AddCardResult G1CardSet::add_to_array(CardSetPtr card_set, uint card_in_region) {
+  G1CardSetArray* array = card_set_ptr<G1CardSetArray>(card_set);
+  return array->add(card_in_region);
+}
+
+G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set,
+                                                uint card_region,
+                                                uint card_in_region,
+                                                bool increment_total) {
+  G1CardSetHowl* howl = card_set_ptr<G1CardSetHowl>(parent_card_set);
+
+  G1AddCardResult add_result;
+  CardSetPtr to_transfer = nullptr;
+  CardSetPtr card_set;
+
+  uint bucket = _config->howl_bucket_index(card_in_region);
+  volatile CardSetPtr* bucket_entry = howl->get_card_set_addr(bucket);
+
+  while (true) {
+    if (Atomic::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) {
+      return Overflow;
+    }
+
+    card_set = acquire_card_set(bucket_entry);
+    add_result = add_to_card_set(bucket_entry, card_set, card_region, card_in_region);
+
+    if (add_result != Overflow) {
+      break;
+    }
+    // Card set has overflown. Coarsen or retry.
+    bool coarsened = coarsen_card_set(bucket_entry, card_set, card_in_region, true /* within_howl */);
+    _coarsen_stats.record_coarsening(card_set_type(card_set) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened);
+    if (coarsened) {
+      // We have been the one coarsening this card set (and in the process added that card).
+      add_result = Added;
+      to_transfer = card_set;
+      break;
+    }
+    // Somebody else beat us to coarsening. Retry.
+    release_and_maybe_free_card_set(card_set);
+  }
+
+  if (increment_total && add_result == Added) {
+    Atomic::inc(&howl->_num_entries, memory_order_relaxed);
+  }
+
+  if (to_transfer != nullptr) {
+    transfer_cards_in_howl(parent_card_set, to_transfer, card_region);
+  }
+
+  release_and_maybe_free_card_set(card_set);
+  return add_result;
+}
+
+G1AddCardResult G1CardSet::add_to_bitmap(CardSetPtr card_set, uint card_in_region) {
+  G1CardSetBitMap* bitmap = card_set_ptr<G1CardSetBitMap>(card_set);
+  uint card_offset = _config->howl_bitmap_offset(card_in_region);
+  return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->num_cards_in_howl_bitmap());
+}
+
+G1AddCardResult G1CardSet::add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region) {
+  G1CardSetInlinePtr value(card_set_addr, card_set);
+  return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->num_cards_in_inline_ptr());
+}
+
+G1CardSet::CardSetPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) {
+  uint8_t* data = nullptr;
+  CardSetPtr new_card_set;
+  if (within_howl) {
+    uint const size_in_bits = _config->num_cards_in_howl_bitmap();
+    uint card_offset = _config->howl_bitmap_offset(card_in_region);
+    data = allocate_mem_object(CardSetBitMap);
+    new (data) G1CardSetBitMap(card_offset, size_in_bits);
+    new_card_set = make_card_set_ptr(data, CardSetBitMap);
+  } else {
+    data = allocate_mem_object(CardSetHowl);
+    new (data) G1CardSetHowl(card_in_region, _config);
+    new_card_set = make_card_set_ptr(data, CardSetHowl);
+  }
+  return new_card_set;
+}
+
+bool G1CardSet::coarsen_card_set(volatile CardSetPtr* card_set_addr,
+                                 CardSetPtr cur_card_set,
+                                 uint card_in_region,
+                                 bool within_howl) {
+  CardSetPtr new_card_set = nullptr;
+
+  switch (card_set_type(cur_card_set)) {
+    case CardSetArrayOfCards : {
+      new_card_set = create_coarsened_array_of_cards(card_in_region, within_howl);
+      break;
+    }
+    case CardSetBitMap: {
+      new_card_set = FullCardSet;
+      break;
+    }
+    case CardSetInlinePtr: {
+      uint const size = _config->num_cards_in_array();
+      uint8_t* data = allocate_mem_object(CardSetArrayOfCards);
+      new (data) G1CardSetArray(card_in_region, size);
+      new_card_set = make_card_set_ptr(data, CardSetArrayOfCards);
+      break;
+    }
+    case CardSetHowl: {
+      new_card_set = FullCardSet; // anything will do at this point.
+      break;
+    }
+    default:
+      ShouldNotReachHere();
+  }
+
+  CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, new_card_set); // Memory order?
+  if (old_value == cur_card_set) {
+    // Success. Indicate that the cards from the current card set must be transferred
+    // by this caller.
+    // Release the hash table reference to the card. The caller still holds the
+    // reference to this card set, so it can never be released (and we do not need to
+    // check its result).
+    bool should_free = release_card_set(cur_card_set);
+    assert(!should_free, "must have had more than one reference");
+    // Free containers if cur_card_set is CardSetHowl
+    if (card_set_type(cur_card_set) == CardSetHowl) {
+      G1ReleaseCardsets rel(this);
+      card_set_ptr<G1CardSetHowl>(cur_card_set)->iterate(rel, _config->num_buckets_in_howl());
+    }
+    return true;
+  } else {
+    // Somebody else beat us to coarsening that card set. Exit, but clean up first.
+    if (new_card_set != FullCardSet) {
+      assert(new_card_set != nullptr, "must not be");
+      release_and_must_free_card_set(new_card_set);
+    }
+    return false;
+  }
+}
+
+class G1TransferCard : public StackObj {
+  G1CardSet* _card_set;
+  uint _region_idx;
+public:
+  G1TransferCard(G1CardSet* card_set, uint region_idx) : _card_set(card_set), _region_idx(region_idx) { }
+
+  void operator ()(uint card_idx) {
+    _card_set->add_card(_region_idx, card_idx, false);
+  }
+};
+
+void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region) {
+  assert(source_card_set != FullCardSet, "Should not need to transfer from full");
+  // Need to transfer old entries unless there is a Full card set in place now, i.e.
+  // the old type has been CardSetBitMap. "Full" contains all elements anyway.
+  if (card_set_type(source_card_set) != CardSetHowl) {
+    G1TransferCard iter(this, card_region);
+    iterate_cards_during_transfer(source_card_set, iter);
+  } else {
+    assert(card_set_type(source_card_set) == CardSetHowl, "must be");
+    // Need to correct for that the Full remembered set occupies more cards than the
+    // AoCS before.
+    Atomic::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed);
+  }
+}
+
+void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
+                                                     CardSetPtr source_card_set,
+                                                     uint card_region) {
+  assert(card_set_type(parent_card_set) == CardSetHowl, "must be");
+  assert(source_card_set != FullCardSet, "Should not need to transfer from full");
+  // Need to transfer old entries unless there is a Full card set in place now, i.e.
+  // the old type has been CardSetBitMap.
+  if (card_set_type(source_card_set) != CardSetBitMap) {
+    // We only need to transfer from anything below CardSetBitMap.
+    G1TransferCard iter(this, card_region);
+    iterate_cards_during_transfer(source_card_set, iter);
+  } else {
+    uint diff = _config->num_cards_in_howl_bitmap() - card_set_ptr<G1CardSetBitMap>(source_card_set)->num_bits_set();
+
+    // Need to correct for that the Full remembered set occupies more cards than the
+    // bitmap before.
+    // We add 1 element less because the values will be incremented
+    // in G1CardSet::add_card for the current addition or where already incremented in
+    // G1CardSet::add_to_howl after coarsening.
+    diff -= 1;
+
+    G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(parent_card_set);
+    Atomic::add(&howling_array->_num_entries, diff, memory_order_relaxed);
+
+    bool should_grow_table = false;
+    G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table);
+    Atomic::add(&table_entry->_num_occupied, diff, memory_order_relaxed);
+
+    Atomic::add(&_num_occupied, diff, memory_order_relaxed);
+  }
+}
+
+G1AddCardResult G1CardSet::add_to_card_set(volatile CardSetPtr* card_set_addr, CardSetPtr card_set,  uint card_region, uint card_in_region, bool increment_total) {
+  assert(card_set_addr != nullptr, "Cannot add to empty cardset");
+
+  G1AddCardResult add_result;
+
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      add_result = add_to_inline_ptr(card_set_addr, card_set, card_in_region);
+      break;
+    }
+    case CardSetArrayOfCards : {
+      add_result = add_to_array(card_set, card_in_region);
+      break;
+    }
+    case CardSetBitMap: {
+      add_result = add_to_bitmap(card_set, card_in_region);
+      break;
+    }
+    case CardSetHowl: {
+      assert(CardSetHowl == card_set_type(FullCardSet), "must be");
+      if (card_set == FullCardSet) {
+        return Found;
+      }
+      add_result = add_to_howl(card_set, card_region, card_in_region, increment_total);
+      break;
+    }
+    default:
+      ShouldNotReachHere();
+  }
+
+  return add_result;
+}
+
+G1CardSetHashTableValue* G1CardSet::get_or_add_card_set(uint card_region, bool* should_grow_table) {
+  return _table->get_or_add(card_region, should_grow_table);
+}
+
+G1CardSet::CardSetPtr G1CardSet::get_card_set(uint card_region) {
+  return _table->get(card_region);
+}
+
+G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool increment_total) {
+  G1AddCardResult add_result;
+  CardSetPtr to_transfer = nullptr;
+  CardSetPtr card_set;
+
+  bool should_grow_table = false;
+  G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table);
+  while (true) {
+    card_set = acquire_card_set(&table_entry->_card_set);
+    add_result = add_to_card_set(&table_entry->_card_set, card_set, card_region, card_in_region, increment_total);
+
+    if (add_result != Overflow) {
+      break;
+    }
+    // Card set has overflown. Coarsen or retry.
+    bool coarsened = coarsen_card_set(&table_entry->_card_set, card_set, card_in_region);
+    _coarsen_stats.record_coarsening(card_set_type(card_set), !coarsened);
+    if (coarsened) {
+      // We have been the one coarsening this card set (and in the process added that card).
+      add_result = Added;
+      to_transfer = card_set;
+      break;
+    }
+    // Somebody else beat us to coarsening. Retry.
+    release_and_maybe_free_card_set(card_set);
+  }
+
+  if (increment_total && add_result == Added) {
+    Atomic::inc(&table_entry->_num_occupied, memory_order_relaxed);
+    Atomic::inc(&_num_occupied, memory_order_relaxed);
+  }
+  if (should_grow_table) {
+    _table->grow();
+  }
+  if (to_transfer != nullptr) {
+    transfer_cards(table_entry, to_transfer, card_region);
+  }
+
+  release_and_maybe_free_card_set(card_set);
+
+  return add_result;
+}
+
+bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
+  assert(card_in_region < _config->max_cards_in_region(),
+         "Card %u is beyond max %u", card_in_region, _config->max_cards_in_region());
+
+  // Protect the card set from reclamation.
+  GlobalCounter::CriticalSection cs(Thread::current());
+  CardSetPtr card_set = get_card_set(card_region);
+  if (card_set == nullptr) {
+    return false;
+  } else if (card_set == FullCardSet) {
+    // contains_card() is not a performance critical method so we do not hide that
+    // case in the switch below.
+    return true;
+  }
+
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      G1CardSetInlinePtr ptr(card_set);
+      return ptr.contains(card_in_region, _config->inline_ptr_bits_per_card());
+    }
+    case CardSetArrayOfCards :  return card_set_ptr<G1CardSetArray>(card_set)->contains(card_in_region);
+    case CardSetBitMap: return card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_in_region, _config->num_cards_in_howl_bitmap());
+    case CardSetHowl: {
+      G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(card_set);
+
+      return howling_array->contains(card_in_region, _config);
+    }
+  }
+  ShouldNotReachHere();
+  return false;
+}
+
+void G1CardSet::print_info(outputStream* st, uint card_region, uint card_in_region) {
+  CardSetPtr card_set = get_card_set(card_region);
+  if (card_set == nullptr) {
+    st->print("NULL card set");
+    return;
+  } else if (card_set == FullCardSet) {
+    st->print("FULL card set)");
+    return;
+  }
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      st->print("InlinePtr not containing %u", card_in_region);
+      break;
+    }
+    case CardSetArrayOfCards :  {
+      st->print("AoC not containing %u", card_in_region);
+      break;
+    }
+    case CardSetBitMap: {
+      st->print("BitMap not containing %u", card_in_region);
+      break;
+    }
+    case CardSetHowl: {
+      st->print("CardSetHowl not containing %u", card_in_region);
+      break;
+    }
+    default: st->print("Unknown card set type %u", card_set_type(card_set)); ShouldNotReachHere(); break;
+  }
+}
+
+template <class CardVisitor>
+void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& found) {
+  uint type = card_set_type(card_set);
+  assert(type == CardSetInlinePtr || type == CardSetArrayOfCards,
+         "invalid card set type %d to transfer from",
+         card_set_type(card_set));
+
+  switch (type) {
+    case CardSetInlinePtr: {
+      G1CardSetInlinePtr ptr(card_set);
+      ptr.iterate(found, _config->inline_ptr_bits_per_card());
+      return;
+    }
+    case CardSetArrayOfCards : {
+      card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
+      return;
+    }
+    default:
+      ShouldNotReachHere();
+  }
+}
+
+void G1CardSet::iterate_containers(G1CardSetPtrIterator* found, bool at_safepoint) {
+  if (at_safepoint) {
+    _table->iterate_safepoint(found);
+  } else {
+    _table->iterate(found);
+  }
+}
+
+template <typename Closure>
+class G1ContainerCards {
+  Closure& _iter;
+  uint _region_idx;
+
+public:
+  G1ContainerCards(Closure& iter, uint region_idx) : _iter(iter), _region_idx(region_idx) { }
+
+  bool start_iterate(uint tag) { return true; }
+
+  void operator()(uint card_idx) {
+    _iter.do_card(_region_idx, card_idx);
+  }
+
+  void operator()(uint card_idx, uint length) {
+    for (uint i = 0; i < length; i++) {
+      _iter.do_card(_region_idx, card_idx);
+    }
+  }
+};
+
+void G1CardSet::iterate_cards(G1CardSetCardIterator& iter) {
+  G1CardSetMergeCardIterator<G1CardSetCardIterator, G1ContainerCards> cl(this, iter);
+  iterate_containers(&cl);
+}
+
+bool G1CardSet::occupancy_less_or_equal_to(size_t limit) const {
+  return occupied() <= limit;
+}
+
+bool G1CardSet::is_empty() const {
+  return _num_occupied == 0;
+}
+
+size_t G1CardSet::occupied() const {
+  return _num_occupied;
+}
+
+size_t G1CardSet::num_containers() {
+  class GetNumberOfContainers : public G1CardSetPtrIterator {
+  public:
+    size_t _count;
+
+    GetNumberOfContainers() : G1CardSetPtrIterator(), _count(0) { }
+
+    void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) override {
+      _count++;
+    }
+  } cl;
+
+  iterate_containers(&cl);
+  return cl._count;
+}
+
+G1CardSetCoarsenStats G1CardSet::coarsen_stats() {
+  return _coarsen_stats;
+}
+
+void G1CardSet::print_coarsen_stats(outputStream* out) {
+  _last_coarsen_stats.subtract_from(_coarsen_stats);
+  out->print("Coarsening (recent): ");
+  _last_coarsen_stats.print_on(out);
+  out->print("Coarsening (all): ");
+  _coarsen_stats.print_on(out);
+}
+
+size_t G1CardSet::mem_size() const {
+  return sizeof(*this) +
+         _table->mem_size() +
+         _mm->mem_size();
+}
+
+size_t G1CardSet::wasted_mem_size() const {
+  return _mm->wasted_mem_size();
+}
+
+size_t G1CardSet::static_mem_size() {
+  return sizeof(FullCardSet) + sizeof(_coarsen_stats);
+}
+
+void G1CardSet::clear() {
+  _table->reset();
+  _num_occupied = 0;
+  _mm->flush();
+}
+
+void G1CardSet::print(outputStream* os) {
+  _table->print(os);
+  _mm->print(os);
+}

src/hotspot/share/gc/g1/g1CardSet.hpp

@@ -0,0 +1,376 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSET_HPP
+#define SHARE_GC_G1_G1CARDSET_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/padded.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/concurrentHashTable.hpp"
+#include "utilities/lockFreeStack.hpp"
+
+class G1CardSetAllocOptions;
+class G1CardSetBufferList;
+class G1CardSetHashTable;
+class G1CardSetHashTableValue;
+class G1CardSetMemoryManager;
+class Mutex;
+
+// The result of an attempt to add a card to that card set.
+enum G1AddCardResult {
+  Overflow,  // The card set is more than full. The entry may have been added. Need
+             // Coarsen and retry.
+  Found,     // The card is already in the set.
+  Added      // The card has been added to the set by this attempt.
+};
+
+class G1CardSetConfiguration {
+  uint _inline_ptr_bits_per_card;
+  uint _num_cards_in_array;
+  uint _num_cards_in_howl_bitmap;
+  uint _num_buckets_in_howl;
+  uint _max_cards_in_card_set;
+  uint _cards_in_howl_threshold;
+  uint _cards_in_howl_bitmap_threshold;
+  uint _log2_num_cards_in_howl_bitmap;
+  size_t _bitmap_hash_mask;
+
+  void log_configuration();
+public:
+
+  // Initialize card set configuration from globals.
+  G1CardSetConfiguration();
+  // Initialize card set configuration from parameters.
+  G1CardSetConfiguration(uint inline_ptr_bits_per_card,
+                         uint num_cards_in_array,
+                         double cards_in_bitmap_threshold,
+                         uint max_buckets_in_howl,
+                         double cards_in_howl_threshold,
+                         uint max_cards_in_cardset);
+
+  // Inline pointer configuration
+  uint inline_ptr_bits_per_card() const { return _inline_ptr_bits_per_card; }
+  uint num_cards_in_inline_ptr() const;
+  static uint num_cards_in_inline_ptr(uint bits_per_card);
+
+  // Array of Cards configuration
+  bool use_cards_in_array() const { return _num_cards_in_array != 0; } // Unused for now
+  // Number of cards in "Array of Cards" set; 0 to disable.
+  // Always coarsen to next level if full, so no specific threshold.
+  uint num_cards_in_array() const { return _num_cards_in_array; }
+
+  // Bitmap within Howl card set container configuration
+  bool use_cards_in_howl_bitmap() const { return _num_cards_in_howl_bitmap != 0; } // Unused for now
+  uint num_cards_in_howl_bitmap() const { return _num_cards_in_howl_bitmap; }
+  // (Approximate) Number of cards in bitmap to coarsen Howl Bitmap to Howl Full.
+  uint cards_in_howl_bitmap_threshold() const { return _cards_in_howl_bitmap_threshold; }
+  uint log2_num_cards_in_howl_bitmap() const {return _log2_num_cards_in_howl_bitmap;}
+
+  // Howl card set container configuration
+  uint num_buckets_in_howl() const { return _num_buckets_in_howl; }
+  // Threshold at which to turn howling arrays into Full.
+  uint cards_in_howl_threshold() const { return _cards_in_howl_threshold; }
+  uint howl_bitmap_offset(uint card_idx) const { return card_idx & _bitmap_hash_mask; }
+  // Given a card index, return the bucket in the array of card sets.
+  uint howl_bucket_index(uint card_idx) { return card_idx >> _log2_num_cards_in_howl_bitmap; }
+
+  // Full card configuration
+  // Maximum number of cards in a non-full card set for a single region. Card sets
+  // with more entries per region are coarsened to Full.
+  uint max_cards_in_region() const { return _max_cards_in_card_set; }
+
+  // Memory object types configuration
+  // Number of distinctly sized memory objects on the card set heap.
+  // Currently contains CHT-Nodes, ArrayOfCards, BitMaps, Howl
+  static constexpr uint num_mem_object_types() { return 4; }
+  // Returns the memory allocation options for the memory objects on the card set heap. The returned
+  // array must be freed by the caller.
+  G1CardSetAllocOptions* mem_object_alloc_options();
+
+  // For a given memory object, get a descriptive name.
+  static const char* mem_object_type_name_str(uint index);
+};
+
+// Collects coarsening statistics: how many attempts of each kind and how many
+// failed due to a competing thread doing the coarsening first.
+class G1CardSetCoarsenStats {
+public:
+  // Number of entries in the statistics tables: since we index with the source
+  // cardset of the coarsening, this is the total number of combinations of
+  // card sets - 1.
+  static constexpr size_t NumCoarsenCategories = 7;
+  // Coarsening statistics for the possible CardSetPtr in the Howl card set
+  // start from this offset.
+  static constexpr size_t CoarsenHowlOffset = 4;
+
+private:
+  // Indices are "from" indices.
+  size_t _coarsen_from[NumCoarsenCategories];
+  size_t _coarsen_collision[NumCoarsenCategories];
+
+public:
+  G1CardSetCoarsenStats() { reset(); }
+
+  void reset();
+
+  void subtract_from(G1CardSetCoarsenStats& other);
+
+  // Record a coarsening for the given tag/category. Collision should be true if
+  // this coarsening lost the race to do the coarsening of that category.
+  void record_coarsening(uint tag, bool collision);
+
+  void print_on(outputStream* out);
+};
+
+// Sparse set of card indexes comprising a remembered set on the Java heap. Card
+// size is assumed to be card table card size.
+//
+// Technically it is implemented using a ConcurrentHashTable that stores a card
+// set container for every region containing at least one card.
+//
+// There are in total five different containers, encoded in the ConcurrentHashTable
+// node as CardSetPtr. A CardSetPtr may cover the whole region or just a part of
+// it.
+// See its description below for more information.
+class G1CardSet : public CHeapObj<mtGCCardSet> {
+  friend class G1CardSetTest;
+  friend class G1CardSetMtTestTask;
+
+  template <typename Closure, template <typename> class CardorRanges>
+  friend class G1CardSetMergeCardIterator;
+
+  friend class G1TransferCard;
+
+  friend class G1ReleaseCardsets;
+
+  static G1CardSetCoarsenStats _coarsen_stats; // Coarsening statistics since VM start.
+  static G1CardSetCoarsenStats _last_coarsen_stats; // Coarsening statistics at last GC.
+public:
+  // Two lower bits are used to encode the card storage types
+  static const uintptr_t CardSetPtrHeaderSize = 2;
+
+  // CardSetPtr represents the card storage type of a given covered area. It encodes
+  // a type in the LSBs, in addition to having a few significant values.
+  //
+  // Possible encodings:
+  //
+  // 0...00000 free               (Empty, should never happen)
+  // 1...11111 full               All card indexes in the whole area this CardSetPtr covers are part of this container.
+  // X...XXX00 inline-ptr-cards   A handful of card indexes covered by this CardSetPtr are encoded within the CardSetPtr.
+  // X...XXX01 array of cards     The container is a contiguous array of card indexes.
+  // X...XXX10 bitmap             The container uses a bitmap to determine whether a given index is part of this set.
+  // X...XXX11 howl               This is a card set container containing an array of CardSetPtr, with each CardSetPtr
+  //                              limited to a sub-range of the original range. Currently only one level of this
+  //                              container is supported.
+  typedef void* CardSetPtr;
+  // Coarsening happens in the order below:
+  // CardSetInlinePtr -> CardSetArrayOfCards -> CardSetHowl -> Full
+  // Corsening of containers inside the CardSetHowl happens in the order:
+  // CardSetInlinePtr -> CardSetArrayOfCards -> CardSetBitMap -> Full
+  static const uintptr_t CardSetInlinePtr      = 0x0;
+  static const uintptr_t CardSetArrayOfCards   = 0x1;
+  static const uintptr_t CardSetBitMap         = 0x2;
+  static const uintptr_t CardSetHowl           = 0x3;
+
+  // The special sentinel values
+  static constexpr CardSetPtr FreeCardSet = nullptr;
+  // Unfortunately we can't make (G1CardSet::CardSetPtr)-1 constexpr because
+  // reinterpret_casts are forbidden in constexprs. Use a regular static instead.
+  static CardSetPtr FullCardSet;
+
+  static const uintptr_t CardSetPtrTypeMask    = ((uintptr_t)1 << CardSetPtrHeaderSize) - 1;
+
+  static CardSetPtr strip_card_set_type(CardSetPtr ptr) { return (CardSetPtr)((uintptr_t)ptr & ~CardSetPtrTypeMask); }
+
+  static uint card_set_type(CardSetPtr ptr) { return (uintptr_t)ptr & CardSetPtrTypeMask; }
+
+  template <class T>
+  static T* card_set_ptr(CardSetPtr ptr);
+
+private:
+  G1CardSetMemoryManager* _mm;
+  G1CardSetConfiguration* _config;
+
+  G1CardSetHashTable* _table;
+
+  // Total number of cards in this card set. This is a best-effort value, i.e. there may
+  // be (slightly) more cards in the card set than this value in reality.
+  size_t _num_occupied;
+
+  CardSetPtr make_card_set_ptr(void* value, uintptr_t type);
+
+  CardSetPtr acquire_card_set(CardSetPtr volatile* card_set_addr);
+  // Returns true if the card set should be released
+  bool release_card_set(CardSetPtr card_set);
+  // Release card set and free if needed.
+  void release_and_maybe_free_card_set(CardSetPtr card_set);
+  // Release card set and free (and it must be freeable).
+  void release_and_must_free_card_set(CardSetPtr card_set);
+
+  // Coarsens the CardSet cur_card_set to the next level; tries to replace the
+  // previous CardSet with a new one which includes the given card_in_region.
+  // coarsen_card_set does not transfer cards from cur_card_set
+  // to the new card_set. Transfer is achieved by transfer_cards.
+  // Returns true if this was the thread that coarsened the CardSet (and added the card).
+  bool coarsen_card_set(CardSetPtr volatile* card_set_addr,
+                        CardSetPtr cur_card_set,
+                        uint card_in_region, bool within_howl = false);
+
+  CardSetPtr create_coarsened_array_of_cards(uint card_in_region, bool within_howl);
+
+  // Transfer entries from source_card_set to a recently installed coarser storage type
+  // We only need to transfer anything finer than CardSetBitMap. "Full" contains
+  // all elements anyway.
+  void transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region);
+  void transfer_cards_in_howl(CardSetPtr parent_card_set, CardSetPtr source_card_set, uint card_region);
+
+  G1AddCardResult add_to_card_set(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_region, uint card, bool increment_total = true);
+
+  G1AddCardResult add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region);
+  G1AddCardResult add_to_array(CardSetPtr card_set, uint card_in_region);
+  G1AddCardResult add_to_bitmap(CardSetPtr card_set, uint card_in_region);
+  G1AddCardResult add_to_howl(CardSetPtr parent_card_set, uint card_region, uint card_in_region, bool increment_total = true);
+
+  G1CardSetHashTableValue* get_or_add_card_set(uint card_region, bool* should_grow_table);
+  CardSetPtr get_card_set(uint card_region);
+
+  // Iterate over cards of a card set container during transfer of the cards from
+  // one container to another. Executes
+  //
+  //     void operator ()(uint card_idx)
+  //
+  // on the given class.
+  template <class CardVisitor>
+  void iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& found);
+
+  // Iterate over the container, calling a method on every card or card range contained
+  // in the card container.
+  // For every container, first calls
+  //
+  //   void start_iterate(uint tag, uint region_idx);
+  //
+  // Then for every card or card range it calls
+  //
+  //   void do_card(uint card_idx);
+  //   void do_card_range(uint card_idx, uint length);
+  //
+  // where card_idx is the card index within that region_idx passed before in
+  // start_iterate().
+  //
+  template <class CardOrRangeVisitor>
+  void iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& found);
+
+  uint card_set_type_to_mem_object_type(uintptr_t type) const;
+  uint8_t* allocate_mem_object(uintptr_t type);
+  void free_mem_object(CardSetPtr card_set);
+
+public:
+  G1CardSetConfiguration* config() const { return _config; }
+
+  // Create a new remembered set for a particular heap region.
+  G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm);
+  virtual ~G1CardSet();
+
+  // Adds the given card to this set, returning an appropriate result. If added,
+  // updates the total count.
+  G1AddCardResult add_card(uint card_region, uint card_in_region, bool increment_total = true);
+
+  bool contains_card(uint card_region, uint card_in_region);
+
+  void print_info(outputStream* st, uint card_region, uint card_in_region);
+
+  // Returns whether this remembered set (and all sub-sets) have an occupancy
+  // that is less or equal to the given occupancy.
+  bool occupancy_less_or_equal_to(size_t limit) const;
+
+  // Returns whether this remembered set (and all sub-sets) does not contain any entry.
+  bool is_empty() const;
+
+  // Returns the number of cards contained in this remembered set.
+  size_t occupied() const;
+
+  size_t num_containers();
+
+  static G1CardSetCoarsenStats coarsen_stats();
+  static void print_coarsen_stats(outputStream* out);
+
+  // Returns size of the actual remembered set containers in bytes.
+  size_t mem_size() const;
+  size_t wasted_mem_size() const;
+  // Returns the size of static data in bytes.
+  static size_t static_mem_size();
+
+  // Clear the entire contents of this remembered set.
+  void clear();
+
+  void print(outputStream* os);
+
+  // Various iterators - should be made inlineable somehow.
+  class G1CardSetPtrIterator {
+  public:
+    virtual void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) = 0;
+  };
+
+  void iterate_containers(G1CardSetPtrIterator* iter, bool safepoint = false);
+
+  class G1CardSetCardIterator {
+  public:
+    virtual void do_card(uint region_idx, uint card_idx) = 0;
+  };
+
+  void iterate_cards(G1CardSetCardIterator& iter);
+
+  // Iterate all cards for card set merging. Must be a CardOrRangeVisitor as
+  // explained above.
+  template <class CardOrRangeVisitor>
+  void iterate_for_merge(CardOrRangeVisitor& cl);
+};
+
+class G1CardSetHashTableValue {
+public:
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  const uint _region_idx;
+  uint volatile _num_occupied;
+  CardSetPtr volatile _card_set;
+
+  G1CardSetHashTableValue(uint region_idx, CardSetPtr card_set) : _region_idx(region_idx), _num_occupied(0), _card_set(card_set) { }
+};
+
+class G1CardSetHashTableConfig : public StackObj {
+public:
+  using Value = G1CardSetHashTableValue;
+
+  static uintx get_hash(Value const& value, bool* is_dead) {
+    *is_dead = false;
+    return value._region_idx;
+  }
+  static void* allocate_node(void* context, size_t size, Value const& value);
+  static void free_node(void* context, void* memory, Value const& value);
+};
+
+typedef ConcurrentHashTable<G1CardSetHashTableConfig, mtGCCardSet> CardSetHash;
+
+#endif // SHARE_GC_G1_G1CARDSET_HPP

src/hotspot/share/gc/g1/g1CardSet.inline.hpp

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSET_INLINE_HPP
+#define SHARE_GC_G1_G1CARDSET_INLINE_HPP
+
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "runtime/atomic.hpp"
+#include "logging/log.hpp"
+
+template <class T>
+inline T* G1CardSet::card_set_ptr(CardSetPtr ptr) {
+  return (T*)strip_card_set_type(ptr);
+}
+
+inline G1CardSet::CardSetPtr G1CardSet::make_card_set_ptr(void* value, uintptr_t type) {
+  assert(card_set_type(value) == 0, "Given ptr " PTR_FORMAT " already has type bits set", p2i(value));
+  return (CardSetPtr)((uintptr_t)value | type);
+}
+
+template <class CardOrRangeVisitor>
+inline void G1CardSet::iterate_cards_or_ranges_in_container(CardSetPtr const card_set, CardOrRangeVisitor& found) {
+  switch (card_set_type(card_set)) {
+    case CardSetInlinePtr: {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedInline)) {
+        G1CardSetInlinePtr ptr(card_set);
+        ptr.iterate(found, _config->inline_ptr_bits_per_card());
+      }
+      return;
+    }
+    case CardSetArrayOfCards : {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedArrayOfCards)) {
+        card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
+      }
+      return;
+    }
+    case CardSetBitMap: {
+      // There is no first-level bitmap spanning the whole area.
+      ShouldNotReachHere();
+      return;
+    }
+    case CardSetHowl: {
+      assert(card_set_type(FullCardSet) == CardSetHowl, "Must be");
+      if (card_set == FullCardSet) {
+        if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedFull)) {
+          found(0, _config->max_cards_in_region());
+        }
+        return;
+      }
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSMergedHowl)) {
+        card_set_ptr<G1CardSetHowl>(card_set)->iterate(found, _config);
+      }
+      return;
+    }
+  }
+  log_error(gc)("Unkown card set type %u", card_set_type(card_set));
+  ShouldNotReachHere();
+}
+
+template <typename Closure>
+class G1ContainerCardsOrRanges {
+  Closure& _iter;
+  uint _region_idx;
+
+public:
+  G1ContainerCardsOrRanges(Closure& iter, uint region_idx) : _iter(iter), _region_idx(region_idx) { }
+
+  bool start_iterate(uint tag) {
+    return _iter.start_iterate(tag, _region_idx);
+  }
+
+  void operator()(uint card_idx) {
+    _iter.do_card(card_idx);
+  }
+
+  void operator()(uint card_idx, uint length) {
+    _iter.do_card_range(card_idx, length);
+  }
+};
+
+template <typename Closure, template <typename> class CardOrRanges>
+class G1CardSetMergeCardIterator : public G1CardSet::G1CardSetPtrIterator {
+  G1CardSet* _card_set;
+  Closure& _iter;
+
+public:
+
+  G1CardSetMergeCardIterator(G1CardSet* card_set, Closure& iter) : _card_set(card_set), _iter(iter) { }
+
+  void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
+    CardOrRanges<Closure> cl(_iter, region_idx);
+    _card_set->iterate_cards_or_ranges_in_container(card_set, cl);
+  }
+};
+
+template <class CardOrRangeVisitor>
+inline void G1CardSet::iterate_for_merge(CardOrRangeVisitor& cl) {
+  G1CardSetMergeCardIterator<CardOrRangeVisitor, G1ContainerCardsOrRanges> cl2(this, cl);
+  iterate_containers(&cl2, true /* at_safepoint */);
+}
+
+#endif // SHARE_GC_G1_G1CARDSET_INLINE_HPP

src/hotspot/share/gc/g1/g1CardSetContainers.hpp

@@ -0,0 +1,294 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETCONTAINERS_HPP
+#define SHARE_GC_G1_G1CARDSETCONTAINERS_HPP
+
+#include "gc/g1/g1CardSet.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/spinYield.hpp"
+
+#include "logging/log.hpp"
+
+#include "runtime/thread.inline.hpp"
+
+class G1CardSetInlinePtr : public StackObj {
+  friend class G1CardSetContainersTest;
+
+  typedef G1CardSet::CardSetPtr CardSetPtr;
+
+  CardSetPtr volatile * _value_addr;
+  CardSetPtr _value;
+
+  static const uint SizeFieldLen = 3;
+  static const uint SizeFieldPos = 2;
+  static const uint HeaderSize = G1CardSet::CardSetPtrHeaderSize + SizeFieldLen;
+
+  static const uint BitsInValue = sizeof(CardSetPtr) * BitsPerByte;
+
+  static const uintptr_t SizeFieldMask = (((uint)1 << SizeFieldLen) - 1) << SizeFieldPos;
+
+  static uint8_t card_pos_for(uint const idx, uint const bits_per_card) {
+    return (idx * bits_per_card + HeaderSize);
+  }
+
+  static CardSetPtr merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card);
+
+  static uint card_at(CardSetPtr value, uint const idx, uint const bits_per_card) {
+    uint8_t card_pos = card_pos_for(idx, bits_per_card);
+    uint result = ((uintptr_t)value >> card_pos) & (((uintptr_t)1 << bits_per_card) - 1);
+    return result;
+  }
+public:
+  G1CardSetInlinePtr() : _value_addr(nullptr), _value((CardSetPtr)G1CardSet::CardSetInlinePtr) { }
+
+  G1CardSetInlinePtr(CardSetPtr value) : _value_addr(nullptr), _value(value) {
+    assert(((uintptr_t)_value & G1CardSet::CardSetInlinePtr) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value));
+  }
+
+  G1CardSetInlinePtr(CardSetPtr volatile* value_addr, CardSetPtr value) : _value_addr(value_addr), _value(value) {
+    assert(((uintptr_t)_value & G1CardSet::CardSetInlinePtr) == G1CardSet::CardSetInlinePtr, "Value " PTR_FORMAT " is not a valid G1CardSetInPtr.", p2i(_value));
+  }
+
+  G1AddCardResult add(uint const card_idx, uint const bits_per_card, uint const max_cards_in_inline_ptr);
+
+  bool contains(uint const card_idx, uint const bits_per_card);
+
+  template <class CardVisitor>
+  void iterate(CardVisitor& found, uint const bits_per_card);
+
+  operator CardSetPtr () { return _value; }
+
+  static uint max_cards_in_inline_ptr(uint bits_per_card) {
+    return (BitsInValue - HeaderSize) / bits_per_card;
+  }
+
+  static uint num_cards_in(CardSetPtr value) {
+    return ((uintptr_t)value & SizeFieldMask) >> SizeFieldPos;
+  }
+};
+
+
+// Common base class for card set containers where the memory for the entries is
+// managed on the (C-)heap. Depending on the current use, one of the two overlapping
+// members are used:
+//
+// While such an object is assigned to a card set container, we utilize the
+// reference count for memory management.
+//
+// In this case the object is one of three states:
+// 1: Live: The object is visible to other threads, thus can
+//    safely be accessed by other threads (_ref_count >= 3).
+// 2: Dead: The object is visible to only a single thread and may be
+//    safely reclaimed (_ref_count == 1).
+// 3: Reclaimed: The object's memory has been reclaimed ((_ref_count & 0x1) == 0).
+// To maintain these constraints, live objects should have ((_ref_count & 0x1) == 1),
+// which requires that we increment the reference counts by 2 starting at _ref_count = 3.
+//
+// When such an object is on a free list, we reuse the same field for linking
+// together those free objects.
+//
+// All but inline pointers are of this kind. For those, card entries are stored
+// directly in the CardSetPtr of the ConcurrentHashTable node.
+class G1CardSetContainer {
+private:
+  union {
+    G1CardSetContainer* _next;
+    uintptr_t _ref_count;
+  };
+
+public:
+  G1CardSetContainer() : _ref_count(3) { }
+
+  uintptr_t refcount() const { return Atomic::load_acquire(&_ref_count); }
+
+  bool try_increment_refcount();
+
+  // Decrement refcount potentially while racing increment, so we need
+  // to check the value after attempting to decrement.
+  uintptr_t decrement_refcount();
+
+  G1CardSetContainer* next() {
+    return _next;
+  }
+
+  G1CardSetContainer** next_addr() {
+    return &_next;
+  }
+
+  void set_next(G1CardSetContainer* next) {
+    _next = next;
+  }
+};
+
+class G1CardSetArray : public G1CardSetContainer {
+public:
+  typedef uint16_t EntryDataType;
+  typedef uint EntryCountType;
+  using CardSetPtr = G1CardSet::CardSetPtr;
+private:
+  EntryCountType _size;
+  EntryCountType volatile _num_entries;
+  EntryDataType _data[2];
+
+  static const EntryCountType LockBitMask = (EntryCountType)1 << (sizeof(EntryCountType) * BitsPerByte - 1);
+  static const EntryCountType EntryMask = LockBitMask - 1;
+
+  class G1CardSetArrayLocker : public StackObj {
+    EntryCountType volatile* _value;
+    EntryCountType volatile _original_value;
+    bool _success;
+  public:
+    G1CardSetArrayLocker(EntryCountType volatile* value);
+
+    EntryCountType num_entries() const { return _original_value; }
+    void inc_num_entries() { _success = true; }
+
+    ~G1CardSetArrayLocker() {
+      assert(((_original_value + _success) & EntryMask) == (EntryCountType)(_original_value + _success), "precondition!" );
+
+      Atomic::release_store(_value, (EntryCountType)(_original_value + _success));
+    }
+  };
+
+  template<typename Derived>
+  static size_t header_size_in_bytes_internal() {
+    return offset_of(Derived, _data);
+  }
+
+public:
+  G1CardSetArray(uint const card_in_region, EntryCountType num_elems);
+
+  G1AddCardResult add(uint card_idx);
+
+  bool contains(uint card_idx);
+
+  template <class CardVisitor>
+  void iterate(CardVisitor& found);
+
+  size_t num_entries() const { return _num_entries & EntryMask; }
+  size_t max_entries() const { return _size; }
+
+  static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetArray>(); }
+
+  static size_t size_in_bytes(size_t num_cards) {
+    return header_size_in_bytes() + sizeof(EntryDataType) * num_cards;
+  }
+};
+
+class G1CardSetBitMap : public G1CardSetContainer {
+  size_t _num_bits_set;
+  BitMap::bm_word_t _bits[1];
+
+  using CardSetPtr = G1CardSet::CardSetPtr;
+
+  template<typename Derived>
+  static size_t header_size_in_bytes_internal() {
+    return offset_of(Derived, _bits);
+  }
+
+public:
+  G1CardSetBitMap(uint const card_in_region, uint const size_in_bits);
+
+  G1AddCardResult add(uint card_idx, size_t threshold, size_t size_in_bits);
+
+  bool contains(uint card_idx, size_t size_in_bits) {
+    BitMapView bm(_bits, size_in_bits);
+    return bm.at(card_idx);
+  }
+
+  uint num_bits_set() const { return (uint)_num_bits_set; }
+
+  template <class CardVisitor>
+  void iterate(CardVisitor& found, size_t const size_in_bits, uint offset);
+
+  uint next(uint const idx, size_t const size_in_bits) {
+    BitMapView bm(_bits, size_in_bits);
+    return static_cast<uint>(bm.get_next_one_offset(idx));
+  }
+
+  static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetBitMap>(); }
+
+  static size_t size_in_bytes(size_t size_in_bits) { return header_size_in_bytes() + BitMap::calc_size_in_words(size_in_bits) * BytesPerWord; }
+};
+
+class G1CardSetHowl : public G1CardSetContainer {
+public:
+  typedef uint EntryCountType;
+  using CardSetPtr = G1CardSet::CardSetPtr;
+  EntryCountType volatile _num_entries;
+private:
+  CardSetPtr _buckets[2];
+  // Do not add class member variables beyond this point
+
+  template<typename Derived>
+  static size_t header_size_in_bytes_internal() {
+    return offset_of(Derived, _buckets);
+  }
+
+  // Iterates over the given CardSetPtr with at index in this Howl card set,
+  // applying a CardOrRangeVisitor on it.
+  template <class CardOrRangeVisitor>
+  void iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config);
+
+public:
+  G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config);
+
+  CardSetPtr* get_card_set_addr(EntryCountType index) {
+    return &_buckets[index];
+  }
+
+  bool contains(uint card_idx, G1CardSetConfiguration* config);
+
+  // Iterates over all CardSetPtrs in this Howl card set, applying a CardOrRangeVisitor
+  // on it.
+  template <class CardOrRangeVisitor>
+  void iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config);
+
+  // Iterates over all CardSetPtrs in this Howl card set. Calls
+  //
+  //   void operator ()(CardSetPtr* card_set_addr);
+  //
+  // on all of them.
+  template <class CardSetPtrVisitor>
+  void iterate(CardSetPtrVisitor& found, uint num_card_sets);
+
+  static EntryCountType num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_buckets);
+
+  static EntryCountType bitmap_size(size_t size_in_bits, uint num_buckets) {
+    EntryCountType num_cards = (EntryCountType)size_in_bits / num_buckets;
+    return round_up_power_of_2(num_cards);
+  }
+
+  static size_t header_size_in_bytes() { return header_size_in_bytes_internal<G1CardSetHowl>(); }
+
+  static size_t size_in_bytes(size_t num_arrays) {
+    return header_size_in_bytes() + sizeof(CardSetPtr) * num_arrays;
+  }
+};
+
+#endif // SHARE_GC_G1_G1CARDSETCONTAINERS_HPP

src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp

@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP
+#define SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP
+
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "gc/g1/g1GCPhaseTimes.hpp"
+
+inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
+  assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx);
+  assert(card_in_region < ((uint)1 << bits_per_card), "Card %u too large to fit into card value field", card_in_region);
+
+  uint8_t card_pos = card_pos_for(idx, bits_per_card);
+  assert(card_pos + bits_per_card < BitsInValue, "Putting card at pos %u with %u bits would extend beyond pointer", card_pos, bits_per_card);
+
+  // Check that we do not touch any fields we do not own.
+  uintptr_t mask = ((((uintptr_t)1 << bits_per_card) - 1) << card_pos);
+  assert(((uintptr_t)orig_value & mask) == 0, "The bits in the new range should be empty; orig_value " PTR_FORMAT " mask " PTR_FORMAT, p2i(orig_value), mask);
+
+  uintptr_t value = ((uintptr_t)(idx + 1) << SizeFieldPos) | ((uintptr_t)card_in_region << card_pos);
+  uintptr_t res = (((uintptr_t)orig_value & ~SizeFieldMask) | value);
+  return (CardSetPtr)res;
+}
+
+inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card, uint max_cards_in_inline_ptr) {
+  assert(_value_addr != nullptr, "No value address available, cannot add to set.");
+
+  while (true) {
+    uint num_elems = num_cards_in(_value);
+    // Check if the card is already stored in the pointer.
+    if (contains(card_idx, bits_per_card)) {
+      return Found;
+    }
+    // Check if there is actually enough space.
+    if (num_elems >= max_cards_in_inline_ptr) {
+      return Overflow;
+    }
+    CardSetPtr new_value = merge(_value, card_idx, num_elems, bits_per_card);
+    CardSetPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
+    if (_value == old_value) {
+      return Added;
+    }
+    // Update values and retry.
+    _value = old_value;
+    // The value of the pointer may have changed to something different than
+    // an inline card set. Exit then instead of overwriting.
+    if (G1CardSet::card_set_type(_value) != G1CardSet::CardSetInlinePtr) {
+      return Overflow;
+    }
+  }
+}
+
+inline bool G1CardSetInlinePtr::contains(uint card_idx, uint bits_per_card) {
+  uint num_elems = num_cards_in(_value);
+  uintptr_t const card_mask = (1 << bits_per_card) - 1;
+
+  uintptr_t value = ((uintptr_t)_value) >> card_pos_for(0, bits_per_card);
+  // Check if the card is already stored in the pointer.
+  for (uint cur_idx = 0; cur_idx < num_elems; cur_idx++) {
+    if ((value & card_mask) == card_idx) {
+      return true;
+    }
+    value >>= bits_per_card;
+  }
+  return false;
+}
+
+template <class CardVisitor>
+inline void G1CardSetInlinePtr::iterate(CardVisitor& found, uint bits_per_card) {
+  uint const num_elems = num_cards_in(_value);
+  uintptr_t const card_mask = (1 << bits_per_card) - 1;
+
+  uintptr_t value = ((uintptr_t)_value) >> card_pos_for(0, bits_per_card);
+  for (uint cur_idx = 0; cur_idx < num_elems; cur_idx++) {
+    found(value & card_mask);
+    value >>= bits_per_card;
+  }
+}
+
+inline bool G1CardSetContainer::try_increment_refcount() {
+  uintptr_t old_value = refcount();
+  while (true) {
+    if (old_value < 3 || (old_value & 0x1) == 0) {  // reclaimed,  reference counts are odd numbers starting at 3
+      return false; // dead, can't revive.
+    }
+
+    uintptr_t new_value = old_value + 2;
+    uintptr_t ref_count = Atomic::cmpxchg(&_ref_count, old_value, new_value);
+    if (ref_count == old_value) {
+      return true;
+    }
+    old_value = ref_count;
+  }
+}
+
+inline uintptr_t G1CardSetContainer::decrement_refcount() {
+  uintptr_t old_value = refcount();
+  assert((old_value & 0x1) != 0 && old_value >= 3, "precondition");
+  return Atomic::sub(&_ref_count, 2u);
+}
+
+inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_elems) :
+  G1CardSetContainer(),
+  _size(num_elems),
+  _num_entries(1) {
+  assert(_size > 0, "CardSetArray of size 0 not supported.");
+  assert(_size < LockBitMask, "Only support CardSetArray of size %u or smaller.", LockBitMask - 1);
+  _data[0] = card_in_region;
+}
+
+inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(EntryCountType volatile* value) :
+  _value(value),
+  _success(false) {
+  SpinYield s;
+  EntryCountType original_value = (*_value) & EntryMask;
+  while (true) {
+    EntryCountType old_value = Atomic::cmpxchg(_value,
+                                               original_value,
+                                               (EntryCountType)(original_value | LockBitMask));
+    if (old_value == original_value) {
+      // Succeeded locking the array.
+      _original_value = original_value;
+      break;
+    }
+    // Failed. Retry (with the lock bit stripped again).
+    original_value = old_value & EntryMask;
+    s.wait();
+  }
+}
+
+inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
+  assert(card_idx < (1u << (sizeof(_data[0]) * BitsPerByte)),
+         "Card index %u does not fit card element.", card_idx);
+  EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
+  EntryCountType idx = 0;
+  for (; idx < num_entries; idx++) {
+    if (_data[idx] == card_idx) {
+      return Found;
+    }
+  }
+
+  // Since we did not find the card, lock.
+  G1CardSetArrayLocker x(&_num_entries);
+
+  // Reload number of entries from the G1CardSetArrayLocker as it might have changed.
+  // It already read the actual value with the necessary synchronization.
+  num_entries = x.num_entries();
+  // Look if the elements added while waiting for the lock are the same as our card.
+  for (; idx < num_entries; idx++) {
+    if (_data[idx] == card_idx) {
+      return Found;
+    }
+  }
+
+  // Check if there is space left.
+  if (num_entries == _size) {
+    return Overflow;
+  }
+
+  _data[num_entries] = card_idx;
+
+  x.inc_num_entries();
+
+  return Added;
+}
+
+inline bool G1CardSetArray::contains(uint card_idx) {
+  EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
+
+  for (EntryCountType idx = 0; idx < num_entries; idx++) {
+    if (_data[idx] == card_idx) {
+      return true;
+    }
+  }
+  return false;
+}
+
+template <class CardVisitor>
+void G1CardSetArray::iterate(CardVisitor& found) {
+  EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
+  for (EntryCountType idx = 0; idx < num_entries; idx++) {
+    found(_data[idx]);
+  }
+}
+
+inline G1CardSetBitMap::G1CardSetBitMap(uint card_in_region, uint size_in_bits) :
+  G1CardSetContainer(), _num_bits_set(1) {
+  assert(size_in_bits % (sizeof(_bits[0]) * BitsPerByte) == 0,
+         "Size %u should be aligned to bitmap word size.", size_in_bits);
+  BitMapView bm(_bits, size_in_bits);
+  bm.clear();
+  bm.set_bit(card_in_region);
+}
+
+inline G1AddCardResult G1CardSetBitMap::add(uint card_idx, size_t threshold, size_t size_in_bits) {
+  BitMapView bm(_bits, size_in_bits);
+  if (_num_bits_set >= threshold) {
+    return bm.at(card_idx) ? Found : Overflow;
+  }
+  if (bm.par_set_bit(card_idx)) {
+    Atomic::inc(&_num_bits_set, memory_order_relaxed);
+    return Added;
+  }
+  return Found;
+}
+
+template <class CardVisitor>
+inline void G1CardSetBitMap::iterate(CardVisitor& found, size_t size_in_bits, uint offset) {
+  BitMapView bm(_bits, size_in_bits);
+  BitMap::idx_t idx = bm.get_next_one_offset(0);
+  while (idx != size_in_bits) {
+    found((offset | (uint)idx));
+    idx = bm.get_next_one_offset(idx + 1);
+  }
+}
+
+inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config) :
+  G1CardSetContainer(),
+  _num_entries((config->num_cards_in_array() + 1)) /* Card Transfer will not increment _num_entries */ {
+  EntryCountType num_buckets = config->num_buckets_in_howl();
+  EntryCountType bucket = config->howl_bucket_index(card_in_region);
+  for (uint i = 0; i < num_buckets; ++i) {
+    _buckets[i] = G1CardSetInlinePtr();
+    if (i == bucket) {
+      G1CardSetInlinePtr value(&_buckets[i], _buckets[i]);
+      value.add(card_in_region, config->inline_ptr_bits_per_card(), config->num_cards_in_inline_ptr());
+    }
+  }
+}
+
+inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) {
+  EntryCountType bucket = config->howl_bucket_index(card_idx);
+  CardSetPtr* array_entry = get_card_set_addr(bucket);
+  CardSetPtr card_set = Atomic::load_acquire(array_entry);
+
+  switch (G1CardSet::card_set_type(card_set)) {
+    case G1CardSet::CardSetArrayOfCards : {
+      return G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->contains(card_idx);
+    }
+    case G1CardSet::CardSetBitMap: {
+      uint card_offset = config->howl_bitmap_offset(card_idx);
+      return G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_offset, config->num_cards_in_howl_bitmap());
+    }
+    case G1CardSet::CardSetInlinePtr: {
+      G1CardSetInlinePtr ptr(card_set);
+      return ptr.contains(card_idx, config->inline_ptr_bits_per_card());
+    }
+    case G1CardSet::CardSetHowl: {// Fullcard set entry
+      assert(card_set == G1CardSet::FullCardSet, "Must be");
+      return true;
+    }
+  }
+  return false;
+}
+
+template <class CardOrRangeVisitor>
+inline void G1CardSetHowl::iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
+  for (uint i = 0; i < config->num_buckets_in_howl(); ++i) {
+    iterate_cardset(_buckets[i], i, found, config);
+  }
+}
+
+template <class CardSetPtrVisitor>
+inline void G1CardSetHowl::iterate(CardSetPtrVisitor& found, uint num_card_sets) {
+  for (uint i = 0; i < num_card_sets; ++i) {
+    found(&_buckets[i]);
+  }
+}
+
+template <class CardOrRangeVisitor>
+inline void G1CardSetHowl::iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
+  switch (G1CardSet::card_set_type(card_set)) {
+    case G1CardSet::CardSetInlinePtr: {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlInline)) {
+        G1CardSetInlinePtr ptr(card_set);
+        ptr.iterate(found, config->inline_ptr_bits_per_card());
+      }
+      return;
+    }
+    case G1CardSet::CardSetArrayOfCards : {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlArrayOfCards)) {
+        G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
+      }
+      return;
+    }
+    case G1CardSet::CardSetBitMap: {
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlBitmap)) {
+        uint offset = index << config->log2_num_cards_in_howl_bitmap();
+        G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->iterate(found, config->num_cards_in_howl_bitmap(), offset);
+      }
+      return;
+    }
+    case G1CardSet::CardSetHowl: { // actually FullCardSet
+      if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlFull)) {
+        assert(card_set == G1CardSet::FullCardSet, "Must be");
+        uint offset = index << config->log2_num_cards_in_howl_bitmap();
+        for (uint i = 0; i < config->max_cards_in_region(); i++) {
+          found((offset | (uint)i));
+        }
+      }
+      return;
+    }
+  }
+}
+
+inline G1CardSetHowl::EntryCountType G1CardSetHowl::num_buckets(size_t size_in_bits, size_t num_cards_in_array, size_t max_num_buckets) {
+  size_t size_bitmap_bytes = BitMap::calc_size_in_words(size_in_bits) * BytesPerWord;
+  // Ensure that in the worst case arrays consume half the memory size
+  // of storing the entire bitmap
+  size_t max_size_arrays_bytes = size_bitmap_bytes / 2;
+  size_t size_array_bytes = num_cards_in_array * sizeof(G1CardSetArray::EntryDataType);
+  size_t num_arrays = max_size_arrays_bytes / size_array_bytes;
+  // We use shifts and masks for indexing the array. So round down to the next
+  // power of two to not use more than expected memory.
+  num_arrays = round_down_power_of_2(MAX2((size_t)1, MIN2(num_arrays, max_num_buckets)));
+  return (EntryCountType)num_arrays;
+}
+
+#endif // SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP

src/hotspot/share/gc/g1/g1CardSetFreeMemoryTask.cpp

@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/g1/g1CardSetFreeMemoryTask.hpp"
+#include "gc/g1/g1CardSetMemory.inline.hpp"
+#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1_globals.hpp"
+#include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "heapRegionRemSet.hpp"
+#include "ci/ciUtilities.hpp"
+
+constexpr const char* G1CardSetFreeMemoryTask::_state_names[];
+
+const char* G1CardSetFreeMemoryTask::get_state_name(State value) const {
+  return _state_names[static_cast<std::underlying_type_t<State>>(value)];
+}
+
+bool G1CardSetFreeMemoryTask::deadline_exceeded(jlong deadline) {
+  return os::elapsed_counter() >= deadline;
+}
+
+static size_t keep_size(size_t free, size_t used, double percent) {
+  size_t to_keep = used * percent;
+  return MIN2(free, to_keep);
+}
+
+bool G1CardSetFreeMemoryTask::calculate_return_infos(jlong deadline) {
+  // Ignore the deadline in this step as it is very short.
+
+  G1CardSetMemoryStats used = _total_used;
+  G1CardSetMemoryStats free = G1CardSetFreePool::free_list_sizes();
+
+  _return_info = new G1ReturnMemoryProcessorSet(used.num_pools());
+  for (uint i = 0; i < used.num_pools(); i++) {
+    size_t return_to_vm_size = keep_size(free._num_mem_sizes[i],
+                                         used._num_mem_sizes[i],
+                                         G1RemSetFreeMemoryKeepExcessRatio);
+    log_trace(gc, task)("Card Set Free Memory: Type %s: Free: %zu (%zu) "
+                        "Used: %zu Keep: %zu",
+                        G1CardSetConfiguration::mem_object_type_name_str(i),
+                        free._num_mem_sizes[i], free._num_buffers[i],
+                        used._num_mem_sizes[i], return_to_vm_size);
+
+    _return_info->append(new G1ReturnMemoryProcessor(return_to_vm_size));
+  }
+
+  G1CardSetFreePool::update_unlink_processors(_return_info);
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::return_memory_to_vm(jlong deadline) {
+  for (int i = 0; i < _return_info->length(); i++) {
+    G1ReturnMemoryProcessor* info = _return_info->at(i);
+    if (!info->finished_return_to_vm()) {
+      if (info->return_to_vm(deadline)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::return_memory_to_os(jlong deadline) {
+  for (int i = 0; i < _return_info->length(); i++) {
+    G1ReturnMemoryProcessor* info = _return_info->at(i);
+    if (!info->finished_return_to_os()) {
+      if (info->return_to_os(deadline)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::cleanup_return_infos() {
+  for (int i = 0; i < _return_info->length(); i++) {
+     G1ReturnMemoryProcessor* info = _return_info->at(i);
+     delete info;
+  }
+  delete _return_info;
+
+  _return_info = nullptr;
+  return false;
+}
+
+bool G1CardSetFreeMemoryTask::free_excess_card_set_memory() {
+  jlong start = os::elapsed_counter();
+  jlong end = start +
+              (os::elapsed_frequency() / 1000) * G1RemSetFreeMemoryStepDurationMillis;
+
+  log_trace(gc, task)("Card Set Free Memory: Step start %1.3f end %1.3f",
+                      TimeHelper::counter_to_millis(start), TimeHelper::counter_to_millis(end));
+
+  State next_state;
+
+  do {
+    switch (_state) {
+      case State::CalculateUsed: {
+        if (calculate_return_infos(end)) {
+          next_state = _state;
+          return true;
+        }
+        next_state = State::ReturnToVM;
+        break;
+      }
+      case State::ReturnToVM: {
+        if (return_memory_to_vm(end)) {
+          next_state = _state;
+          return true;
+        }
+        next_state = State::ReturnToOS;
+        break;
+      }
+      case State::ReturnToOS: {
+        if (return_memory_to_os(end)) {
+          next_state = _state;
+          return true;
+        }
+        next_state = State::Cleanup;
+        break;
+      }
+      case State::Cleanup: {
+        cleanup_return_infos();
+        next_state = State::Inactive;
+        break;
+      }
+      default:
+        log_error(gc, task)("Should not try to free excess card set memory in %s state", get_state_name(_state));
+        ShouldNotReachHere();
+        break;
+    }
+
+    set_state(next_state);
+  } while (_state != State::Inactive && !deadline_exceeded(end));
+
+  log_trace(gc, task)("Card Set Free Memory: Step took %1.3fms, done %s",
+                      TimeHelper::counter_to_millis(os::elapsed_counter() - start),
+                      bool_to_str(_state == State::CalculateUsed));
+
+  return is_active();
+}
+
+void G1CardSetFreeMemoryTask::set_state(State new_state) {
+  log_trace(gc, task)("Card Set Free Memory: State change from %s to %s",
+                      get_state_name(_state),
+                      get_state_name(new_state));
+  _state = new_state;
+}
+
+bool G1CardSetFreeMemoryTask::is_active() const {
+  return _state != State::Inactive;
+}
+
+jlong G1CardSetFreeMemoryTask::reschedule_delay_ms() const {
+  return G1RemSetFreeMemoryRescheduleDelayMillis;
+}
+
+G1CardSetFreeMemoryTask::G1CardSetFreeMemoryTask(const char* name) :
+  G1ServiceTask(name), _state(State::CalculateUsed), _return_info(nullptr) { }
+
+void G1CardSetFreeMemoryTask::execute() {
+  SuspendibleThreadSetJoiner sts;
+
+  if (free_excess_card_set_memory()) {
+    schedule(reschedule_delay_ms());
+  }
+}
+
+void G1CardSetFreeMemoryTask::notify_new_stats(G1CardSetMemoryStats* young_gen_stats,
+                                               G1CardSetMemoryStats* collection_set_candidate_stats) {
+  assert_at_safepoint_on_vm_thread();
+
+  _total_used = *young_gen_stats;
+  _total_used.add(*collection_set_candidate_stats);
+
+  if (!is_active()) {
+    set_state(State::CalculateUsed);
+    G1CollectedHeap::heap()->service_thread()->schedule_task(this, 0);
+  }
+}

src/hotspot/share/gc/g1/g1CardSetFreeMemoryTask.hpp

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETFREEMEMORYTASK_HPP
+#define SHARE_GC_G1_G1CARDSETFREEMEMORYTASK_HPP
+
+#include "gc/g1/g1ServiceThread.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/heapRegionRemSet.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/ticks.hpp"
+
+class G1CardSetBuffer;
+
+// Task handling deallocation of free card set memory.
+class G1CardSetFreeMemoryTask : public G1ServiceTask {
+
+  enum class State : uint {
+    Inactive,
+    CalculateUsed,
+    ReturnToVM,
+    ReturnToOS,
+    Cleanup
+  };
+
+  static constexpr const char* _state_names[] = { "Invalid",
+                                                  "CalculateUsed",
+                                                  "ReturnToVM",
+                                                  "ReturnToOS",
+                                                  "Cleanup" };
+
+  const char* get_state_name(State value) const;
+
+  State _state;
+
+  // Current total card set memory usage.
+  G1CardSetMemoryStats _total_used;
+
+  typedef G1CardSetFreePool::G1ReturnMemoryProcessor G1ReturnMemoryProcessor;
+  typedef G1CardSetFreePool::G1ReturnMemoryProcessorSet G1ReturnMemoryProcessorSet;
+
+  G1ReturnMemoryProcessorSet* _return_info;
+
+  // Returns whether the given deadline has passed.
+  bool deadline_exceeded(jlong deadline);
+
+  // Methods for the tasks to be done. They all return true if that step has
+  // completed.
+  bool calculate_return_infos(jlong deadline);
+  bool return_memory_to_vm(jlong deadline);
+  bool return_memory_to_os(jlong deadline);
+  bool cleanup_return_infos();
+
+  // Free excess card set memory, main method. Returns true if there is more work
+  // to do.
+  bool free_excess_card_set_memory();
+
+  void set_state(State new_state);
+  // Returns whether we are currently processing a recent request.
+  bool is_active() const;
+
+  // The delay used to reschedule this task if not all work has been completed.
+  jlong reschedule_delay_ms() const;
+
+public:
+  explicit G1CardSetFreeMemoryTask(const char* name);
+
+  void execute() override;
+
+  // Notify the task of new used remembered set memory statistics for the young
+  // generation and the collection set candidate sets.
+  void notify_new_stats(G1CardSetMemoryStats* young_gen_stats,
+                        G1CardSetMemoryStats* collection_set_candidate_stats);
+};
+
+#endif // SHARE_GC_G1_G1CARDSETFREEMEMORYTASK_HPP

src/hotspot/share/gc/g1/g1CardSetMemory.cpp

@@ -0,0 +1,478 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/g1/g1CardSetMemory.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/formatBuffer.hpp"
+#include "utilities/ostream.hpp"
+
+G1CardSetBuffer::G1CardSetBuffer(uint elem_size, uint num_instances, G1CardSetBuffer* next) :
+    _elem_size(elem_size), _num_elems(num_instances), _next(next), _next_allocate(0) {
+
+  _buffer = NEW_C_HEAP_ARRAY(char, (size_t)_num_elems * elem_size, mtGCCardSet);
+}
+
+G1CardSetBuffer::~G1CardSetBuffer() {
+  FREE_C_HEAP_ARRAY(mtGCCardSet, _buffer);
+}
+
+void* G1CardSetBuffer::get_new_buffer_elem() {
+  if (_next_allocate >= _num_elems) {
+    return nullptr;
+  }
+  uint result = Atomic::fetch_and_add(&_next_allocate, 1u, memory_order_relaxed);
+  if (result >= _num_elems) {
+    return nullptr;
+  }
+  void* r = _buffer + (uint)result * _elem_size;
+  return r;
+}
+
+void G1CardSetBufferList::bulk_add(G1CardSetBuffer& first, G1CardSetBuffer& last, size_t num, size_t mem_size) {
+  _list.prepend(first, last);
+  Atomic::add(&_num_buffers, num, memory_order_relaxed);
+  Atomic::add(&_mem_size, mem_size, memory_order_relaxed);
+}
+
+void G1CardSetBufferList::print_on(outputStream* out, const char* prefix) {
+  out->print_cr("%s: buffers %zu size %zu", prefix, Atomic::load(&_num_buffers), Atomic::load(&_mem_size));
+}
+
+G1CardSetBuffer* G1CardSetBufferList::get() {
+  GlobalCounter::CriticalSection cs(Thread::current());
+
+  G1CardSetBuffer* result = _list.pop();
+  if (result != nullptr) {
+    Atomic::dec(&_num_buffers, memory_order_relaxed);
+    Atomic::sub(&_mem_size, result->mem_size(), memory_order_relaxed);
+  }
+  return result;
+}
+
+G1CardSetBuffer* G1CardSetBufferList::get_all(size_t& num_buffers, size_t& mem_size) {
+  GlobalCounter::CriticalSection cs(Thread::current());
+
+  G1CardSetBuffer* result = _list.pop_all();
+  num_buffers = Atomic::load(&_num_buffers);
+  mem_size = Atomic::load(&_mem_size);
+
+  if (result != nullptr) {
+    Atomic::sub(&_num_buffers, num_buffers, memory_order_relaxed);
+    Atomic::sub(&_mem_size, mem_size, memory_order_relaxed);
+  }
+  return result;
+}
+
+void G1CardSetBufferList::free_all() {
+  size_t num_freed = 0;
+  size_t mem_size_freed = 0;
+  G1CardSetBuffer* cur;
+
+  while ((cur = _list.pop()) != nullptr) {
+    mem_size_freed += cur->mem_size();
+    num_freed++;
+    delete cur;
+  }
+
+  Atomic::sub(&_num_buffers, num_freed, memory_order_relaxed);
+  Atomic::sub(&_mem_size, mem_size_freed, memory_order_relaxed);
+}
+
+template <class Elem>
+G1CardSetAllocator<Elem>::G1CardSetAllocator(const char* name,
+                                             const G1CardSetAllocOptions& buffer_options,
+                                             G1CardSetBufferList* free_buffer_list) :
+  _alloc_options(buffer_options),
+  _first(nullptr),
+  _last(nullptr),
+  _num_buffers(0),
+  _mem_size(0),
+  _free_buffer_list(free_buffer_list),
+  _transfer_lock(false),
+  _free_nodes_list(),
+  _pending_nodes_list(),
+  _num_pending_nodes(0),
+  _num_free_nodes(0),
+  _num_allocated_nodes(0),
+  _num_available_nodes(0)
+{
+  assert(elem_size() >= sizeof(G1CardSetContainer), "Element instance size %u for allocator %s too small",
+         elem_size(), name);
+  assert(_free_buffer_list != nullptr, "precondition!");
+}
+
+template <class Elem>
+bool G1CardSetAllocator<Elem>::try_transfer_pending() {
+  // Attempt to claim the lock.
+  if (Atomic::load_acquire(&_transfer_lock) || // Skip CAS if likely to fail.
+      Atomic::cmpxchg(&_transfer_lock, false, true)) {
+    return false;
+  }
+  // Have the lock; perform the transfer.
+
+  // Claim all the pending nodes.
+  G1CardSetContainer* first = _pending_nodes_list.pop_all();
+
+  if (first != nullptr) {
+    // Prepare to add the claimed nodes, and update _num_pending_nodes.
+    G1CardSetContainer* last = first;
+    Atomic::load_acquire(&_num_pending_nodes);
+
+    uint count = 1;
+    for (G1CardSetContainer* next = first->next(); next != nullptr; next = next->next()) {
+      last = next;
+      ++count;
+    }
+
+    Atomic::sub(&_num_pending_nodes, count);
+
+    // Wait for any in-progress pops to avoid ABA for them.
+    GlobalCounter::write_synchronize();
+    // Add synchronized nodes to _free_node_list.
+    // Update count first so there can be no underflow in allocate().
+    Atomic::add(&_num_free_nodes, count);
+    _free_nodes_list.prepend(*first, *last);
+  }
+  Atomic::release_store(&_transfer_lock, false);
+  return true;
+}
+
+template <class Elem>
+void G1CardSetAllocator<Elem>::free(Elem* elem) {
+  assert(elem != nullptr, "precondition");
+  assert(elem_size() >= sizeof(G1CardSetContainer), "size mismatch");
+  // Desired minimum transfer batch size.  There is relatively little
+  // importance to the specific number.  It shouldn't be too big, else
+  // we're wasting space when the release rate is low.  If the release
+  // rate is high, we might accumulate more than this before being
+  // able to start a new transfer, but that's okay.  Also note that
+  // the allocation rate and the release rate are going to be fairly
+  // similar, due to how the buffers are used. - kbarret
+  uint const trigger_transfer = 10;
+
+  uint pending_count = Atomic::add(&_num_pending_nodes, 1u, memory_order_relaxed);
+
+  G1CardSetContainer* node =  reinterpret_cast<G1CardSetContainer*>(reinterpret_cast<char*>(elem));
+
+  node->set_next(nullptr);
+  assert(node->next() == nullptr, "precondition");
+
+  _pending_nodes_list.push(*node);
+
+  if (pending_count > trigger_transfer) {
+    try_transfer_pending();
+  }
+}
+
+template <class Elem>
+void G1CardSetAllocator<Elem>::drop_all() {
+  _free_nodes_list.pop_all();
+  _pending_nodes_list.pop_all();
+  G1CardSetBuffer* cur = Atomic::load_acquire(&_first);
+
+  if (cur != nullptr) {
+    assert(_last != nullptr, "If there is at least one element, there must be a last one.");
+
+    G1CardSetBuffer* first = cur;
+#ifdef ASSERT
+    // Check list consistency.
+    G1CardSetBuffer* last = cur;
+    uint num_buffers = 0;
+    size_t mem_size = 0;
+    while (cur != nullptr) {
+      mem_size += cur->mem_size();
+      num_buffers++;
+
+      G1CardSetBuffer* next = cur->next();
+      last = cur;
+      cur = next;
+    }
+#endif
+    assert(num_buffers == _num_buffers, "Buffer count inconsistent %u %u", num_buffers, _num_buffers);
+    assert(mem_size == _mem_size, "Memory size inconsistent");
+    assert(last == _last, "Inconsistent last element");
+
+    _free_buffer_list->bulk_add(*first, *_last, _num_buffers, _mem_size);
+  }
+
+  _first = nullptr;
+  _last = nullptr;
+  _num_available_nodes = 0;
+  _num_allocated_nodes = 0;
+  _num_pending_nodes = 0;
+  _num_buffers = 0;
+  _mem_size = 0;
+  _num_free_nodes = 0;
+}
+
+template <class Elem>
+void G1CardSetAllocator<Elem>::print(outputStream* os) {
+  os->print("MA " PTR_FORMAT ": %u elems pending (allocated %u available %u) used %.3f highest %u buffers %u size %zu ",
+                p2i(this), _num_pending_nodes, _num_allocated_nodes, _num_available_nodes, percent_of(_num_allocated_nodes - _num_pending_nodes, _num_available_nodes), _first != nullptr ? _first->num_elems() : 0, _num_buffers, mem_size());
+}
+
+G1CardSetMemoryStats::G1CardSetMemoryStats() {
+  clear();
+}
+
+G1CardSetMemoryStats::G1CardSetMemoryStats(void(*fn)(const void*,uint,size_t&,size_t&), const void* context) {
+  clear();
+  for (uint i = 0; i < num_pools(); i++) {
+    fn(context, i, _num_mem_sizes[i], _num_buffers[i]);
+  }
+}
+
+void G1CardSetMemoryStats::clear() {
+  for (uint i = 0; i < num_pools(); i++) {
+    _num_mem_sizes[i] = 0;
+    _num_buffers[i] = 0;
+  }
+}
+
+void G1CardSetFreePool::update_unlink_processors(G1ReturnMemoryProcessorSet* unlink_processor) {
+  uint num_free_lists = _freelist_pool.num_free_lists();
+
+  for (uint i = 0; i < num_free_lists; i++) {
+    unlink_processor->at(i)->visit_free_list(_freelist_pool.free_list(i));
+  }
+}
+
+void G1CardSetFreePool::G1ReturnMemoryProcessor::visit_free_list(G1CardSetBufferList* source) {
+  assert(_source == nullptr, "already visited");
+  if (_return_to_vm_size > 0) {
+    _source = source;
+  } else {
+    assert(_source == nullptr, "must be");
+  }
+  if (source->mem_size() > _return_to_vm_size) {
+    _first = source->get_all(_num_unlinked, _unlinked_bytes);
+  } else {
+    assert(_first == nullptr, "must be");
+  }
+  // Above we were racing with other threads getting the contents of the free list,
+  // so while we might have been asked to return something to the OS initially,
+  // the free list might be empty anyway. In this case just reset internal values
+  // used for checking whether there is work available.
+  if (_first == nullptr) {
+    _source = nullptr;
+    _return_to_vm_size = 0;
+  }
+}
+
+bool G1CardSetFreePool::G1ReturnMemoryProcessor::return_to_vm(jlong deadline) {
+  assert(!finished_return_to_vm(), "already returned everything to the VM");
+  assert(_first != nullptr, "must have element to return");
+
+  size_t keep_size = 0;
+  size_t keep_num = 0;
+
+  G1CardSetBuffer* cur = _first;
+  G1CardSetBuffer* last = nullptr;
+
+  while (cur != nullptr && _return_to_vm_size > 0) {
+    size_t cur_size = cur->mem_size();
+    _return_to_vm_size -= MIN2(_return_to_vm_size, cur_size);
+
+    keep_size += cur_size;
+    keep_num++;
+
+    last = cur;
+    cur = cur->next();
+    // To ensure progress, perform the deadline check here.
+    if (os::elapsed_counter() > deadline) {
+      break;
+    }
+  }
+
+  assert(_first != nullptr, "must be");
+  assert(last != nullptr, "must be");
+
+  last->set_next(nullptr);
+
+  // Wait for any in-progress pops to avoid ABA for them.
+  GlobalCounter::write_synchronize();
+  _source->bulk_add(*_first, *last, keep_num, keep_size);
+  _first = cur;
+
+  log_trace(gc, task)("Card Set Free Memory: Returned to VM %zu buffers size %zu", keep_num, keep_size);
+
+  // _return_to_vm_size may be larger than what is available in the list at the
+  // time we actually get the list. I.e. the list and _return_to_vm_size may be
+  // inconsistent.
+  // So also check if we actually already at the end of the list for the exit
+  // condition.
+  if (_return_to_vm_size == 0 || _first == nullptr) {
+    _source = nullptr;
+    _return_to_vm_size = 0;
+  }
+  return _source != nullptr;
+}
+
+bool G1CardSetFreePool::G1ReturnMemoryProcessor::return_to_os(jlong deadline) {
+  assert(finished_return_to_vm(), "not finished returning to VM");
+  assert(!finished_return_to_os(), "already returned everything to the OS");
+
+  // Now delete the rest.
+  size_t num_delete = 0;
+  size_t mem_size_deleted = 0;
+
+  while (_first != nullptr) {
+    G1CardSetBuffer* next = _first->next();
+    num_delete++;
+    mem_size_deleted += _first->mem_size();
+    delete _first;
+    _first = next;
+
+    // To ensure progress, perform the deadline check here.
+    if (os::elapsed_counter() > deadline) {
+      break;
+    }
+  }
+
+  log_trace(gc, task)("Card Set Free Memory: Return to OS %zu buffers size %zu", num_delete, mem_size_deleted);
+
+  return _first != nullptr;
+}
+
+G1CardSetFreePool G1CardSetFreePool::_freelist_pool(G1CardSetConfiguration::num_mem_object_types());
+
+G1CardSetFreePool::G1CardSetFreePool(uint num_free_lists) :
+  _num_free_lists(num_free_lists) {
+
+  _free_lists = NEW_C_HEAP_ARRAY(G1CardSetBufferList, _num_free_lists, mtGC);
+  for (uint i = 0; i < _num_free_lists; i++) {
+    new (&_free_lists[i]) G1CardSetBufferList();
+  }
+}
+
+G1CardSetFreePool::~G1CardSetFreePool() {
+  for (uint i = 0; i < _num_free_lists; i++) {
+    _free_lists[i].~G1CardSetBufferList();
+  }
+  FREE_C_HEAP_ARRAY(mtGC, _free_lists);
+}
+
+static void collect_mem_sizes(const void* context, uint i, size_t& mem_size, size_t& num_buffers) {
+  ((G1CardSetFreePool*)context)->get_size(i, mem_size, num_buffers);
+}
+
+void G1CardSetFreePool::get_size(uint i, size_t& mem_size, size_t& num_buffers) const {
+  mem_size = _free_lists[i].mem_size();
+  num_buffers = _free_lists[i].num_buffers();
+}
+
+G1CardSetMemoryStats G1CardSetFreePool::memory_sizes() const {
+  return G1CardSetMemoryStats(collect_mem_sizes, this);
+}
+
+size_t G1CardSetFreePool::mem_size() const {
+  size_t result = 0;
+  for (uint i = 0; i < _num_free_lists; i++) {
+    result += _free_lists[i].mem_size();
+  }
+  return result;
+}
+
+void G1CardSetFreePool::print_on(outputStream* out) {
+  out->print_cr("  Free Pool: size %zu", free_list_pool()->mem_size());
+  for (uint i = 0; i < _num_free_lists; i++) {
+    FormatBuffer<> fmt("    %s", G1CardSetConfiguration::mem_object_type_name_str(i));
+    _free_lists[i].print_on(out, fmt);
+  }
+}
+
+G1CardSetMemoryManager::G1CardSetMemoryManager(G1CardSetConfiguration* config,
+                                               G1CardSetFreePool* free_list_pool) : _config(config) {
+
+  _allocators = NEW_C_HEAP_ARRAY(G1CardSetAllocator<G1CardSetContainer>,
+                                 _config->num_mem_object_types(),
+                                 mtGC);
+  G1CardSetAllocOptions* alloc_options = _config->mem_object_alloc_options();
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    new (&_allocators[i]) G1CardSetAllocator<G1CardSetContainer>(_config->mem_object_type_name_str(i),
+                                                                 alloc_options[i],
+                                                                 free_list_pool->free_list(i));
+  }
+  FREE_C_HEAP_ARRAY(size_t, alloc_options);
+}
+
+uint G1CardSetMemoryManager::num_mem_object_types() const {
+  return _config->num_mem_object_types();
+}
+
+
+G1CardSetMemoryManager::~G1CardSetMemoryManager() {
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    _allocators[i].~G1CardSetAllocator();
+  }
+  FREE_C_HEAP_ARRAY(G1CardSetAllocator<G1CardSetContainer>, _allocators);
+}
+
+void G1CardSetMemoryManager::free(uint type, void* value) {
+  assert(type < num_mem_object_types(), "must be");
+  _allocators[type].free((G1CardSetContainer*)value);
+}
+
+void G1CardSetMemoryManager::flush() {
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    _allocators[i].drop_all();
+  }
+}
+
+void G1CardSetMemoryManager::print(outputStream* os) {
+  os->print_cr("MM " PTR_FORMAT " size %zu", p2i(this), sizeof(*this));
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    _allocators[i].print(os);
+  }
+}
+
+size_t G1CardSetMemoryManager::mem_size() const {
+  size_t result = 0;
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    result += _allocators[i].mem_size();
+  }
+  return sizeof(*this) -
+    (sizeof(G1CardSetAllocator<G1CardSetContainer>) * num_mem_object_types()) +
+    result;
+}
+
+size_t G1CardSetMemoryManager::wasted_mem_size() const {
+  size_t result = 0;
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    result += _allocators[i].wasted_mem_size();
+  }
+  return result;
+}
+
+G1CardSetMemoryStats G1CardSetMemoryManager::memory_stats() const {
+  G1CardSetMemoryStats result;
+  for (uint i = 0; i < num_mem_object_types(); i++) {
+    result._num_mem_sizes[i] += _allocators[i].mem_size();
+    result._num_buffers[i] += _allocators[i].num_buffers();
+  }
+  return result;
+}

src/hotspot/share/gc/g1/g1CardSetMemory.hpp

@@ -0,0 +1,384 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETMEMORY_HPP
+#define SHARE_GC_G1_G1CARDSETMEMORY_HPP
+
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/lockFreeStack.hpp"
+
+class G1CardSetConfiguration;
+class outputStream;
+
+// Collects G1CardSetAllocator options/heuristics. Called by G1CardSetAllocator
+// to determine the next size of the allocated G1CardSetBuffer.
+class G1CardSetAllocOptions {
+  uint _elem_size;
+  uint _initial_num_elems;
+  // Defines a limit to the number of elements in the buffer
+  uint _max_num_elems;
+
+  uint exponential_expand(uint prev_num_elems) {
+    return clamp(prev_num_elems * 2, _initial_num_elems, _max_num_elems);
+  }
+
+public:
+  static const uint BufferAlignment = 8;
+  static const uint MinimumBufferSize = 8;
+  static const uint MaximumBufferSize =  UINT_MAX / 2;
+
+  G1CardSetAllocOptions(uint elem_size, uint initial_num_elems = MinimumBufferSize, uint max_num_elems = MaximumBufferSize) :
+    _elem_size(align_up(elem_size, BufferAlignment)),
+    _initial_num_elems(initial_num_elems),
+    _max_num_elems(max_num_elems) {
+  }
+
+  uint next_num_elems(uint prev_num_elems) {
+    return exponential_expand(prev_num_elems);
+  }
+
+  uint elem_size () const {return _elem_size;}
+};
+
+// A single buffer/arena containing _num_elems blocks of memory of _elem_size.
+// G1CardSetBuffers can be linked together using a singly linked list.
+class G1CardSetBuffer : public CHeapObj<mtGCCardSet> {
+  uint _elem_size;
+  uint _num_elems;
+
+  G1CardSetBuffer* volatile _next;
+
+  char* _buffer;  // Actual data.
+
+  // Index into the next free block to allocate into. Full if equal (or larger)
+  // to _num_elems (can be larger because we atomically increment this value and
+  // check only afterwards if the allocation has been successful).
+  uint volatile _next_allocate;
+
+public:
+  G1CardSetBuffer(uint elem_size, uint num_elems, G1CardSetBuffer* next);
+  ~G1CardSetBuffer();
+
+  G1CardSetBuffer* volatile* next_addr() { return &_next; }
+
+  void* get_new_buffer_elem();
+
+  uint num_elems() const { return _num_elems; }
+
+  G1CardSetBuffer* next() const { return _next; }
+
+  void set_next(G1CardSetBuffer* next) {
+    assert(next != this, " loop condition");
+    _next = next;
+  }
+
+  void reset(G1CardSetBuffer* next) {
+    _next_allocate = 0;
+    assert(next != this, " loop condition");
+    set_next(next);
+    memset((void*)_buffer, 0, (size_t)_num_elems * _elem_size);
+  }
+
+  uint elem_size() const { return _elem_size; }
+
+  size_t mem_size() const { return sizeof(*this) + (size_t)_num_elems * _elem_size; }
+
+  bool is_full() const { return _next_allocate >= _num_elems; }
+};
+
+// Set of (free) G1CardSetBuffers. The assumed usage is that allocation
+// to it and removal of elements is strictly separate, but every action may be
+// performed by multiple threads at the same time.
+// Counts and memory usage are current on a best-effort basis if accessed concurrently.
+class G1CardSetBufferList {
+  static G1CardSetBuffer* volatile* next_ptr(G1CardSetBuffer& node) {
+    return node.next_addr();
+  }
+  typedef LockFreeStack<G1CardSetBuffer, &next_ptr> NodeStack;
+
+  NodeStack _list;
+
+  volatile size_t _num_buffers;
+  volatile size_t _mem_size;
+
+public:
+  G1CardSetBufferList() : _list(), _num_buffers(0), _mem_size(0) { }
+  ~G1CardSetBufferList() { free_all(); }
+
+  void bulk_add(G1CardSetBuffer& first, G1CardSetBuffer& last, size_t num, size_t mem_size);
+  void add(G1CardSetBuffer& elem) { _list.prepend(elem); }
+
+  G1CardSetBuffer* get();
+  G1CardSetBuffer* get_all(size_t& num_buffers, size_t& mem_size);
+
+  // Give back all memory to the OS.
+  void free_all();
+
+  void print_on(outputStream* out, const char* prefix = "");
+
+  size_t num_buffers() const { return Atomic::load(&_num_buffers); }
+  size_t mem_size() const { return Atomic::load(&_mem_size); }
+};
+
+// Arena-like allocator for (card set) heap memory objects (Elem elements).
+//
+// Actual allocation from the C heap occurs on G1CardSetBuffer basis, i.e. sets
+// of elements. The assumed allocation pattern for these G1CardSetBuffer elements
+// is assumed to be strictly two-phased:
+//
+// - in the first phase, G1CardSetBuffers are allocated from the C heap (or a free
+// list given at initialization time). This allocation may occur in parallel. This
+// typically corresponds to a single mutator phase, but may extend over multiple.
+//
+// - in the second phase, G1CardSetBuffers are given back in bulk to the free list.
+// This is typically done during a GC pause.
+//
+// Some third party is responsible for giving back memory from the free list to
+// the operating system.
+//
+// Allocation and deallocation in the first phase on G1CardSetContainer basis
+// may occur by multiple threads at once.
+//
+// Allocation occurs from an internal free list of G1CardSetContainers first,
+// only then trying to bump-allocate from the current G1CardSetBuffer. If there is
+// none, this class allocates a new G1CardSetBuffer (allocated from the C heap,
+// asking the G1CardSetAllocOptions instance about sizes etc) and uses that one.
+//
+// The G1CardSetContainerOnHeaps free list is a linked list of G1CardSetContainers
+// within all G1CardSetBuffer instances allocated so far. It uses a separate
+// pending list and global synchronization to avoid the ABA problem when the
+// user frees a memory object.
+//
+// The class also manages a few counters for statistics using atomic operations.
+// Their values are only consistent within each other with extra global
+// synchronization.
+//
+// Since it is expected that every CardSet (and in extension each region) has its
+// own set of allocators, there is intentionally no padding between them to save
+// memory.
+template <class Elem>
+class G1CardSetAllocator {
+  // G1CardSetBuffer management.
+
+  // G1CardSetAllocOptions provides parameters for allocation buffer
+  // sizing and expansion.
+  G1CardSetAllocOptions _alloc_options;
+
+  G1CardSetBuffer* volatile _first;       // The (start of the) list of all buffers.
+  G1CardSetBuffer* _last;                 // The last element of the list of all buffers.
+  volatile uint _num_buffers;             // Number of assigned buffers to this allocator.
+  volatile size_t _mem_size;              // Memory used by all buffers.
+
+  G1CardSetBufferList* _free_buffer_list; // The global free buffer list to
+                                          // preferentially get new buffers from.
+
+  // G1CardSetContainer node management within the G1CardSetBuffers allocated
+  // by this allocator.
+
+  static G1CardSetContainer* volatile* next_ptr(G1CardSetContainer& node);
+  typedef LockFreeStack<G1CardSetContainer, &next_ptr> NodeStack;
+
+  volatile bool _transfer_lock;
+  NodeStack _free_nodes_list;
+  NodeStack _pending_nodes_list;
+
+  volatile uint _num_pending_nodes;   // Number of nodes in the pending list.
+  volatile uint _num_free_nodes;      // Number of nodes in the free list.
+
+  volatile uint _num_allocated_nodes; // Number of total nodes allocated and in use.
+  volatile uint _num_available_nodes; // Number of nodes available in all buffers (allocated + free + pending + not yet used).
+
+  // Try to transfer nodes from _pending_nodes_list to _free_nodes_list, with a
+  // synchronization delay for any in-progress pops from the _free_nodes_list
+  // to solve ABA here.
+  bool try_transfer_pending();
+
+  uint num_free_elems() const;
+
+  G1CardSetBuffer* create_new_buffer(G1CardSetBuffer* const prev);
+
+  uint elem_size() const { return _alloc_options.elem_size(); }
+
+public:
+  G1CardSetAllocator(const char* name,
+                     const G1CardSetAllocOptions& buffer_options,
+                     G1CardSetBufferList* free_buffer_list);
+  ~G1CardSetAllocator() {
+    drop_all();
+  }
+
+  Elem* allocate();
+  void free(Elem* elem);
+
+  // Deallocate all buffers to the free buffer list and reset this allocator. Must
+  // be called in a globally synchronized area.
+  void drop_all();
+
+  uint num_buffers() const;
+
+  size_t mem_size() const {
+    return sizeof(*this) +
+      num_buffers() * sizeof(G1CardSetBuffer) + (size_t)_num_available_nodes * elem_size();
+  }
+
+  size_t wasted_mem_size() const {
+    return ((size_t)_num_available_nodes - (_num_allocated_nodes - _num_pending_nodes)) * elem_size();
+  }
+
+  void print(outputStream* os);
+};
+
+// Statistics for a fixed set of buffer lists. Contains the number of buffers and memory
+// used for each. Note that statistics are typically not taken atomically so there
+// can be inconsistencies. The user must be prepared for them.
+class G1CardSetMemoryStats {
+public:
+
+  size_t _num_mem_sizes[G1CardSetConfiguration::num_mem_object_types()];
+  size_t _num_buffers[G1CardSetConfiguration::num_mem_object_types()];
+
+  // Returns all-zero statistics.
+  G1CardSetMemoryStats();
+  // For every element in the set (indicated by i), call fn to provide the
+  // memory size and number of buffers for that i'th buffer list.
+  G1CardSetMemoryStats(void (*fn)(const void* context, uint i, size_t& mem_size, size_t& num_buffers), const void* context);
+
+  void add(G1CardSetMemoryStats const other) {
+    STATIC_ASSERT(ARRAY_SIZE(_num_buffers) == ARRAY_SIZE(_num_mem_sizes));
+    for (uint i = 0; i < ARRAY_SIZE(_num_mem_sizes); i++) {
+      _num_mem_sizes[i] += other._num_mem_sizes[i];
+      _num_buffers[i] += other._num_buffers[i];
+    }
+  }
+
+  void clear();
+
+  uint num_pools() const { return G1CardSetConfiguration::num_mem_object_types(); }
+};
+
+// A set of free lists holding memory buffers for use by G1CardSetAllocators.
+class G1CardSetFreePool {
+  // The global free pool.
+  static G1CardSetFreePool _freelist_pool;
+
+  uint _num_free_lists;
+  G1CardSetBufferList* _free_lists;
+
+public:
+  static G1CardSetFreePool* free_list_pool() { return &_freelist_pool; }
+  static G1CardSetMemoryStats free_list_sizes() { return _freelist_pool.memory_sizes(); }
+
+  class G1ReturnMemoryProcessor;
+  typedef GrowableArrayCHeap<G1ReturnMemoryProcessor*, mtGC> G1ReturnMemoryProcessorSet;
+
+  static void update_unlink_processors(G1ReturnMemoryProcessorSet* unlink_processors);
+
+  explicit G1CardSetFreePool(uint num_free_lists);
+  ~G1CardSetFreePool();
+
+  G1CardSetBufferList* free_list(uint i) {
+    assert(i < _num_free_lists, "must be");
+    return &_free_lists[i];
+  }
+
+  uint num_free_lists() const { return _num_free_lists; }
+
+  // Return sizes for free list i in this free list pool.
+  void get_size(uint i, size_t& mem_size, size_t& num_buffers) const;
+
+  G1CardSetMemoryStats memory_sizes() const;
+  size_t mem_size() const;
+
+  void print_on(outputStream* out);
+};
+
+// Data structure containing current in-progress state for returning memory to the
+// operating system for a single G1CardSetBufferList.
+class G1CardSetFreePool::G1ReturnMemoryProcessor : public CHeapObj<mtGC> {
+  G1CardSetBufferList* _source;
+  size_t _return_to_vm_size;
+
+  G1CardSetBuffer* _first;
+  size_t _unlinked_bytes;
+  size_t _num_unlinked;
+
+public:
+  explicit G1ReturnMemoryProcessor(size_t return_to_vm) :
+    _source(nullptr), _return_to_vm_size(return_to_vm), _first(nullptr), _unlinked_bytes(0), _num_unlinked(0) {
+  }
+
+  // Updates the instance members about the given card set buffer list for the purpose
+  // of giving back memory. Only necessary members are updated, e.g. if there is
+  // nothing to return to the VM, do not set the source list.
+  void visit_free_list(G1CardSetBufferList* source);
+
+  bool finished_return_to_vm() const { return _return_to_vm_size == 0; }
+  bool finished_return_to_os() const { return _first == nullptr; }
+
+  // Returns memory to the VM until the given deadline expires. Returns true if
+  // there is no more work. Guarantees forward progress, i.e. at least one buffer
+  // has been processed after returning.
+  // return_to_vm() re-adds buffers to the respective free list.
+  bool return_to_vm(jlong deadline);
+  // Returns memory to the VM until the given deadline expires. Returns true if
+  // there is no more work. Guarantees forward progress, i.e. at least one buffer
+  // has been processed after returning.
+  // return_to_os() gives back buffers to the OS.
+  bool return_to_os(jlong deadline);
+};
+
+class G1CardSetMemoryManager : public CHeapObj<mtGCCardSet> {
+  G1CardSetConfiguration* _config;
+
+  G1CardSetAllocator<G1CardSetContainer>* _allocators;
+
+  uint num_mem_object_types() const;
+public:
+  G1CardSetMemoryManager(G1CardSetConfiguration* config,
+                         G1CardSetFreePool* free_list_pool);
+
+  virtual ~G1CardSetMemoryManager();
+
+  // Allocate and free a memory object of given type.
+  inline uint8_t* allocate(uint type);
+  void free(uint type, void* value);
+
+  // Allocate and free a hash table node.
+  inline uint8_t* allocate_node();
+  inline void free_node(void* value);
+
+  void flush();
+
+  void print(outputStream* os);
+
+  size_t mem_size() const;
+  size_t wasted_mem_size() const;
+
+  G1CardSetMemoryStats memory_stats() const;
+};
+
+#endif // SHARE_GC_G1_G1CARDSETMEMORY_HPP

src/hotspot/share/gc/g1/g1CardSetMemory.inline.hpp

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_G1_G1CARDSETMEMORY_INLINE_HPP
+#define SHARE_GC_G1_G1CARDSETMEMORY_INLINE_HPP
+
+#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/g1CardSetContainers.hpp"
+#include "utilities/ostream.hpp"
+
+#include "gc/g1/g1CardSetContainers.inline.hpp"
+#include "utilities/globalCounter.inline.hpp"
+
+template <class Elem>
+G1CardSetContainer* volatile* G1CardSetAllocator<Elem>::next_ptr(G1CardSetContainer& node) {
+  return node.next_addr();
+}
+
+template <class Elem>
+G1CardSetBuffer* G1CardSetAllocator<Elem>::create_new_buffer(G1CardSetBuffer* const prev) {
+
+  // Take an existing buffer if available.
+  G1CardSetBuffer* next = _free_buffer_list->get();
+  if (next == nullptr) {
+    uint prev_num_elems = (prev != nullptr) ? prev->num_elems() : 0;
+    uint num_elems = _alloc_options.next_num_elems(prev_num_elems);
+    next = new G1CardSetBuffer(elem_size(), num_elems, prev);
+  } else {
+    assert(elem_size() == next->elem_size() , "Mismatch %d != %d Elem %zu", elem_size(), next->elem_size(), sizeof(Elem));
+    next->reset(prev);
+  }
+
+  // Install it as current allocation buffer.
+  G1CardSetBuffer* old = Atomic::cmpxchg(&_first, prev, next);
+  if (old != prev) {
+    // Somebody else installed the buffer, use that one.
+    delete next;
+    return old;
+  } else {
+    // Did we install the first element in the list? If so, this is also the last.
+    if (prev == nullptr) {
+      _last = next;
+    }
+    // Successfully installed the buffer into the list.
+    Atomic::inc(&_num_buffers, memory_order_relaxed);
+    Atomic::add(&_mem_size, next->mem_size(), memory_order_relaxed);
+    Atomic::add(&_num_available_nodes, next->num_elems(), memory_order_relaxed);
+    return next;
+  }
+}
+
+template <class Elem>
+Elem* G1CardSetAllocator<Elem>::allocate() {
+  assert(elem_size() > 0, "instance size not set.");
+
+  if (num_free_elems() > 0) {
+    // Pop under critical section to deal with ABA problem
+    // Other solutions to the same problem are more complicated (ref counting, HP)
+    GlobalCounter::CriticalSection cs(Thread::current());
+
+    G1CardSetContainer* node = _free_nodes_list.pop();
+    if (node != nullptr) {
+      Elem* elem = reinterpret_cast<Elem*>(reinterpret_cast<char*>(node));
+      Atomic::sub(&_num_free_nodes, 1u);
+      guarantee(is_aligned(elem, 8), "result " PTR_FORMAT " not aligned", p2i(elem));
+      return elem;
+    }
+  }
+
+  G1CardSetBuffer* cur = Atomic::load_acquire(&_first);
+  if (cur == nullptr) {
+    cur = create_new_buffer(cur);
+  }
+
+  while (true) {
+    Elem* elem = (Elem*)cur->get_new_buffer_elem();
+    if (elem != nullptr) {
+      Atomic::inc(&_num_allocated_nodes, memory_order_relaxed);
+      guarantee(is_aligned(elem, 8), "result " PTR_FORMAT " not aligned", p2i(elem));
+      return elem;
+    }
+    // The buffer is full. Next round.
+    assert(cur->is_full(), "must be");
+    cur = create_new_buffer(cur);
+  }
+}
+
+inline uint8_t* G1CardSetMemoryManager::allocate(uint type) {
+  assert(type < num_mem_object_types(), "must be");
+  return (uint8_t*)_allocators[type].allocate();
+}
+
+inline uint8_t* G1CardSetMemoryManager::allocate_node() {
+  return allocate(0);
+}
+
+inline void G1CardSetMemoryManager::free_node(void* value) {
+  free(0, value);
+}
+
+template <class Elem>
+inline uint G1CardSetAllocator<Elem>::num_buffers() const {
+  return Atomic::load(&_num_buffers);
+}
+
+template <class Elem>
+inline uint G1CardSetAllocator<Elem>::num_free_elems() const {
+  return Atomic::load(&_num_free_nodes);
+}
+
+#endif // SHARE_GC_G1_G1CARDSETMEMORY_INLINE_HPP

src/hotspot/share/gc/g1/g1CardTable.hpp

@@ -102,7 +102,7 @@ public:
   // already scanned cards. Assumes that most cards in that area are Clean.
   // Returns the number of dirtied cards that were not yet dirty. This result may
   // be inaccurate as it does not perform the dirtying atomically.
-  inline size_t mark_region_dirty(size_t start_card_index, size_t num_cards);
+  inline size_t mark_range_dirty(size_t start_card_index, size_t num_cards);
 
   // Change the given range of dirty cards to "which". All of these cards must be Dirty.
   inline void change_dirty_cards_to(size_t start_card_index, size_t num_cards, CardValue which);

src/hotspot/share/gc/g1/g1CardTable.inline.hpp

@@ -43,7 +43,7 @@ inline bool G1CardTable::mark_clean_as_dirty(CardValue* card) {
   return false;
 }
 
-inline size_t G1CardTable::mark_region_dirty(size_t start_card_index, size_t num_cards) {
+inline size_t G1CardTable::mark_range_dirty(size_t start_card_index, size_t num_cards) {
   assert(is_aligned(start_card_index, sizeof(size_t)), "Start card index must be aligned.");
   assert(is_aligned(num_cards, sizeof(size_t)), "Number of cards to change must be evenly divisible.");
 

src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp

@@ -28,7 +28,7 @@
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oop.inline.hpp"

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -34,6 +34,7 @@
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectionSet.hpp"
+#include "gc/g1/g1CollectionSetCandidates.hpp"
 #include "gc/g1/g1CollectorState.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/g1ConcurrentRefineThread.hpp"
@@ -68,7 +69,7 @@
 #include "gc/g1/g1VMOperations.hpp"
 #include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcBehaviours.hpp"
@@ -147,7 +148,7 @@ void G1CollectedHeap::run_batch_task(G1BatchedGangTask* cl) {
 
 HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index,
                                              MemRegion mr) {
-  return new HeapRegion(hrs_index, bot(), mr);
+  return new HeapRegion(hrs_index, bot(), mr, &_card_set_config);
 }
 
 // Private methods.
@@ -1436,6 +1437,7 @@ G1CollectedHeap::G1CollectedHeap() :
   CollectedHeap(),
   _service_thread(NULL),
   _periodic_gc_task(NULL),
+  _free_card_set_memory_task(NULL),
   _workers(NULL),
   _card_table(NULL),
   _collection_pause_end(Ticks::now()),
@@ -1472,6 +1474,7 @@ G1CollectedHeap::G1CollectedHeap() :
   _collection_set(this, _policy),
   _hot_card_cache(NULL),
   _rem_set(NULL),
+  _card_set_config(),
   _cm(NULL),
   _cm_thread(NULL),
   _cr(NULL),
@@ -1741,6 +1744,9 @@ jint G1CollectedHeap::initialize() {
   _periodic_gc_task = new G1PeriodicGCTask("Periodic GC Task");
   _service_thread->register_task(_periodic_gc_task);
 
+  _free_card_set_memory_task = new G1CardSetFreeMemoryTask("Card Set Free Memory Task");
+  _service_thread->register_task(_free_card_set_memory_task);
+
   {
     G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
     dcqs.set_process_cards_threshold(concurrent_refine()->yellow_zone());
@@ -2621,6 +2627,9 @@ void G1CollectedHeap::gc_epilogue(bool full) {
   _numa->print_statistics();
 
   _collection_pause_end = Ticks::now();
+
+  _free_card_set_memory_task->notify_new_stats(&_young_gen_card_set_stats,
+                                               &_collection_set_candidates_card_set_stats);
 }
 
 uint G1CollectedHeap::uncommit_regions(uint region_limit) {
@@ -2842,6 +2851,9 @@ void G1CollectedHeap::verify_before_young_collection(G1HeapVerifier::G1VerifyTyp
 }
 
 void G1CollectedHeap::verify_after_young_collection(G1HeapVerifier::G1VerifyType type) {
+  if (evacuation_failed()) {
+    type = (G1HeapVerifier::G1VerifyType)(type | G1HeapVerifier::G1VerifyYoungEvacFail);
+  }
   if (VerifyRememberedSets) {
     log_info(gc, verify)("[Verifying RemSets after GC]");
     VerifyRegionRemSetClosure v_cl;
@@ -2958,7 +2970,6 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
                                                             Threads::number_of_non_daemon_threads());
     active_workers = workers()->update_active_workers(active_workers);
     log_info(gc,task)("Using %u workers of %u for evacuation", active_workers, workers()->total_workers());
-
     G1MonitoringScope ms(g1mm(),
                          false /* full_gc */,
                          collector_state()->in_mixed_phase() /* all_memory_pools_affected */);
@@ -2967,12 +2978,10 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
 
     {
       IsGCActiveMark x;
-
       gc_prologue(false);
 
       G1HeapVerifier::G1VerifyType verify_type = young_collection_verify_type();
       verify_before_young_collection(verify_type);
-
       {
         // The elapsed time induced by the start time below deliberately elides
         // the possible verification above.
@@ -3061,12 +3070,6 @@ void G1CollectedHeap::do_collection_pause_at_safepoint_helper(double target_paus
     print_heap_regions();
     trace_heap_after_gc(_gc_tracer_stw);
 
-    // We must call G1MonitoringSupport::update_sizes() in the same scoping level
-    // as an active TraceMemoryManagerStats object (i.e. before the destructor for the
-    // TraceMemoryManagerStats is called) so that the G1 memory pools are updated
-    // before any GC notifications are raised.
-    g1mm()->update_sizes();
-
     gc_tracer_report_gc_end(concurrent_operation_is_full_mark, evacuation_info);
   }
   // It should now be safe to tell the concurrent mark thread to start
@@ -3319,6 +3322,15 @@ bool G1CollectedHeap::should_do_eager_reclaim() const {
           (has_humongous_reclaim_candidates() || do_humongous_object_logging()));
 }
 
+bool G1CollectedHeap::should_sample_collection_set_candidates() const {
+  G1CollectionSetCandidates* candidates = G1CollectedHeap::heap()->collection_set()->candidates();
+  return candidates != NULL && candidates->num_remaining() > 0;
+}
+
+void G1CollectedHeap::set_collection_set_candidates_stats(G1CardSetMemoryStats& stats) {
+  _collection_set_candidates_card_set_stats = stats;
+}
+
 class G1PrepareEvacuationTask : public AbstractGangTask {
   class G1PrepareRegionsClosure : public HeapRegionClosure {
     G1CollectedHeap* _g1h;
@@ -3326,6 +3338,17 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
     uint _worker_humongous_total;
     uint _worker_humongous_candidates;
 
+    G1CardSetMemoryStats _card_set_stats;
+
+    void sample_card_set_size(HeapRegion* hr) {
+      // Sample card set sizes for young gen and humongous before GC: this makes
+      // the policy to give back memory to the OS keep the most recent amount of
+      // memory for these regions.
+      if (hr->is_young() || hr->is_starts_humongous()) {
+        _card_set_stats.add(hr->rem_set()->card_set_memory_stats());
+      }
+    }
+
     bool humongous_region_is_candidate(HeapRegion* region) const {
       assert(region->is_starts_humongous(), "Must start a humongous object");
 
@@ -3398,6 +3421,8 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
       // First prepare the region for scanning
       _g1h->rem_set()->prepare_region_for_scan(hr);
 
+      sample_card_set_size(hr);
+
       // Now check if region is a humongous candidate
       if (!hr->is_starts_humongous()) {
         _g1h->register_region_with_region_attr(hr);
@@ -3428,12 +3453,19 @@ class G1PrepareEvacuationTask : public AbstractGangTask {
 
       return false;
     }
+
+    G1CardSetMemoryStats card_set_stats() const {
+      return _card_set_stats;
+    }
   };
 
   G1CollectedHeap* _g1h;
   HeapRegionClaimer _claimer;
   volatile uint _humongous_total;
   volatile uint _humongous_candidates;
+
+  G1CardSetMemoryStats _all_card_set_stats;
+
 public:
   G1PrepareEvacuationTask(G1CollectedHeap* g1h) :
     AbstractGangTask("Prepare Evacuation"),
@@ -3445,6 +3477,9 @@ public:
   void work(uint worker_id) {
     G1PrepareRegionsClosure cl(_g1h, this);
     _g1h->heap_region_par_iterate_from_worker_offset(&cl, &_claimer, worker_id);
+
+    MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
+    _all_card_set_stats.add(cl.card_set_stats());
   }
 
   void add_humongous_candidates(uint candidates) {
@@ -3462,6 +3497,10 @@ public:
   uint humongous_total() {
     return _humongous_total;
   }
+
+  G1CardSetMemoryStats all_card_set_stats() const {
+    return _all_card_set_stats;
+  }
 };
 
 void G1CollectedHeap::pre_evacuate_collection_set(G1EvacuationInfo& evacuation_info, G1ParScanThreadStateSet* per_thread_states) {
@@ -3489,6 +3528,8 @@ void G1CollectedHeap::pre_evacuate_collection_set(G1EvacuationInfo& evacuation_i
     G1PrepareEvacuationTask g1_prep_task(this);
     Tickspan task_time = run_task_timed(&g1_prep_task);
 
+    _young_gen_card_set_stats = g1_prep_task.all_card_set_stats();
+
     phase_times()->record_register_regions(task_time.seconds() * 1000.0);
     _num_humongous_objects = g1_prep_task.humongous_total();
     _num_humongous_reclaim_candidates = g1_prep_task.humongous_candidates();
@@ -3763,6 +3804,7 @@ void G1CollectedHeap::post_evacuate_collection_set(G1EvacuationInfo& evacuation_
   evacuation_info.set_bytes_used(_bytes_used_during_gc);
 
   policy()->print_age_table();
+  rem_set()->print_coarsen_stats();
 }
 
 void G1CollectedHeap::record_obj_copy_mem_stats() {

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -27,6 +27,8 @@
 
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1BiasedArray.hpp"
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetFreeMemoryTask.hpp"
 #include "gc/g1/g1CardTable.hpp"
 #include "gc/g1/g1CollectionSet.hpp"
 #include "gc/g1/g1CollectorState.hpp"
@@ -66,6 +68,7 @@
 // Forward declarations
 class HeapRegion;
 class GenerationSpec;
+class G1CardSetFreeMemoryTask;
 class G1ParScanThreadState;
 class G1ParScanThreadStateSet;
 class G1ParScanThreadState;
@@ -158,6 +161,7 @@ class G1CollectedHeap : public CollectedHeap {
 private:
   G1ServiceThread* _service_thread;
   G1ServiceTask* _periodic_gc_task;
+  G1CardSetFreeMemoryTask* _free_card_set_memory_task;
 
   WorkGang* _workers;
   G1CardTable* _card_table;
@@ -173,6 +177,11 @@ private:
   HeapRegionSet _archive_set;
   HeapRegionSet _humongous_set;
 
+  // Young gen memory statistics before GC.
+  G1CardSetMemoryStats _young_gen_card_set_stats;
+  // Collection set candidates memory statistics after GC.
+  G1CardSetMemoryStats _collection_set_candidates_card_set_stats;
+
   void rebuild_free_region_list();
   // Start a new incremental collection set for the next pause.
   void start_new_collection_set();
@@ -267,6 +276,9 @@ public:
 
   bool should_do_eager_reclaim() const;
 
+  bool should_sample_collection_set_candidates() const;
+  void set_collection_set_candidates_stats(G1CardSetMemoryStats& stats);
+
 private:
 
   G1HRPrinter _hr_printer;
@@ -840,6 +852,8 @@ public:
 
   // The g1 remembered set of the heap.
   G1RemSet* _rem_set;
+  // Global card set configuration
+  G1CardSetConfiguration _card_set_config;
 
   void post_evacuate_cleanup_1(G1ParScanThreadStateSet* per_thread_states,
                                G1RedirtyCardsQueueSet* rdcqs);

src/hotspot/share/gc/g1/g1CollectionSet.cpp

@@ -31,7 +31,7 @@
 #include "gc/g1/g1ParScanThreadState.hpp"
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.hpp"
 #include "logging/logStream.hpp"
 #include "runtime/orderAccess.hpp"

src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp

@@ -26,7 +26,7 @@
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectionSetCandidates.hpp"
 #include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/space.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/quickSort.hpp"

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -27,6 +27,7 @@
 #include "classfile/systemDictionary.hpp"
 #include "code/codeCache.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1CollectorState.hpp"
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
@@ -39,7 +40,7 @@
 #include "gc/g1/g1ThreadLocalData.hpp"
 #include "gc/g1/g1Trace.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/shared/gcTimer.hpp"
@@ -1692,7 +1693,7 @@ class G1RemarkThreadsClosure : public ThreadClosure {
         // * Weakly reachable otherwise
         // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
         // live by the SATB invariant but other oops recorded in nmethods may behave differently.
-        thread->as_Java_thread()->nmethods_do(&_code_cl);
+        JavaThread::cast(thread)->nmethods_do(&_code_cl);
       }
     }
   }
@@ -2947,7 +2948,7 @@ G1PrintRegionLivenessInfoClosure::~G1PrintRegionLivenessInfoClosure() {
   }
 
   // add static memory usages to remembered set sizes
-  _total_remset_bytes += HeapRegionRemSet::fl_mem_size() + HeapRegionRemSet::static_mem_size();
+  _total_remset_bytes += G1CardSetFreePool::free_list_pool()->mem_size() + HeapRegionRemSet::static_mem_size();
   // Print the footer of the output.
   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX);
   log_trace(gc, liveness)(G1PPRL_LINE_PREFIX

src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp

@@ -34,7 +34,7 @@
 #include "gc/g1/g1Policy.hpp"
 #include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
 #include "gc/g1/g1RemSetTrackingPolicy.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegion.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/taskqueue.inline.hpp"

src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp

@@ -34,7 +34,7 @@
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "memory/iterator.hpp"
 #include "runtime/atomic.hpp"
@@ -45,8 +45,8 @@
 #include "runtime/thread.inline.hpp"
 #include "runtime/threadSMR.hpp"
 #include "utilities/globalCounter.inline.hpp"
-#include "utilities/lockFreeQueue.inline.hpp"
 #include "utilities/macros.hpp"
+#include "utilities/nonblockingQueue.inline.hpp"
 #include "utilities/pair.hpp"
 #include "utilities/quickSort.hpp"
 #include "utilities/ticks.hpp"
@@ -131,12 +131,12 @@ void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) {
 }
 
 // Thread-safe attempt to remove and return the first buffer from
-// the _completed queue, using the LockFreeQueue::try_pop() underneath.
-// It has a restriction that it may return NULL when there are objects
+// the _completed queue, using the NonblockingQueue::try_pop() underneath.
+// It has a limitation that it may return NULL when there are objects
 // in the queue if there is a concurrent push/append operation.
 BufferNode* G1DirtyCardQueueSet::dequeue_completed_buffer() {
-  using Status = LockFreeQueuePopStatus;
   Thread* current_thread = Thread::current();
+  BufferNode* result = NULL;
   while (true) {
     // Use GlobalCounter critical section to avoid ABA problem.
     // The release of a buffer to its allocator's free list uses
@@ -147,19 +147,7 @@ BufferNode* G1DirtyCardQueueSet::dequeue_completed_buffer() {
     // one CS could defer releasing buffer to the free list for reuse,
     // leading to excessive allocations.
     GlobalCounter::CriticalSection cs(current_thread);
-    Pair<Status, BufferNode*> pop_result = _completed.try_pop();
-    switch (pop_result.first) {
-      case Status::success:
-        return pop_result.second;
-      case Status::operation_in_progress:
-        // Returning NULL instead retrying, in order to mitigate the
-        // chance of spinning for a long time. In the case of getting a
-        // buffer to refine, it is also OK to return NULL when there is
-        // an interfering concurrent push/append operation.
-        return NULL;
-      case Status::lost_race:
-        break;  // Try again.
-    }
+    if (_completed.try_pop(&result)) return result;
   }
 }
 
@@ -177,8 +165,9 @@ BufferNode* G1DirtyCardQueueSet::get_completed_buffer() {
 #ifdef ASSERT
 void G1DirtyCardQueueSet::verify_num_cards() const {
   size_t actual = 0;
-  BufferNode* cur = _completed.top();
-  for ( ; cur != NULL; cur = cur->next()) {
+  for (BufferNode* cur = _completed.first();
+       !_completed.is_end(cur);
+       cur = cur->next()) {
     actual += buffer_size() - cur->index();
   }
   assert(actual == Atomic::load(&_num_cards),

src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp

@@ -32,7 +32,7 @@
 #include "gc/shared/ptrQueue.hpp"
 #include "memory/allocation.hpp"
 #include "memory/padded.hpp"
-#include "utilities/lockFreeQueue.hpp"
+#include "utilities/nonblockingQueue.hpp"
 
 class G1ConcurrentRefineThread;
 class G1DirtyCardQueueSet;
@@ -164,9 +164,9 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
   volatile size_t _num_cards;
   DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
   // Buffers ready for refinement.
-  // LockFreeQueue has inner padding of one cache line.
-  LockFreeQueue<BufferNode, &BufferNode::next_ptr> _completed;
-  // Add a trailer padding after LockFreeQueue.
+  // NonblockingQueue has inner padding of one cache line.
+  NonblockingQueue<BufferNode, &BufferNode::next_ptr> _completed;
+  // Add a trailer padding after NonblockingQueue.
   DEFINE_PAD_MINUS_SIZE(3, DEFAULT_CACHE_LINE_SIZE, sizeof(BufferNode*));
   // Buffers for which refinement is temporarily paused.
   // PausedBuffers has inner padding, including trailer.

src/hotspot/share/gc/g1/g1EvacFailure.cpp

@@ -31,7 +31,7 @@
 #include "gc/g1/g1OopClosures.inline.hpp"
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -31,7 +31,7 @@
 #include "gc/g1/g1FullCollector.inline.hpp"
 #include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
 #include "gc/g1/g1FullGCMarker.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"

src/hotspot/share/gc/g1/g1FullGCScope.cpp

@@ -51,11 +51,6 @@ G1FullGCScope::G1FullGCScope(G1MonitoringSupport* monitoring_support,
 }
 
 G1FullGCScope::~G1FullGCScope() {
-  // We must call G1MonitoringSupport::update_sizes() in the same scoping level
-  // as an active TraceMemoryManagerStats object (i.e. before the destructor for the
-  // TraceMemoryManagerStats is called) so that the G1 memory pools are updated
-  // before any GC notifications are raised.
-  _g1h->g1mm()->update_sizes();
   _g1h->trace_heap_after_gc(&_tracer);
   _g1h->post_full_gc_dump(&_timer);
   _timer.register_gc_end();

src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp

@@ -41,6 +41,8 @@
 #include "utilities/enumIterator.hpp"
 #include "utilities/macros.hpp"
 
+constexpr const char* G1GCPhaseTimes::GCMergeRSWorkItemsStrings[];
+
 G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _max_gc_threads(max_gc_threads),
   _gc_start_counter(0),
@@ -71,16 +73,14 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _gc_par_phases[MergeER] = new WorkerDataArray<double>("MergeER", "Eager Reclaim (ms):", max_gc_threads);
 
   _gc_par_phases[MergeRS] = new WorkerDataArray<double>("MergeRS", "Remembered Sets (ms):", max_gc_threads);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Merged Sparse:", MergeRSMergedSparse);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Merged Fine:", MergeRSMergedFine);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Merged Coarse:", MergeRSMergedCoarse);
-  _gc_par_phases[MergeRS]->create_thread_work_items("Dirty Cards:", MergeRSDirtyCards);
+  for (uint i = 0; i < MergeRSContainersSentinel; i++) {
+    _gc_par_phases[MergeRS]->create_thread_work_items(GCMergeRSWorkItemsStrings[i], i);
+  }
 
   _gc_par_phases[OptMergeRS] = new WorkerDataArray<double>("OptMergeRS", "Optional Remembered Sets (ms):", max_gc_threads);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Merged Sparse:", MergeRSMergedSparse);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Merged Fine:", MergeRSMergedFine);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Merged Coarse:", MergeRSMergedCoarse);
-  _gc_par_phases[OptMergeRS]->create_thread_work_items("Dirty Cards:", MergeRSDirtyCards);
+  for (uint i = 0; i < MergeRSContainersSentinel; i++) {
+    _gc_par_phases[OptMergeRS]->create_thread_work_items(GCMergeRSWorkItemsStrings[i], i);
+  }
 
   _gc_par_phases[MergeLB] = new WorkerDataArray<double>("MergeLB", "Log Buffers (ms):", max_gc_threads);
   if (G1HotCardCache::default_use_cache()) {
@@ -134,6 +134,8 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
   _gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Candidates", EagerlyReclaimNumCandidates);
   _gc_par_phases[EagerlyReclaimHumongousObjects]->create_thread_work_items("Humongous Reclaimed", EagerlyReclaimNumReclaimed);
 
+  _gc_par_phases[SampleCollectionSetCandidates] = new WorkerDataArray<double>("SampleCandidates", "Sample CSet Candidates (ms):", max_gc_threads);
+
   _gc_par_phases[Termination]->create_thread_work_items("Termination Attempts:");
 
   _gc_par_phases[OptTermination]->create_thread_work_items("Optional Termination Attempts:");
@@ -170,6 +172,7 @@ void G1GCPhaseTimes::reset() {
   _recorded_clear_claimed_marks_time_ms = 0.0;
   _recorded_young_cset_choice_time_ms = 0.0;
   _recorded_non_young_cset_choice_time_ms = 0.0;
+  _recorded_sample_collection_set_candidates_time_ms = 0.0;
   _recorded_preserve_cm_referents_time_ms = 0.0;
   _recorded_start_new_cset_time_ms = 0.0;
   _recorded_serial_free_cset_time_ms = 0.0;
@@ -449,6 +452,7 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
                         _recorded_preserve_cm_referents_time_ms +
                         _cur_ref_proc_time_ms +
                         (_weak_phase_times.total_time_sec() * MILLIUNITS) +
+                        _recorded_sample_collection_set_candidates_time_ms +
                         _cur_post_evacuate_cleanup_1_time_ms +
                         _cur_post_evacuate_cleanup_2_time_ms +
                         _recorded_total_rebuild_freelist_time_ms +
@@ -472,6 +476,8 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
     debug_phase(_gc_par_phases[RemoveSelfForwardingPtr], 1);
   }
 
+  debug_time("Sample Collection Set Candidates", _recorded_sample_collection_set_candidates_time_ms);
+  trace_phase(_gc_par_phases[RedirtyCards]);
   debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
   if (G1CollectedHeap::heap()->evacuation_failed()) {
     debug_phase(_gc_par_phases[RecalculateUsed], 1);
@@ -485,6 +491,9 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set() const {
   if (G1CollectedHeap::heap()->should_do_eager_reclaim()) {
     debug_phase(_gc_par_phases[EagerlyReclaimHumongousObjects], 1);
   }
+  if (G1CollectedHeap::heap()->should_sample_collection_set_candidates()) {
+    debug_phase(_gc_par_phases[SampleCollectionSetCandidates], 1);
+  }
   debug_phase(_gc_par_phases[RedirtyCards], 1);
   debug_phase(_gc_par_phases[FreeCollectionSet], 1);
   trace_phase(_gc_par_phases[YoungFreeCSet], true, 1);

src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp

@@ -75,6 +75,7 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     YoungFreeCSet,
     NonYoungFreeCSet,
     RebuildFreeList,
+    SampleCollectionSetCandidates,
     MergePSS,
     RemoveSelfForwardingPtr,
     ClearCardTable,
@@ -97,13 +98,24 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     return GCParPhases(StrongOopStorageSetRoots + index);
   }
 
-  enum GCMergeRSWorkTimes {
-    MergeRSMergedSparse,
-    MergeRSMergedFine,
-    MergeRSMergedCoarse,
-    MergeRSDirtyCards
+  enum GCMergeRSWorkItems : uint {
+    MergeRSMergedInline = 0,
+    MergeRSMergedArrayOfCards,
+    MergeRSMergedHowl,
+    MergeRSMergedFull,
+    MergeRSHowlInline,
+    MergeRSHowlArrayOfCards,
+    MergeRSHowlBitmap,
+    MergeRSHowlFull,
+    MergeRSDirtyCards,
+    MergeRSContainersSentinel
   };
 
+  static constexpr const char* GCMergeRSWorkItemsStrings[MergeRSContainersSentinel] =
+    { "Merged Inline", "Merged ArrayOfCards", "Merged Howl", "Merged Full",
+      "Merged Howl Inline", "Merged Howl ArrayOfCards", "Merged Howl BitMap", "Merged Howl Full",
+      "Dirty Cards" };
+
   enum GCScanHRWorkItems {
     ScanHRScannedCards,
     ScanHRScannedBlocks,
@@ -173,6 +185,8 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
   double _recorded_young_cset_choice_time_ms;
   double _recorded_non_young_cset_choice_time_ms;
 
+  double _recorded_sample_collection_set_candidates_time_ms;
+
   double _recorded_preserve_cm_referents_time_ms;
 
   double _recorded_start_new_cset_time_ms;
@@ -329,6 +343,10 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
     _recorded_non_young_cset_choice_time_ms = time_ms;
   }
 
+  void record_sample_collection_set_candidates_time_ms(double time_ms) {
+    _recorded_sample_collection_set_candidates_time_ms = time_ms;
+  }
+
   void record_preserve_cm_referents_time_ms(double time_ms) {
     _recorded_preserve_cm_referents_time_ms = time_ms;
   }

src/hotspot/share/gc/g1/g1HeapVerifier.cpp

@@ -467,7 +467,7 @@ void G1HeapVerifier::enable_verification_type(G1VerifyType type) {
 }
 
 bool G1HeapVerifier::should_verify(G1VerifyType type) {
-  return (_enabled_verification_types & type) == type;
+  return (_enabled_verification_types & type) != 0;
 }
 
 void G1HeapVerifier::verify(VerifyOption vo) {

src/hotspot/share/gc/g1/g1HeapVerifier.hpp

@@ -45,9 +45,10 @@ public:
     G1VerifyYoungNormal     =  1, // -XX:VerifyGCType=young-normal
     G1VerifyConcurrentStart =  2, // -XX:VerifyGCType=concurrent-start
     G1VerifyMixed           =  4, // -XX:VerifyGCType=mixed
-    G1VerifyRemark          =  8, // -XX:VerifyGCType=remark
-    G1VerifyCleanup         = 16, // -XX:VerifyGCType=cleanup
-    G1VerifyFull            = 32, // -XX:VerifyGCType=full
+    G1VerifyYoungEvacFail   =  8, // -XX:VerifyGCType=young-evac-fail
+    G1VerifyRemark          = 16, // -XX:VerifyGCType=remark
+    G1VerifyCleanup         = 32, // -XX:VerifyGCType=cleanup
+    G1VerifyFull            = 64, // -XX:VerifyGCType=full
     G1VerifyAll             = -1
   };
 

src/hotspot/share/gc/g1/g1MonitoringSupport.cpp

@@ -343,7 +343,12 @@ MemoryUsage G1MonitoringSupport::old_gen_memory_usage(size_t initial_size, size_
 }
 
 G1MonitoringScope::G1MonitoringScope(G1MonitoringSupport* g1mm, bool full_gc, bool all_memory_pools_affected) :
+  _g1mm(g1mm),
   _tcs(full_gc ? g1mm->_full_collection_counters : g1mm->_incremental_collection_counters),
   _tms(full_gc ? &g1mm->_full_gc_memory_manager : &g1mm->_incremental_memory_manager,
        G1CollectedHeap::heap()->gc_cause(), all_memory_pools_affected) {
 }
+
+G1MonitoringScope::~G1MonitoringScope() {
+  _g1mm->update_sizes();
+}

src/hotspot/share/gc/g1/g1MonitoringSupport.hpp

@@ -238,10 +238,12 @@ public:
 
 // Scope object for java.lang.management support.
 class G1MonitoringScope : public StackObj {
+  G1MonitoringSupport* _g1mm;
   TraceCollectorStats _tcs;
   TraceMemoryManagerStats _tms;
 public:
   G1MonitoringScope(G1MonitoringSupport* g1mm, bool full_gc, bool all_memory_pools_affected);
+  ~G1MonitoringScope();
 };
 
 #endif // SHARE_GC_G1_G1MONITORINGSUPPORT_HPP

src/hotspot/share/gc/g1/g1OopClosures.inline.hpp

@@ -32,7 +32,7 @@
 #include "gc/g1/g1ParScanThreadState.inline.hpp"
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"

src/hotspot/share/gc/g1/g1ParScanThreadState.hpp

@@ -30,7 +30,7 @@
 #include "gc/g1/g1RedirtyCardsQueue.hpp"
 #include "gc/g1/g1OopClosures.hpp"
 #include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/ageTable.hpp"
 #include "gc/shared/partialArrayTaskStepper.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"

src/hotspot/share/gc/g1/g1Policy.cpp

@@ -40,7 +40,7 @@
 #include "gc/g1/g1SurvivorRegions.hpp"
 #include "gc/g1/g1YoungGenSizer.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/shared/concurrentGCBreakpoints.hpp"
 #include "gc/shared/gcPolicyCounters.hpp"
 #include "logging/log.hpp"
@@ -50,6 +50,8 @@
 #include "utilities/growableArray.hpp"
 #include "utilities/pair.hpp"
 
+#include "gc/shared/gcTraceTime.inline.hpp"
+
 G1Policy::G1Policy(STWGCTimer* gc_timer) :
   _predictor(G1ConfidencePercent / 100.0),
   _analytics(new G1Analytics(&_predictor)),

src/hotspot/share/gc/g1/g1RemSet.cpp

@@ -26,6 +26,7 @@
 #include "gc/g1/g1BarrierSet.hpp"
 #include "gc/g1/g1BatchedGangTask.hpp"
 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
+#include "gc/g1/g1CardSet.inline.hpp"
 #include "gc/g1/g1CardTable.inline.hpp"
 #include "gc/g1/g1CardTableEntryClosure.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
@@ -45,7 +46,6 @@
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/sparsePRT.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/ptrQueue.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
@@ -381,9 +381,7 @@ public:
            "Trying to access index " SIZE_FORMAT " out of bounds " SIZE_FORMAT,
            card_idx >> _scan_chunks_shift, _num_total_scan_chunks);
     size_t const chunk_idx = card_idx >> _scan_chunks_shift;
-    if (!_region_scan_chunks[chunk_idx]) {
-      _region_scan_chunks[chunk_idx] = true;
-    }
+    _region_scan_chunks[chunk_idx] = true;
   }
 
   G1AbstractSubTask* create_cleanup_after_scan_heap_roots_task() {
@@ -1061,9 +1059,9 @@ void G1RemSet::prepare_for_scan_heap_roots() {
   _scan_state->prepare();
 }
 
-// Small ring buffer used for prefetching cards for read/write from the card
+// Small ring buffer used for prefetching cards for write from the card
 // table during GC.
-template <class T, bool for_write>
+template <class T>
 class G1MergeHeapRootsPrefetchCache {
 public:
   static const uint CacheSize = G1MergeHeapRootsPrefetchCacheSize;
@@ -1096,11 +1094,7 @@ public:
   }
 
   T* push(T* elem) {
-    if (for_write) {
-      Prefetch::write(elem, 0);
-    } else {
-      Prefetch::read(elem, 0);
-    }
+    Prefetch::write(elem, 0);
     T* result = _cache[_cur_cache_idx];
     _cache[_cur_cache_idx++] = elem;
     _cur_cache_idx &= (CacheSize - 1);
@@ -1111,7 +1105,33 @@ public:
 
 class G1MergeHeapRootsTask : public AbstractGangTask {
 
-  // Visitor for remembered sets, dropping entries onto the card table.
+  class G1MergeCardSetStats {
+    size_t _merged[G1GCPhaseTimes::MergeRSContainersSentinel];
+
+  public:
+    G1MergeCardSetStats() {
+      for (uint i = 0; i < ARRAY_SIZE(_merged); i++) {
+        _merged[i] = 0;
+      }
+    }
+
+    void inc_card_set_merged(uint tag) {
+      assert(tag < ARRAY_SIZE(_merged), "tag out of bounds %u", tag);
+      _merged[tag]++;
+    }
+
+    void inc_cards_dirty(size_t increment = 1) {
+      _merged[G1GCPhaseTimes::MergeRSDirtyCards] += increment;
+    }
+
+    size_t merged(uint i) const { return _merged[i]; }
+  };
+
+  // Visitor for remembered sets. Several methods of it are called by a region's
+  // card set iterator to drop card set remembered set entries onto the card.
+  // table. This is in addition to being the HeapRegionClosure to iterate over
+  // all region's remembered sets.
+  //
   // We add a small prefetching cache in front of the actual work as dropping
   // onto the card table is basically random memory access. This improves
   // performance of this operation significantly.
@@ -1121,24 +1141,20 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
     G1RemSetScanState* _scan_state;
     G1CardTable* _ct;
 
-    uint _merged_sparse;
-    uint _merged_fine;
-    uint _merged_coarse;
-
-    size_t _cards_dirty;
+    G1MergeCardSetStats _stats;
 
     // Cached card table index of the currently processed region to avoid constant
     // recalculation as our remembered set containers are per region.
     size_t _region_base_idx;
 
-    class G1MergeCardSetCache : public G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, true> {
+    class G1MergeCardSetCache : public G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue> {
       G1MergeCardSetClosure* const _merge_card_cl;
 
     public:
       G1MergeCardSetCache(G1MergeCardSetClosure* const merge_card_cl) :
         // Initially set dummy card value to Dirty to avoid any actual mark work if we
         // try to process it.
-        G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, true>(G1CardTable::dirty_card_val()),
+        G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue>(G1CardTable::dirty_card_val()),
         _merge_card_cl(merge_card_cl) { }
 
       ~G1MergeCardSetCache() {
@@ -1160,26 +1176,19 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
 
     void mark_card(G1CardTable::CardValue* value) {
       if (_ct->mark_clean_as_dirty(value)) {
-        _cards_dirty++;
+        _stats.inc_cards_dirty();
         _scan_state->set_chunk_dirty(_ct->index_for_cardvalue(value));
       }
     }
 
-    void start_iterate(uint const region_idx) {
-      _region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
-    }
-
   public:
+
     G1MergeCardSetClosure(G1RemSetScanState* scan_state) :
       _scan_state(scan_state),
       _ct(G1CollectedHeap::heap()->card_table()),
-      _merged_sparse(0),
-      _merged_fine(0),
-      _merged_coarse(0),
-      _cards_dirty(0),
+      _stats(),
       _region_base_idx(0),
-      _merge_card_set_cache(this) {
-    }
+      _merge_card_set_cache(this) { }
 
     void do_card(uint const card_idx) {
       G1CardTable::CardValue* to_prefetch = _ct->byte_for_index(_region_base_idx + card_idx);
@@ -1188,47 +1197,26 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       mark_card(to_process);
     }
 
-    void next_coarse_prt(uint const region_idx) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
+    // Returns whether the given region actually needs iteration.
+    bool start_iterate(uint const tag, uint const region_idx) {
+      assert(tag < G1GCPhaseTimes::MergeRSDirtyCards, "invalid tag %u", tag);
+      if (remember_if_interesting(region_idx)) {
+        _region_base_idx = (size_t)region_idx << HeapRegion::LogCardsPerRegion;
+        _stats.inc_card_set_merged(tag);
+        return true;
       }
+      return false;
+    }
 
-      _merged_coarse++;
-
-      start_iterate(region_idx);
-      _cards_dirty += _ct->mark_region_dirty(_region_base_idx, HeapRegion::CardsPerRegion);
+    void do_card_range(uint const start_card_idx, uint const length) {
+      assert(start_card_idx == 0, "must be");
+      assert(length == HeapRegion::CardsPerRegion, "must be");
+      size_t num_dirtied = _ct->mark_range_dirty(_region_base_idx, HeapRegion::CardsPerRegion);
+      _stats.inc_cards_dirty(num_dirtied);
       _scan_state->set_chunk_region_dirty(_region_base_idx);
     }
 
-    void next_fine_prt(uint const region_idx, BitMap* bm) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_fine++;
-
-      start_iterate(region_idx);
-      BitMap::idx_t cur = bm->get_next_one_offset(0);
-      while (cur != bm->size()) {
-        do_card((uint)cur);
-        cur = bm->get_next_one_offset(cur + 1);
-      }
-    }
-
-    void next_sparse_prt(uint const region_idx, SparsePRTEntry::card_elem_t* cards, uint const num_cards) {
-      if (!remember_if_interesting(region_idx)) {
-        return;
-      }
-
-      _merged_sparse++;
-
-      start_iterate(region_idx);
-      for (uint i = 0; i < num_cards; i++) {
-        do_card(cards[i]);
-      }
-    }
-
-    // Helper to put the remembered set cards for these regions onto the card
+    // Helper to merge the cards in the card set for the given region onto the card
     // table.
     //
     // Called directly for humongous starts regions because we should not add
@@ -1242,12 +1230,12 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
     // pass) or the "dirty" list will be merged with the "all" list later otherwise.
     // (And there is no problem either way if the region does not contain dirty
     // cards).
-    void dump_rem_set_for_region(HeapRegion* r) {
+    void merge_card_set_for_region(HeapRegion* r) {
       assert(r->in_collection_set() || r->is_starts_humongous(), "must be");
 
       HeapRegionRemSet* rem_set = r->rem_set();
       if (!rem_set->is_empty()) {
-        rem_set->iterate_prts(*this);
+        rem_set->iterate_for_merge(*this);
       }
     }
 
@@ -1255,25 +1243,22 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       assert(r->in_collection_set(), "must be");
 
       _scan_state->add_all_dirty_region(r->hrm_index());
-      dump_rem_set_for_region(r);
+      merge_card_set_for_region(r);
 
       return false;
     }
 
-    size_t merged_sparse() const { return _merged_sparse; }
-    size_t merged_fine() const { return _merged_fine; }
-    size_t merged_coarse() const { return _merged_coarse; }
-
-    size_t cards_dirty() const { return _cards_dirty; }
+    G1MergeCardSetStats stats() const { return _stats; }
   };
 
   // Visitor for the remembered sets of humongous candidate regions to merge their
   // remembered set into the card table.
   class G1FlushHumongousCandidateRemSets : public HeapRegionClosure {
-    G1MergeCardSetClosure _cl;
+    G1RemSetScanState* _scan_state;
+    G1MergeCardSetStats _merge_stats;
 
   public:
-    G1FlushHumongousCandidateRemSets(G1RemSetScanState* scan_state) : _cl(scan_state) { }
+    G1FlushHumongousCandidateRemSets(G1RemSetScanState* scan_state) : _scan_state(scan_state), _merge_stats() { }
 
     virtual bool do_heap_region(HeapRegion* r) {
       G1CollectedHeap* g1h = G1CollectedHeap::heap();
@@ -1287,7 +1272,12 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       guarantee(r->rem_set()->occupancy_less_or_equal_than(G1EagerReclaimRemSetThreshold),
                 "Found a not-small remembered set here. This is inconsistent with previous assumptions.");
 
-      _cl.dump_rem_set_for_region(r);
+      G1MergeCardSetStats stats;
+      {
+        G1MergeCardSetClosure cl(_scan_state);
+        cl.merge_card_set_for_region(r);
+        stats = cl.stats();
+      }
 
       // We should only clear the card based remembered set here as we will not
       // implicitly rebuild anything else during eager reclaim. Note that at the moment
@@ -1307,16 +1297,11 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       return false;
     }
 
-    size_t merged_sparse() const { return _cl.merged_sparse(); }
-    size_t merged_fine() const { return _cl.merged_fine(); }
-    size_t merged_coarse() const { return _cl.merged_coarse(); }
-
-    size_t cards_dirty() const { return _cl.cards_dirty(); }
+    size_t merged(uint i) const { return _merge_stats.merged(i); }
   };
 
   // Visitor for the log buffer entries to merge them into the card table.
   class G1MergeLogBufferCardsClosure : public G1CardTableEntryClosure {
-    friend class G1MergeLogBufferCardsCache;
 
     G1RemSetScanState* _scan_state;
     G1CardTable* _ct;
@@ -1324,23 +1309,6 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
     size_t _cards_dirty;
     size_t _cards_skipped;
 
-    class G1MergeLogBufferCardsCache : public G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, false> {
-      G1MergeLogBufferCardsClosure* const _merge_log_buffer_cl;
-
-    public:
-      G1MergeLogBufferCardsCache(G1MergeLogBufferCardsClosure* const merge_log_buffer_cl) :
-        // Initially set dummy card value to Clean to avoid any actual work if we
-        // try to process it.
-        G1MergeHeapRootsPrefetchCache<G1CardTable::CardValue, false>(G1CardTable::clean_card_val()),
-        _merge_log_buffer_cl(merge_log_buffer_cl) { }
-
-      ~G1MergeLogBufferCardsCache() {
-        for (uint i = 0; i < CacheSize; i++) {
-          _merge_log_buffer_cl->process_card(push(&_dummy_card));
-        }
-      }
-    } _merge_log_buffer_cache;
-
     void process_card(CardValue* card_ptr) {
       if (*card_ptr == G1CardTable::dirty_card_val()) {
         uint const region_idx = _ct->region_idx_for(card_ptr);
@@ -1355,8 +1323,7 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       _scan_state(scan_state),
       _ct(g1h->card_table()),
       _cards_dirty(0),
-      _cards_skipped(0),
-      _merge_log_buffer_cache(this)
+      _cards_skipped(0)
     {}
 
     void do_card_ptr(CardValue* card_ptr, uint worker_id) {
@@ -1373,8 +1340,7 @@ class G1MergeHeapRootsTask : public AbstractGangTask {
       // This code may count duplicate entries in the log buffers (even if rare) multiple
       // times.
       if (_scan_state->contains_cards_to_process(region_idx)) {
-        CardValue* to_process = _merge_log_buffer_cache.push(card_ptr);
-        process_card(to_process);
+        process_card(card_ptr);
       } else {
         // We may have had dirty cards in the (initial) collection set (or the
         // young regions which are always in the initial collection set). We do
@@ -1442,22 +1408,24 @@ public:
       G1FlushHumongousCandidateRemSets cl(_scan_state);
       g1h->heap_region_iterate(&cl);
 
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeRSDirtyCards);
+      for (uint i = 0; i < G1GCPhaseTimes::MergeRSContainersSentinel; i++) {
+        p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged(i), i);
+      }
     }
 
     // Merge remembered sets of current candidates.
     {
       G1GCParPhaseTimesTracker x(p, merge_remset_phase, worker_id, _initial_evacuation /* must_record */);
-      G1MergeCardSetClosure cl(_scan_state);
-      g1h->collection_set_iterate_increment_from(&cl, &_hr_claimer, worker_id);
+      G1MergeCardSetStats stats;
+      {
+        G1MergeCardSetClosure cl(_scan_state);
+        g1h->collection_set_iterate_increment_from(&cl, &_hr_claimer, worker_id);
+        stats = cl.stats();
+      }
 
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_sparse(), G1GCPhaseTimes::MergeRSMergedSparse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_fine(), G1GCPhaseTimes::MergeRSMergedFine);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
-      p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeRSDirtyCards);
+      for (uint i = 0; i < G1GCPhaseTimes::MergeRSContainersSentinel; i++) {
+        p->record_or_add_thread_work_item(merge_remset_phase, worker_id, stats.merged(i), i);
+      }
     }
 
     // Apply closure to log entries in the HCC.
@@ -1486,20 +1454,25 @@ public:
 };
 
 void G1RemSet::print_merge_heap_roots_stats() {
-  size_t num_visited_cards = _scan_state->num_visited_cards();
+  LogTarget(Debug, gc, remset) lt;
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
 
-  size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
+    size_t num_visited_cards = _scan_state->num_visited_cards();
 
-  G1CollectedHeap* g1h = G1CollectedHeap::heap();
-  size_t total_old_region_cards =
-    (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
+    size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
 
-  log_debug(gc,remset)("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
-                       num_visited_cards,
-                       total_dirty_region_cards,
-                       percent_of(num_visited_cards, total_dirty_region_cards),
-                       total_old_region_cards,
-                       percent_of(num_visited_cards, total_old_region_cards));
+    G1CollectedHeap* g1h = G1CollectedHeap::heap();
+    size_t total_old_region_cards =
+      (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
+
+    ls.print_cr("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
+                num_visited_cards,
+                total_dirty_region_cards,
+                percent_of(num_visited_cards, total_dirty_region_cards),
+                total_old_region_cards,
+                percent_of(num_visited_cards, total_old_region_cards));
+  }
 }
 
 void G1RemSet::merge_heap_roots(bool initial_evacuation) {
@@ -1531,9 +1504,7 @@ void G1RemSet::merge_heap_roots(bool initial_evacuation) {
     workers->run_task(&cl, num_workers);
   }
 
-  if (log_is_enabled(Debug, gc, remset)) {
-    print_merge_heap_roots_stats();
-  }
+  print_merge_heap_roots_stats();
 }
 
 void G1RemSet::complete_evac_phase(bool has_more_than_one_evacuation_phase) {
@@ -1548,6 +1519,15 @@ G1AbstractSubTask* G1RemSet::create_cleanup_after_scan_heap_roots_task() {
   return _scan_state->create_cleanup_after_scan_heap_roots_task();
 }
 
+void G1RemSet::print_coarsen_stats() {
+  LogTarget(Debug, gc, remset) lt;
+  if (lt.is_enabled()) {
+    LogStream ls(lt);
+
+    G1CardSet::print_coarsen_stats(&ls);
+  }
+}
+
 inline void check_card_ptr(CardTable::CardValue* card_ptr, G1CardTable* ct) {
 #ifdef ASSERT
   G1CollectedHeap* g1h = G1CollectedHeap::heap();

src/hotspot/share/gc/g1/g1RemSet.hpp

@@ -107,6 +107,10 @@ public:
   // Prepare for and cleanup after scanning the heap roots. Must be called
   // once before and after in sequential code.
   void prepare_for_scan_heap_roots();
+  // Cleans the card table from temporary duplicate detection information.
+  void cleanup_after_scan_heap_roots();
+  // Print coarsening stats.
+  void print_coarsen_stats();
   // Creates a gang task for cleaining up temporary data structures and the
   // card table, removing temporary duplicate detection information.
   G1AbstractSubTask* create_cleanup_after_scan_heap_roots_task();

src/hotspot/share/gc/g1/g1RemSetSummary.cpp

@@ -24,6 +24,7 @@
 
 #include "precompiled.hpp"
 #include "gc/g1/g1BarrierSet.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/g1ConcurrentRefineThread.hpp"
@@ -31,7 +32,7 @@
 #include "gc/g1/g1RemSet.hpp"
 #include "gc/g1/g1RemSetSummary.hpp"
 #include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/iterator.hpp"
 #include "runtime/thread.inline.hpp"
@@ -48,9 +49,10 @@ void G1RemSetSummary::update() {
       _counter++;
     }
   } collector(this);
+
   G1CollectedHeap* g1h = G1CollectedHeap::heap();
   g1h->concurrent_refine()->threads_do(&collector);
-  _num_coarsenings = HeapRegionRemSet::n_coarsenings();
+  _coarsenings = HeapRegionRemSet::coarsen_stats();
 
   set_sampling_task_vtime(g1h->rem_set()->sampling_task_vtime());
 }
@@ -68,7 +70,7 @@ double G1RemSetSummary::rs_thread_vtime(uint thread) const {
 }
 
 G1RemSetSummary::G1RemSetSummary(bool should_update) :
-  _num_coarsenings(0),
+  _coarsenings(),
   _num_vtimes(G1ConcurrentRefine::max_num_threads()),
   _rs_threads_vtimes(NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC)),
   _sampling_task_vtime(0.0f) {
@@ -88,7 +90,7 @@ void G1RemSetSummary::set(G1RemSetSummary* other) {
   assert(other != NULL, "just checking");
   assert(_num_vtimes == other->_num_vtimes, "just checking");
 
-  _num_coarsenings = other->num_coarsenings();
+  _coarsenings = other->_coarsenings;
 
   memcpy(_rs_threads_vtimes, other->_rs_threads_vtimes, sizeof(double) * _num_vtimes);
 
@@ -99,7 +101,7 @@ void G1RemSetSummary::subtract_from(G1RemSetSummary* other) {
   assert(other != NULL, "just checking");
   assert(_num_vtimes == other->_num_vtimes, "just checking");
 
-  _num_coarsenings = other->num_coarsenings() - _num_coarsenings;
+  _coarsenings.subtract_from(other->_coarsenings);
 
   for (uint i = 0; i < _num_vtimes; i++) {
     set_rs_thread_vtime(i, other->rs_thread_vtime(i) - rs_thread_vtime(i));
@@ -112,9 +114,11 @@ class RegionTypeCounter {
 private:
   const char* _name;
 
+  size_t _rs_wasted_mem_size;
   size_t _rs_mem_size;
   size_t _cards_occupied;
   size_t _amount;
+  size_t _amount_tracked;
 
   size_t _code_root_mem_size;
   size_t _code_root_elems;
@@ -136,21 +140,25 @@ private:
   }
 
   size_t amount() const { return _amount; }
+  size_t amount_tracked() const { return _amount_tracked; }
 
 public:
 
-  RegionTypeCounter(const char* name) : _name(name), _rs_mem_size(0), _cards_occupied(0),
-    _amount(0), _code_root_mem_size(0), _code_root_elems(0) { }
+  RegionTypeCounter(const char* name) : _name(name), _rs_wasted_mem_size(0), _rs_mem_size(0), _cards_occupied(0),
+    _amount(0), _amount_tracked(0), _code_root_mem_size(0), _code_root_elems(0) { }
 
-  void add(size_t rs_mem_size, size_t cards_occupied, size_t code_root_mem_size,
-    size_t code_root_elems) {
+  void add(size_t rs_wasted_mem_size, size_t rs_mem_size, size_t cards_occupied,
+           size_t code_root_mem_size, size_t code_root_elems, bool tracked) {
+    _rs_wasted_mem_size += rs_wasted_mem_size;
     _rs_mem_size += rs_mem_size;
     _cards_occupied += cards_occupied;
     _code_root_mem_size += code_root_mem_size;
     _code_root_elems += code_root_elems;
     _amount++;
+    _amount_tracked += tracked ? 1 : 0;
   }
 
+  size_t rs_wasted_mem_size() const { return _rs_wasted_mem_size; }
   size_t rs_mem_size() const { return _rs_mem_size; }
   size_t cards_occupied() const { return _cards_occupied; }
 
@@ -158,15 +166,18 @@ public:
   size_t code_root_elems() const { return _code_root_elems; }
 
   void print_rs_mem_info_on(outputStream * out, size_t total) {
-    out->print_cr("    " SIZE_FORMAT_W(8) "%s (%5.1f%%) by " SIZE_FORMAT " %s regions",
-        byte_size_in_proper_unit(rs_mem_size()),
-        proper_unit_for_byte_size(rs_mem_size()),
-        rs_mem_size_percent_of(total), amount(), _name);
+    out->print_cr("    " SIZE_FORMAT_W(8) " (%5.1f%%) by " SIZE_FORMAT " "
+                  "(" SIZE_FORMAT ") %s regions wasted " SIZE_FORMAT,
+                  rs_mem_size(), rs_mem_size_percent_of(total),
+                  amount_tracked(), amount(),
+                  _name, rs_wasted_mem_size());
   }
 
   void print_cards_occupied_info_on(outputStream * out, size_t total) {
-    out->print_cr("     " SIZE_FORMAT_W(8) " (%5.1f%%) entries by " SIZE_FORMAT " %s regions",
-        cards_occupied(), cards_occupied_percent_of(total), amount(), _name);
+    out->print_cr("     " SIZE_FORMAT_W(8) " (%5.1f%%) entries by " SIZE_FORMAT " "
+                  "(" SIZE_FORMAT ") %s regions",
+                  cards_occupied(), cards_occupied_percent_of(total),
+                  amount_tracked(), amount(), _name);
   }
 
   void print_code_root_mem_info_on(outputStream * out, size_t total) {
@@ -195,6 +206,7 @@ private:
   size_t _max_rs_mem_sz;
   HeapRegion* _max_rs_mem_sz_region;
 
+  size_t total_rs_wasted_mem_sz() const     { return _all.rs_wasted_mem_size(); }
   size_t total_rs_mem_sz() const            { return _all.rs_mem_size(); }
   size_t total_cards_occupied() const       { return _all.cards_occupied(); }
 
@@ -222,6 +234,7 @@ public:
 
     // HeapRegionRemSet::mem_size() includes the
     // size of the strong code roots
+    size_t rs_wasted_mem_sz = hrrs->wasted_mem_size();
     size_t rs_mem_sz = hrrs->mem_size();
     if (rs_mem_sz > _max_rs_mem_sz) {
       _max_rs_mem_sz = rs_mem_sz;
@@ -249,8 +262,10 @@ public:
     } else {
       ShouldNotReachHere();
     }
-    current->add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems);
-    _all.add(rs_mem_sz, occupied_cards, code_root_mem_sz, code_root_elems);
+    current->add(rs_wasted_mem_sz, rs_mem_sz, occupied_cards,
+                 code_root_mem_sz, code_root_elems, r->rem_set()->is_tracked());
+    _all.add(rs_wasted_mem_sz, rs_mem_sz, occupied_cards,
+             code_root_mem_sz, code_root_elems, r->rem_set()->is_tracked());
 
     return false;
   }
@@ -259,23 +274,15 @@ public:
     RegionTypeCounter* counters[] = { &_young, &_humongous, &_free, &_old, &_archive, NULL };
 
     out->print_cr(" Current rem set statistics");
-    out->print_cr("  Total per region rem sets sizes = " SIZE_FORMAT "%s."
-                  " Max = " SIZE_FORMAT "%s.",
-                  byte_size_in_proper_unit(total_rs_mem_sz()),
-                  proper_unit_for_byte_size(total_rs_mem_sz()),
-                  byte_size_in_proper_unit(max_rs_mem_sz()),
-                  proper_unit_for_byte_size(max_rs_mem_sz()));
+    out->print_cr("  Total per region rem sets sizes = " SIZE_FORMAT
+                  " Max = " SIZE_FORMAT " wasted = " SIZE_FORMAT,
+                  total_rs_mem_sz(),
+                  max_rs_mem_sz(),
+                  total_rs_wasted_mem_sz());
     for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) {
       (*current)->print_rs_mem_info_on(out, total_rs_mem_sz());
     }
 
-    out->print_cr("   Static structures = " SIZE_FORMAT "%s,"
-                  " free_lists = " SIZE_FORMAT "%s.",
-                  byte_size_in_proper_unit(HeapRegionRemSet::static_mem_size()),
-                  proper_unit_for_byte_size(HeapRegionRemSet::static_mem_size()),
-                  byte_size_in_proper_unit(HeapRegionRemSet::fl_mem_size()),
-                  proper_unit_for_byte_size(HeapRegionRemSet::fl_mem_size()));
-
     out->print_cr("    " SIZE_FORMAT " occupied cards represented.",
                   total_cards_occupied());
     for (RegionTypeCounter** current = &counters[0]; *current != NULL; current++) {
@@ -285,12 +292,14 @@ public:
     // Largest sized rem set region statistics
     HeapRegionRemSet* rem_set = max_rs_mem_sz_region()->rem_set();
     out->print_cr("    Region with largest rem set = " HR_FORMAT ", "
-                  "size = " SIZE_FORMAT "%s, occupied = " SIZE_FORMAT "%s.",
+                  "size = " SIZE_FORMAT " occupied = " SIZE_FORMAT,
                   HR_FORMAT_PARAMS(max_rs_mem_sz_region()),
-                  byte_size_in_proper_unit(rem_set->mem_size()),
-                  proper_unit_for_byte_size(rem_set->mem_size()),
-                  byte_size_in_proper_unit(rem_set->occupied()),
-                  proper_unit_for_byte_size(rem_set->occupied()));
+                  rem_set->mem_size(),
+                  rem_set->occupied());
+
+    HeapRegionRemSet::print_static_mem_size(out);
+    G1CardSetFreePool::free_list_pool()->print_on(out);
+
     // Strong code root statistics
     HeapRegionRemSet* max_code_root_rem_set = max_code_root_mem_sz_region()->rem_set();
     out->print_cr("  Total heap region code root sets sizes = " SIZE_FORMAT "%s."
@@ -319,7 +328,8 @@ public:
 };
 
 void G1RemSetSummary::print_on(outputStream* out) {
-  out->print_cr("  Did " SIZE_FORMAT " coarsenings.", num_coarsenings());
+  out->print("Coarsening: ");
+  _coarsenings.print_on(out);
   out->print_cr("  Concurrent refinement threads times (s)");
   out->print("     ");
   for (uint i = 0; i < _num_vtimes; i++) {

src/hotspot/share/gc/g1/g1RemSetSummary.hpp

@@ -25,6 +25,7 @@
 #ifndef SHARE_GC_G1_G1REMSETSUMMARY_HPP
 #define SHARE_GC_G1_G1REMSETSUMMARY_HPP
 
+#include "gc/g1/g1CardSet.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/ostream.hpp"
 
@@ -33,7 +34,7 @@ class G1RemSet;
 // A G1RemSetSummary manages statistical information about the G1RemSet
 
 class G1RemSetSummary {
-  size_t _num_coarsenings;
+  G1CardSetCoarsenStats _coarsenings;
 
   size_t _num_vtimes;
   double* _rs_threads_vtimes;
@@ -65,10 +66,6 @@ public:
   double sampling_task_vtime() const {
     return _sampling_task_vtime;
   }
-
-  size_t num_coarsenings() const {
-    return _num_coarsenings;
-  }
 };
 
 #endif // SHARE_GC_G1_G1REMSETSUMMARY_HPP

src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp

@@ -26,7 +26,7 @@
 #include "gc/g1/g1CollectionSetChooser.hpp"
 #include "gc/g1/g1RemSetTrackingPolicy.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "runtime/safepoint.hpp"
 
 bool G1RemSetTrackingPolicy::needs_scan_for_rebuild(HeapRegion* r) const {

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp

@@ -25,8 +25,10 @@
 #include "precompiled.hpp"
 
 #include "compiler/oopMap.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CardTableEntryClosure.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1CollectionSetCandidates.hpp"
 #include "gc/g1/g1ConcurrentMark.inline.hpp"
 #include "gc/g1/g1EvacStats.inline.hpp"
 #include "gc/g1/g1ParScanThreadState.hpp"
@@ -41,6 +43,9 @@ G1PostEvacuateCollectionSetCleanupTask1::G1PostEvacuateCollectionSetCleanupTask1
 {
   add_serial_task(new MergePssTask(per_thread_states));
   add_serial_task(new RecalculateUsedTask());
+  if (SampleCollectionSetCandidatesTask::should_execute()) {
+    add_serial_task(new SampleCollectionSetCandidatesTask());
+  }
   if (RemoveSelfForwardPtrsTask::should_execute()) {
     add_parallel_task(new RemoveSelfForwardPtrsTask(rdcqs));
   }
@@ -64,6 +69,32 @@ void G1PostEvacuateCollectionSetCleanupTask1::RecalculateUsedTask::do_work(uint
   G1CollectedHeap::heap()->update_used_after_gc();
 }
 
+bool G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask::should_execute() {
+  return G1CollectedHeap::heap()->should_sample_collection_set_candidates();
+}
+
+double G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask::worker_cost() const {
+  return should_execute() ? 1.0 : AlmostNoWork;
+}
+
+class G1SampleCollectionSetCandidatesClosure : public HeapRegionClosure {
+public:
+  G1CardSetMemoryStats _total;
+
+  bool do_heap_region(HeapRegion* r) override {
+    _total.add(r->rem_set()->card_set_memory_stats());
+    return false;
+  }
+};
+
+void G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask::do_work(uint worker_id) {
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+  G1SampleCollectionSetCandidatesClosure cl;
+  g1h->collection_set()->candidates()->iterate(&cl);
+  g1h->set_collection_set_candidates_stats(cl._total);
+}
+
 bool G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask::should_execute() {
   return G1CollectedHeap::heap()->evacuation_failed();
 }

src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp

@@ -38,11 +38,13 @@ class G1RedirtyCardsQueueSet;
 // First set of post evacuate collection set tasks containing ("s" means serial):
 // - Merge PSS (s)
 // - Recalculate Used (s)
+// - Sample Collection Set Candidates (s)
 // - Remove Self Forwards (on evacuation failure)
 // - Clear Card Table
 class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedGangTask {
   class MergePssTask;
   class RecalculateUsedTask;
+  class SampleCollectionSetCandidatesTask;
   class RemoveSelfForwardPtrsTask;
 
 public:
@@ -68,6 +70,16 @@ public:
   void do_work(uint worker_id) override;
 };
 
+class G1PostEvacuateCollectionSetCleanupTask1::SampleCollectionSetCandidatesTask : public G1AbstractSubTask {
+public:
+  SampleCollectionSetCandidatesTask() : G1AbstractSubTask(G1GCPhaseTimes::SampleCollectionSetCandidates) { }
+
+  static bool should_execute();
+
+  double worker_cost() const override;
+  void do_work(uint worker_id) override;
+};
+
 class G1PostEvacuateCollectionSetCleanupTask1::RemoveSelfForwardPtrsTask : public G1AbstractSubTask {
   G1ParRemoveSelfForwardPtrsTask _task;
 

src/hotspot/share/gc/g1/g1_globals.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -130,15 +130,16 @@
           range(0, max_jint)                                                \
                                                                             \
   product(size_t, G1ConcRefinementThresholdStep, 2,                         \
-          "Each time the rset update queue increases by this amount "       \
-          "activate the next refinement thread if available. "              \
+          "Each time the remembered set update queue increases by this "    \
+          "amount activate the next refinement thread if available. "       \
           "The actual step size will be selected ergonomically by "         \
           "default, with this value used to determine a lower bound.")      \
           range(1, SIZE_MAX)                                                \
                                                                             \
   product(intx, G1RSetUpdatingPauseTimePercent, 10,                         \
           "A target percentage of time that is allowed to be spend on "     \
-          "process RS update buffers during the collection pause.")         \
+          "processing remembered set update buffers during the collection " \
+          "pause.")                                                         \
           range(0, 100)                                                     \
                                                                             \
   product(bool, G1UseAdaptiveConcRefinement, true,                          \
@@ -153,26 +154,40 @@
           "The threshold that defines (>=) a hot card.")                    \
           range(0, max_jubyte)                                              \
                                                                             \
-  develop(intx, G1RSetRegionEntriesBase, 256,                               \
-          "Max number of regions in a fine-grain table per MB.")            \
-          range(1, max_jint/wordSize)                                       \
+  develop(uint, G1RemSetArrayOfCardsEntriesBase, 4,                         \
+          "Maximum number of entries per region in the Array of Cards "     \
+          "card set container per MB of a heap region.")                    \
+          range(1, 65536)                                                   \
                                                                             \
-  product(intx, G1RSetRegionEntries, 0,                                     \
-          "Max number of regions for which we keep bitmaps."                \
-          "Will be set ergonomically by default")                           \
-          range(0, max_jint/wordSize)                                       \
-          constraint(G1RSetRegionEntriesConstraintFunc,AfterErgo)           \
+  product(uint, G1RemSetArrayOfCardsEntries, 0,  EXPERIMENTAL,              \
+          "Maximum number of entries per Array of Cards card set "          \
+          "container. Will be set ergonomically by default.")               \
+          range(0, 65536)                                                   \
+          constraint(G1RemSetArrayOfCardsEntriesConstraintFunc,AfterErgo)   \
                                                                             \
-  develop(intx, G1RSetSparseRegionEntriesBase, 4,                           \
-          "Max number of entries per region in a sparse table "             \
-          "per MB.")                                                        \
-          range(1, max_jint/wordSize)                                       \
+  product(uint, G1RemSetHowlMaxNumBuckets, 8, EXPERIMENTAL,                 \
+          "Maximum number of buckets per Howl card set container. The "     \
+          "default gives at worst bitmaps of size 8k. This showed to be a " \
+          "good tradeoff between bitmap size (waste) and cacheability of "  \
+          "the bucket array. Must be a power of two.")                      \
+          range(1, 1024)                                                    \
+          constraint(G1RemSetHowlMaxNumBucketsConstraintFunc,AfterErgo)     \
                                                                             \
-  product(intx, G1RSetSparseRegionEntries, 0,                               \
-          "Max number of entries per region in a sparse table."             \
-          "Will be set ergonomically by default.")                          \
-          range(0, max_jint/wordSize)                                       \
-          constraint(G1RSetSparseRegionEntriesConstraintFunc,AfterErgo)     \
+  product(uint, G1RemSetHowlNumBuckets, 0, EXPERIMENTAL,                    \
+          "Number of buckets per Howl card set container. Must be a power " \
+          "of two. Will be set ergonomically by default.")                  \
+          range(0, 1024)                                                    \
+          constraint(G1RemSetHowlNumBucketsConstraintFunc,AfterErgo)        \
+                                                                            \
+  product(uint, G1RemSetCoarsenHowlBitmapToHowlFullPercent, 90, EXPERIMENTAL, \
+          "Percentage at which to coarsen a Howl bitmap to Howl full card " \
+          "set container.")                                                 \
+          range(1, 100)                                                     \
+                                                                            \
+  product(uint, G1RemSetCoarsenHowlToFullPercent, 90, EXPERIMENTAL,         \
+          "Percentage at which to coarsen a Howl card set to Full card "    \
+          "set container.")                                                 \
+          range(1, 100)                                                     \
                                                                             \
   develop(intx, G1MaxVerifyFailures, -1,                                    \
           "The maximum number of verification failures to print.  "         \
@@ -190,7 +205,7 @@
           constraint(G1HeapRegionSizeConstraintFunc,AfterMemoryInit)        \
                                                                             \
   product(uint, G1ConcRefinementThreads, 0,                                 \
-          "The number of parallel rem set update threads. "                 \
+          "The number of parallel remembered set update threads. "          \
           "Will be set ergonomically by default.")                          \
           range(0, (max_jint-1)/wordSize)                                   \
                                                                             \
@@ -309,7 +324,23 @@
           "disables this check.")                                           \
           range(0.0, (double)max_uintx)                                     \
                                                                             \
-  product(bool, G1AllowPreventiveGC, true, DIAGNOSTIC,                       \
+  product(uint, G1RemSetFreeMemoryRescheduleDelayMillis, 10, EXPERIMENTAL,  \
+          "Time after which the card set free memory task reschedules "     \
+          "itself if there is work remaining.")                             \
+          range(1, UINT_MAX)                                                \
+                                                                            \
+  product(double, G1RemSetFreeMemoryStepDurationMillis, 1, EXPERIMENTAL,    \
+          "The amount of time that the free memory task should spend "      \
+          "before a pause of G1RemSetFreeMemoryRescheduleDelayMillis "      \
+          "length.")                                                        \
+          range(1e-3, 1e+6)                                                 \
+                                                                            \
+  product(double, G1RemSetFreeMemoryKeepExcessRatio, 0.1, EXPERIMENTAL,     \
+          "The percentage of free card set memory that G1 should keep as "  \
+          "percentage of the currently used memory.")                       \
+          range(0.0, 1.0)                                                   \
+                                                                            \
+  product(bool, G1AllowPreventiveGC, true, DIAGNOSTIC,                      \
           "Allows collections to be triggered proactively based on the      \
            number of free regions and the expected survival rates in each   \
            section of the heap.")

src/hotspot/share/gc/g1/heapRegion.cpp

@@ -34,7 +34,7 @@
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/g1/heapRegionBounds.inline.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionTracer.hpp"
 #include "gc/shared/genOopClosures.inline.hpp"
 #include "logging/log.hpp"
@@ -226,7 +226,8 @@ void HeapRegion::clear_humongous() {
 
 HeapRegion::HeapRegion(uint hrm_index,
                        G1BlockOffsetTable* bot,
-                       MemRegion mr) :
+                       MemRegion mr,
+                       G1CardSetConfiguration* config) :
   _bottom(mr.start()),
   _end(mr.end()),
   _top(NULL),
@@ -252,7 +253,7 @@ HeapRegion::HeapRegion(uint hrm_index,
   assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()),
          "invalid space boundaries");
 
-  _rem_set = new HeapRegionRemSet(bot, this);
+  _rem_set = new HeapRegionRemSet(this, config);
   initialize();
 }
 
@@ -610,11 +611,12 @@ public:
           if (!_failures) {
             log.error("----------");
           }
+          LogStream ls(log.error());
+          to->rem_set()->print_info(&ls, p);
           log.error("Missing rem set entry:");
           log.error("Field " PTR_FORMAT " of obj " PTR_FORMAT " in region " HR_FORMAT,
                     p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
           ResourceMark rm;
-          LogStream ls(log.error());
           _containing_obj->print_on(&ls);
           log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT " remset %s",
                     p2i(obj), HR_FORMAT_PARAMS(to), to->rem_set()->get_state_str());

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -36,6 +36,8 @@
 #include "runtime/mutex.hpp"
 #include "utilities/macros.hpp"
 
+class G1CardSetConfiguration;
+class G1CardSetMemoryManager;
 class G1CollectedHeap;
 class G1CMBitMap;
 class G1Predictions;
@@ -281,7 +283,10 @@ private:
   // starting at p extending to at most the prev TAMS using the given mark bitmap.
   inline size_t block_size_using_bitmap(const HeapWord* p, const G1CMBitMap* const prev_bitmap) const;
 public:
-  HeapRegion(uint hrm_index, G1BlockOffsetTable* bot, MemRegion mr);
+  HeapRegion(uint hrm_index,
+             G1BlockOffsetTable* bot,
+             MemRegion mr,
+             G1CardSetConfiguration* config);
 
   // If this region is a member of a HeapRegionManager, the index in that
   // sequence, otherwise -1.
@@ -445,6 +450,7 @@ public:
   // Unsets the humongous-related fields on the region.
   void clear_humongous();
 
+  void set_rem_set(HeapRegionRemSet* rem_set) { _rem_set = rem_set; }
   // If the region has a remembered set, return a pointer to it.
   HeapRegionRemSet* rem_set() const {
     return _rem_set;

src/hotspot/share/gc/g1/heapRegionRemSet.cpp

@@ -22,13 +22,14 @@
  *
  */
 
+#include <cstdio>
+
 #include "precompiled.hpp"
 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1ConcurrentRefine.hpp"
 #include "gc/g1/heapRegionManager.inline.hpp"
 #include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/sparsePRT.inline.hpp"
 #include "memory/allocation.hpp"
 #include "memory/padded.inline.hpp"
 #include "oops/oop.inline.hpp"
@@ -44,371 +45,19 @@
 const char* HeapRegionRemSet::_state_strings[] =  {"Untracked", "Updating", "Complete"};
 const char* HeapRegionRemSet::_short_state_strings[] =  {"UNTRA", "UPDAT", "CMPLT"};
 
-PerRegionTable* PerRegionTable::alloc(HeapRegion* hr) {
-  PerRegionTable* fl = _free_list;
-  while (fl != NULL) {
-    PerRegionTable* nxt = fl->next();
-    PerRegionTable* res = Atomic::cmpxchg(&_free_list, fl, nxt);
-    if (res == fl) {
-      fl->init(hr, true);
-      return fl;
-    } else {
-      fl = _free_list;
-    }
-  }
-  assert(fl == NULL, "Loop condition.");
-  return new PerRegionTable(hr);
-}
-
-PerRegionTable* volatile PerRegionTable::_free_list = NULL;
-
-size_t OtherRegionsTable::_max_fine_entries = 0;
-size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0;
-size_t OtherRegionsTable::_fine_eviction_stride = 0;
-size_t OtherRegionsTable::_fine_eviction_sample_size = 0;
-
-OtherRegionsTable::OtherRegionsTable(Mutex* m) :
-  _g1h(G1CollectedHeap::heap()),
-  _m(m),
-  _num_occupied(0),
-  _coarse_map(mtGC),
-  _has_coarse_entries(false),
-  _fine_grain_regions(NULL),
-  _n_fine_entries(0),
-  _first_all_fine_prts(NULL),
-  _last_all_fine_prts(NULL),
-  _fine_eviction_start(0),
-  _sparse_table()
-{
-  typedef PerRegionTable* PerRegionTablePtr;
-
-  if (_max_fine_entries == 0) {
-    assert(_mod_max_fine_entries_mask == 0, "Both or none.");
-    size_t max_entries_log = (size_t)log2i(G1RSetRegionEntries);
-    _max_fine_entries = (size_t)1 << max_entries_log;
-    _mod_max_fine_entries_mask = _max_fine_entries - 1;
-
-    assert(_fine_eviction_sample_size == 0
-           && _fine_eviction_stride == 0, "All init at same time.");
-    _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log);
-    _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size;
-  }
-
-  _fine_grain_regions = NEW_C_HEAP_ARRAY(PerRegionTablePtr, _max_fine_entries, mtGC);
-  for (size_t i = 0; i < _max_fine_entries; i++) {
-    _fine_grain_regions[i] = NULL;
-  }
-}
-
-void OtherRegionsTable::link_to_all(PerRegionTable* prt) {
-  // We always append to the beginning of the list for convenience;
-  // the order of entries in this list does not matter.
-  if (_first_all_fine_prts != NULL) {
-    prt->set_next(_first_all_fine_prts);
-  } else {
-    // this is the first element we insert. Adjust the "last" pointer
-    _last_all_fine_prts = prt;
-    assert(prt->next() == NULL, "just checking");
-  }
-  _first_all_fine_prts = prt;
-
-  assert(_first_all_fine_prts == prt, "just checking");
-  assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) ||
-         (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL),
-         "just checking");
-  assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL,
-         "just checking");
-}
-
-CardIdx_t OtherRegionsTable::card_within_region(OopOrNarrowOopStar within_region, HeapRegion* hr) {
-  assert(hr->is_in_reserved(within_region),
-         "HeapWord " PTR_FORMAT " is outside of region %u [" PTR_FORMAT ", " PTR_FORMAT ")",
-         p2i(within_region), hr->hrm_index(), p2i(hr->bottom()), p2i(hr->end()));
-  CardIdx_t result = (CardIdx_t)(pointer_delta((HeapWord*)within_region, hr->bottom()) >> (CardTable::card_shift - LogHeapWordSize));
-  return result;
-}
-
-void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, uint tid) {
-  // Note that this may be a continued H region.
-  HeapRegion* from_hr = _g1h->heap_region_containing(from);
-  RegionIdx_t from_hrm_ind = (RegionIdx_t) from_hr->hrm_index();
-
-  // If the region is already coarsened, return.
-  if (is_region_coarsened(from_hrm_ind)) {
-    assert(contains_reference(from), "We just found " PTR_FORMAT " in the Coarse table", p2i(from));
-    return;
-  }
-
-  size_t num_added_by_coarsening = 0;
-  // Otherwise find a per-region table to add it to.
-  size_t ind = from_hrm_ind & _mod_max_fine_entries_mask;
-  PerRegionTable* prt = find_region_table(ind, from_hr);
-  if (prt == NULL) {
-    MutexLocker x(_m, Mutex::_no_safepoint_check_flag);
-
-    // Rechecking if the region is coarsened, while holding the lock.
-    if (is_region_coarsened(from_hrm_ind)) {
-      assert(contains_reference_locked(from), "We just found " PTR_FORMAT " in the Coarse table", p2i(from));
-      return;
-    }
-
-    // Confirm that it's really not there...
-    prt = find_region_table(ind, from_hr);
-    if (prt == NULL) {
-
-      CardIdx_t card_index = card_within_region(from, from_hr);
-
-      SparsePRT::AddCardResult result = _sparse_table.add_card(from_hrm_ind, card_index);
-      if (result != SparsePRT::overflow) {
-        if (result == SparsePRT::added) {
-          Atomic::inc(&_num_occupied, memory_order_relaxed);
-        }
-        assert(contains_reference_locked(from), "We just added " PTR_FORMAT " to the Sparse table", p2i(from));
-        return;
-      }
-
-      // Sparse PRT returned overflow (sparse table is full)
-
-      if (_n_fine_entries == _max_fine_entries) {
-        prt = delete_region_table(num_added_by_coarsening);
-        // There is no need to clear the links to the 'all' list here:
-        // prt will be reused immediately, i.e. remain in the 'all' list.
-        prt->init(from_hr, false /* clear_links_to_all_list */);
-      } else {
-        prt = PerRegionTable::alloc(from_hr);
-        link_to_all(prt);
-      }
-
-      PerRegionTable* first_prt = _fine_grain_regions[ind];
-      prt->set_collision_list_next(first_prt);
-      // The assignment into _fine_grain_regions allows the prt to
-      // start being used concurrently. In addition to
-      // collision_list_next which must be visible (else concurrent
-      // parsing of the list, if any, may fail to see other entries),
-      // the content of the prt must be visible (else for instance
-      // some mark bits may not yet seem cleared or a 'later' update
-      // performed by a concurrent thread could be undone when the
-      // zeroing becomes visible). This requires store ordering.
-      Atomic::release_store(&_fine_grain_regions[ind], prt);
-      _n_fine_entries++;
-
-      // Transfer from sparse to fine-grain. The cards from the sparse table
-      // were already added to the total in _num_occupied.
-      SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrm_ind);
-      assert(sprt_entry != NULL, "There should have been an entry");
-      for (int i = 0; i < sprt_entry->num_valid_cards(); i++) {
-        CardIdx_t c = sprt_entry->card(i);
-        prt->add_card(c);
-      }
-      // Now we can delete the sparse entry.
-      bool res = _sparse_table.delete_entry(from_hrm_ind);
-      assert(res, "It should have been there.");
-    }
-    assert(prt != NULL && prt->hr() == from_hr, "consequence");
-  }
-  // Note that we can't assert "prt->hr() == from_hr", because of the
-  // possibility of concurrent reuse.  But see head comment of
-  // OtherRegionsTable for why this is OK.
-  assert(prt != NULL, "Inv");
-
-  if (prt->add_reference(from)) {
-    num_added_by_coarsening++;
-  }
-  Atomic::add(&_num_occupied, num_added_by_coarsening, memory_order_relaxed);
-  assert(contains_reference(from), "We just added " PTR_FORMAT " to the PRT (%d)", p2i(from), prt->contains_reference(from));
-}
-
-PerRegionTable*
-OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const {
-  assert(ind < _max_fine_entries, "Preconditions.");
-  PerRegionTable* prt = _fine_grain_regions[ind];
-  while (prt != NULL && prt->hr() != hr) {
-    prt = prt->collision_list_next();
-  }
-  // Loop postcondition is the method postcondition.
-  return prt;
-}
-
-jint OtherRegionsTable::_n_coarsenings = 0;
-
-PerRegionTable* OtherRegionsTable::delete_region_table(size_t& added_by_deleted) {
-  assert(_m->owned_by_self(), "Precondition");
-  assert(_n_fine_entries == _max_fine_entries, "Precondition");
-  PerRegionTable* max = NULL;
-  jint max_occ = 0;
-  PerRegionTable** max_prev = NULL;
-
-  size_t i = _fine_eviction_start;
-  for (size_t k = 0; k < _fine_eviction_sample_size; k++) {
-    size_t ii = i;
-    // Make sure we get a non-NULL sample.
-    while (_fine_grain_regions[ii] == NULL) {
-      ii++;
-      if (ii == _max_fine_entries) ii = 0;
-      guarantee(ii != i, "We must find one.");
-    }
-    PerRegionTable** prev = &_fine_grain_regions[ii];
-    PerRegionTable* cur = *prev;
-    while (cur != NULL) {
-      jint cur_occ = cur->occupied();
-      if (max == NULL || cur_occ > max_occ) {
-        max = cur;
-        max_prev = prev;
-        max_occ = cur_occ;
-      }
-      prev = cur->collision_list_next_addr();
-      cur = cur->collision_list_next();
-    }
-    i = i + _fine_eviction_stride;
-    if (i >= _n_fine_entries) i = i - _n_fine_entries;
-  }
-
-  _fine_eviction_start++;
-
-  if (_fine_eviction_start >= _n_fine_entries) {
-    _fine_eviction_start -= _n_fine_entries;
-  }
-
-  guarantee(max != NULL, "Since _n_fine_entries > 0");
-  guarantee(max_prev != NULL, "Since max != NULL.");
-
-  // Ensure the corresponding coarse bit is set.
-  size_t max_hrm_index = (size_t) max->hr()->hrm_index();
-  if (Atomic::load(&_has_coarse_entries)) {
-    _coarse_map.at_put(max_hrm_index, true);
-  } else {
-    // This will lazily initialize an uninitialized bitmap
-    _coarse_map.reinitialize(G1CollectedHeap::heap()->max_reserved_regions());
-    assert(!_coarse_map.at(max_hrm_index), "No coarse entries");
-    _coarse_map.at_put(max_hrm_index, true);
-    // Release store guarantees that the bitmap has initialized before any
-    // concurrent reader will ever see _has_coarse_entries is true
-    // (when read with load_acquire)
-    Atomic::release_store(&_has_coarse_entries, true);
-  }
-
-  added_by_deleted = HeapRegion::CardsPerRegion - max_occ;
-  // Unsplice.
-  *max_prev = max->collision_list_next();
-  Atomic::inc(&_n_coarsenings);
-  _n_fine_entries--;
-  return max;
-}
-
-bool OtherRegionsTable::occupancy_less_or_equal_than(size_t limit) const {
-  return occupied() <= limit;
-}
-
-bool OtherRegionsTable::is_empty() const {
-  return occupied() == 0;
-}
-
-size_t OtherRegionsTable::occupied() const {
-  return _num_occupied;
-}
-
-size_t OtherRegionsTable::mem_size() const {
-  size_t sum = 0;
-  // all PRTs are of the same size so it is sufficient to query only one of them.
-  if (_first_all_fine_prts != NULL) {
-    assert(_last_all_fine_prts != NULL &&
-      _first_all_fine_prts->mem_size() == _last_all_fine_prts->mem_size(), "check that mem_size() is constant");
-    sum += _first_all_fine_prts->mem_size() * _n_fine_entries;
-  }
-  sum += (sizeof(PerRegionTable*) * _max_fine_entries);
-  sum += (_coarse_map.size_in_words() * HeapWordSize);
-  sum += (_sparse_table.mem_size());
-  sum += sizeof(OtherRegionsTable) - sizeof(_sparse_table); // Avoid double counting above.
-  return sum;
-}
-
-size_t OtherRegionsTable::static_mem_size() {
-  return G1FromCardCache::static_mem_size();
-}
-
-size_t OtherRegionsTable::fl_mem_size() {
-  return PerRegionTable::fl_mem_size();
-}
-
-void OtherRegionsTable::clear() {
-  // if there are no entries, skip this step
-  if (_first_all_fine_prts != NULL) {
-    guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking");
-    PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts);
-    memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0]));
-  } else {
-    guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking");
-  }
-
-  _first_all_fine_prts = _last_all_fine_prts = NULL;
-  _sparse_table.clear();
-  if (Atomic::load(&_has_coarse_entries)) {
-    _coarse_map.clear();
-  }
-  _n_fine_entries = 0;
-  Atomic::store(&_has_coarse_entries, false);
-
-  _num_occupied = 0;
-}
-
-bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const {
-  // Cast away const in this case.
-  MutexLocker x((Mutex*)_m, Mutex::_no_safepoint_check_flag);
-  return contains_reference_locked(from);
-}
-
-bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const {
-  HeapRegion* hr = _g1h->heap_region_containing(from);
-  RegionIdx_t hr_ind = (RegionIdx_t) hr->hrm_index();
-  // Is this region in the coarse map?
-  if (is_region_coarsened(hr_ind)) return true;
-
-  PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask,
-                                          hr);
-  if (prt != NULL) {
-    return prt->contains_reference(from);
-  } else {
-    CardIdx_t card_index = card_within_region(from, hr);
-    return _sparse_table.contains_card(hr_ind, card_index);
-  }
-}
-
-// A load_acquire on _has_coarse_entries - coupled with the release_store in
-// delete_region_table - guarantees we don't access _coarse_map before
-// it's been properly initialized.
-bool OtherRegionsTable::is_region_coarsened(RegionIdx_t from_hrm_ind) const {
-  return Atomic::load_acquire(&_has_coarse_entries) && _coarse_map.at(from_hrm_ind);
-}
-
-HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetTable* bot,
-                                   HeapRegion* hr)
-  : _bot(bot),
-    _code_roots(),
-    _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Mutex::_safepoint_check_never),
-    _other_regions(&_m),
-    _hr(hr),
-    _state(Untracked)
-{
-}
+HeapRegionRemSet::HeapRegionRemSet(HeapRegion* hr,
+                                   G1CardSetConfiguration* config) :
+  _m(Mutex::leaf + 1, FormatBuffer<128>("HeapRegionRemSet lock #%u", hr->hrm_index()), true, Monitor::_safepoint_check_never),
+  _code_roots(),
+  _card_set_mm(config, G1CardSetFreePool::free_list_pool()),
+  _card_set(config, &_card_set_mm),
+  _hr(hr),
+  _state(Untracked) { }
 
 void HeapRegionRemSet::clear_fcc() {
   G1FromCardCache::clear(_hr->hrm_index());
 }
 
-void HeapRegionRemSet::setup_remset_size() {
-  const int LOG_M = 20;
-  guarantee(HeapRegion::LogOfHRGrainBytes >= LOG_M, "Code assumes the region size >= 1M, but is " SIZE_FORMAT "B", HeapRegion::GrainBytes);
-
-  int region_size_log_mb = HeapRegion::LogOfHRGrainBytes - LOG_M;
-  if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) {
-    G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * ((size_t)1 << (region_size_log_mb + 1));
-  }
-  if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) {
-    G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1);
-  }
-  guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity");
-}
-
 void HeapRegionRemSet::clear(bool only_cardset) {
   MutexLocker x(&_m, Mutex::_no_safepoint_check_flag);
   clear_locked(only_cardset);
@@ -419,11 +68,15 @@ void HeapRegionRemSet::clear_locked(bool only_cardset) {
     _code_roots.clear();
   }
   clear_fcc();
-  _other_regions.clear();
+  _card_set.clear();
   set_state_empty();
   assert(occupied() == 0, "Should be clear.");
 }
 
+void HeapRegionRemSet::print_static_mem_size(outputStream* out) {
+  out->print_cr("  Static structures = " SIZE_FORMAT, HeapRegionRemSet::static_mem_size());
+}
+
 // Code roots support
 //
 // The code root set is protected by two separate locking schemes

src/hotspot/share/gc/g1/heapRegionRemSet.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,299 +25,62 @@
 #ifndef SHARE_GC_G1_HEAPREGIONREMSET_HPP
 #define SHARE_GC_G1_HEAPREGIONREMSET_HPP
 
+#include "gc/g1/g1CardSet.hpp"
+#include "gc/g1/g1CardSetMemory.hpp"
 #include "gc/g1/g1CodeCacheRemSet.hpp"
 #include "gc/g1/g1FromCardCache.hpp"
-#include "gc/g1/sparsePRT.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/safepoint.hpp"
 #include "utilities/bitMap.hpp"
 
-// Remembered set for a heap region.  Represent a set of "cards" that
-// contain pointers into the owner heap region.  Cards are defined somewhat
-// abstractly, in terms of what the "BlockOffsetTable" in use can parse.
-
-class G1CollectedHeap;
-class G1BlockOffsetTable;
-class G1CardLiveData;
-class HeapRegion;
-class PerRegionTable;
-class SparsePRT;
-class nmethod;
-
-// The "_coarse_map" is a bitmap with one bit for each region, where set
-// bits indicate that the corresponding region may contain some pointer
-// into the owning region.
-
-// The "_fine_grain_entries" array is an open hash table of PerRegionTables
-// (PRTs), indicating regions for which we're keeping the RS as a set of
-// cards.  The strategy is to cap the size of the fine-grain table,
-// deleting an entry and setting the corresponding coarse-grained bit when
-// we would overflow this cap.
-
-// We use a mixture of locking and lock-free techniques here.  We allow
-// threads to locate PRTs without locking, but threads attempting to alter
-// a bucket list obtain a lock.  This means that any failing attempt to
-// find a PRT must be retried with the lock.  It might seem dangerous that
-// a read can find a PRT that is concurrently deleted.  This is all right,
-// because:
-//
-//   1) We only actually free PRT's at safe points (though we reuse them at
-//      other times).
-//   2) We find PRT's in an attempt to add entries.  If a PRT is deleted,
-//      it's _coarse_map bit is set, so the that we were attempting to add
-//      is represented.  If a deleted PRT is re-used, a thread adding a bit,
-//      thinking the PRT is for a different region, does no harm.
-
-class OtherRegionsTable {
-  G1CollectedHeap* _g1h;
-  Mutex*           _m;
-
-  size_t volatile _num_occupied;
-
-  // These are protected by "_m".
-  CHeapBitMap   _coarse_map;
-  bool volatile _has_coarse_entries;
-  static jint   _n_coarsenings;
-
-  PerRegionTable** _fine_grain_regions;
-  size_t           _n_fine_entries;
-
-  // The fine grain remembered sets are linked together using
-  // their 'next' fields.
-  // This allows fast bulk freeing of all the fine grain remembered
-  // set entries, and fast finding of all of them without iterating
-  // over the _fine_grain_regions table.
-  PerRegionTable * _first_all_fine_prts;
-  PerRegionTable * _last_all_fine_prts;
-
-  // Used to sample a subset of the fine grain PRTs to determine which
-  // PRT to evict and coarsen.
-  size_t        _fine_eviction_start;
-  static size_t _fine_eviction_stride;
-  static size_t _fine_eviction_sample_size;
-
-  SparsePRT   _sparse_table;
-
-  // These are static after init.
-  static size_t _max_fine_entries;
-  static size_t _mod_max_fine_entries_mask;
-
-  // Requires "prt" to be the first element of the bucket list appropriate
-  // for "hr".  If this list contains an entry for "hr", return it,
-  // otherwise return "NULL".
-  PerRegionTable* find_region_table(size_t ind, HeapRegion* hr) const;
-
-  // Find, delete, and return a candidate PerRegionTable, if any exists,
-  // adding the deleted region to the coarse bitmap.  Requires the caller
-  // to hold _m, and the fine-grain table to be full.
-  PerRegionTable* delete_region_table(size_t& added_by_deleted);
-
-  // link/add the given fine grain remembered set into the "all" list
-  void link_to_all(PerRegionTable * prt);
-
-  bool contains_reference_locked(OopOrNarrowOopStar from) const;
-
-public:
-  // Create a new remembered set. The given mutex is used to ensure consistency.
-  OtherRegionsTable(Mutex* m);
-
-  template <class Closure>
-  void iterate(Closure& v);
-
-  // Returns the card index of the given within_region pointer relative to the bottom
-  // of the given heap region.
-  static CardIdx_t card_within_region(OopOrNarrowOopStar within_region, HeapRegion* hr);
-  // Adds the reference from "from to this remembered set.
-  void add_reference(OopOrNarrowOopStar from, uint tid);
-
-  // Returns whether the remembered set contains the given reference.
-  bool contains_reference(OopOrNarrowOopStar from) const;
-
-  // Returns whether this remembered set (and all sub-sets) have an occupancy
-  // that is less or equal than the given occupancy.
-  bool occupancy_less_or_equal_than(size_t limit) const;
-
-  // Returns whether this remembered set (and all sub-sets) does not contain any entry.
-  bool is_empty() const;
-
-  // Returns the number of cards contained in this remembered set.
-  size_t occupied() const;
-
-  static jint n_coarsenings() { return _n_coarsenings; }
-
-  // Returns size of the actual remembered set containers in bytes.
-  size_t mem_size() const;
-  // Returns the size of static data in bytes.
-  static size_t static_mem_size();
-  // Returns the size of the free list content in bytes.
-  static size_t fl_mem_size();
-
-  // Clear the entire contents of this remembered set.
-  void clear();
-
-  // Safe for use by concurrent readers outside _m
-  bool is_region_coarsened(RegionIdx_t from_hrm_ind) const;
-};
-
-class PerRegionTable: public CHeapObj<mtGC> {
-  friend class OtherRegionsTable;
-
-  HeapRegion*     _hr;
-  CHeapBitMap     _bm;
-  jint            _occupied;
-
-  // next pointer for free/allocated 'all' list
-  PerRegionTable* _next;
-
-  // next pointer in collision list
-  PerRegionTable * _collision_list_next;
-
-  // Global free list of PRTs
-  static PerRegionTable* volatile _free_list;
-
-protected:
-  PerRegionTable(HeapRegion* hr) :
-    _hr(hr),
-    _bm(HeapRegion::CardsPerRegion, mtGC),
-    _occupied(0),
-    _next(NULL),
-    _collision_list_next(NULL)
-  {}
-
-public:
-  // We need access in order to union things into the base table.
-  BitMap* bm() { return &_bm; }
-
-  HeapRegion* hr() const { return Atomic::load_acquire(&_hr); }
-
-  jint occupied() const {
-    return _occupied;
-  }
-
-  void init(HeapRegion* hr, bool clear_links_to_all_list);
-
-  inline bool add_reference(OopOrNarrowOopStar from);
-
-  inline bool add_card(CardIdx_t from_card_index);
-
-  // (Destructively) union the bitmap of the current table into the given
-  // bitmap (which is assumed to be of the same size.)
-  void union_bitmap_into(BitMap* bm) {
-    bm->set_union(_bm);
-  }
-
-  // Mem size in bytes.
-  size_t mem_size() const {
-    return sizeof(PerRegionTable) + _bm.size_in_words() * HeapWordSize;
-  }
-
-  // Requires "from" to be in "hr()".
-  bool contains_reference(OopOrNarrowOopStar from) const {
-    assert(hr()->is_in_reserved(from), "Precondition.");
-    size_t card_ind = pointer_delta(from, hr()->bottom(),
-                                    G1CardTable::card_size);
-    return _bm.at(card_ind);
-  }
-
-  // Bulk-free the PRTs from prt to last, assumes that they are
-  // linked together using their _next field.
-  static void bulk_free(PerRegionTable* prt, PerRegionTable* last) {
-    while (true) {
-      PerRegionTable* fl = _free_list;
-      last->set_next(fl);
-      PerRegionTable* res = Atomic::cmpxchg(&_free_list, fl, prt);
-      if (res == fl) {
-        return;
-      }
-    }
-    ShouldNotReachHere();
-  }
-
-  static void free(PerRegionTable* prt) {
-    bulk_free(prt, prt);
-  }
-
-  // Returns an initialized PerRegionTable instance.
-  static PerRegionTable* alloc(HeapRegion* hr);
-
-  PerRegionTable* next() const { return _next; }
-  void set_next(PerRegionTable* next) { _next = next; }
-
-  // Accessor and Modification routines for the pointer for the
-  // singly linked collision list that links the PRTs within the
-  // OtherRegionsTable::_fine_grain_regions hash table.
-  //
-
-  PerRegionTable* collision_list_next() const {
-    return _collision_list_next;
-  }
-
-  void set_collision_list_next(PerRegionTable* next) {
-    _collision_list_next = next;
-  }
-
-  PerRegionTable** collision_list_next_addr() {
-    return &_collision_list_next;
-  }
-
-  static size_t fl_mem_size() {
-    PerRegionTable* cur = _free_list;
-    size_t res = 0;
-    while (cur != NULL) {
-      res += cur->mem_size();
-      cur = cur->next();
-    }
-    return res;
-  }
-
-  static void test_fl_mem_size();
-};
+class outputStream;
 
 class HeapRegionRemSet : public CHeapObj<mtGC> {
   friend class VMStructs;
 
-private:
-  G1BlockOffsetTable* _bot;
-
+  Mutex _m;
   // A set of code blobs (nmethods) whose code contains pointers into
   // the region that owns this RSet.
   G1CodeRootSet _code_roots;
 
-  Mutex _m;
+  G1CardSetMemoryManager _card_set_mm;
 
-  OtherRegionsTable _other_regions;
+  // The set of cards in the Java heap
+  G1CardSet _card_set;
 
   HeapRegion* _hr;
 
+  inline void split_card(OopOrNarrowOopStar from, uint& card_region, uint& card_within_region) const;
   void clear_fcc();
 
 public:
-  HeapRegionRemSet(G1BlockOffsetTable* bot, HeapRegion* hr);
+  HeapRegionRemSet(HeapRegion* hr, G1CardSetConfiguration* config);
 
   // Setup sparse and fine-grain tables sizes.
-  static void setup_remset_size();
+  bool cardset_is_empty() const {
+    return _card_set.is_empty();
+  }
 
   bool is_empty() const {
-    return (strong_code_roots_list_length() == 0) && _other_regions.is_empty();
+    return (strong_code_roots_list_length() == 0) && cardset_is_empty();
   }
 
   bool occupancy_less_or_equal_than(size_t occ) const {
-    return (strong_code_roots_list_length() == 0) && _other_regions.occupancy_less_or_equal_than(occ);
+    return (strong_code_roots_list_length() == 0) && _card_set.occupancy_less_or_equal_to(occ);
   }
 
-  // For each PRT in the card (remembered) set call one of the following methods
-  // of the given closure:
-  //
-  // set_full_region_dirty(uint region_idx) - pass the region index for coarse PRTs
-  // set_bitmap_dirty(uint region_idx, BitMap* bitmap) - pass the region index and bitmap for fine PRTs
-  // set_cards_dirty(uint region_idx, elem_t* cards, uint num_cards) - pass region index and cards for sparse PRTs
-  template <class Closure>
-  inline void iterate_prts(Closure& cl);
+  // Iterate the card based remembered set for merging them into the card table.
+  // The passed closure must be a CardOrRangeVisitor; we use a template parameter
+  // to pass it in to facilitate inlining as much as possible.
+  template <class CardOrRangeVisitor>
+  inline void iterate_for_merge(CardOrRangeVisitor& cl);
 
   size_t occupied() {
-    return _other_regions.occupied();
+    return _card_set.occupied();
   }
 
-  static jint n_coarsenings() { return OtherRegionsTable::n_coarsenings(); }
+  // Coarsening statistics since VM start.
+  static G1CardSetCoarsenStats coarsen_stats() { return G1CardSet::coarsen_stats(); }
 
 private:
   enum RemSetState {
@@ -339,74 +102,42 @@ public:
   bool is_updating() { return _state == Updating; }
   bool is_complete() { return _state == Complete; }
 
-  void set_state_empty() {
-    guarantee(SafepointSynchronize::is_at_safepoint() || !is_tracked(), "Should only set to Untracked during safepoint but is %s.", get_state_str());
-    if (_state == Untracked) {
-      return;
-    }
-    clear_fcc();
-    _state = Untracked;
-  }
+  inline void set_state_empty();
+  inline void set_state_updating();
+  inline void set_state_complete();
 
-  void set_state_updating() {
-    guarantee(SafepointSynchronize::is_at_safepoint() && !is_tracked(), "Should only set to Updating from Untracked during safepoint but is %s", get_state_str());
-    clear_fcc();
-    _state = Updating;
-  }
-
-  void set_state_complete() {
-    clear_fcc();
-    _state = Complete;
-  }
-
-  void add_reference(OopOrNarrowOopStar from, uint tid) {
-    RemSetState state = _state;
-    if (state == Untracked) {
-      return;
-    }
-
-    uint cur_idx = _hr->hrm_index();
-    uintptr_t from_card = uintptr_t(from) >> CardTable::card_shift;
-
-    if (G1FromCardCache::contains_or_replace(tid, cur_idx, from_card)) {
-      assert(contains_reference(from), "We just found " PTR_FORMAT " in the FromCardCache", p2i(from));
-      return;
-    }
-
-    _other_regions.add_reference(from, tid);
-  }
+  inline void add_reference(OopOrNarrowOopStar from, uint tid);
 
   // The region is being reclaimed; clear its remset, and any mention of
   // entries for this region in other remsets.
   void clear(bool only_cardset = false);
   void clear_locked(bool only_cardset = false);
 
-  // The actual # of bytes this hr_remset takes up.
-  // Note also includes the strong code root set.
+  G1CardSetMemoryStats card_set_memory_stats() const { return _card_set_mm.memory_stats(); }
+
+  // The actual # of bytes this hr_remset takes up. Also includes the strong code
+  // root set.
   size_t mem_size() {
-    MutexLocker x(&_m, Mutex::_no_safepoint_check_flag);
-    return _other_regions.mem_size()
-      // This correction is necessary because the above includes the second
-      // part.
-      + (sizeof(HeapRegionRemSet) - sizeof(OtherRegionsTable))
+    return _card_set.mem_size()
+      + (sizeof(HeapRegionRemSet) - sizeof(G1CardSet)) // Avoid double-counting G1CardSet.
       + strong_code_roots_mem_size();
   }
 
+  size_t wasted_mem_size() {
+    return _card_set.wasted_mem_size();
+  }
+
   // Returns the memory occupancy of all static data structures associated
   // with remembered sets.
   static size_t static_mem_size() {
-    return OtherRegionsTable::static_mem_size() + G1CodeRootSet::static_mem_size();
+    return G1CardSet::static_mem_size() + G1CodeRootSet::static_mem_size() + sizeof(G1CardSetFreePool);
   }
 
-  // Returns the memory occupancy of all free_list data structures associated
-  // with remembered sets.
-  static size_t fl_mem_size() {
-    return OtherRegionsTable::fl_mem_size();
-  }
+  static void print_static_mem_size(outputStream* out);
 
-  bool contains_reference(OopOrNarrowOopStar from) const {
-    return _other_regions.contains_reference(from);
-  }
+  inline bool contains_reference(OopOrNarrowOopStar from);
+
+  inline void print_info(outputStream* st, OopOrNarrowOopStar from);
 
   // Routines for managing the list of code roots that point into
   // the heap region that owns this RSet.

src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,74 +27,86 @@
 
 #include "gc/g1/heapRegionRemSet.hpp"
 
+#include "gc/g1/g1CardSet.inline.hpp"
+#include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/sparsePRT.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/bitMap.inline.hpp"
 
-template <class Closure>
-inline void HeapRegionRemSet::iterate_prts(Closure& cl) {
-  _other_regions.iterate(cl);
+void HeapRegionRemSet::set_state_empty() {
+  guarantee(SafepointSynchronize::is_at_safepoint() || !is_tracked(),
+            "Should only set to Untracked during safepoint but is %s.", get_state_str());
+  if (_state == Untracked) {
+    return;
+  }
+  clear_fcc();
+  _state = Untracked;
 }
 
-inline bool PerRegionTable::add_card(CardIdx_t from_card_index) {
-  if (_bm.par_set_bit(from_card_index)) {
-    Atomic::inc(&_occupied, memory_order_relaxed);
-    return true;
-  }
-  return false;
+void HeapRegionRemSet::set_state_updating() {
+  guarantee(SafepointSynchronize::is_at_safepoint() && !is_tracked(),
+            "Should only set to Updating from Untracked during safepoint but is %s", get_state_str());
+  clear_fcc();
+  _state = Updating;
 }
 
-inline bool PerRegionTable::add_reference(OopOrNarrowOopStar from) {
-  // Must make this robust in case "from" is not in "_hr", because of
-  // concurrency.
-
-  HeapRegion* loc_hr = hr();
-  // If the test below fails, then this table was reused concurrently
-  // with this operation.  This is OK, since the old table was coarsened,
-  // and adding a bit to the new table is never incorrect.
-  if (loc_hr->is_in_reserved(from)) {
-    CardIdx_t from_card = OtherRegionsTable::card_within_region(from, loc_hr);
-    return add_card(from_card);
-  }
-  return false;
+void HeapRegionRemSet::set_state_complete() {
+  clear_fcc();
+  _state = Complete;
 }
 
-inline void PerRegionTable::init(HeapRegion* hr, bool clear_links_to_all_list) {
-  if (clear_links_to_all_list) {
-    set_next(NULL);
-  }
-  _collision_list_next = NULL;
-  _occupied = 0;
-  _bm.clear();
-  // Make sure that the bitmap clearing above has been finished before publishing
-  // this PRT to concurrent threads.
-  Atomic::release_store(&_hr, hr);
+
+template <class CardOrRangeVisitor>
+inline void HeapRegionRemSet::iterate_for_merge(CardOrRangeVisitor& cl) {
+  _card_set.iterate_for_merge(cl);
 }
 
-template <class Closure>
-void OtherRegionsTable::iterate(Closure& cl) {
-  if (Atomic::load(&_has_coarse_entries)) {
-    BitMap::idx_t cur = _coarse_map.get_next_one_offset(0);
-    while (cur != _coarse_map.size()) {
-      cl.next_coarse_prt((uint)cur);
-      cur = _coarse_map.get_next_one_offset(cur + 1);
-    }
+void HeapRegionRemSet::split_card(OopOrNarrowOopStar from, uint& card_region, uint& card_within_region) const {
+  HeapRegion* hr = G1CollectedHeap::heap()->heap_region_containing(from);
+  card_region = hr->hrm_index();
+  card_within_region = (uint)(pointer_delta((HeapWord*)from, hr->bottom()) >> (CardTable::card_shift - LogHeapWordSize));
+}
+
+void HeapRegionRemSet::add_reference(OopOrNarrowOopStar from, uint tid) {
+  RemSetState state = _state;
+  if (state == Untracked) {
+    return;
   }
-  {
-    PerRegionTable* cur = _first_all_fine_prts;
-    while (cur != NULL) {
-      cl.next_fine_prt(cur->hr()->hrm_index(), cur->bm());
-      cur = cur->next();
-    }
-  }
-  {
-    SparsePRTBucketIter iter(&_sparse_table);
-    SparsePRTEntry* cur;
-    while (iter.has_next(cur)) {
-      cl.next_sparse_prt(cur->r_ind(), cur->cards(), cur->num_valid_cards());
-    }
+
+  uint cur_idx = _hr->hrm_index();
+  uintptr_t from_card = uintptr_t(from) >> CardTable::card_shift;
+
+  if (G1FromCardCache::contains_or_replace(tid, cur_idx, from_card)) {
+    // We can't check whether the card is in the remembered set - the card container
+    // may be coarsened just now.
+    //assert(contains_reference(from), "We just found " PTR_FORMAT " in the FromCardCache", p2i(from));
+   return;
   }
+
+  uint card_region;
+  uint card_within_region;
+
+  split_card(from, card_region, card_within_region);
+
+  _card_set.add_card(card_region, card_within_region);
+}
+
+bool HeapRegionRemSet::contains_reference(OopOrNarrowOopStar from) {
+  uint card_region;
+  uint card_within_region;
+
+  split_card(from, card_region, card_within_region);
+
+  return _card_set.contains_card(card_region, card_within_region);
+}
+
+void HeapRegionRemSet::print_info(outputStream* st, OopOrNarrowOopStar from) {
+  uint card_region;
+  uint card_within_region;
+
+  split_card(from, card_region, card_within_region);
+
+  _card_set.print_info(st, card_region, card_within_region);
 }
 
 #endif // SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP

src/hotspot/share/gc/g1/heapRegionSet.cpp

@@ -25,7 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/g1/g1CollectedHeap.inline.hpp"
 #include "gc/g1/g1NUMA.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/heapRegionRemSet.inline.hpp"
 #include "gc/g1/heapRegionSet.inline.hpp"
 
 uint FreeRegionList::_unrealistically_long_length = 0;

src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp

@@ -28,15 +28,15 @@
 #include "runtime/globals_extension.hpp"
 #include "utilities/globalDefinitions.hpp"
 
-JVMFlag::Error G1RSetRegionEntriesConstraintFunc(intx value, bool verbose) {
+JVMFlag::Error G1RemSetArrayOfCardsEntriesConstraintFunc(uint value, bool verbose) {
   if (!UseG1GC) return JVMFlag::SUCCESS;
 
-  // Default value of G1RSetRegionEntries=0 means will be set ergonomically.
+  // Default value of G1RemSetArrayOfCardsEntries=0 means will be set ergonomically.
   // Minimum value is 1.
-  if (FLAG_IS_CMDLINE(G1RSetRegionEntries) && (value < 1)) {
+  if (FLAG_IS_CMDLINE(G1RemSetArrayOfCardsEntries) && (value < 1)) {
     JVMFlag::printError(verbose,
-                        "G1RSetRegionEntries (" INTX_FORMAT ") must be "
-                        "greater than or equal to 1\n",
+                        "G1RemSetArrayOfCardsEntries (%u) must be "
+                        "greater than or equal to 1.\n",
                         value);
     return JVMFlag::VIOLATES_CONSTRAINT;
   } else {
@@ -44,20 +44,35 @@ JVMFlag::Error G1RSetRegionEntriesConstraintFunc(intx value, bool verbose) {
   }
 }
 
-JVMFlag::Error G1RSetSparseRegionEntriesConstraintFunc(intx value, bool verbose) {
+JVMFlag::Error G1RemSetHowlNumBucketsConstraintFunc(uint value, bool verbose) {
   if (!UseG1GC) return JVMFlag::SUCCESS;
 
-  // Default value of G1RSetSparseRegionEntries=0 means will be set ergonomically.
-  // Minimum value is 1.
-  if (FLAG_IS_CMDLINE(G1RSetSparseRegionEntries) && (value < 1)) {
-    JVMFlag::printError(verbose,
-                        "G1RSetSparseRegionEntries (" INTX_FORMAT ") must be "
-                        "greater than or equal to 1\n",
-                        value);
-    return JVMFlag::VIOLATES_CONSTRAINT;
-  } else {
+  if (!FLAG_IS_CMDLINE(G1RemSetHowlNumBuckets)) {
     return JVMFlag::SUCCESS;
   }
+  if (value == 0 || !is_power_of_2(G1RemSetHowlNumBuckets)) {
+    JVMFlag::printError(verbose,
+                        "G1RemSetHowlNumBuckets (%u) must be a power of two "
+                        "and greater than or equal to 1.\n",
+                        value);
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  }
+  return JVMFlag::SUCCESS;
+}
+
+JVMFlag::Error G1RemSetHowlMaxNumBucketsConstraintFunc(uint value, bool verbose) {
+  if (!UseG1GC) return JVMFlag::SUCCESS;
+
+  if (!FLAG_IS_CMDLINE(G1RemSetHowlMaxNumBuckets)) {
+    return JVMFlag::SUCCESS;
+  }
+  if (!is_power_of_2(G1RemSetHowlMaxNumBuckets)) {
+    JVMFlag::printError(verbose,
+                        "G1RemSetMaxHowlNumBuckets (%u) must be a power of two.\n",
+                        value);
+    return JVMFlag::VIOLATES_CONSTRAINT;
+  }
+  return JVMFlag::SUCCESS;
 }
 
 JVMFlag::Error G1HeapRegionSizeConstraintFunc(size_t value, bool verbose) {

src/hotspot/share/gc/g1/jvmFlagConstraintsG1.hpp

@@ -30,9 +30,12 @@
 
 #define G1_GC_CONSTRAINTS(f)                          \
                                                       \
-  /* G1 Flag Constraints */                           \
-  f(intx,   G1RSetRegionEntriesConstraintFunc)        \
-  f(intx,   G1RSetSparseRegionEntriesConstraintFunc)  \
+  /* G1 Remembered Sets Constraints */                \
+  f(uint,   G1RemSetArrayOfCardsEntriesConstraintFunc)\
+  f(uint,   G1RemSetHowlMaxNumBucketsConstraintFunc)  \
+  f(uint,   G1RemSetHowlNumBucketsConstraintFunc)     \
+                                                      \
+  /* G1 Heap Size Constraints */                      \
   f(size_t, G1HeapRegionSizeConstraintFunc)           \
   f(uintx,  G1NewSizePercentConstraintFunc)           \
   f(uintx,  G1MaxNewSizePercentConstraintFunc)        \

src/hotspot/share/gc/g1/sparsePRT.cpp

@@ -1,311 +0,0 @@
-/*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
-#include "gc/g1/sparsePRT.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
-#include "gc/shared/space.inline.hpp"
-#include "memory/allocation.inline.hpp"
-
-// Check that the size of the SparsePRTEntry is evenly divisible by the maximum
-// member type to avoid SIGBUS when accessing them.
-STATIC_ASSERT(sizeof(SparsePRTEntry) % sizeof(int) == 0);
-
-void SparsePRTEntry::init(RegionIdx_t region_ind) {
-  // Check that the card array element type can represent all cards in the region.
-  // Choose a large SparsePRTEntry::card_elem_t (e.g. CardIdx_t) if required.
-  assert(((size_t)1 << (sizeof(SparsePRTEntry::card_elem_t) * BitsPerByte)) *
-         G1CardTable::card_size >= HeapRegionBounds::max_size(), "precondition");
-  assert(G1RSetSparseRegionEntries > 0, "precondition");
-  _region_ind = region_ind;
-  _next_index = RSHashTable::NullEntry;
-  _next_null = 0;
-}
-
-bool SparsePRTEntry::contains_card(CardIdx_t card_index) const {
-  for (int i = 0; i < num_valid_cards(); i++) {
-    if (card(i) == card_index) {
-      return true;
-    }
-  }
-  return false;
-}
-
-SparsePRT::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) {
-  for (int i = 0; i < num_valid_cards(); i++) {
-    if (card(i) == card_index) {
-      return SparsePRT::found;
-    }
-  }
-  if (num_valid_cards() < cards_num() - 1) {
-    _cards[_next_null] = (card_elem_t)card_index;
-    _next_null++;
-    return SparsePRT::added;
-   }
-  // Otherwise, we're full.
-  return SparsePRT::overflow;
-}
-
-void SparsePRTEntry::copy_cards(card_elem_t* cards) const {
-  memcpy(cards, _cards, cards_num() * sizeof(card_elem_t));
-}
-
-void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const {
-  copy_cards(e->_cards);
-  assert(_next_null >= 0, "invariant");
-  assert(_next_null <= cards_num(), "invariant");
-  e->_next_null = _next_null;
-}
-
-// ----------------------------------------------------------------------
-
-float RSHashTable::TableOccupancyFactor = 0.5f;
-
-// The empty table can't hold any entries and is effectively immutable
-// This means it can be used as an initial sentinel value
-static int empty_buckets[] = { RSHashTable::NullEntry };
-RSHashTable RSHashTable::empty_table;
-
-RSHashTable::RSHashTable() :
-  _num_entries(0),
-  _capacity(0),
-  _capacity_mask(0),
-  _occupied_entries(0),
-  _entries(NULL),
-  _buckets(empty_buckets),
-  _free_region(0),
-  _free_list(NullEntry) { }
-
-RSHashTable::RSHashTable(size_t capacity) :
-  _num_entries((capacity * TableOccupancyFactor) + 1),
-  _capacity(capacity),
-  _capacity_mask(capacity - 1),
-  _occupied_entries(0),
-  _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, _num_entries * SparsePRTEntry::size(), mtGC)),
-  _buckets(NEW_C_HEAP_ARRAY(int, capacity, mtGC)),
-  _free_region(0),
-  _free_list(NullEntry)
-{
-  clear();
-}
-
-RSHashTable::~RSHashTable() {
-  // Nothing to free for empty RSHashTable
-  if (_buckets != empty_buckets) {
-    assert(_entries != NULL, "invariant");
-    FREE_C_HEAP_ARRAY(SparsePRTEntry, _entries);
-    FREE_C_HEAP_ARRAY(int, _buckets);
-  }
-}
-
-void RSHashTable::clear() {
-  assert(_buckets != empty_buckets, "Shouldn't call this for the empty_table");
-  _occupied_entries = 0;
-  guarantee(_entries != NULL, "invariant");
-  guarantee(_buckets != NULL, "invariant");
-
-  guarantee(_capacity <= ((size_t)1 << (sizeof(int)*BitsPerByte-1)) - 1,
-                "_capacity too large");
-
-  // This will put -1 == NullEntry in the key field of all entries.
-  memset((void*)_entries, NullEntry, _num_entries * SparsePRTEntry::size());
-  memset((void*)_buckets, NullEntry, _capacity * sizeof(int));
-  _free_list = NullEntry;
-  _free_region = 0;
-}
-
-SparsePRT::AddCardResult RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) {
-  assert(this != &empty_table, "can't add a card to the empty table");
-  SparsePRTEntry* e = entry_for_region_ind_create(region_ind);
-  assert(e != NULL && e->r_ind() == region_ind,
-         "Postcondition of call above.");
-  SparsePRT::AddCardResult res = e->add_card(card_index);
-  assert(e->num_valid_cards() > 0, "Postcondition");
-  return res;
-}
-
-SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) const {
-  int ind = (int) (region_ind & capacity_mask());
-  int cur_ind = _buckets[ind];
-  SparsePRTEntry* cur;
-  while (cur_ind != NullEntry &&
-         (cur = entry(cur_ind))->r_ind() != region_ind) {
-    cur_ind = cur->next_index();
-  }
-
-  if (cur_ind == NullEntry) return NULL;
-  // Otherwise...
-  assert(cur->r_ind() == region_ind, "Postcondition of loop + test above.");
-  assert(cur->num_valid_cards() > 0, "Inv");
-  return cur;
-}
-
-bool RSHashTable::delete_entry(RegionIdx_t region_ind) {
-  int ind = (int) (region_ind & capacity_mask());
-  int* prev_loc = &_buckets[ind];
-  int cur_ind = *prev_loc;
-  SparsePRTEntry* cur;
-  while (cur_ind != NullEntry &&
-         (cur = entry(cur_ind))->r_ind() != region_ind) {
-    prev_loc = cur->next_index_addr();
-    cur_ind = *prev_loc;
-  }
-
-  if (cur_ind == NullEntry) return false;
-  // Otherwise, splice out "cur".
-  *prev_loc = cur->next_index();
-  free_entry(cur_ind);
-  _occupied_entries--;
-  return true;
-}
-
-SparsePRTEntry*
-RSHashTable::entry_for_region_ind_create(RegionIdx_t region_ind) {
-  SparsePRTEntry* res = get_entry(region_ind);
-  if (res == NULL) {
-    int new_ind = alloc_entry();
-    res = entry(new_ind);
-    res->init(region_ind);
-    // Insert at front.
-    int ind = (int) (region_ind & capacity_mask());
-    res->set_next_index(_buckets[ind]);
-    _buckets[ind] = new_ind;
-    _occupied_entries++;
-  }
-  return res;
-}
-
-int RSHashTable::alloc_entry() {
-  int res;
-  if (_free_list != NullEntry) {
-    res = _free_list;
-    _free_list = entry(res)->next_index();
-    return res;
-  } else if ((size_t)_free_region < _num_entries) {
-    res = _free_region;
-    _free_region++;
-    return res;
-  } else {
-    return NullEntry;
-  }
-}
-
-void RSHashTable::free_entry(int fi) {
-  entry(fi)->set_next_index(_free_list);
-  _free_list = fi;
-}
-
-void RSHashTable::add_entry(SparsePRTEntry* e) {
-  assert(e->num_valid_cards() > 0, "Precondition.");
-  SparsePRTEntry* e2 = entry_for_region_ind_create(e->r_ind());
-  e->copy_cards(e2);
-  assert(e2->num_valid_cards() > 0, "Postcondition.");
-}
-
-bool RSHashTableBucketIter::has_next(SparsePRTEntry*& entry) {
-  while (_bl_ind == RSHashTable::NullEntry)  {
-    if (_tbl_ind + 1 >= _rsht->capacity()) {
-      return false;
-    }
-    _tbl_ind++;
-    _bl_ind = _rsht->_buckets[_tbl_ind];
-  }
-  entry = _rsht->entry(_bl_ind);
-  _bl_ind = entry->next_index();
-  return true;
-}
-
-bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) const {
-  SparsePRTEntry* e = get_entry(region_index);
-  return (e != NULL && e->contains_card(card_index));
-}
-
-size_t RSHashTable::mem_size() const {
-  return sizeof(RSHashTable) +
-    _num_entries * (SparsePRTEntry::size() + sizeof(int));
-}
-
-// ----------------------------------------------------------------------
-
-SparsePRT::SparsePRT() :
-  _table(&RSHashTable::empty_table) {
-}
-
-
-SparsePRT::~SparsePRT() {
-  if (_table != &RSHashTable::empty_table) {
-    delete _table;
-  }
-}
-
-
-size_t SparsePRT::mem_size() const {
-  // We ignore "_cur" here, because it either = _next, or else it is
-  // on the deleted list.
-  return sizeof(SparsePRT) + _table->mem_size();
-}
-
-SparsePRT::AddCardResult SparsePRT::add_card(RegionIdx_t region_id, CardIdx_t card_index) {
-  if (_table->should_expand()) {
-    expand();
-  }
-  return _table->add_card(region_id, card_index);
-}
-
-SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) {
-  return _table->get_entry(region_id);
-}
-
-bool SparsePRT::delete_entry(RegionIdx_t region_id) {
-  return _table->delete_entry(region_id);
-}
-
-void SparsePRT::clear() {
-  // If the entry table not at initial capacity, just reset to the empty table.
-  if (_table->capacity() == InitialCapacity) {
-    _table->clear();
-  } else if (_table != &RSHashTable::empty_table) {
-    delete _table;
-    _table = &RSHashTable::empty_table;
-  }
-}
-
-void SparsePRT::expand() {
-  RSHashTable* last = _table;
-  if (last != &RSHashTable::empty_table) {
-    _table = new RSHashTable(last->capacity() * 2);
-    for (size_t i = 0; i < last->num_entries(); i++) {
-      SparsePRTEntry* e = last->entry((int)i);
-      if (e->valid_entry()) {
-        _table->add_entry(e);
-      }
-    }
-    delete last;
-  } else {
-    _table = new RSHashTable(InitialCapacity);
-  }
-}

src/hotspot/share/gc/g1/sparsePRT.hpp

@@ -1,246 +0,0 @@
-/*
- * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_G1_SPARSEPRT_HPP
-#define SHARE_GC_G1_SPARSEPRT_HPP
-
-#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
-#include "memory/allocation.hpp"
-#include "runtime/mutex.hpp"
-#include "utilities/align.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-class RSHashTable;
-class SparsePRTEntry;
-
-// Sparse remembered set for a heap region (the "owning" region).  Maps
-// indices of other regions to short sequences of cards in the other region
-// that might contain pointers into the owner region.
-// Concurrent access to a SparsePRT must be serialized by some external mutex.
-class SparsePRT {
-  friend class SparsePRTBucketIter;
-
-  RSHashTable* _table;
-
-  static const size_t InitialCapacity = 8;
-
-  void expand();
-
-public:
-  SparsePRT();
-  ~SparsePRT();
-
-  size_t mem_size() const;
-
-  enum AddCardResult {
-    overflow, // The table is full, could not add the card to the table.
-    found,    // The card is already in the PRT.
-    added     // The card has been added.
-  };
-
-  // Attempts to ensure that the given card_index in the given region is in
-  // the sparse table.  If successful (because the card was already
-  // present, or because it was successfully added) returns "true".
-  // Otherwise, returns "false" to indicate that the addition would
-  // overflow the entry for the region.  The caller must transfer these
-  // entries to a larger-capacity representation.
-  AddCardResult add_card(RegionIdx_t region_id, CardIdx_t card_index);
-
-  // Return the pointer to the entry associated with the given region.
-  SparsePRTEntry* get_entry(RegionIdx_t region_ind);
-
-  // If there is an entry for "region_ind", removes it and return "true";
-  // otherwise returns "false."
-  bool delete_entry(RegionIdx_t region_ind);
-
-  // Clear the table, and reinitialize to initial capacity.
-  void clear();
-
-  bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const;
-};
-
-class SparsePRTEntry: public CHeapObj<mtGC> {
-public:
-  // The type of a card entry.
-  typedef uint16_t card_elem_t;
-
-private:
-  // We need to make sizeof(SparsePRTEntry) an even multiple of maximum member size,
-  // in order to force correct alignment that could otherwise cause SIGBUS errors
-  // when reading the member variables. This calculates the minimum number of card
-  // array elements required to get that alignment.
-  static const size_t card_array_alignment = sizeof(int) / sizeof(card_elem_t);
-
-  RegionIdx_t _region_ind;
-  int         _next_index;
-  int         _next_null;
-  // The actual cards stored in this array.
-  // WARNING: Don't put any data members beyond this line. Card array has, in fact, variable length.
-  // It should always be the last data member.
-  card_elem_t _cards[card_array_alignment];
-
-  // Copy the current entry's cards into "cards".
-  inline void copy_cards(card_elem_t* cards) const;
-public:
-  // Returns the size of the entry, used for entry allocation.
-  static size_t size() { return sizeof(SparsePRTEntry) + sizeof(card_elem_t) * (cards_num() - card_array_alignment); }
-  // Returns the size of the card array.
-  static int cards_num() {
-    return align_up((int)G1RSetSparseRegionEntries, (int)card_array_alignment);
-  }
-
-  // Set the region_ind to the given value, and delete all cards.
-  inline void init(RegionIdx_t region_ind);
-
-  RegionIdx_t r_ind() const { return _region_ind; }
-  bool valid_entry() const { return r_ind() >= 0; }
-
-  int next_index() const { return _next_index; }
-  int* next_index_addr() { return &_next_index; }
-  void set_next_index(int ni) { _next_index = ni; }
-
-  // Returns "true" iff the entry contains the given card index.
-  inline bool contains_card(CardIdx_t card_index) const;
-
-  // Returns the number of non-NULL card entries.
-  inline int num_valid_cards() const { return _next_null; }
-
-  inline SparsePRT::AddCardResult add_card(CardIdx_t card_index);
-
-  // Copy the current entry's cards into the "_card" array of "e."
-  inline void copy_cards(SparsePRTEntry* e) const;
-
-  card_elem_t* cards() { return _cards; }
-
-  inline CardIdx_t card(int i) const {
-    assert(i >= 0, "must be nonnegative");
-    assert(i < cards_num(), "range checking");
-    return (CardIdx_t)_cards[i];
-  }
-};
-
-class RSHashTable : public CHeapObj<mtGC> {
-
-  friend class RSHashTableBucketIter;
-
-  // Inverse maximum hash table occupancy used.
-  static float TableOccupancyFactor;
-
-  size_t _num_entries;
-
-  size_t _capacity;
-  size_t _capacity_mask;
-  size_t _occupied_entries;
-
-  SparsePRTEntry* _entries;
-  int* _buckets;
-  int  _free_region;
-  int  _free_list;
-
-  // Requires that the caller hold a lock preventing parallel modifying
-  // operations, and that the the table be less than completely full.  If
-  // an entry for "region_ind" is already in the table, finds it and
-  // returns its address; otherwise allocates, initializes, inserts and
-  // returns a new entry for "region_ind".
-  SparsePRTEntry* entry_for_region_ind_create(RegionIdx_t region_ind);
-
-  // Returns the index of the next free entry in "_entries".
-  int alloc_entry();
-  // Declares the entry "fi" to be free.  (It must have already been
-  // deleted from any bucket lists.
-  void free_entry(int fi);
-
-  // For the empty sentinel created at static initialization time
-  RSHashTable();
-
-public:
-  RSHashTable(size_t capacity);
-  ~RSHashTable();
-
-  static const int NullEntry = -1;
-  static RSHashTable empty_table;
-
-  bool should_expand() const { return _occupied_entries == _num_entries; }
-
-  // Attempts to ensure that the given card_index in the given region is in
-  // the sparse table.  If successful (because the card was already
-  // present, or because it was successfully added) returns "true".
-  // Otherwise, returns "false" to indicate that the addition would
-  // overflow the entry for the region.  The caller must transfer these
-  // entries to a larger-capacity representation.
-  SparsePRT::AddCardResult add_card(RegionIdx_t region_id, CardIdx_t card_index);
-
-  bool delete_entry(RegionIdx_t region_id);
-
-  bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const;
-
-  void add_entry(SparsePRTEntry* e);
-
-  SparsePRTEntry* get_entry(RegionIdx_t region_id) const;
-
-  void clear();
-
-  size_t capacity() const      { return _capacity; }
-  size_t capacity_mask() const { return _capacity_mask;  }
-  size_t mem_size() const;
-  // The number of SparsePRTEntry instances available.
-  size_t num_entries() const { return _num_entries; }
-
-  SparsePRTEntry* entry(int i) const {
-    assert(i >= 0 && (size_t)i < _num_entries, "precondition");
-    return (SparsePRTEntry*)((char*)_entries + SparsePRTEntry::size() * i);
-  }
-
-  void print();
-};
-
-// This is embedded in HRRS iterator.
-class RSHashTableBucketIter {
-  uint _tbl_ind;        // [0.._rsht->_capacity)
-  int  _bl_ind;         // [-1, 0.._rsht->_capacity)
-
-  RSHashTable* _rsht;
-
-public:
-  RSHashTableBucketIter(RSHashTable* rsht) :
-    _tbl_ind(0),
-    _bl_ind(rsht->_buckets[_tbl_ind]),
-    _rsht(rsht) { }
-
-  bool has_next(SparsePRTEntry*& entry);
-};
-
-class SparsePRTBucketIter: public RSHashTableBucketIter {
-public:
-  SparsePRTBucketIter(const SparsePRT* sprt) :
-    RSHashTableBucketIter(sprt->_table) {}
-
-  bool has_next(SparsePRTEntry*& entry) {
-    return RSHashTableBucketIter::has_next(entry);
-  }
-};
-
-#endif // SHARE_GC_G1_SPARSEPRT_HPP

src/hotspot/share/gc/shared/cardTable.hpp

@@ -235,8 +235,8 @@ public:
     card_size_in_words          = card_size / sizeof(HeapWord)
   };
 
-  static CardValue clean_card_val()          { return clean_card; }
-  static CardValue dirty_card_val()          { return dirty_card; }
+  static constexpr CardValue clean_card_val()          { return clean_card; }
+  static constexpr CardValue dirty_card_val()          { return dirty_card; }
   static intptr_t clean_card_row_val()   { return clean_card_row; }
 
   // Card marking array base (adjusted for heap low boundary)

src/hotspot/share/gc/shared/cardTableBarrierSet.cpp

@@ -187,7 +187,7 @@ void CardTableBarrierSet::on_thread_detach(Thread* thread) {
   // card-table (or other remembered set structure) before GC starts
   // processing the card-table (or other remembered set).
   if (thread->is_Java_thread()) { // Only relevant for Java threads.
-    flush_deferred_card_mark_barrier(thread->as_Java_thread());
+    flush_deferred_card_mark_barrier(JavaThread::cast(thread));
   }
 }
 

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -172,7 +172,7 @@ void MemAllocator::Allocation::check_for_valid_allocation_state() const {
   assert(!_thread->has_pending_exception(),
          "shouldn't be allocating with pending exception");
   // Allocation of an oop can always invoke a safepoint.
-  _thread->as_Java_thread()->check_for_valid_safepoint_state();
+  JavaThread::cast(_thread)->check_for_valid_safepoint_state();
 }
 #endif
 

src/hotspot/share/gc/shared/referenceProcessor.hpp

@@ -397,7 +397,6 @@ public:
 
   // whether discovery is atomic wrt other collectors
   bool discovery_is_atomic() const { return _discovery_is_atomic; }
-  void set_atomic_discovery(bool atomic) { _discovery_is_atomic = atomic; }
 
   // whether discovery is done by multiple threads same-old-timeously
   bool discovery_is_mt() const { return _discovery_is_mt; }
@@ -561,27 +560,6 @@ class ReferenceProcessorIsAliveMutator: StackObj {
   }
 };
 
-// A utility class to temporarily change the disposition
-// of the "discovery_is_atomic" field of the
-// given ReferenceProcessor in the scope that contains it.
-class ReferenceProcessorAtomicMutator: StackObj {
- private:
-  ReferenceProcessor* _rp;
-  bool                _saved_atomic_discovery;
-
- public:
-  ReferenceProcessorAtomicMutator(ReferenceProcessor* rp,
-                                  bool atomic):
-    _rp(rp) {
-    _saved_atomic_discovery = _rp->discovery_is_atomic();
-    _rp->set_atomic_discovery(atomic);
-  }
-
-  ~ReferenceProcessorAtomicMutator() {
-    _rp->set_atomic_discovery(_saved_atomic_discovery);
-  }
-};
-
 enum class RefProcThreadModel { Multi, Single };
 
 /*

src/hotspot/share/gc/shared/workerDataArray.hpp

@@ -34,7 +34,7 @@ template <class T>
 class WorkerDataArray  : public CHeapObj<mtGC> {
   friend class WDAPrinter;
 public:
-  static const uint MaxThreadWorkItems = 6;
+  static const uint MaxThreadWorkItems = 9;
 private:
   T*          _data;
   uint        _length;

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -102,7 +102,7 @@ void ShenandoahBarrierSet::on_thread_attach(Thread *thread) {
     ShenandoahThreadLocalData::set_disarmed_value(thread, ShenandoahCodeRoots::disarmed_value());
 
     if (ShenandoahStackWatermarkBarrier) {
-      JavaThread* const jt = thread->as_Java_thread();
+      JavaThread* const jt = JavaThread::cast(thread);
       StackWatermark* const watermark = new ShenandoahStackWatermark(jt);
       StackWatermarkSet::add_watermark(jt, watermark);
     }
@@ -122,10 +122,10 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
     if (ShenandoahStackWatermarkBarrier) {
       if (_heap->is_concurrent_mark_in_progress()) {
         ShenandoahKeepAliveClosure oops;
-        StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc);
+        StackWatermarkSet::finish_processing(JavaThread::cast(thread), &oops, StackWatermarkKind::gc);
       } else if (_heap->is_concurrent_weak_root_in_progress() && _heap->is_evacuation_in_progress()) {
         ShenandoahContextEvacuateUpdateRootsClosure oops;
-        StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc);
+        StackWatermarkSet::finish_processing(JavaThread::cast(thread), &oops, StackWatermarkKind::gc);
       }
     }
   }

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -621,7 +621,7 @@ ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(Oop
 }
 
 void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
-  JavaThread* const jt = thread->as_Java_thread();
+  JavaThread* const jt = JavaThread::cast(thread);
   StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
 }
 
@@ -934,7 +934,7 @@ ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
 
 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
   if (thread->is_Java_thread()) {
-    JavaThread* jt = thread->as_Java_thread();
+    JavaThread* jt = JavaThread::cast(thread);
     ResourceMark rm;
     jt->oops_do(&_cl, NULL);
   }

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1750,7 +1750,7 @@ bool ShenandoahHeap::try_cancel_gc() {
     if (thread->is_Java_thread()) {
       // We need to provide a safepoint here, otherwise we might
       // spin forever if a SP is pending.
-      ThreadBlockInVM sp(thread->as_Java_thread());
+      ThreadBlockInVM sp(JavaThread::cast(thread));
       SpinPause();
     }
   }