8319548: Unexpected internal name for Filler array klass causes error in VisualVM

Co-authored-by: Tomáš Hůrka <tomas.hurka@oracle.com>
Reviewed-by: ayang, dholmes

make/autoconf/spec.gmk.template

@@ -191,7 +191,7 @@ PRODUCT_NAME := @PRODUCT_NAME@
 PRODUCT_SUFFIX := @PRODUCT_SUFFIX@
 JDK_RC_PLATFORM_NAME := @JDK_RC_PLATFORM_NAME@
 JDK_RC_NAME := @JDK_RC_NAME@
-JDK_RC_COMPANY_NAME:=@JDK_RC_COMPANY_NAME@
+JDK_RC_COMPANY_NAME := @JDK_RC_COMPANY_NAME@
 COMPANY_NAME := @COMPANY_NAME@
 HOTSPOT_VM_DISTRO := @HOTSPOT_VM_DISTRO@
 MACOSX_BUNDLE_NAME_BASE := @MACOSX_BUNDLE_NAME_BASE@

make/data/hotspot-symbols/symbols-unix

@@ -223,6 +223,7 @@ JVM_VirtualThreadEnd
 JVM_VirtualThreadMount
 JVM_VirtualThreadUnmount
 JVM_VirtualThreadHideFrames
+JVM_VirtualThreadDisableSuspend
 
 # Scoped values
 JVM_EnsureMaterializedForStackWalk_func

make/jdk/src/classes/build/tools/generatecurrencydata/GenerateCurrencyData.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.text.SimpleDateFormat;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Locale;
@@ -339,9 +340,15 @@ public class GenerateCurrencyData {
                 validCurrencyCodes.substring(i * 7 + 3, i * 7 + 6));
             checkCurrencyCode(currencyCode);
             int tableEntry = mainTable[(currencyCode.charAt(0) - 'A') * A_TO_Z + (currencyCode.charAt(1) - 'A')];
-            if (tableEntry == INVALID_COUNTRY_ENTRY ||
-                    (tableEntry & SPECIAL_CASE_COUNTRY_MASK) != 0 ||
-                    (tableEntry & SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK) != (currencyCode.charAt(2) - 'A')) {
+
+            // Do not allow a future currency to be classified as an otherCurrency,
+            // otherwise it will leak out into Currency:getAvailableCurrencies
+            boolean futureCurrency = Arrays.asList(specialCaseNewCurrencies).contains(currencyCode);
+            boolean simpleCurrency = (tableEntry & SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK) == (currencyCode.charAt(2) - 'A');
+
+            // If neither a simple currency, or one defined in the future
+            // then the current currency is applicable to be added to the otherTable
+            if (!futureCurrency && !simpleCurrency) {
                 if (otherCurrenciesCount == maxOtherCurrencies) {
                     throw new RuntimeException("too many other currencies");
                 }

src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp

@@ -28,6 +28,7 @@
 #include "code/compiledIC.hpp"
 #include "code/icBuffer.hpp"
 #include "code/nmethod.hpp"
+#include "logging/log.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/safepoint.hpp"
@@ -90,9 +91,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
   address stub = find_stub();
   guarantee(stub != nullptr, "stub not found");
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+    log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
                   p2i(instruction_address()),
                   callee->name_and_sig_as_C_string());
   }

src/hotspot/cpu/arm/compiledIC_arm.cpp

@@ -28,6 +28,7 @@
 #include "code/icBuffer.hpp"
 #include "code/nativeInst.hpp"
 #include "code/nmethod.hpp"
+#include "logging/log.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/safepoint.hpp"
@@ -105,9 +106,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
   address stub = find_stub();
   guarantee(stub != nullptr, "stub not found");
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+    log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
                   p2i(instruction_address()),
                   callee->name_and_sig_as_C_string());
   }

src/hotspot/cpu/ppc/compiledIC_ppc.cpp

@@ -167,9 +167,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
   address stub = find_stub();
   guarantee(stub != nullptr, "stub not found");
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+    log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
                   p2i(instruction_address()),
                   callee->name_and_sig_as_C_string());
   }

src/hotspot/cpu/riscv/assembler_riscv.hpp

@@ -758,6 +758,8 @@ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
   INSN(amomax_d , 0b0101111, 0b011, 0b10100);
   INSN(amominu_d, 0b0101111, 0b011, 0b11000);
   INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
+  INSN(amocas_w,  0b0101111, 0b010, 0b00101);
+  INSN(amocas_d,  0b0101111, 0b011, 0b00101);
 #undef INSN
 
 enum operand_size { int8, int16, int32, uint32, int64 };
@@ -2940,6 +2942,17 @@ public:
     return uabs(target - branch) < branch_range;
   }
 
+  // Decode the given instruction, checking if it's a 16-bit compressed
+  // instruction and return the address of the next instruction.
+  static address locate_next_instruction(address inst) {
+    // Instruction wider than 16 bits has the two least-significant bits set.
+    if ((0x3 & *inst) == 0x3) {
+      return inst + instruction_size;
+    } else {
+      return inst + compressed_instruction_size;
+    }
+  }
+
   Assembler(CodeBuffer* code) : AbstractAssembler(code), _in_compressible_region(true) {}
 };
 

src/hotspot/cpu/riscv/compiledIC_riscv.cpp

@@ -29,6 +29,7 @@
 #include "code/compiledIC.hpp"
 #include "code/icBuffer.hpp"
 #include "code/nmethod.hpp"
+#include "logging/log.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/safepoint.hpp"
@@ -88,9 +89,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
   address stub = find_stub();
   guarantee(stub != nullptr, "stub not found");
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+    log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
                   p2i(instruction_address()),
                   callee->name_and_sig_as_C_string());
   }

src/hotspot/cpu/riscv/gc/x/x_riscv.ad

@@ -52,11 +52,11 @@ static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node
 %}
 
 // Load Pointer
-instruct xLoadP(iRegPNoSp dst, memory mem)
+instruct xLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
 %{
   match(Set dst (LoadP mem));
   predicate(UseZGC && !ZGenerational && (n->as_Load()->barrier_data() != 0));
-  effect(TEMP dst);
+  effect(TEMP dst, TEMP tmp);
 
   ins_cost(4 * DEFAULT_COST);
 
@@ -65,17 +65,17 @@ instruct xLoadP(iRegPNoSp dst, memory mem)
   ins_encode %{
     const Address ref_addr (as_Register($mem$$base), $mem$$disp);
     __ ld($dst$$Register, ref_addr);
-    x_load_barrier(_masm, this, ref_addr, $dst$$Register, t0 /* tmp */, barrier_data());
+    x_load_barrier(_masm, this, ref_addr, $dst$$Register, $tmp$$Register /* tmp */, barrier_data());
   %}
 
   ins_pipe(iload_reg_mem);
 %}
 
-instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
   predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
-  effect(KILL cr, TEMP_DEF res);
+  effect(TEMP_DEF res, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -86,17 +86,15 @@ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
     Label failed;
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-               Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
-               true /* result_as_bool */);
-    __ beqz($res$$Register, failed);
-    __ mv(t0, $oldval$$Register);
-    __ bind(failed);
+               Assembler::relaxed /* acquire */, Assembler::rl /* release */, $tmp$$Register);
+    __ sub(t0, $tmp$$Register, $oldval$$Register);
+    __ seqz($res$$Register, t0);
     if (barrier_data() != XLoadBarrierElided) {
       Label good;
-      __ ld(t1, Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
-      __ andr(t1, t1, t0);
-      __ beqz(t1, good);
-      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
+      __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(t0, t0, $tmp$$Register);
+      __ beqz(t0, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
       __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
                  Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
                  true /* result_as_bool */);
@@ -107,11 +105,11 @@ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
   ins_pipe(pipe_slow);
 %}
 
-instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
   predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
-  effect(KILL cr, TEMP_DEF res);
+  effect(TEMP_DEF res, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -122,17 +120,15 @@ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
     Label failed;
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-               Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
-               true /* result_as_bool */);
-    __ beqz($res$$Register, failed);
-    __ mv(t0, $oldval$$Register);
-    __ bind(failed);
+               Assembler::aq /* acquire */, Assembler::rl /* release */, $tmp$$Register);
+    __ sub(t0, $tmp$$Register, $oldval$$Register);
+    __ seqz($res$$Register, t0);
     if (barrier_data() != XLoadBarrierElided) {
       Label good;
-      __ ld(t1, Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
-      __ andr(t1, t1, t0);
-      __ beqz(t1, good);
-      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
+      __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(t0, t0, $tmp$$Register);
+      __ beqz(t0, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
       __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
                  Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
                  true /* result_as_bool */);
@@ -143,10 +139,10 @@ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
   ins_pipe(pipe_slow);
 %}
 
-instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
-  effect(TEMP_DEF res);
+  effect(TEMP_DEF res, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -161,7 +157,7 @@ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
       __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
       __ andr(t0, t0, $res$$Register);
       __ beqz(t0, good);
-      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
       __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
                  Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
       __ bind(good);
@@ -171,10 +167,10 @@ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
   ins_pipe(pipe_slow);
 %}
 
-instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
-  effect(TEMP_DEF res);
+  effect(TEMP_DEF res, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -189,7 +185,7 @@ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
       __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
       __ andr(t0, t0, $res$$Register);
       __ beqz(t0, good);
-      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
       __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
                  Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
       __ bind(good);
@@ -199,10 +195,10 @@ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
   ins_pipe(pipe_slow);
 %}
 
-instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
   match(Set prev (GetAndSetP mem newv));
   predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP_DEF prev, KILL cr);
+  effect(TEMP_DEF prev, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -210,16 +206,16 @@ instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
 
   ins_encode %{
     __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
-    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
   %}
 
   ins_pipe(pipe_serial);
 %}
 
-instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
   match(Set prev (GetAndSetP mem newv));
   predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0));
-  effect(TEMP_DEF prev, KILL cr);
+  effect(TEMP_DEF prev, TEMP tmp);
 
   ins_cost(VOLATILE_REF_COST);
 
@@ -227,7 +223,7 @@ instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr)
 
   ins_encode %{
     __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
-    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
   %}
   ins_pipe(pipe_serial);
 %}

src/hotspot/cpu/riscv/gc/z/z_riscv.ad

@@ -79,7 +79,7 @@ static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address
 
 static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
   if (node->barrier_data() == ZBarrierElided) {
-    z_color(_masm, node, rnew_zpointer, rnew_zaddress, t0);
+    z_color(_masm, node, rnew_zpointer, rnew_zaddress, tmp);
   } else {
     bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
     ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
@@ -90,11 +90,11 @@ static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address
 %}
 
 // Load Pointer
-instruct zLoadP(iRegPNoSp dst, memory mem)
+instruct zLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
 %{
   match(Set dst (LoadP mem));
   predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
-  effect(TEMP dst);
+  effect(TEMP dst, TEMP tmp);
 
   ins_cost(4 * DEFAULT_COST);
 
@@ -103,34 +103,35 @@ instruct zLoadP(iRegPNoSp dst, memory mem)
   ins_encode %{
     const Address ref_addr(as_Register($mem$$base), $mem$$disp);
     __ ld($dst$$Register, ref_addr);
-    z_load_barrier(_masm, this, ref_addr, $dst$$Register, t0);
+    z_load_barrier(_masm, this, ref_addr, $dst$$Register, $tmp$$Register);
   %}
 
   ins_pipe(iload_reg_mem);
 %}
 
 // Store Pointer
-instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2)
 %{
   predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
   match(Set mem (StoreP mem src));
-  effect(TEMP tmp, KILL cr);
+  effect(TEMP tmp1, TEMP tmp2);
 
   ins_cost(125); // XXX
   format %{ "sd    $mem, $src\t# ptr" %}
   ins_encode %{
     const Address ref_addr(as_Register($mem$$base), $mem$$disp);
-    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, t1, false /* is_atomic */);
-    __ sd($tmp$$Register, ref_addr);
+    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp1$$Register, $tmp2$$Register, false /* is_atomic */);
+    __ sd($tmp1$$Register, ref_addr);
   %}
   ins_pipe(pipe_serial);
 %}
 
-instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval,
+                          iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
   predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -140,19 +141,20 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
-    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval,
+                             iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
   predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -162,18 +164,19 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
-    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval,
+                              iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -182,8 +185,8 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
-    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
     z_uncolor(_masm, this, $res$$Register);
   %}
@@ -191,10 +194,11 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval,
+                                 iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -203,8 +207,8 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
-    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
     z_uncolor(_masm, this, $res$$Register);
   %}
@@ -212,17 +216,17 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
   ins_pipe(pipe_slow);
 %}
 
-instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
   match(Set prev (GetAndSetP mem newv));
   predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP_DEF prev, KILL cr);
+  effect(TEMP_DEF prev, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "atomic_xchg  $prev, $newv, [$mem], #@zGetAndSetP" %}
 
   ins_encode %{
-    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, t1, true /* is_atomic */);
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
     __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
     z_uncolor(_masm, this, $prev$$Register);
   %}
@@ -230,17 +234,17 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   ins_pipe(pipe_serial);
 %}
 
-instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
   match(Set prev (GetAndSetP mem newv));
   predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP_DEF prev, KILL cr);
+  effect(TEMP_DEF prev, TEMP tmp);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "atomic_xchg_acq  $prev, $newv, [$mem], #@zGetAndSetPAcq" %}
 
   ins_encode %{
-    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, t1, true /* is_atomic */);
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
     __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
     z_uncolor(_masm, this, $prev$$Register);
   %}

src/hotspot/cpu/riscv/globals_riscv.hpp

@@ -105,6 +105,7 @@ define_pd_global(intx, InlineSmallCode,          1000);
   product(bool, UseZba, false, "Use Zba instructions")                           \
   product(bool, UseZbb, false, "Use Zbb instructions")                           \
   product(bool, UseZbs, false, "Use Zbs instructions")                           \
+  product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions")         \
   product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions")       \
   product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions")       \
   product(bool, UseZicbop, false, EXPERIMENTAL, "Use Zicbop instructions")       \

src/hotspot/cpu/riscv/interp_masm_riscv.cpp

@@ -777,7 +777,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
       assert(lock_offset == 0,
              "displached header must be first word in BasicObjectLock");
 
-      cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+      cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, tmp, count, /*fallthrough*/nullptr);
 
       // Test if the oopMark is an obvious stack pointer, i.e.,
       //  1) (mark & 7) == 0, and
@@ -891,7 +891,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
       beqz(header_reg, count);
 
       // Atomic swap back the old header
-      cmpxchg_obj_header(swap_reg, header_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+      cmpxchg_obj_header(swap_reg, header_reg, obj_reg, tmp_reg, count, /*fallthrough*/nullptr);
     }
 
     // Call the runtime routine for slow case.

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -2725,27 +2725,36 @@ void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acqui
 
 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
                                 Label &succeed, Label *fail) {
-  assert_different_registers(addr, tmp);
-  assert_different_registers(newv, tmp);
-  assert_different_registers(oldv, tmp);
+  assert_different_registers(addr, tmp, t0);
+  assert_different_registers(newv, tmp, t0);
+  assert_different_registers(oldv, tmp, t0);
 
   // oldv holds comparison value
   // newv holds value to write in exchange
   // addr identifies memory word to compare against/update
-  Label retry_load, nope;
-  bind(retry_load);
-  // Load reserved from the memory location
-  load_reserved(tmp, addr, int64, Assembler::aqrl);
-  // Fail and exit if it is not what we expect
-  bne(tmp, oldv, nope);
-  // If the store conditional succeeds, tmp will be zero
-  store_conditional(tmp, newv, addr, int64, Assembler::rl);
-  beqz(tmp, succeed);
-  // Retry only when the store conditional failed
-  j(retry_load);
+  if (UseZacas) {
+    mv(tmp, oldv);
+    atomic_cas(tmp, newv, addr, Assembler::int64, Assembler::aq, Assembler::rl);
+    beq(tmp, oldv, succeed);
+  } else {
+    Label retry_load, nope;
+    bind(retry_load);
+    // Load reserved from the memory location
+    load_reserved(tmp, addr, int64, Assembler::aqrl);
+    // Fail and exit if it is not what we expect
+    bne(tmp, oldv, nope);
+    // If the store conditional succeeds, tmp will be zero
+    store_conditional(tmp, newv, addr, int64, Assembler::rl);
+    beqz(tmp, succeed);
+    // Retry only when the store conditional failed
+    j(retry_load);
 
-  bind(nope);
+    bind(nope);
+  }
+
+  // neither amocas nor lr/sc have an implied barrier in the failing case
   membar(AnyAny);
+
   mv(oldv, tmp);
   if (fail != nullptr) {
     j(*fail);
@@ -2771,7 +2780,7 @@ void MacroAssembler::load_reserved(Register dst,
       break;
     case uint32:
       lr_w(dst, addr, acquire);
-      zero_extend(t0, t0, 32);
+      zero_extend(dst, dst, 32);
       break;
     default:
       ShouldNotReachHere();
@@ -2819,7 +2828,7 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
   }
   sll(mask, mask, shift);
 
-  xori(not_mask, mask, -1);
+  notr(not_mask, mask);
 
   sll(expected, expected, shift);
   andr(expected, expected, mask);
@@ -2829,7 +2838,7 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
 }
 
 // cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps.
-// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w,
+// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w or amocas.w,
 // which are forced to work with 4-byte aligned address.
 void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
                                           Register new_val,
@@ -2844,14 +2853,29 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
   Label retry, fail, done;
 
   bind(retry);
-  lr_w(old, aligned_addr, acquire);
-  andr(tmp, old, mask);
-  bne(tmp, expected, fail);
 
-  andr(tmp, old, not_mask);
-  orr(tmp, tmp, new_val);
-  sc_w(tmp, tmp, aligned_addr, release);
-  bnez(tmp, retry);
+  if (UseZacas) {
+    lw(old, aligned_addr);
+
+    // if old & mask != expected
+    andr(tmp, old, mask);
+    bne(tmp, expected, fail);
+
+    andr(tmp, old, not_mask);
+    orr(tmp, tmp, new_val);
+
+    atomic_cas(old, tmp, aligned_addr, operand_size::int32, acquire, release);
+    bne(tmp, old, retry);
+  } else {
+    lr_w(old, aligned_addr, acquire);
+    andr(tmp, old, mask);
+    bne(tmp, expected, fail);
+
+    andr(tmp, old, not_mask);
+    orr(tmp, tmp, new_val);
+    sc_w(tmp, tmp, aligned_addr, release);
+    bnez(tmp, retry);
+  }
 
   if (result_as_bool) {
     mv(result, 1);
@@ -2891,14 +2915,28 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
 
   Label fail, done;
 
-  lr_w(old, aligned_addr, acquire);
-  andr(tmp, old, mask);
-  bne(tmp, expected, fail);
+  if (UseZacas) {
+    lw(old, aligned_addr);
 
-  andr(tmp, old, not_mask);
-  orr(tmp, tmp, new_val);
-  sc_w(tmp, tmp, aligned_addr, release);
-  bnez(tmp, fail);
+    // if old & mask != expected
+    andr(tmp, old, mask);
+    bne(tmp, expected, fail);
+
+    andr(tmp, old, not_mask);
+    orr(tmp, tmp, new_val);
+
+    atomic_cas(tmp, new_val, addr, operand_size::int32, acquire, release);
+    bne(tmp, old, fail);
+  } else {
+    lr_w(old, aligned_addr, acquire);
+    andr(tmp, old, mask);
+    bne(tmp, expected, fail);
+
+    andr(tmp, old, not_mask);
+    orr(tmp, tmp, new_val);
+    sc_w(tmp, tmp, aligned_addr, release);
+    bnez(tmp, fail);
+  }
 
   // Success
   mv(result, 1);
@@ -2921,6 +2959,19 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
   assert_different_registers(expected, t0);
   assert_different_registers(new_val, t0);
 
+  if (UseZacas) {
+    if (result_as_bool) {
+      mv(t0, expected);
+      atomic_cas(t0, new_val, addr, size, acquire, release);
+      xorr(t0, t0, expected);
+      seqz(result, t0);
+    } else {
+      mv(result, expected);
+      atomic_cas(result, new_val, addr, size, acquire, release);
+    }
+    return;
+  }
+
   Label retry_load, done, ne_done;
   bind(retry_load);
   load_reserved(t0, addr, size, acquire);
@@ -2952,6 +3003,11 @@ void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
                                   enum operand_size size,
                                   Assembler::Aqrl acquire, Assembler::Aqrl release,
                                   Register result) {
+  if (UseZacas) {
+    cmpxchg(addr, expected, new_val, size, acquire, release, result, true);
+    return;
+  }
+
   assert_different_registers(addr, t0);
   assert_different_registers(expected, t0);
   assert_different_registers(new_val, t0);
@@ -3018,6 +3074,89 @@ ATOMIC_XCHGU(xchgalwu, xchgalw)
 
 #undef ATOMIC_XCHGU
 
+#define ATOMIC_CAS(OP, AOP, ACQUIRE, RELEASE)                                        \
+void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) {      \
+  assert(UseZacas, "invariant");                                                     \
+  prev = prev->is_valid() ? prev : zr;                                               \
+  AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE));                       \
+  return;                                                                            \
+}
+
+ATOMIC_CAS(cas, amocas_d, Assembler::relaxed, Assembler::relaxed)
+ATOMIC_CAS(casw, amocas_w, Assembler::relaxed, Assembler::relaxed)
+ATOMIC_CAS(casl, amocas_d, Assembler::relaxed, Assembler::rl)
+ATOMIC_CAS(caslw, amocas_w, Assembler::relaxed, Assembler::rl)
+ATOMIC_CAS(casal, amocas_d, Assembler::aq, Assembler::rl)
+ATOMIC_CAS(casalw, amocas_w, Assembler::aq, Assembler::rl)
+
+#undef ATOMIC_CAS
+
+#define ATOMIC_CASU(OP1, OP2)                                                        \
+void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) {     \
+  atomic_##OP2(prev, newv, addr);                                                    \
+  zero_extend(prev, prev, 32);                                                       \
+  return;                                                                            \
+}
+
+ATOMIC_CASU(caswu, casw)
+ATOMIC_CASU(caslwu, caslw)
+ATOMIC_CASU(casalwu, casalw)
+
+#undef ATOMIC_CASU
+
+void MacroAssembler::atomic_cas(
+    Register prev, Register newv, Register addr, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
+  switch (size) {
+    case int64:
+      switch ((Assembler::Aqrl)(acquire | release)) {
+        case Assembler::relaxed:
+          atomic_cas(prev, newv, addr);
+          break;
+        case Assembler::rl:
+          atomic_casl(prev, newv, addr);
+          break;
+        case Assembler::aqrl:
+          atomic_casal(prev, newv, addr);
+          break;
+        default:
+          ShouldNotReachHere();
+      }
+      break;
+    case int32:
+      switch ((Assembler::Aqrl)(acquire | release)) {
+        case Assembler::relaxed:
+          atomic_casw(prev, newv, addr);
+          break;
+        case Assembler::rl:
+          atomic_caslw(prev, newv, addr);
+          break;
+        case Assembler::aqrl:
+          atomic_casalw(prev, newv, addr);
+          break;
+        default:
+          ShouldNotReachHere();
+      }
+      break;
+    case uint32:
+      switch ((Assembler::Aqrl)(acquire | release)) {
+        case Assembler::relaxed:
+          atomic_caswu(prev, newv, addr);
+          break;
+        case Assembler::rl:
+          atomic_caslwu(prev, newv, addr);
+          break;
+        case Assembler::aqrl:
+          atomic_casalwu(prev, newv, addr);
+          break;
+        default:
+          ShouldNotReachHere();
+      }
+      break;
+    default:
+      ShouldNotReachHere();
+  }
+}
+
 void MacroAssembler::far_jump(const Address &entry, Register tmp) {
   assert(ReservedCodeCacheSize < 4*G, "branch out of range");
   assert(CodeCache::find_blob(entry.target()) != nullptr,

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -1063,6 +1063,19 @@ public:
   void atomic_xchgwu(Register prev, Register newv, Register addr);
   void atomic_xchgalwu(Register prev, Register newv, Register addr);
 
+  void atomic_cas(Register prev, Register newv, Register addr);
+  void atomic_casw(Register prev, Register newv, Register addr);
+  void atomic_casl(Register prev, Register newv, Register addr);
+  void atomic_caslw(Register prev, Register newv, Register addr);
+  void atomic_casal(Register prev, Register newv, Register addr);
+  void atomic_casalw(Register prev, Register newv, Register addr);
+  void atomic_caswu(Register prev, Register newv, Register addr);
+  void atomic_caslwu(Register prev, Register newv, Register addr);
+  void atomic_casalwu(Register prev, Register newv, Register addr);
+
+  void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size,
+              Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);
+
   // Emit a far call/jump. Only invalidates the tmp register which
   // is used to keep the entry address for jalr.
   // The address must be inside the code cache.

src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp

@@ -1675,7 +1675,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
       __ sd(swap_reg, Address(lock_reg, mark_word_offset));
 
       // src -> dest if dest == x10 else x10 <- dest
-      __ cmpxchg_obj_header(x10, lock_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+      __ cmpxchg_obj_header(x10, lock_reg, obj_reg, lock_tmp, count, /*fallthrough*/nullptr);
 
       // Test if the oopMark is an obvious stack pointer, i.e.,
       //  1) (mark & 3) == 0, and
@@ -1815,7 +1815,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
       // Atomic swap old header if oop still contains the stack lock
       Label count;
-      __ cmpxchg_obj_header(x10, old_hdr, obj_reg, t0, count, &slow_path_unlock);
+      __ cmpxchg_obj_header(x10, old_hdr, obj_reg, lock_tmp, count, &slow_path_unlock);
       __ bind(count);
       __ decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
     } else {

src/hotspot/cpu/riscv/stubGenerator_riscv.cpp

@@ -4155,14 +4155,18 @@ class StubGenerator: public StubCodeGenerator {
 
     // to minimize the number of memory operations:
     // read the 4 state 4-byte values in pairs, with a single ld,
-    // and split them into 2 registers
-    __ mv(t0, mask32);
+    // and split them into 2 registers.
+    //
+    // And, as the core algorithm of md5 works on 32-bits words, so
+    // in the following code, it does not care about the content of
+    // higher 32-bits in state[x]. Based on this observation,
+    // we can apply further optimization, which is to just ignore the
+    // higher 32-bits in state0/state2, rather than set the higher
+    // 32-bits of state0/state2 to zero explicitly with extra instructions.
     __ ld(state0, Address(state));
     __ srli(state1, state0, 32);
-    __ andr(state0, state0, t0);
     __ ld(state2, Address(state, 8));
     __ srli(state3, state2, 32);
-    __ andr(state2, state2, t0);
 
     Label md5_loop;
     __ BIND(md5_loop);

src/hotspot/cpu/riscv/vm_version_riscv.hpp

@@ -142,6 +142,7 @@ class VM_Version : public Abstract_VM_Version {
   decl(ext_Zic64b      , "Zic64b"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b))      \
   decl(ext_Ztso        , "Ztso"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso))        \
   decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
+  decl(ext_Zacas       , "Zacas"       , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas))       \
   decl(mvendorid       , "VendorId"    , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)              \
   decl(marchid         , "ArchId"      , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)              \
   decl(mimpid          , "ImpId"       , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)              \

src/hotspot/cpu/s390/compiledIC_s390.cpp

@@ -95,9 +95,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
   address stub = find_stub();
   guarantee(stub != nullptr, "stub not found");
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+    log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
                   p2i(instruction_address()),
                   callee->name_and_sig_as_C_string());
   }

src/hotspot/cpu/x86/assembler_x86.cpp

@@ -2846,6 +2846,13 @@ void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
   emit_int16(0x47, (0xC0 | encode));
 }
 
+void Assembler::kxnorwl(KRegister dst, KRegister src1, KRegister src2) {
+  assert(VM_Version::supports_evex(), "");
+  InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
+  int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
+  emit_int16(0x46, (0xC0 | encode));
+}
+
 void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
   assert(VM_Version::supports_evex(), "");
   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -10771,7 +10778,7 @@ void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int v
   assert(src.isxmmindex(),"expected to be xmm index");
   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
   InstructionMark im(this);
-  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
   emit_int8((unsigned char)0x90);
   emit_operand(dst, src, 0);
@@ -10784,7 +10791,7 @@ void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int v
   assert(src.isxmmindex(),"expected to be xmm index");
   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
   InstructionMark im(this);
-  InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
   emit_int8((unsigned char)0x90);
   emit_operand(dst, src, 0);
@@ -10797,7 +10804,7 @@ void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int v
   assert(src.isxmmindex(),"expected to be xmm index");
   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
   InstructionMark im(this);
-  InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
   emit_int8((unsigned char)0x92);
   emit_operand(dst, src, 0);
@@ -10810,7 +10817,7 @@ void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int v
   assert(src.isxmmindex(),"expected to be xmm index");
   assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
   InstructionMark im(this);
-  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
+  InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
   vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
   emit_int8((unsigned char)0x92);
   emit_operand(dst, src, 0);

src/hotspot/cpu/x86/assembler_x86.hpp

@@ -1524,6 +1524,8 @@ private:
   void kordl(KRegister dst, KRegister src1, KRegister src2);
   void korql(KRegister dst, KRegister src1, KRegister src2);
 
+  void kxnorwl(KRegister dst, KRegister src1, KRegister src2);
+
   void kxorbl(KRegister dst, KRegister src1, KRegister src2);
   void kxorwl(KRegister dst, KRegister src1, KRegister src2);
   void kxordl(KRegister dst, KRegister src1, KRegister src2);

src/hotspot/cpu/x86/compiledIC_x86.cpp

@@ -28,6 +28,7 @@
 #include "code/compiledIC.hpp"
 #include "code/icBuffer.hpp"
 #include "code/nmethod.hpp"
+#include "logging/log.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/safepoint.hpp"
@@ -84,9 +85,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
   address stub = find_stub();
   guarantee(stub != nullptr, "stub not found");
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+    log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
                   p2i(instruction_address()),
                   callee->name_and_sig_as_C_string());
   }

src/hotspot/cpu/x86/x86.ad

@@ -4053,39 +4053,26 @@ instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
   effect(TEMP dst, TEMP tmp, TEMP mask);
   format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
   ins_encode %{
-    assert(UseAVX >= 2, "sanity");
-
     int vlen_enc = vector_length_encoding(this);
     BasicType elem_bt = Matcher::vector_element_basic_type(this);
-
-    assert(Matcher::vector_length_in_bytes(this) >= 16, "sanity");
     assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
-
-    if (vlen_enc == Assembler::AVX_128bit) {
-      __ movdqu($mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), noreg);
-    } else {
-      __ vmovdqu($mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), noreg);
-    }
+    __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
     __ lea($tmp$$Register, $mem$$Address);
     __ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
   %}
   ins_pipe( pipe_slow );
 %}
 
+
 instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
   predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
   match(Set dst (LoadVectorGather mem idx));
   effect(TEMP dst, TEMP tmp, TEMP ktmp);
   format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
   ins_encode %{
-    assert(UseAVX > 2, "sanity");
-
     int vlen_enc = vector_length_encoding(this);
     BasicType elem_bt = Matcher::vector_element_basic_type(this);
-
-    assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
-
-    __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
+    __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
     __ lea($tmp$$Register, $mem$$Address);
     __ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
   %}
@@ -4093,6 +4080,7 @@ instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
 %}
 
 instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
+  predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
   match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
   effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
   format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}

src/hotspot/os/bsd/os_bsd.cpp

@@ -979,7 +979,7 @@ bool os::dll_address_to_library_name(address addr, char* buf,
 // in case of error it checks if .dll/.so was built for the
 // same architecture as Hotspot is running on
 
-void *os::Bsd::dlopen_helper(const char *filename, int mode) {
+void *os::Bsd::dlopen_helper(const char *filename, int mode, char *ebuf, int ebuflen) {
 #ifndef IA32
   bool ieee_handling = IEEE_subnormal_handling_OK();
   if (!ieee_handling) {
@@ -1005,27 +1005,44 @@ void *os::Bsd::dlopen_helper(const char *filename, int mode) {
   assert(rtn == 0, "fegetenv must succeed");
 #endif // IA32
 
-  void * result= ::dlopen(filename, RTLD_LAZY);
-
-#ifndef IA32
-  if (result  != nullptr && ! IEEE_subnormal_handling_OK()) {
-    // We just dlopen()ed a library that mangled the floating-point
-    // flags. Silently fix things now.
-    int rtn = fesetenv(&default_fenv);
-    assert(rtn == 0, "fesetenv must succeed");
-    bool ieee_handling_after_issue = IEEE_subnormal_handling_OK();
-
-    if (ieee_handling_after_issue) {
-      Events::log_dll_message(nullptr, "IEEE subnormal handling had to be corrected after loading %s", filename);
-      log_info(os)("IEEE subnormal handling had to be corrected after loading %s", filename);
-    } else {
-      Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
-      log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+  void* result;
+  JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
+  result = ::dlopen(filename, RTLD_LAZY);
+  if (result == nullptr) {
+    const char* error_report = ::dlerror();
+    if (error_report == nullptr) {
+      error_report = "dlerror returned no error description";
     }
+    if (ebuf != nullptr && ebuflen > 0) {
+      ::strncpy(ebuf, error_report, ebuflen-1);
+      ebuf[ebuflen-1]='\0';
+    }
+    Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
+    log_info(os)("shared library load of %s failed, %s", filename, error_report);
+    JFR_ONLY(load_event.set_error_msg(error_report);)
+  } else {
+    Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
+    log_info(os)("shared library load of %s was successful", filename);
+#ifndef IA32
+    if (! IEEE_subnormal_handling_OK()) {
+      // We just dlopen()ed a library that mangled the floating-point
+      // flags. Silently fix things now.
+      JFR_ONLY(load_event.set_fp_env_correction_attempt(true);)
+      int rtn = fesetenv(&default_fenv);
+      assert(rtn == 0, "fesetenv must succeed");
 
-    assert(ieee_handling_after_issue, "fesetenv didn't work");
-  }
+      if (IEEE_subnormal_handling_OK()) {
+        Events::log_dll_message(nullptr, "IEEE subnormal handling had to be corrected after loading %s", filename);
+        log_info(os)("IEEE subnormal handling had to be corrected after loading %s", filename);
+        JFR_ONLY(load_event.set_fp_env_correction_success(true);)
+      } else {
+        Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
+        log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+        assert(false, "fesetenv didn't work");
+      }
+    }
 #endif // IA32
+  }
 
   return result;
 }
@@ -1037,30 +1054,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
 #else
   log_info(os)("attempting shared library load of %s", filename);
 
-  void* result;
-  JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
-  result = os::Bsd::dlopen_helper(filename, RTLD_LAZY);
-  if (result != nullptr) {
-    Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
-    // Successful loading
-    log_info(os)("shared library load of %s was successful", filename);
-    return result;
-  }
-
-  const char* error_report = ::dlerror();
-  if (error_report == nullptr) {
-    error_report = "dlerror returned no error description";
-  }
-  if (ebuf != nullptr && ebuflen > 0) {
-    // Read system error message into ebuf
-    ::strncpy(ebuf, error_report, ebuflen-1);
-    ebuf[ebuflen-1]='\0';
-  }
-  Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
-  log_info(os)("shared library load of %s failed, %s", filename, error_report);
-  JFR_ONLY(load_event.set_error_msg(error_report);)
-
-  return nullptr;
+  return os::Bsd::dlopen_helper(filename, RTLD_LAZY, ebuf, ebuflen);
 #endif // STATIC_BUILD
 }
 #else
@@ -1071,29 +1065,13 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
   log_info(os)("attempting shared library load of %s", filename);
 
   void* result;
-  JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
-  result = os::Bsd::dlopen_helper(filename, RTLD_LAZY);
+  result = os::Bsd::dlopen_helper(filename, RTLD_LAZY, ebuf, ebuflen);
   if (result != nullptr) {
-    Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
-    // Successful loading
-    log_info(os)("shared library load of %s was successful", filename);
     return result;
   }
 
-  Elf32_Ehdr elf_head;
-
-  const char* const error_report = ::dlerror();
-  if (error_report == nullptr) {
-    error_report = "dlerror returned no error description";
-  }
-  if (ebuf != nullptr && ebuflen > 0) {
-    // Read system error message into ebuf
-    ::strncpy(ebuf, error_report, ebuflen-1);
-    ebuf[ebuflen-1]='\0';
-  }
   Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
   log_info(os)("shared library load of %s failed, %s", filename, error_report);
-  JFR_ONLY(load_event.set_error_msg(error_report);)
   int diag_msg_max_length=ebuflen-strlen(ebuf);
   char* diag_msg_buf=ebuf+strlen(ebuf);
 
@@ -1102,7 +1080,6 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
     return nullptr;
   }
 
-
   int file_descriptor= ::open(filename, O_RDONLY | O_NONBLOCK);
 
   if (file_descriptor < 0) {
@@ -1110,6 +1087,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
     return nullptr;
   }
 
+  Elf32_Ehdr elf_head;
   bool failed_to_read_elf_head=
     (sizeof(elf_head)!=
      (::read(file_descriptor, &elf_head,sizeof(elf_head))));

src/hotspot/os/bsd/os_bsd.hpp

@@ -70,7 +70,7 @@ class os::Bsd {
   // Real-time clock functions
   static void clock_init(void);
 
-  static void *dlopen_helper(const char *path, int mode);
+  static void *dlopen_helper(const char *path, int mode, char *ebuf, int ebuflen);
 
   // Stack repair handling
 

src/hotspot/os/linux/os_linux.cpp

@@ -1856,18 +1856,19 @@ void * os::Linux::dlopen_helper(const char *filename, char *ebuf, int ebuflen) {
     if (! IEEE_subnormal_handling_OK()) {
       // We just dlopen()ed a library that mangled the floating-point flags.
       // Attempt to fix things now.
+      JFR_ONLY(load_event.set_fp_env_correction_attempt(true);)
       int rtn = fesetenv(&default_fenv);
       assert(rtn == 0, "fesetenv must succeed");
-      bool ieee_handling_after_issue = IEEE_subnormal_handling_OK();
 
-      if (ieee_handling_after_issue) {
+      if (IEEE_subnormal_handling_OK()) {
         Events::log_dll_message(nullptr, "IEEE subnormal handling had to be corrected after loading %s", filename);
         log_info(os)("IEEE subnormal handling had to be corrected after loading %s", filename);
+        JFR_ONLY(load_event.set_fp_env_correction_success(true);)
       } else {
         Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
         log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+        assert(false, "fesetenv didn't work");
       }
-      assert(ieee_handling_after_issue, "fesetenv didn't work");
     }
 #endif // IA32
   }
@@ -2215,6 +2216,8 @@ void os::Linux::print_system_memory_info(outputStream* st) {
   // https://www.kernel.org/doc/Documentation/vm/transhuge.txt
   _print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/enabled",
                       "/sys/kernel/mm/transparent_hugepage/enabled", st);
+  _print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/hpage_pdm_size",
+                      "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", st);
   _print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/shmem_enabled",
                       "/sys/kernel/mm/transparent_hugepage/shmem_enabled", st);
   _print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/defrag (defrag/compaction efforts parameter)",

src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp

@@ -232,7 +232,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
         CompiledMethod* nm = (cb != nullptr) ? cb->as_compiled_method_or_null() : nullptr;
         bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc));
         if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) {
-          address next_pc = pc + NativeCall::instruction_size;
+          address next_pc = Assembler::locate_next_instruction(pc);
           if (is_unsafe_arraycopy) {
             next_pc = UnsafeCopyMemory::page_error_continue_pc(pc);
           }
@@ -271,7 +271,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
                 thread->thread_state() == _thread_in_native) &&
                 sig == SIGBUS && /* info->si_code == BUS_OBJERR && */
                 thread->doing_unsafe_access()) {
-      address next_pc = pc + NativeCall::instruction_size;
+      address next_pc = Assembler::locate_next_instruction(pc);
       if (UnsafeCopyMemory::contains_pc(pc)) {
         next_pc = UnsafeCopyMemory::page_error_continue_pc(pc);
       }

src/hotspot/share/c1/c1_globals.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -275,9 +275,11 @@
   develop(bool, InstallMethods, true,                                       \
           "Install methods at the end of successful compilations")          \
                                                                             \
+  /* The compiler assumes, in many places, that methods are at most 1MB. */ \
+  /* Therefore, we restrict this flag to at most 1MB.                    */ \
   develop(intx, NMethodSizeLimit, (64*K)*wordSize,                          \
           "Maximum size of a compiled method.")                             \
-          range(0, max_jint)                                                \
+          range(0, 1*M)                                                     \
                                                                             \
   develop(bool, TraceFPUStack, false,                                       \
           "Trace emulation of the FPU stack (intel only)")                  \

src/hotspot/share/cds/cdsHeapVerifier.cpp

@@ -92,11 +92,6 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
   //
   //       class                                         field                     type
   ADD_EXCL("java/lang/ClassLoader",                      "scl");                   // A
-  ADD_EXCL("java/lang/invoke/InvokerBytecodeGenerator",  "DONTINLINE_SIG",         // B
-                                                         "FORCEINLINE_SIG",        // B
-                                                         "HIDDEN_SIG",             // B
-                                                         "INJECTEDPROFILE_SIG",    // B
-                                                         "LF_COMPILED_SIG");       // B
   ADD_EXCL("java/lang/Module",                           "ALL_UNNAMED_MODULE",     // A
                                                          "ALL_UNNAMED_MODULE_SET", // A
                                                          "EVERYONE_MODULE",        // A
@@ -106,10 +101,6 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
   ADD_EXCL("java/lang/reflect/AccessFlag$Location",      "EMPTY_SET");             // E
 
   ADD_EXCL("java/lang/System",                           "bootLayer");             // A
-  ADD_EXCL("java/lang/VersionProps",                     "VENDOR_URL_BUG",         // C
-                                                         "VENDOR_URL_VM_BUG",      // C
-                                                         "VENDOR_VERSION");        // C
-  ADD_EXCL("java/net/URL$DefaultFactory",                "PREFIX");                // B FIXME: JDK-8276561
 
   // A dummy object used by HashSet. The value doesn't matter and it's never
   // tested for equality.
@@ -118,7 +109,6 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
   ADD_EXCL("jdk/internal/loader/ClassLoaders",           "BOOT_LOADER",            // A
                                                          "APP_LOADER",             // A
                                                          "PLATFORM_LOADER");       // A
-  ADD_EXCL("jdk/internal/loader/URLClassPath",           "JAVA_VERSION");          // B
   ADD_EXCL("jdk/internal/module/Builder",                "cachedVersion");         // D
   ADD_EXCL("jdk/internal/module/ModuleLoaderMap$Mapper", "APP_CLASSLOADER",        // A
                                                          "APP_LOADER_INDEX",       // A
@@ -128,30 +118,10 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
 
   // This just points to an empty Map
   ADD_EXCL("jdk/internal/reflect/Reflection",            "methodFilterMap");       // E
-  ADD_EXCL("jdk/internal/util/StaticProperty",           "FILE_ENCODING",          // C
-                                                 "JAVA_LOCALE_USE_OLD_ISO_CODES",  // C
-                                                 "USER_LANGUAGE",                  // C
-                                                 "USER_LANGUAGE_DISPLAY",          // C
-                                                 "USER_LANGUAGE_FORMAT",           // C
-                                                 "USER_SCRIPT",                    // C
-                                                 "USER_SCRIPT_DISPLAY",            // C
-                                                 "USER_SCRIPT_FORMAT",             // C
-                                                 "USER_COUNTRY",                   // C
-                                                 "USER_COUNTRY_DISPLAY",           // C
-                                                 "USER_COUNTRY_FORMAT",            // C
-                                                 "USER_VARIANT",                   // C
-                                                 "USER_VARIANT_DISPLAY",           // C
-                                                 "USER_VARIANT_FORMAT",            // C
-                                                 "USER_EXTENSIONS",                // C
-                                                 "USER_EXTENSIONS_DISPLAY",        // C
-                                                 "USER_EXTENSIONS_FORMAT",         // C
-                                                 "USER_REGION");                   // C
 
   // Integer for 0 and 1 are in java/lang/Integer$IntegerCache and are archived
   ADD_EXCL("sun/invoke/util/ValueConversions",           "ONE_INT",                // E
                                                          "ZERO_INT");              // E
-  ADD_EXCL("sun/security/util/SecurityConstants",        "PROVIDER_VER");          // C
-
 
 # undef ADD_EXCL
 
@@ -245,6 +215,12 @@ inline bool CDSHeapVerifier::do_entry(oop& orig_obj, HeapShared::CachedOopInfo&
 
   StaticFieldInfo* info = _table.get(orig_obj);
   if (info != nullptr) {
+    if (value.orig_referrer() == nullptr && java_lang_String::is_instance(orig_obj)) {
+      // This string object is not referenced by any of the archived object graphs. It's archived
+      // only because it's in the interned string table. So we are not in a condition that
+      // should be flagged by CDSHeapVerifier.
+      return true; /* keep on iterating */
+    }
     ResourceMark rm;
     LogStream ls(Log(cds, heap)::warning());
     ls.print_cr("Archive heap points to a static field that may be reinitialized at runtime:");

src/hotspot/share/classfile/vmIntrinsics.hpp

@@ -597,6 +597,7 @@ class methodHandle;
   do_intrinsic(_notifyJvmtiVThreadMount, java_lang_VirtualThread, notifyJvmtiMount_name, bool_void_signature, F_RN)     \
   do_intrinsic(_notifyJvmtiVThreadUnmount, java_lang_VirtualThread, notifyJvmtiUnmount_name, bool_void_signature, F_RN) \
   do_intrinsic(_notifyJvmtiVThreadHideFrames, java_lang_VirtualThread, notifyJvmtiHideFrames_name, bool_void_signature, F_RN) \
+  do_intrinsic(_notifyJvmtiVThreadDisableSuspend, java_lang_VirtualThread, notifyJvmtiDisableSuspend_name, bool_void_signature, F_RN) \
                                                                                                                         \
   /* support for UnsafeConstants */                                                                                     \
   do_class(jdk_internal_misc_UnsafeConstants,      "jdk/internal/misc/UnsafeConstants")                                 \

src/hotspot/share/classfile/vmSymbols.hpp

@@ -421,6 +421,7 @@ class SerializeClosure;
   template(notifyJvmtiMount_name,                     "notifyJvmtiMount")                         \
   template(notifyJvmtiUnmount_name,                   "notifyJvmtiUnmount")                       \
   template(notifyJvmtiHideFrames_name,                "notifyJvmtiHideFrames")                    \
+  template(notifyJvmtiDisableSuspend_name,            "notifyJvmtiDisableSuspend")                \
   template(doYield_name,                              "doYield")                                  \
   template(enter_name,                                "enter")                                    \
   template(enterSpecial_name,                         "enterSpecial")                             \

src/hotspot/share/code/codeBlob.cpp

@@ -164,7 +164,7 @@ RuntimeBlob::RuntimeBlob(
 void RuntimeBlob::free(RuntimeBlob* blob) {
   assert(blob != nullptr, "caller must check for nullptr");
   ThreadInVMfromUnknown __tiv;  // get to VM state in case we block on CodeCache_lock
-  blob->purge();
+  blob->purge(true /* free_code_cache_data */, true /* unregister_nmethod */);
   {
     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
     CodeCache::free(blob);
@@ -173,7 +173,7 @@ void RuntimeBlob::free(RuntimeBlob* blob) {
   MemoryService::track_code_cache_memory_usage();
 }
 
-void CodeBlob::purge(bool free_code_cache_data) {
+void CodeBlob::purge(bool free_code_cache_data, bool unregister_nmethod) {
   if (_oop_maps != nullptr) {
     delete _oop_maps;
     _oop_maps = nullptr;

src/hotspot/share/code/codeBlob.hpp

@@ -143,7 +143,7 @@ public:
   static unsigned int align_code_offset(int offset);
 
   // Deletion
-  virtual void purge(bool free_code_cache_data = true);
+  virtual void purge(bool free_code_cache_data, bool unregister_nmethod);
 
   // Typing
   virtual bool is_buffer_blob() const                 { return false; }

src/hotspot/share/code/codeCache.cpp

@@ -1819,16 +1819,20 @@ void CodeCache::log_state(outputStream* st) {
 }
 
 #ifdef LINUX
-void CodeCache::write_perf_map() {
+void CodeCache::write_perf_map(const char* filename) {
   MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 
-  // Perf expects to find the map file at /tmp/perf-<pid>.map.
+  // Perf expects to find the map file at /tmp/perf-<pid>.map
+  // if the file name is not specified.
   char fname[32];
-  jio_snprintf(fname, sizeof(fname), "/tmp/perf-%d.map", os::current_process_id());
+  if (filename == nullptr) {
+    jio_snprintf(fname, sizeof(fname), "/tmp/perf-%d.map", os::current_process_id());
+    filename = fname;
+  }
 
-  fileStream fs(fname, "w");
+  fileStream fs(filename, "w");
   if (!fs.is_open()) {
-    log_warning(codecache)("Failed to create %s for perf map", fname);
+    log_warning(codecache)("Failed to create %s for perf map", filename);
     return;
   }
 

src/hotspot/share/code/codeCache.hpp

@@ -227,7 +227,7 @@ class CodeCache : AllStatic {
   static void print_trace(const char* event, CodeBlob* cb, uint size = 0) PRODUCT_RETURN;
   static void print_summary(outputStream* st, bool detailed = true); // Prints a summary of the code cache usage
   static void log_state(outputStream* st);
-  LINUX_ONLY(static void write_perf_map();)
+  LINUX_ONLY(static void write_perf_map(const char* filename = nullptr);)
   static const char* get_code_heap_name(CodeBlobType code_blob_type)  { return (heap_available(code_blob_type) ? get_code_heap(code_blob_type)->name() : "Unused"); }
   static void report_codemem_full(CodeBlobType code_blob_type, bool print);
 

src/hotspot/share/code/compiledIC.cpp

@@ -291,10 +291,10 @@ bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecod
     }
   }
 
-  if (TraceICs) {
+  {
     ResourceMark rm;
     assert(call_info->selected_method() != nullptr, "Unexpected null selected method");
-    tty->print_cr ("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
+    log_trace(inlinecache)("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
                    p2i(instruction_address()), call_info->selected_method()->print_value_string(), p2i(entry));
   }
 
@@ -364,10 +364,11 @@ bool CompiledIC::is_call_to_interpreted() const {
 
 bool CompiledIC::set_to_clean(bool in_use) {
   assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
-  if (TraceInlineCacheClearing || TraceICs) {
+  if (TraceInlineCacheClearing) {
     tty->print_cr("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
     print();
   }
+  log_trace(inlinecache)("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
 
   address entry = _call->get_resolve_call_stub(is_optimized());
 
@@ -433,9 +434,9 @@ bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
       methodHandle method (thread, (Method*)info.cached_metadata());
       _call->set_to_interpreted(method, info);
 
-      if (TraceICs) {
-         ResourceMark rm(thread);
-         tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to interpreter: %s",
+      {
+        ResourceMark rm(thread);
+        log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter: %s",
            p2i(instruction_address()),
            method->print_value_string());
       }
@@ -449,9 +450,9 @@ bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
       // LSan appears unable to follow malloc-based memory consistently when embedded as an
       // immediate in generated machine code. So we have to ignore it.
       LSAN_IGNORE_OBJECT(holder);
-      if (TraceICs) {
+      {
          ResourceMark rm(thread);
-         tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address()));
+         log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address()));
       }
     }
   } else {
@@ -479,10 +480,10 @@ bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
       }
     }
 
-    if (TraceICs) {
+    {
       ResourceMark rm(thread);
       assert(info.cached_metadata() == nullptr || info.cached_metadata()->is_klass(), "must be");
-      tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to compiled (rcvr klass = %s) %s",
+      log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to compiled (rcvr klass = %s) %s",
         p2i(instruction_address()),
         (info.cached_metadata() != nullptr) ? ((Klass*)info.cached_metadata())->print_value_string() : "nullptr",
         (safe) ? "" : " via stub");
@@ -606,9 +607,9 @@ bool CompiledDirectStaticCall::is_call_to_interpreted() const {
 }
 
 void CompiledStaticCall::set_to_compiled(address entry) {
-  if (TraceICs) {
+  {
     ResourceMark rm;
-    tty->print_cr("%s@" INTPTR_FORMAT ": set_to_compiled " INTPTR_FORMAT,
+    log_trace(inlinecache)("%s@" INTPTR_FORMAT ": set_to_compiled " INTPTR_FORMAT,
         name(),
         p2i(instruction_address()),
         p2i(entry));

src/hotspot/share/code/compiledMethod.hpp

@@ -174,7 +174,7 @@ protected:
 
   void* _gc_data;
 
-  virtual void purge(bool free_code_cache_data = true) = 0;
+  virtual void purge(bool free_code_cache_data, bool unregister_nmethod) = 0;
 
 private:
   DeoptimizationStatus deoptimization_status() const {

src/hotspot/share/code/nmethod.cpp

@@ -1444,7 +1444,9 @@ void nmethod::unlink() {
   ClassUnloadingContext::context()->register_unlinked_nmethod(this);
 }
 
-void nmethod::purge(bool free_code_cache_data) {
+void nmethod::purge(bool free_code_cache_data, bool unregister_nmethod) {
+  assert(!free_code_cache_data, "must only call not freeing code cache data");
+
   MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 
   // completely deallocate this method
@@ -1464,13 +1466,13 @@ void nmethod::purge(bool free_code_cache_data) {
     ec = next;
   }
 
-  Universe::heap()->unregister_nmethod(this);
+  if (unregister_nmethod) {
+    Universe::heap()->unregister_nmethod(this);
+  }
+
   CodeCache::unregister_old_nmethod(this);
 
-  CodeBlob::purge();
-  if (free_code_cache_data) {
-    CodeCache::free(this);
-  }
+  CodeBlob::purge(free_code_cache_data, unregister_nmethod);
 }
 
 oop nmethod::oop_at(int index) const {

src/hotspot/share/code/nmethod.hpp

@@ -522,7 +522,7 @@ public:
   void unlink();
 
   // Deallocate this nmethod - called by the GC
-  void purge(bool free_code_cache_data = true);
+  void purge(bool free_code_cache_data, bool unregister_nmethod);
 
   // See comment at definition of _last_seen_on_stack
   void mark_as_maybe_on_stack();

src/hotspot/share/compiler/compileBroker.cpp

@@ -1788,7 +1788,7 @@ bool CompileBroker::init_compiler_runtime() {
 void CompileBroker::free_buffer_blob_if_allocated(CompilerThread* thread) {
   BufferBlob* blob = thread->get_buffer_blob();
   if (blob != nullptr) {
-    blob->purge();
+    blob->purge(true /* free_code_cache_data */, true /* unregister_nmethod */);
     MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
     CodeCache::free(blob);
   }

src/hotspot/share/gc/g1/g1Arguments.cpp

@@ -126,7 +126,6 @@ void G1Arguments::initialize_mark_stack_size() {
     FLAG_SET_ERGO(MarkStackSize, mark_stack_size);
   }
 
-  log_trace(gc)("MarkStackSize: %uk  MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
 }
 
 

src/hotspot/share/gc/g1/g1CodeRootSet.cpp

@@ -187,6 +187,15 @@ public:
     }
   }
 
+  // Removes dead/unlinked entries.
+  void bulk_remove() {
+    auto delete_check = [&] (nmethod** value) {
+      return (*value)->is_unlinked();
+    };
+
+    clean(delete_check);
+  }
+
   // Calculate the log2 of the table size we want to shrink to.
   size_t log2_target_shrink_size(size_t current_size) const {
     // A table with the new size should be at most filled by this factor. Otherwise
@@ -255,6 +264,11 @@ bool G1CodeRootSet::remove(nmethod* method) {
   return _table->remove(method);
 }
 
+void G1CodeRootSet::bulk_remove() {
+  assert(!_is_iterating, "should not mutate while iterating the table");
+  _table->bulk_remove();
+}
+
 bool G1CodeRootSet::contains(nmethod* method) {
   return _table->contains(method);
 }

src/hotspot/share/gc/g1/g1CodeRootSet.hpp

@@ -44,6 +44,7 @@ class G1CodeRootSet {
 
   void add(nmethod* method);
   bool remove(nmethod* method);
+  void bulk_remove();
   bool contains(nmethod* method);
   void clear();
 

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -2517,6 +2517,7 @@ void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjec
   GCTraceTime(Debug, gc, phases) debug(description, timer);
 
   ClassUnloadingContext ctx(workers()->active_workers(),
+                            false /* unregister_nmethods_during_purge */,
                             false /* lock_codeblob_free_separately */);
   {
     CodeCache::UnlinkingScope scope(is_alive);
@@ -2528,6 +2529,10 @@ void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjec
     GCTraceTime(Debug, gc, phases) t("Purge Unlinked NMethods", timer);
     ctx.purge_nmethods();
   }
+  {
+    GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", timer);
+    G1CollectedHeap::heap()->bulk_unregister_nmethods();
+  }
   {
     GCTraceTime(Debug, gc, phases) t("Free Code Blobs", timer);
     ctx.free_code_blobs();
@@ -2539,6 +2544,33 @@ void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjec
   }
 }
 
+class G1BulkUnregisterNMethodTask : public WorkerTask {
+  HeapRegionClaimer _hrclaimer;
+
+  class UnregisterNMethodsHeapRegionClosure : public HeapRegionClosure {
+  public:
+
+    bool do_heap_region(HeapRegion* hr) {
+      hr->rem_set()->bulk_remove_code_roots();
+      return false;
+    }
+  } _cl;
+
+public:
+  G1BulkUnregisterNMethodTask(uint num_workers)
+  : WorkerTask("G1 Remove Unlinked NMethods From Code Root Set Task"),
+    _hrclaimer(num_workers) { }
+
+  void work(uint worker_id) {
+    G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hrclaimer, worker_id);
+  }
+};
+
+void G1CollectedHeap::bulk_unregister_nmethods() {
+  uint num_workers = workers()->active_workers();
+  G1BulkUnregisterNMethodTask t(num_workers);
+  workers()->run_task(&t);
+}
 
 bool G1STWSubjectToDiscoveryClosure::do_object_b(oop obj) {
   assert(obj != nullptr, "must not be null");
@@ -2963,31 +2995,6 @@ public:
   void do_oop(narrowOop* p) { ShouldNotReachHere(); }
 };
 
-class UnregisterNMethodOopClosure: public OopClosure {
-  G1CollectedHeap* _g1h;
-  nmethod* _nm;
-
-public:
-  UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
-    _g1h(g1h), _nm(nm) {}
-
-  void do_oop(oop* p) {
-    oop heap_oop = RawAccess<>::oop_load(p);
-    if (!CompressedOops::is_null(heap_oop)) {
-      oop obj = CompressedOops::decode_not_null(heap_oop);
-      HeapRegion* hr = _g1h->heap_region_containing(obj);
-      assert(!hr->is_continues_humongous(),
-             "trying to remove code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT
-             " starting at " HR_FORMAT,
-             p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()));
-
-      hr->remove_code_root(_nm);
-    }
-  }
-
-  void do_oop(narrowOop* p) { ShouldNotReachHere(); }
-};
-
 void G1CollectedHeap::register_nmethod(nmethod* nm) {
   guarantee(nm != nullptr, "sanity");
   RegisterNMethodOopClosure reg_cl(this, nm);
@@ -2995,9 +3002,8 @@ void G1CollectedHeap::register_nmethod(nmethod* nm) {
 }
 
 void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
-  guarantee(nm != nullptr, "sanity");
-  UnregisterNMethodOopClosure reg_cl(this, nm);
-  nm->oops_do(&reg_cl, true);
+  // We always unregister nmethods in bulk during code unloading only.
+  ShouldNotReachHere();
 }
 
 void G1CollectedHeap::update_used_after_gc(bool evacuation_failed) {

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -1270,6 +1270,8 @@ public:
 
   void unload_classes_and_code(const char* description, BoolObjectClosure* cl, GCTimer* timer);
 
+  void bulk_unregister_nmethods();
+
   // Verification
 
   // Perform any cleanup actions necessary before allowing a verification.

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -75,6 +75,7 @@
 #include "utilities/align.hpp"
 #include "utilities/formatBuffer.hpp"
 #include "utilities/growableArray.hpp"
+#include "utilities/powerOfTwo.hpp"
 
 bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
   assert(addr < _cm->finger(), "invariant");
@@ -94,80 +95,173 @@ bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
 }
 
 G1CMMarkStack::G1CMMarkStack() :
-  _max_chunk_capacity(0),
-  _base(nullptr),
-  _chunk_capacity(0) {
+  _chunk_allocator() {
   set_empty();
 }
 
-bool G1CMMarkStack::resize(size_t new_capacity) {
-  assert(is_empty(), "Only resize when stack is empty.");
-  assert(new_capacity <= _max_chunk_capacity,
-         "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
-
-  TaskQueueEntryChunk* new_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(new_capacity, mtGC);
-
-  if (new_base == nullptr) {
-    log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
-    return false;
-  }
-  // Release old mapping.
-  if (_base != nullptr) {
-    MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
-  }
-
-  _base = new_base;
-  _chunk_capacity = new_capacity;
-  set_empty();
-
-  return true;
-}
-
 size_t G1CMMarkStack::capacity_alignment() {
   return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
 }
 
-bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) {
-  guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized.");
+bool G1CMMarkStack::initialize() {
+  guarantee(_chunk_allocator.capacity() == 0, "G1CMMarkStack already initialized.");
+
+  size_t initial_capacity = MarkStackSize;
+  size_t max_capacity = MarkStackSizeMax;
 
   size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
 
-  _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
-  size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
+  size_t max_num_chunks = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
+  size_t initial_num_chunks = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
 
-  guarantee(initial_chunk_capacity <= _max_chunk_capacity,
-            "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT,
-            _max_chunk_capacity,
-            initial_chunk_capacity);
+  initial_num_chunks = round_up_power_of_2(initial_num_chunks);
+  max_num_chunks = MAX2(initial_num_chunks, max_num_chunks);
+
+  size_t limit = (INT_MAX - 1);
+  max_capacity = MIN2((max_num_chunks * TaskEntryChunkSizeInVoidStar), limit);
+  initial_capacity = MIN2((initial_num_chunks * TaskEntryChunkSizeInVoidStar), limit);
+
+  FLAG_SET_ERGO(MarkStackSizeMax, max_capacity);
+  FLAG_SET_ERGO(MarkStackSize, initial_capacity);
+
+  log_trace(gc)("MarkStackSize: %uk  MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
 
   log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
-                initial_chunk_capacity, _max_chunk_capacity);
+                initial_num_chunks, max_capacity);
 
-  return resize(initial_chunk_capacity);
+  return _chunk_allocator.initialize(initial_num_chunks, max_num_chunks);
+}
+
+G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_chunk() {
+  if (_size >= _max_capacity) {
+    return nullptr;
+  }
+
+  size_t cur_idx = Atomic::fetch_then_add(&_size, 1u);
+
+  if (cur_idx >= _max_capacity) {
+    return nullptr;
+  }
+
+  size_t bucket = get_bucket(cur_idx);
+  if (Atomic::load_acquire(&_buckets[bucket]) == nullptr) {
+    if (!_should_grow) {
+      // Prefer to restart the CM.
+      return nullptr;
+    }
+
+    MutexLocker x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
+    if (Atomic::load_acquire(&_buckets[bucket]) == nullptr) {
+      if (!expand()) {
+        return nullptr;
+      }
+    }
+  }
+
+  size_t bucket_idx = get_bucket_index(cur_idx);
+  TaskQueueEntryChunk* result = ::new (&_buckets[bucket][bucket_idx]) TaskQueueEntryChunk;
+  result->next = nullptr;
+  return result;
+}
+
+G1CMMarkStack::ChunkAllocator::ChunkAllocator() :
+  _min_capacity(0),
+  _max_capacity(0),
+  _capacity(0),
+  _num_buckets(0),
+  _should_grow(false),
+  _buckets(nullptr),
+  _size(0)
+{ }
+
+bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t max_capacity) {
+  guarantee(is_power_of_2(initial_capacity), "Invalid initial_capacity");
+
+  _min_capacity = initial_capacity;
+  _max_capacity = max_capacity;
+  _num_buckets  = get_bucket(_max_capacity) + 1;
+
+  _buckets = NEW_C_HEAP_ARRAY(TaskQueueEntryChunk*, _num_buckets, mtGC);
+
+  for (size_t i = 0; i < _num_buckets; i++) {
+    _buckets[i] = nullptr;
+  }
+
+  size_t new_capacity = bucket_size(0);
+
+  if (!reserve(new_capacity)) {
+    log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
+    return false;
+  }
+  return true;
+}
+
+bool G1CMMarkStack::ChunkAllocator::expand() {
+  if (_capacity == _max_capacity) {
+    log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _capacity);
+    return false;
+  }
+  size_t old_capacity = _capacity;
+  // Double capacity if possible.
+  size_t new_capacity = MIN2(old_capacity * 2, _max_capacity);
+
+  if (reserve(new_capacity)) {
+    log_debug(gc)("Expanded the mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
+                  old_capacity, new_capacity);
+    return true;
+  }
+  return false;
+}
+
+G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
+  if (_buckets == nullptr) {
+    return;
+  }
+
+  for (size_t i = 0; i < _num_buckets; i++) {
+    if (_buckets[i] != nullptr) {
+      MmapArrayAllocator<TaskQueueEntryChunk>::free(_buckets[i],  bucket_size(i));
+      _buckets[i] = nullptr;
+    }
+  }
+
+  FREE_C_HEAP_ARRAY(TaskQueueEntryChunk*, _buckets);
+}
+
+bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
+  assert(new_capacity <= _max_capacity, "Cannot expand overflow mark stack beyond the max_capacity" SIZE_FORMAT " chunks.", _max_capacity);
+
+  size_t highest_bucket = get_bucket(new_capacity - 1);
+  size_t i = get_bucket(_capacity);
+
+  for (; i <= highest_bucket; i++) {
+    if (Atomic::load_acquire(&_buckets[i]) != nullptr) {
+      continue; // Skip over already allocated buckets.
+    }
+
+    size_t bucket_capacity = bucket_size(i);
+
+    // Trim bucket size so that we do not exceed the _max_capacity.
+    bucket_capacity = (_capacity + bucket_capacity) <= _max_capacity ?
+                      bucket_capacity :
+                      _max_capacity - _capacity;
+
+
+    TaskQueueEntryChunk* bucket_base = MmapArrayAllocator<TaskQueueEntryChunk>::allocate_or_null(bucket_capacity, mtGC);
+
+    if (bucket_base == nullptr) {
+      log_warning(gc)("Failed to reserve memory for increasing the overflow mark stack capacity with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.",
+                      bucket_capacity, bucket_capacity * sizeof(TaskQueueEntryChunk));
+      return false;
+    }
+    _capacity += bucket_capacity;
+    Atomic::release_store(&_buckets[i], bucket_base);
+  }
+  return true;
 }
 
 void G1CMMarkStack::expand() {
-  if (_chunk_capacity == _max_chunk_capacity) {
-    log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity);
-    return;
-  }
-  size_t old_capacity = _chunk_capacity;
-  // Double capacity if possible
-  size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity);
-
-  if (resize(new_capacity)) {
-    log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
-                  old_capacity, new_capacity);
-  } else {
-    log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
-                    old_capacity, new_capacity);
-  }
-}
-
-G1CMMarkStack::~G1CMMarkStack() {
-  if (_base != nullptr) {
-    MmapArrayAllocator<TaskQueueEntryChunk>::free(_base, _chunk_capacity);
-  }
+  _chunk_allocator.expand();
 }
 
 void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
@@ -208,31 +302,13 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list()
   return remove_chunk_from_list(&_free_list);
 }
 
-G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() {
-  // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code.
-  // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding
-  // wraparound of _hwm.
-  if (_hwm >= _chunk_capacity) {
-    return nullptr;
-  }
-
-  size_t cur_idx = Atomic::fetch_then_add(&_hwm, 1u);
-  if (cur_idx >= _chunk_capacity) {
-    return nullptr;
-  }
-
-  TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk;
-  result->next = nullptr;
-  return result;
-}
-
 bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
   // Get a new chunk.
   TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
 
   if (new_chunk == nullptr) {
     // Did not get a chunk from the free list. Allocate from backing memory.
-    new_chunk = allocate_new_chunk();
+    new_chunk = _chunk_allocator.allocate_new_chunk();
 
     if (new_chunk == nullptr) {
       return false;
@@ -261,9 +337,9 @@ bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
 
 void G1CMMarkStack::set_empty() {
   _chunks_in_chunk_list = 0;
-  _hwm = 0;
   _chunk_list = nullptr;
   _free_list = nullptr;
+  _chunk_allocator.reset();
 }
 
 G1CMRootMemRegions::G1CMRootMemRegions(uint const max_regions) :
@@ -440,7 +516,7 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
   _concurrent_workers->initialize_workers();
   _num_concurrent_workers = _concurrent_workers->active_workers();
 
-  if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) {
+  if (!_global_mark_stack.initialize()) {
     vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
   }
 
@@ -1635,6 +1711,9 @@ void G1ConcurrentMark::weak_refs_work() {
 
     assert(_global_mark_stack.is_empty(), "mark stack should be empty");
 
+    // Prefer to grow the stack until the max capacity.
+    _global_mark_stack.set_should_grow();
+
     // We need at least one active thread. If reference processing
     // is not multi-threaded we use the current (VMThread) thread,
     // otherwise we use the workers from the G1CollectedHeap and

src/hotspot/share/gc/g1/g1ConcurrentMark.hpp

@@ -136,10 +136,101 @@ private:
     G1TaskQueueEntry data[EntriesPerChunk];
   };
 
-  size_t _max_chunk_capacity;    // Maximum number of TaskQueueEntryChunk elements on the stack.
+  class ChunkAllocator {
+    // The chunk allocator relies on a growable array data structure that allows resizing without the
+    // need to copy existing items. The basic approach involves organizing the array into chunks,
+    // essentially creating an "array of arrays"; referred to as buckets in this implementation. To
+    // facilitate efficient indexing, the size of the first bucket is set to a power of 2. This choice
+    // allows for quick conversion of an array index into a bucket index and the corresponding offset
+    // within the bucket. Additionally, each new bucket added to the growable array doubles the capacity of
+    // the growable array.
+    //
+    // Illustration of the Growable Array data structure.
+    //
+    //        +----+        +----+----+
+    //        |    |------->|    |    |
+    //        |    |        +----+----+
+    //        +----+        +----+----+
+    //        |    |------->|    |    |
+    //        |    |        +----+----+
+    //        +----+        +-----+-----+-----+-----+
+    //        |    |------->|     |     |     |     |
+    //        |    |        +-----+-----+-----+-----+
+    //        +----+        +-----+-----+-----+-----+-----+-----+-----+----+
+    //        |    |------->|     |     |     |     |     |     |     |    |
+    //        |    |        +-----+-----+-----+-----+-----+-----+-----+----+
+    //        +----+
+    //
+    size_t _min_capacity;
+    size_t _max_capacity;
+    size_t _capacity;
+    size_t _num_buckets;
+    bool _should_grow;
+    TaskQueueEntryChunk* volatile* _buckets;
+    char _pad0[DEFAULT_CACHE_LINE_SIZE];
+    volatile size_t _size;
+    char _pad4[DEFAULT_CACHE_LINE_SIZE - sizeof(size_t)];
 
-  TaskQueueEntryChunk* _base;    // Bottom address of allocated memory area.
-  size_t _chunk_capacity;        // Current maximum number of TaskQueueEntryChunk elements.
+    size_t bucket_size(size_t bucket) {
+      return (bucket == 0) ?
+              _min_capacity :
+              _min_capacity * ( 1ULL << (bucket -1));
+    }
+
+    static unsigned int find_highest_bit(uintptr_t mask) {
+      return count_leading_zeros(mask) ^ (BitsPerWord - 1U);
+    }
+
+    size_t get_bucket(size_t array_idx) {
+      if (array_idx < _min_capacity) {
+        return 0;
+      }
+
+      return find_highest_bit(array_idx) - find_highest_bit(_min_capacity) + 1;
+    }
+
+    size_t get_bucket_index(size_t array_idx) {
+      if (array_idx < _min_capacity) {
+        return array_idx;
+      }
+      return array_idx - (1ULL << find_highest_bit(array_idx));
+    }
+
+    bool reserve(size_t new_capacity);
+
+  public:
+    ChunkAllocator();
+
+    ~ChunkAllocator();
+
+    bool initialize(size_t initial_capacity, size_t max_capacity);
+
+    void reset() {
+      _size = 0;
+      _should_grow = false;
+    }
+
+    // During G1CMConcurrentMarkingTask or finalize_marking phases, we prefer to restart the marking when
+    // the G1CMMarkStack overflows. Attempts to expand the G1CMMarkStack should be followed with a restart
+    // of the marking. On failure to allocate a new chuck, the caller just returns and forces a restart.
+    // This approach offers better memory utilization for the G1CMMarkStack, as each iteration of the
+    // marking potentially involves traversing fewer unmarked nodes in the graph.
+
+    // However, during the reference processing phase, instead of restarting the marking process, the
+    // G1CMMarkStack is expanded upon failure to allocate a new chunk. The decision between these two
+    // modes of expansion is determined by the _should_grow parameter.
+    void set_should_grow() {
+      _should_grow = true;
+    }
+
+    size_t capacity() const { return _capacity; }
+
+    bool expand();
+
+    TaskQueueEntryChunk* allocate_new_chunk();
+  };
+
+  ChunkAllocator _chunk_allocator;
 
   char _pad0[DEFAULT_CACHE_LINE_SIZE];
   TaskQueueEntryChunk* volatile _free_list;  // Linked list of free chunks that can be allocated by users.
@@ -148,13 +239,6 @@ private:
   volatile size_t _chunks_in_chunk_list;
   char _pad2[DEFAULT_CACHE_LINE_SIZE - sizeof(TaskQueueEntryChunk*) - sizeof(size_t)];
 
-  volatile size_t _hwm;          // High water mark within the reserved space.
-  char _pad4[DEFAULT_CACHE_LINE_SIZE - sizeof(size_t)];
-
-  // Allocate a new chunk from the reserved memory, using the high water mark. Returns
-  // null if out of memory.
-  TaskQueueEntryChunk* allocate_new_chunk();
-
   // Atomically add the given chunk to the list.
   void add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem);
   // Atomically remove and return a chunk from the given list. Returns null if the
@@ -167,19 +251,15 @@ private:
   TaskQueueEntryChunk* remove_chunk_from_chunk_list();
   TaskQueueEntryChunk* remove_chunk_from_free_list();
 
-  // Resizes the mark stack to the given new capacity. Releases any previous
-  // memory if successful.
-  bool resize(size_t new_capacity);
-
  public:
   G1CMMarkStack();
-  ~G1CMMarkStack();
+  ~G1CMMarkStack() = default;
 
   // Alignment and minimum capacity of this mark stack in number of oops.
   static size_t capacity_alignment();
 
-  // Allocate and initialize the mark stack with the given number of oops.
-  bool initialize(size_t initial_capacity, size_t max_capacity);
+  // Allocate and initialize the mark stack.
+  bool initialize();
 
   // Pushes the given buffer containing at most EntriesPerChunk elements on the mark
   // stack. If less than EntriesPerChunk elements are to be pushed, the array must
@@ -197,7 +277,11 @@ private:
   // _chunk_list.
   bool is_empty() const { return _chunk_list == nullptr; }
 
-  size_t capacity() const  { return _chunk_capacity; }
+  size_t capacity() const  { return _chunk_allocator.capacity(); }
+
+  void set_should_grow() {
+    _chunk_allocator.set_should_grow();
+  }
 
   // Expand the stack, typically in response to an overflow condition
   void expand();

src/hotspot/share/gc/g1/heapRegionRemSet.cpp

@@ -115,6 +115,10 @@ void HeapRegionRemSet::remove_code_root(nmethod* nm) {
   guarantee(!_code_roots.contains(nm), "duplicate entry found");
 }
 
+void HeapRegionRemSet::bulk_remove_code_roots() {
+  _code_roots.bulk_remove();
+}
+
 void HeapRegionRemSet::code_roots_do(CodeBlobClosure* blk) const {
   _code_roots.nmethods_do(blk);
 }

src/hotspot/share/gc/g1/heapRegionRemSet.hpp

@@ -150,6 +150,7 @@ public:
   // the heap region that owns this RSet.
   void add_code_root(nmethod* nm);
   void remove_code_root(nmethod* nm);
+  void bulk_remove_code_roots();
 
   // Applies blk->do_code_blob() to each of the entries in _code_roots
   void code_roots_do(CodeBlobClosure* blk) const;

src/hotspot/share/gc/parallel/gcAdaptivePolicyCounters.hpp

@@ -60,7 +60,6 @@ class GCAdaptivePolicyCounters : public GCPolicyCounters {
   PerfVariable*         _decrease_for_footprint_counter;
 
   PerfVariable*         _minor_pause_young_slope_counter;
-  PerfVariable*         _major_pause_old_slope_counter;
 
   PerfVariable*         _decide_at_full_gc_counter;
 

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -532,6 +532,14 @@ void ParallelScavengeHeap::resize_all_tlabs() {
   CollectedHeap::resize_all_tlabs();
 }
 
+void ParallelScavengeHeap::prune_scavengable_nmethods() {
+  ScavengableNMethods::prune_nmethods_not_into_young();
+}
+
+void ParallelScavengeHeap::prune_unlinked_nmethods() {
+  ScavengableNMethods::prune_unlinked_nmethods();
+}
+
 // This method is used by System.gc() and JVMTI.
 void ParallelScavengeHeap::collect(GCCause::Cause cause) {
   assert(!Heap_lock->owned_by_self(),
@@ -863,10 +871,6 @@ void ParallelScavengeHeap::verify_nmethod(nmethod* nm) {
   ScavengableNMethods::verify_nmethod(nm);
 }
 
-void ParallelScavengeHeap::prune_scavengable_nmethods() {
-  ScavengableNMethods::prune_nmethods();
-}
-
 GrowableArray<GCMemoryManager*> ParallelScavengeHeap::memory_managers() {
   GrowableArray<GCMemoryManager*> memory_managers(2);
   memory_managers.append(_young_manager);

src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp

@@ -176,6 +176,7 @@ class ParallelScavengeHeap : public CollectedHeap {
   void verify_nmethod(nmethod* nm) override;
 
   void prune_scavengable_nmethods();
+  void prune_unlinked_nmethods();
 
   size_t max_capacity() const override;
 

src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp

@@ -51,7 +51,6 @@ PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t init_eden_size,
      _avg_major_pause(new AdaptivePaddedAverage(AdaptiveTimeWeight, PausePadding)),
      _avg_base_footprint(new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight)),
      _gc_stats(),
-     _collection_cost_margin_fraction(AdaptiveSizePolicyCollectionCostMargin / 100.0),
      _major_pause_old_estimator(new LinearLeastSquareFit(AdaptiveSizePolicyWeight)),
      _major_pause_young_estimator(new LinearLeastSquareFit(AdaptiveSizePolicyWeight)),
      _latest_major_mutator_interval_seconds(0),
@@ -281,11 +280,11 @@ void PSAdaptiveSizePolicy::compute_eden_space_size(
     //
     // Make changes only to affect one of the pauses (the larger)
     // at a time.
-    adjust_eden_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size);
+    adjust_eden_for_pause_time(&desired_eden_size);
 
   } else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) {
     // Adjust only for the minor pause time goal
-    adjust_eden_for_minor_pause_time(is_full_gc, &desired_eden_size);
+    adjust_eden_for_minor_pause_time(&desired_eden_size);
 
   } else if(adjusted_mutator_cost() < _throughput_goal) {
     // This branch used to require that (mutator_cost() > 0.0 in 1.4.2.
@@ -456,7 +455,7 @@ void PSAdaptiveSizePolicy::compute_old_gen_free_space(
     // at a time.
     if (is_full_gc) {
       set_decide_at_full_gc(decide_at_full_gc_true);
-      adjust_promo_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size);
+      adjust_promo_for_pause_time(&desired_promo_size);
     }
   } else if (adjusted_mutator_cost() < _throughput_goal) {
     // This branch used to require that (mutator_cost() > 0.0 in 1.4.2.
@@ -571,9 +570,7 @@ void PSAdaptiveSizePolicy::decay_supplemental_growth(bool is_full_gc) {
   }
 }
 
-void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(bool is_full_gc,
-    size_t* desired_eden_size_ptr) {
-
+void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(size_t* desired_eden_size_ptr) {
   // Adjust the young generation size to reduce pause time of
   // of collections.
   //
@@ -586,25 +583,23 @@ void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(bool is_full_gc,
           decrease_young_gen_for_min_pauses_true);
     *desired_eden_size_ptr = *desired_eden_size_ptr -
       eden_decrement_aligned_down(*desired_eden_size_ptr);
-    } else {
-      // EXPERIMENTAL ADJUSTMENT
-      // Only record that the estimator indicated such an action.
-      // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta;
-      set_change_young_gen_for_min_pauses(
-          increase_young_gen_for_min_pauses_true);
+  } else {
+    // EXPERIMENTAL ADJUSTMENT
+    // Only record that the estimator indicated such an action.
+    // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta;
+    set_change_young_gen_for_min_pauses(
+      increase_young_gen_for_min_pauses_true);
   }
 }
 
-void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(bool is_full_gc,
-                                             size_t* desired_promo_size_ptr,
-                                             size_t* desired_eden_size_ptr) {
+void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(size_t* desired_promo_size_ptr) {
 
   size_t promo_heap_delta = 0;
   // Add some checks for a threshold for a change.  For example,
   // a change less than the required alignment is probably not worth
   // attempting.
 
-  if (_avg_minor_pause->padded_average() <= _avg_major_pause->padded_average() && is_full_gc) {
+  if (_avg_minor_pause->padded_average() <= _avg_major_pause->padded_average()) {
     // Adjust for the major pause time only at full gc's because the
     // affects of a change can only be seen at full gc's.
 
@@ -631,16 +626,14 @@ void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(bool is_full_gc,
     *desired_promo_size_ptr, promo_heap_delta);
 }
 
-void PSAdaptiveSizePolicy::adjust_eden_for_pause_time(bool is_full_gc,
-                                             size_t* desired_promo_size_ptr,
-                                             size_t* desired_eden_size_ptr) {
+void PSAdaptiveSizePolicy::adjust_eden_for_pause_time(size_t* desired_eden_size_ptr) {
 
   size_t eden_heap_delta = 0;
   // Add some checks for a threshold for a change.  For example,
   // a change less than the required alignment is probably not worth
   // attempting.
   if (_avg_minor_pause->padded_average() > _avg_major_pause->padded_average()) {
-    adjust_eden_for_minor_pause_time(is_full_gc, desired_eden_size_ptr);
+    adjust_eden_for_minor_pause_time(desired_eden_size_ptr);
   }
   log_trace(gc, ergo)(
     "PSAdaptiveSizePolicy::adjust_eden_for_pause_time "

src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp

@@ -75,8 +75,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
   // Statistical data gathered for GC
   GCStats _gc_stats;
 
-  const double _collection_cost_margin_fraction;
-
   // Variable for estimating the major and minor pause times.
   // These variables represent linear least-squares fits of
   // the data.
@@ -116,19 +114,13 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
  private:
 
   // Accessors
-  AdaptivePaddedAverage* avg_major_pause() const { return _avg_major_pause; }
   double gc_minor_pause_goal_sec() const { return _gc_minor_pause_goal_sec; }
 
-  void adjust_eden_for_minor_pause_time(bool is_full_gc,
-                                   size_t* desired_eden_size_ptr);
+  void adjust_eden_for_minor_pause_time(size_t* desired_eden_size_ptr);
   // Change the generation sizes to achieve a GC pause time goal
   // Returned sizes are not necessarily aligned.
-  void adjust_promo_for_pause_time(bool is_full_gc,
-                         size_t* desired_promo_size_ptr,
-                         size_t* desired_eden_size_ptr);
-  void adjust_eden_for_pause_time(bool is_full_gc,
-                         size_t* desired_promo_size_ptr,
-                         size_t* desired_eden_size_ptr);
+  void adjust_promo_for_pause_time(size_t* desired_promo_size_ptr);
+  void adjust_eden_for_pause_time(size_t* desired_eden_size_ptr);
   // Change the generation sizes to achieve an application throughput goal
   // Returned sizes are not necessarily aligned.
   void adjust_promo_for_throughput(bool is_full_gc,
@@ -160,7 +152,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
   size_t scale_down(size_t change, double part, double total);
 
  protected:
-  // Time accessors
 
   // Footprint accessors
   size_t live_space() const {
@@ -175,9 +166,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
   void set_promo_size(size_t new_size) {
     _promo_size = new_size;
   }
-  void set_survivor_size(size_t new_size) {
-    _survivor_size = new_size;
-  }
 
   // Update estimators
   void update_minor_pause_old_estimator(double minor_pause_in_ms);
@@ -226,10 +214,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
 
   size_t calculated_old_free_size_in_bytes() const;
 
-  size_t average_old_live_in_bytes() const {
-    return (size_t) avg_old_live()->average();
-  }
-
   size_t average_promoted_in_bytes() const {
     return (size_t)avg_promoted()->average();
   }
@@ -252,40 +236,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
     _change_old_gen_for_min_pauses = v;
   }
 
-  // Return true if the old generation size was changed
-  // to try to reach a pause time goal.
-  bool old_gen_changed_for_pauses() {
-    bool result = _change_old_gen_for_maj_pauses != 0 ||
-                  _change_old_gen_for_min_pauses != 0;
-    return result;
-  }
-
-  // Return true if the young generation size was changed
-  // to try to reach a pause time goal.
-  bool young_gen_changed_for_pauses() {
-    bool result = _change_young_gen_for_min_pauses != 0 ||
-                  _change_young_gen_for_maj_pauses != 0;
-    return result;
-  }
-  // end flags for pause goal
-
-  // Return true if the old generation size was changed
-  // to try to reach a throughput goal.
-  bool old_gen_changed_for_throughput() {
-    bool result = _change_old_gen_for_throughput != 0;
-    return result;
-  }
-
-  // Return true if the young generation size was changed
-  // to try to reach a throughput goal.
-  bool young_gen_changed_for_throughput() {
-    bool result = _change_young_gen_for_throughput != 0;
-    return result;
-  }
-
-  int decrease_for_footprint() { return _decrease_for_footprint; }
-
-
   // Accessors for estimators.  The slope of the linear fit is
   // currently all that is used for making decisions.
 
@@ -293,18 +243,12 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
     return _major_pause_old_estimator;
   }
 
-  LinearLeastSquareFit* major_pause_young_estimator() {
-    return _major_pause_young_estimator;
-  }
-
-
   virtual void clear_generation_free_space_flags();
 
   double major_pause_old_slope() { return _major_pause_old_estimator->slope(); }
   double major_pause_young_slope() {
     return _major_pause_young_estimator->slope();
   }
-  double major_collection_slope() { return _major_collection_estimator->slope();}
 
   // Calculates optimal (free) space sizes for both the young and old
   // generations.  Stores results in _eden_size and _promo_size.

src/hotspot/share/gc/parallel/psCardTable.cpp

@@ -111,7 +111,7 @@ void PSCardTable::scan_obj_with_limit(PSPromotionManager* pm,
   }
 }
 
-void PSCardTable::pre_scavenge(HeapWord* old_gen_bottom, uint active_workers) {
+void PSCardTable::pre_scavenge(uint active_workers) {
   _preprocessing_active_workers = active_workers;
 }
 

src/hotspot/share/gc/parallel/psCardTable.hpp

@@ -73,7 +73,7 @@ class PSCardTable: public CardTable {
                                       _preprocessing_active_workers(0) {}
 
   // Scavenge support
-  void pre_scavenge(HeapWord* old_gen_bottom, uint active_workers);
+  void pre_scavenge(uint active_workers);
   // Scavenge contents of stripes with the given index.
   void scavenge_contents_parallel(ObjectStartArray* start_array,
                                   HeapWord* old_gen_bottom,

src/hotspot/share/gc/parallel/psParallelCompact.cpp

@@ -1769,6 +1769,7 @@ bool PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) {
     ref_processor()->start_discovery(maximum_heap_compaction);
 
     ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
+                              false /* unregister_nmethods_during_purge */,
                               false /* lock_codeblob_free_separately */);
 
     marking_phase(&_gc_tracer);
@@ -2078,6 +2079,10 @@ void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
       // Release unloaded nmethod's memory.
       ctx->purge_nmethods();
     }
+    {
+      GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", &_gc_timer);
+      ParallelScavengeHeap::heap()->prune_unlinked_nmethods();
+    }
     {
       GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
       ctx->free_code_blobs();

src/hotspot/share/gc/parallel/psScavenge.cpp

@@ -302,7 +302,7 @@ public:
 
     if (!_is_old_gen_empty) {
       PSCardTable* card_table = ParallelScavengeHeap::heap()->card_table();
-      card_table->pre_scavenge(_old_gen->object_space()->bottom(), active_workers);
+      card_table->pre_scavenge(active_workers);
     }
   }
 

src/hotspot/share/gc/serial/cardTableRS.cpp

@@ -382,20 +382,17 @@ CardTable::CardValue* CardTableRS::find_first_dirty_card(CardValue* const start_
   return end_card;
 }
 
-// Because non-objArray objs can be imprecisely-marked (only obj-start card is
-// dirty instead of the part containing old-to-young pointers), if the
-// obj-start of a non-objArray is dirty, all cards that obj completely resides
-// on are considered as dirty, since that obj will be iterated (scanned for
-// old-to-young pointers) as a whole.
+// Because non-objArray objs can be imprecisely marked (only the obj-start card
+// is dirty instead of the part containing old-to-young pointers), if the
+// obj-start of a non-objArray is dirty, all cards that the obj resides on,
+// except the final one, are unconditionally considered as dirty. This is
+// because that obj will be iterated (scanned for old-to-young pointers) as a
+// whole.
 template<typename Func>
 CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_card,
                                                          CardValue* const end_card,
                                                          CardTableRS* ct,
                                                          Func& object_start) {
-
-  // end_card might be just beyond the heap, so need to use the _raw variant.
-  HeapWord* end_address = ct->addr_for_raw(end_card);
-
   for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
     if (is_dirty(current_card)) {
       current_card++;
@@ -418,21 +415,14 @@ CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_
       return current_card;
     }
 
-    // This might be the last object in this area, avoid trying to access the
-    // card beyond the allowed area.
-    HeapWord* next_address = obj_start_addr + obj->size();
-    if (next_address >= end_address) {
-      break;
-    }
-
-    // Card occupied by next obj.
-    CardValue* next_obj_card = ct->byte_for(next_address);
-    if (is_clean(next_obj_card)) {
-      return next_obj_card;
+    // Final card occupied by obj.
+    CardValue* obj_final_card = ct->byte_for(obj_start_addr + obj->size() - 1);
+    if (is_clean(obj_final_card)) {
+      return obj_final_card;
     }
 
     // Continue the search after this known-dirty card...
-    current_card = next_obj_card + 1;
+    current_card = obj_final_card + 1;
   }
 
   return end_card;

src/hotspot/share/gc/serial/genMarkSweep.cpp

@@ -219,6 +219,10 @@ void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) {
       // Release unloaded nmethod's memory.
       ctx->purge_nmethods();
     }
+    {
+      GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", gc_timer());
+      gch->prune_unlinked_nmethods();
+    }
     {
       GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
       ctx->free_code_blobs();

src/hotspot/share/gc/serial/serialHeap.cpp

@@ -28,6 +28,7 @@
 #include "gc/serial/tenuredGeneration.inline.hpp"
 #include "gc/shared/gcLocker.inline.hpp"
 #include "gc/shared/genMemoryPools.hpp"
+#include "gc/shared/scavengableNMethods.hpp"
 #include "gc/shared/strongRootsScope.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "memory/universe.hpp"

src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp

@@ -51,15 +51,11 @@ class AdaptiveSizePolicy : public CHeapObj<mtGC> {
     decrease_old_gen_for_throughput_true = -7,
     decrease_young_gen_for_througput_true = -6,
 
-    increase_old_gen_for_min_pauses_true = -5,
-    decrease_old_gen_for_min_pauses_true = -4,
-    decrease_young_gen_for_maj_pauses_true = -3,
     increase_young_gen_for_min_pauses_true = -2,
     increase_old_gen_for_maj_pauses_true = -1,
 
     decrease_young_gen_for_min_pauses_true = 1,
     decrease_old_gen_for_maj_pauses_true = 2,
-    increase_young_gen_for_maj_pauses_true = 3,
 
     increase_old_gen_for_throughput_true = 4,
     increase_young_gen_for_througput_true = 5,

src/hotspot/share/gc/shared/cardTable.hpp

@@ -83,14 +83,6 @@ protected:
     return cards_required(_whole_heap.word_size()) - 1;
   }
 
-  // Mapping from card marking array entry to address of first word without checks.
-  HeapWord* addr_for_raw(const CardValue* p) const {
-    // As _byte_map_base may be "negative" (the card table has been allocated before
-    // the heap in memory), do not use pointer_delta() to avoid the assertion failure.
-    size_t delta = p - _byte_map_base;
-    return (HeapWord*) (delta << _card_shift);
-  }
-
 private:
   void initialize_covered_region(void* region0_start, void* region1_start);
 
@@ -152,13 +144,16 @@ public:
     return byte_after(p);
   }
 
-  // Mapping from card marking array entry to address of first word.
+  // Mapping from card marking array entry to address of first word
   HeapWord* addr_for(const CardValue* p) const {
     assert(p >= _byte_map && p < _byte_map + _byte_map_size,
            "out of bounds access to card marking array. p: " PTR_FORMAT
            " _byte_map: " PTR_FORMAT " _byte_map + _byte_map_size: " PTR_FORMAT,
            p2i(p), p2i(_byte_map), p2i(_byte_map + _byte_map_size));
-    HeapWord* result = addr_for_raw(p);
+    // As _byte_map_base may be "negative" (the card table has been allocated before
+    // the heap in memory), do not use pointer_delta() to avoid the assertion failure.
+    size_t delta = p - _byte_map_base;
+    HeapWord* result = (HeapWord*) (delta << _card_shift);
     assert(_whole_heap.contains(result),
            "Returning result = " PTR_FORMAT " out of bounds of "
            " card marking array's _whole_heap = [" PTR_FORMAT "," PTR_FORMAT ")",

src/hotspot/share/gc/shared/classUnloadingContext.cpp

@@ -32,10 +32,13 @@
 
 ClassUnloadingContext* ClassUnloadingContext::_context = nullptr;
 
-ClassUnloadingContext::ClassUnloadingContext(uint num_workers, bool lock_codeblob_free_separately) :
+ClassUnloadingContext::ClassUnloadingContext(uint num_workers,
+                                             bool unregister_nmethods_during_purge,
+                                             bool lock_codeblob_free_separately) :
   _cld_head(nullptr),
   _num_nmethod_unlink_workers(num_workers),
   _unlinked_nmethods(nullptr),
+  _unregister_nmethods_during_purge(unregister_nmethods_during_purge),
   _lock_codeblob_free_separately(lock_codeblob_free_separately) {
 
   assert(_context == nullptr, "context already set");
@@ -113,7 +116,7 @@ void ClassUnloadingContext::purge_nmethods() {
     NMethodSet* set = _unlinked_nmethods[i];
     for (nmethod* nm : *set) {
       freed_memory += nm->size();
-      nm->purge(false /* free_code_cache_data */);
+      nm->purge(false /* free_code_cache_data */, _unregister_nmethods_during_purge);
     }
   }
 

src/hotspot/share/gc/shared/classUnloadingContext.hpp

@@ -42,6 +42,7 @@ class ClassUnloadingContext : public CHeapObj<mtGC> {
   using NMethodSet = GrowableArrayCHeap<nmethod*, mtGC>;
   NMethodSet** _unlinked_nmethods;
 
+  bool _unregister_nmethods_during_purge;
   bool _lock_codeblob_free_separately;
 
 public:
@@ -49,10 +50,14 @@ public:
 
   // Num_nmethod_unlink_workers configures the maximum numbers of threads unlinking
   //     nmethods.
+  // unregister_nmethods_during_purge determines whether unloaded nmethods should
+  //     be unregistered from the garbage collector during purge. If not, ,the caller
+  //     is responsible to do that later.
   // lock_codeblob_free_separately determines whether freeing the code blobs takes
   //     the CodeCache_lock during the whole operation (=false) or per code blob
   //     free operation (=true).
   ClassUnloadingContext(uint num_nmethod_unlink_workers,
+                        bool unregister_nmethods_during_purge,
                         bool lock_codeblob_free_separately);
   ~ClassUnloadingContext();
 

src/hotspot/share/gc/shared/genCollectedHeap.cpp

@@ -524,6 +524,7 @@ void GenCollectedHeap::do_collection(bool           full,
     CodeCache::on_gc_marking_cycle_start();
 
     ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
+                              false /* unregister_nmethods_during_purge */,
                               false /* lock_codeblob_free_separately */);
 
     collect_generation(_old_gen,
@@ -582,7 +583,11 @@ void GenCollectedHeap::verify_nmethod(nmethod* nm) {
 }
 
 void GenCollectedHeap::prune_scavengable_nmethods() {
-  ScavengableNMethods::prune_nmethods();
+  ScavengableNMethods::prune_nmethods_not_into_young();
+}
+
+void GenCollectedHeap::prune_unlinked_nmethods() {
+  ScavengableNMethods::prune_unlinked_nmethods();
 }
 
 HeapWord* GenCollectedHeap::satisfy_failed_allocation(size_t size, bool is_tlab) {

src/hotspot/share/gc/shared/genCollectedHeap.hpp

@@ -182,6 +182,7 @@ public:
   void verify_nmethod(nmethod* nm) override;
 
   void prune_scavengable_nmethods();
+  void prune_unlinked_nmethods();
 
   // Iteration functions.
   void object_iterate(ObjectClosure* cl) override;

src/hotspot/share/gc/shared/scavengableNMethods.cpp

@@ -59,18 +59,8 @@ void ScavengableNMethods::register_nmethod(nmethod* nm) {
 }
 
 void ScavengableNMethods::unregister_nmethod(nmethod* nm) {
-  assert_locked_or_safepoint(CodeCache_lock);
-
-  if (gc_data(nm).on_list()) {
-    nmethod* prev = nullptr;
-    for (nmethod* cur = _head; cur != nullptr; cur = gc_data(cur).next()) {
-      if (cur == nm) {
-        unlist_nmethod(cur, prev);
-        return;
-      }
-      prev = cur;
-    }
-  }
+  // All users of this method only unregister in bulk during code unloading.
+  ShouldNotReachHere();
 }
 
 #ifndef PRODUCT
@@ -172,10 +162,37 @@ void ScavengableNMethods::nmethods_do_and_prune(CodeBlobToOopClosure* cl) {
   debug_only(verify_unlisted_nmethods(nullptr));
 }
 
-void ScavengableNMethods::prune_nmethods() {
+void ScavengableNMethods::prune_nmethods_not_into_young() {
   nmethods_do_and_prune(nullptr /* No closure */);
 }
 
+void ScavengableNMethods::prune_unlinked_nmethods() {
+  assert_locked_or_safepoint(CodeCache_lock);
+
+  debug_only(mark_on_list_nmethods());
+
+  nmethod* prev = nullptr;
+  nmethod* cur = _head;
+  while (cur != nullptr) {
+    ScavengableNMethodsData data = gc_data(cur);
+    debug_only(data.clear_marked());
+    assert(data.on_list(), "else shouldn't be on this list");
+
+    nmethod* const next = data.next();
+
+    if (cur->is_unlinked()) {
+      unlist_nmethod(cur, prev);
+    } else {
+      prev = cur;
+    }
+
+    cur = next;
+  }
+
+  // Check for stray marks.
+  debug_only(verify_unlisted_nmethods(nullptr));
+}
+
 // Walk the list of methods which might contain oops to the java heap.
 void ScavengableNMethods::nmethods_do(CodeBlobToOopClosure* cl) {
   nmethods_do_and_prune(cl);
@@ -218,8 +235,9 @@ void ScavengableNMethods::mark_on_list_nmethods() {
     nmethod* nm = iter.method();
     ScavengableNMethodsData data = gc_data(nm);
     assert(data.not_marked(), "clean state");
-    if (data.on_list())
+    if (data.on_list()) {
       data.set_marked();
+    }
   }
 }
 
@@ -230,7 +248,10 @@ void ScavengableNMethods::verify_unlisted_nmethods(CodeBlobClosure* cl) {
   while(iter.next()) {
     nmethod* nm = iter.method();
 
-    verify_nmethod(nm);
+    // Can not verify already unlinked nmethods as they are partially invalid already.
+    if (!nm->is_unlinked()) {
+      verify_nmethod(nm);
+    }
 
     if (cl != nullptr && !gc_data(nm).on_list()) {
       cl->do_code_blob(nm);

src/hotspot/share/gc/shared/scavengableNMethods.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -46,8 +46,10 @@ public:
   static void unregister_nmethod(nmethod* nm);
   static void verify_nmethod(nmethod* nm);
 
-  // Remove nmethods that no longer have scavengable oops.
-  static void prune_nmethods();
+  // Remove nmethods that no longer have oops into young gen.
+  static void prune_nmethods_not_into_young();
+  // Remvoe unlinked (dead) nmethods.
+  static void prune_unlinked_nmethods();
 
   // Apply closure to every scavengable nmethod.
   // Remove nmethods that no longer have scavengable oops.

src/hotspot/share/gc/shared/workerPolicy.cpp

@@ -73,11 +73,7 @@ uint WorkerPolicy::calc_parallel_worker_threads() {
 
 uint WorkerPolicy::parallel_worker_threads() {
   if (!_parallel_worker_threads_initialized) {
-    if (FLAG_IS_DEFAULT(ParallelGCThreads)) {
-      _parallel_worker_threads = WorkerPolicy::calc_parallel_worker_threads();
-    } else {
-      _parallel_worker_threads = ParallelGCThreads;
-    }
+    _parallel_worker_threads = WorkerPolicy::calc_parallel_worker_threads();
     _parallel_worker_threads_initialized = true;
   }
   return _parallel_worker_threads;

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -504,6 +504,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
   _num_regions(0),
   _regions(nullptr),
   _update_refs_iterator(this),
+  _gc_state_changed(false),
   _gc_no_progress_count(0),
   _control_thread(nullptr),
   _shenandoah_policy(policy),
@@ -1741,16 +1742,21 @@ void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
   _update_refs_iterator.reset();
 }
 
-void ShenandoahHeap::set_gc_state_all_threads(char state) {
-  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
-    ShenandoahThreadLocalData::set_gc_state(t, state);
+void ShenandoahHeap::set_gc_state_all_threads() {
+  assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
+  if (_gc_state_changed) {
+    _gc_state_changed = false;
+    char state = gc_state();
+    for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
+      ShenandoahThreadLocalData::set_gc_state(t, state);
+    }
   }
 }
 
 void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
-  assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
+  assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
   _gc_state.set_cond(mask, value);
-  set_gc_state_all_threads(_gc_state.raw_value());
+  _gc_state_changed = true;
 }
 
 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
@@ -1822,6 +1828,7 @@ void ShenandoahHeap::stop() {
 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
   if (!unload_classes()) return;
   ClassUnloadingContext ctx(_workers->active_workers(),
+                            true /* unregister_nmethods_during_purge */,
                             false /* lock_codeblob_free_separately */);
 
   // Unload classes and purge SystemDictionary.

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -124,6 +124,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
   friend class ShenandoahGCStateResetter;
   friend class ShenandoahParallelObjectIterator;
   friend class ShenandoahSafepoint;
+
   // Supported GC
   friend class ShenandoahConcurrentGC;
   friend class ShenandoahDegenGC;
@@ -283,6 +284,7 @@ public:
   };
 
 private:
+  bool _gc_state_changed;
   ShenandoahSharedBitmap _gc_state;
   ShenandoahSharedFlag   _degenerated_gc_in_progress;
   ShenandoahSharedFlag   _full_gc_in_progress;
@@ -291,11 +293,12 @@ private:
 
   size_t _gc_no_progress_count;
 
-  void set_gc_state_all_threads(char state);
   void set_gc_state_mask(uint mask, bool value);
 
 public:
   char gc_state() const;
+  void set_gc_state_all_threads();
+  bool has_gc_state_changed() { return _gc_state_changed; }
 
   void set_concurrent_mark_in_progress(bool in_progress);
   void set_evacuation_in_progress(bool in_progress);

src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp

@@ -140,6 +140,7 @@ void ShenandoahUnload::unload() {
   assert(heap->is_concurrent_weak_root_in_progress(), "Filtered by caller");
 
   ClassUnloadingContext ctx(heap->workers()->active_workers(),
+                            true /* unregister_nmethods_during_purge */,
                             true /* lock_codeblob_free_separately */);
 
   // Unlink stale metadata and nmethods

src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp

@@ -35,12 +35,23 @@
 #include "interpreter/oopMapCache.hpp"
 #include "memory/universe.hpp"
 
+bool VM_ShenandoahOperation::doit_prologue() {
+  assert(!ShenandoahHeap::heap()->has_gc_state_changed(), "GC State can only be changed on a safepoint.");
+  return true;
+}
+
+void VM_ShenandoahOperation::doit_epilogue() {
+  assert(!ShenandoahHeap::heap()->has_gc_state_changed(), "GC State was not synchronized to java threads.");
+}
+
 bool VM_ShenandoahReferenceOperation::doit_prologue() {
+  VM_ShenandoahOperation::doit_prologue();
   Heap_lock->lock();
   return true;
 }
 
 void VM_ShenandoahReferenceOperation::doit_epilogue() {
+  VM_ShenandoahOperation::doit_epilogue();
   OopMapCache::cleanup_old_entries();
   if (Universe::has_reference_pending_list()) {
     Heap_lock->notify_all();
@@ -51,34 +62,41 @@ void VM_ShenandoahReferenceOperation::doit_epilogue() {
 void VM_ShenandoahInitMark::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Init Mark", SvcGCMarker::CONCURRENT);
   _gc->entry_init_mark();
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }
 
 void VM_ShenandoahFinalMarkStartEvac::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Final Mark", SvcGCMarker::CONCURRENT);
   _gc->entry_final_mark();
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }
 
 void VM_ShenandoahFullGC::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Full GC", SvcGCMarker::FULL);
   _full_gc->entry_full(_gc_cause);
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }
 
 void VM_ShenandoahDegeneratedGC::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Degenerated GC", SvcGCMarker::CONCURRENT);
   _gc->entry_degenerated();
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }
 
 void VM_ShenandoahInitUpdateRefs::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Init Update Refs", SvcGCMarker::CONCURRENT);
   _gc->entry_init_updaterefs();
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }
 
 void VM_ShenandoahFinalUpdateRefs::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Final Update Refs", SvcGCMarker::CONCURRENT);
   _gc->entry_final_updaterefs();
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }
 
 void VM_ShenandoahFinalRoots::doit() {
   ShenandoahGCPauseMark mark(_gc_id, "Final Roots", SvcGCMarker::CONCURRENT);
   _gc->entry_final_roots();
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
 }

src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp

@@ -47,14 +47,16 @@ protected:
   uint         _gc_id;
 public:
   VM_ShenandoahOperation() : _gc_id(GCId::current()) {};
-  virtual bool skip_thread_oop_barriers() const { return true; }
+  bool skip_thread_oop_barriers() const override { return true; }
+  bool doit_prologue() override;
+  void doit_epilogue() override;
 };
 
 class VM_ShenandoahReferenceOperation : public VM_ShenandoahOperation {
 public:
   VM_ShenandoahReferenceOperation() : VM_ShenandoahOperation() {};
-  bool doit_prologue();
-  void doit_epilogue();
+  bool doit_prologue() override;
+  void doit_epilogue() override;
 };
 
 class VM_ShenandoahInitMark: public VM_ShenandoahOperation {

src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp

@@ -620,6 +620,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label,
   guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
   guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
 
+  ShenandoahHeap::heap()->set_gc_state_all_threads();
+
   // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
   ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
 

src/hotspot/share/gc/x/xHeap.cpp

@@ -322,6 +322,7 @@ void XHeap::process_non_strong_references() {
   _weak_roots_processor.process_weak_roots();
 
   ClassUnloadingContext ctx(_workers.active_workers(),
+                            true /* unregister_nmethods_during_purge */,
                             true /* lock_codeblob_free_separately */);
 
   // Unlink stale metadata and nmethods

src/hotspot/share/gc/z/zArguments.cpp

@@ -143,6 +143,9 @@ void ZArguments::initialize() {
     FLAG_SET_DEFAULT(ZFragmentationLimit, 5.0);
   }
 
+  // Set medium page size here because MaxTenuringThreshold may use it.
+  ZHeuristics::set_medium_page_size();
+
   if (!FLAG_IS_DEFAULT(ZTenuringThreshold) && ZTenuringThreshold != -1) {
     FLAG_SET_ERGO_IF_DEFAULT(MaxTenuringThreshold, ZTenuringThreshold);
     if (MaxTenuringThreshold == 0) {

src/hotspot/share/gc/z/zBarrierSet.cpp

@@ -152,6 +152,19 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
   deoptimize_allocation(thread);
 }
 
+void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj, size_t size) {
+  volatile zpointer* src = (volatile zpointer*)src_obj->base();
+  volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
+
+  for (const zpointer* const end = cast_from_oop<const zpointer*>(src_obj) + size; src < end; src++, dst++) {
+    zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
+    // We avoid healing here because the store below colors the pointer store good,
+    // hence avoiding the cost of a CAS.
+    ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
+    Atomic::store(dst, ZAddress::store_good(elem));
+  }
+}
+
 void ZBarrierSet::print_on(outputStream* st) const {
   st->print_cr("ZBarrierSet");
 }

src/hotspot/share/gc/z/zBarrierSet.hpp

@@ -39,6 +39,8 @@ public:
   static ZBarrierSetAssembler* assembler();
   static bool barrier_needed(DecoratorSet decorators, BasicType type);
 
+  static void clone_obj_array(objArrayOop src, objArrayOop dst, size_t size);
+
   virtual void on_thread_create(Thread* thread);
   virtual void on_thread_destroy(Thread* thread);
   virtual void on_thread_attach(Thread* thread);

src/hotspot/share/gc/z/zBarrierSet.inline.hpp

@@ -403,14 +403,13 @@ inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_i
   return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
 }
 
-class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
+class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
 public:
   virtual void do_oop(oop* p_) {
     volatile zpointer* const p = (volatile zpointer*)p_;
     const zpointer ptr = ZBarrier::load_atomic(p);
     const zaddress addr = ZPointer::uncolor(ptr);
-    ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
-    *p = ZAddress::store_good(addr);
+    Atomic::store(p, ZAddress::store_good(addr));
   }
 
   virtual void do_oop(narrowOop* p) {
@@ -433,6 +432,17 @@ template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
   assert_is_valid(to_zaddress(src));
 
+  if (dst->is_objArray()) {
+    // Cloning an object array is similar to performing array copy.
+    // If an array is large enough to have its allocation segmented,
+    // this operation might require GC barriers. However, the intrinsics
+    // for cloning arrays transform the clone to an optimized allocation
+    // and arraycopy sequence, so the performance of this runtime call
+    // does not matter for object arrays.
+    clone_obj_array(objArrayOop(src), objArrayOop(dst), size);
+    return;
+  }
+
   // Fix the oops
   ZLoadBarrierOopClosure cl;
   ZIterator::oop_iterate(src, &cl);
@@ -440,10 +450,10 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(o
   // Clone the object
   Raw::clone_in_heap(src, dst, size);
 
-  assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
+  assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
 
   // Color store good before handing out
-  ZStoreBarrierOopClosure cl_sg;
+  ZColorStoreGoodOopClosure cl_sg;
   ZIterator::oop_iterate(dst, &cl_sg);
 }
 

src/hotspot/share/gc/z/zGeneration.cpp

@@ -1318,6 +1318,7 @@ void ZGenerationOld::process_non_strong_references() {
   _weak_roots_processor.process_weak_roots();
 
   ClassUnloadingContext ctx(_workers.active_workers(),
+                            true /* unregister_nmethods_during_purge */,
                             true /* lock_codeblob_free_separately */);
 
   // Unlink stale metadata and nmethods

src/hotspot/share/gc/z/zInitialize.cpp

@@ -28,7 +28,6 @@
 #include "gc/z/zDriver.hpp"
 #include "gc/z/zGCIdPrinter.hpp"
 #include "gc/z/zGlobals.hpp"
-#include "gc/z/zHeuristics.hpp"
 #include "gc/z/zInitialize.hpp"
 #include "gc/z/zJNICritical.hpp"
 #include "gc/z/zLargePages.hpp"
@@ -54,7 +53,6 @@ ZInitialize::ZInitialize(ZBarrierSet* barrier_set) {
   ZThreadLocalAllocBuffer::initialize();
   ZTracer::initialize();
   ZLargePages::initialize();
-  ZHeuristics::set_medium_page_size();
   ZBarrierSet::set_barrier_set(barrier_set);
   ZJNICritical::initialize();
   ZDriver::initialize();

src/hotspot/share/include/jvm.h

@@ -1154,6 +1154,9 @@ JVM_VirtualThreadUnmount(JNIEnv* env, jobject vthread, jboolean hide);
 JNIEXPORT void JNICALL
 JVM_VirtualThreadHideFrames(JNIEnv* env, jobject vthread, jboolean hide);
 
+JNIEXPORT void JNICALL
+JVM_VirtualThreadDisableSuspend(JNIEnv* env, jobject vthread, jboolean enter);
+
 /*
  * Core reflection support.
  */

src/hotspot/share/jfr/jni/jfrJavaSupport.cpp

@@ -525,6 +525,12 @@ const char* JfrJavaSupport::c_str(jstring string, Thread* thread, bool c_heap /*
   return string != nullptr ? c_str(resolve_non_null(string), thread, c_heap) : nullptr;
 }
 
+void JfrJavaSupport::free_c_str(const char* str, bool c_heap) {
+  if (c_heap) {
+    FREE_C_HEAP_ARRAY(char, str);
+  }
+}
+
 static Symbol** allocate_symbol_array(bool c_heap, int length, Thread* thread) {
   return c_heap ?
            NEW_C_HEAP_ARRAY(Symbol*, length, mtTracing) :
@@ -546,6 +552,7 @@ Symbol** JfrJavaSupport::symbol_array(jobjectArray string_array, JavaThread* thr
     if (object != nullptr) {
       const char* text = c_str(arrayOop->obj_at(i), thread, c_heap);
       symbol = SymbolTable::new_symbol(text);
+      free_c_str(text, c_heap);
     }
     result_array[i] = symbol;
   }

src/hotspot/share/jfr/jni/jfrJavaSupport.hpp

@@ -86,6 +86,7 @@ class JfrJavaSupport : public AllStatic {
   static Klass* klass(const jobject handle);
   static const char* c_str(jstring string, Thread* thread, bool c_heap = false);
   static const char* c_str(oop string, Thread* thread, bool c_heap = false);
+  static void free_c_str(const char* str, bool c_heap);
   static Symbol** symbol_array(jobjectArray string_array, JavaThread* thread, intptr_t* result_size, bool c_heap = false);
 
   // exceptions

src/hotspot/share/jfr/metadata/metadata.xml

@@ -957,6 +957,8 @@
     <Field type="string" name="name" label="Name" />
     <Field type="boolean" name="success" label="Success" description="Success or failure of the load operation" />
     <Field type="string" name="errorMessage" label="Error Message" description="In case of a load error, error description" />
+    <Field type="boolean" name="fpEnvCorrectionAttempt" label="FPU Environment correction" description="In case of IEEE conformance issues we might reset the FP environment" />
+    <Field type="boolean" name="fpEnvCorrectionSuccess" label="FPU Environment correction result" description="Stores the result in the case of an FP environment correction" />
   </Event>
 
   <Event name="NativeLibraryUnload" category="Java Virtual Machine, Runtime" label="Native Library Unload" thread="true" stackTrace="true" startTime="true"

src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.cpp

@@ -69,7 +69,7 @@ static inline JfrTicksWrapper* allocate_start_time() {
   return EventType::is_enabled() ? new JfrTicksWrapper() : nullptr;
 }
 
-NativeLibraryLoadEvent::NativeLibraryLoadEvent(const char* name, void** result) : JfrNativeLibraryEventBase(name), _result(result) {
+NativeLibraryLoadEvent::NativeLibraryLoadEvent(const char* name, void** result) : JfrNativeLibraryEventBase(name), _result(result), _fp_env_correction_attempt(false), _fp_env_correction_success(false) {
   assert(_result != nullptr, "invariant");
   _start_time = allocate_start_time<EventNativeLibraryLoad>();
 }
@@ -90,8 +90,17 @@ void NativeLibraryUnloadEvent::set_result(bool result) {
   _result = result;
 }
 
+static void set_additional_data(EventNativeLibraryLoad& event, const NativeLibraryLoadEvent& helper) {
+  event.set_fpEnvCorrectionAttempt(helper.get_fp_env_correction_attempt());
+  event.set_fpEnvCorrectionSuccess(helper.get_fp_env_correction_success());
+}
+
+static void set_additional_data(EventNativeLibraryUnload& event, const NativeLibraryUnloadEvent& helper) {
+  // no additional entries atm. for the unload event
+}
+
 template <typename EventType, typename HelperType>
-static void commit(HelperType& helper) {
+static void commit(const HelperType& helper) {
   if (!helper.has_start_time()) {
     return;
   }
@@ -101,6 +110,7 @@ static void commit(HelperType& helper) {
   event.set_name(helper.name());
   event.set_errorMessage(helper.error_msg());
   event.set_success(helper.success());
+  set_additional_data(event, helper);
   Thread* thread = Thread::current();
   assert(thread != nullptr, "invariant");
   if (thread->is_Java_thread()) {

src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.hpp

@@ -52,10 +52,16 @@ class JfrNativeLibraryEventBase : public StackObj {
 class NativeLibraryLoadEvent : public JfrNativeLibraryEventBase {
  private:
   void** _result;
+  bool _fp_env_correction_attempt;
+  bool _fp_env_correction_success;
  public:
   NativeLibraryLoadEvent(const char* name, void** result);
   ~NativeLibraryLoadEvent();
   bool success() const;
+  bool get_fp_env_correction_attempt() const { return _fp_env_correction_attempt; }
+  bool get_fp_env_correction_success() const { return _fp_env_correction_success; }
+  void set_fp_env_correction_attempt(bool v) { _fp_env_correction_attempt = v; }
+  void set_fp_env_correction_success(bool v) { _fp_env_correction_success = v; }
 };
 
 class NativeLibraryUnloadEvent : public JfrNativeLibraryEventBase {

src/hotspot/share/jvmci/vmStructs_jvmci.cpp

@@ -220,6 +220,7 @@
   nonstatic_field(JavaThread,                  _lock_stack,                                   LockStack)                             \
   JVMTI_ONLY(nonstatic_field(JavaThread,       _is_in_VTMS_transition,                        bool))                                 \
   JVMTI_ONLY(nonstatic_field(JavaThread,       _is_in_tmp_VTMS_transition,                    bool))                                 \
+  JVMTI_ONLY(nonstatic_field(JavaThread,       _is_disable_suspend,                           bool))                                 \
                                                                                                                                      \
   nonstatic_field(LockStack,                   _top,                                          uint32_t)                              \
                                                                                                                                      \

src/hotspot/share/logging/logTag.hpp

@@ -97,6 +97,7 @@ class outputStream;
   LOG_TAG(iklass) \
   LOG_TAG(indy) \
   LOG_TAG(init) \
+  LOG_TAG(inlinecache)\
   LOG_TAG(inlining) \
   LOG_TAG(install) \
   LOG_TAG(interpreter) \

src/hotspot/share/memory/universe.cpp

@@ -338,7 +338,7 @@ void Universe::genesis(TRAPS) {
       // Initialization of the fillerArrayKlass must come before regular
       // int-TypeArrayKlass so that the int-Array mirror points to the
       // int-TypeArrayKlass.
-      _fillerArrayKlassObj = TypeArrayKlass::create_klass(T_INT, "Ljdk/internal/vm/FillerArray;", CHECK);
+      _fillerArrayKlassObj = TypeArrayKlass::create_klass(T_INT, "[Ljdk/internal/vm/FillerElement;", CHECK);
       for (int i = T_BOOLEAN; i < T_LONG+1; i++) {
         _typeArrayKlassObjs[i] = TypeArrayKlass::create_klass((BasicType)i, CHECK);
       }

src/hotspot/share/nmt/mallocTracker.cpp

@@ -59,6 +59,23 @@ void MemoryCounter::update_peak(size_t size, size_t cnt) {
   }
 }
 
+void MallocMemorySnapshot::copy_to(MallocMemorySnapshot* s) {
+  // Use ThreadCritical to make sure that mtChunks don't get deallocated while the
+  // copy is going on, because their size is adjusted using this
+  // buffer in make_adjustment().
+  ThreadCritical tc;
+  s->_all_mallocs = _all_mallocs;
+  size_t total_size = 0;
+  size_t total_count = 0;
+  for (int index = 0; index < mt_number_of_types; index ++) {
+    s->_malloc[index] = _malloc[index];
+    total_size += s->_malloc[index].malloc_size();
+    total_count += s->_malloc[index].malloc_count();
+  }
+  // malloc counters may be updated concurrently
+  s->_all_mallocs.set_size_and_count(total_size, total_count);
+}
+
 // Total malloc'd memory used by arenas
 size_t MallocMemorySnapshot::total_arena() const {
   size_t amount = 0;

src/hotspot/share/nmt/mallocTracker.hpp

@@ -55,6 +55,12 @@ class MemoryCounter {
  public:
   MemoryCounter() : _count(0), _size(0), _peak_count(0), _peak_size(0) {}
 
+  inline void set_size_and_count(size_t size, size_t count) {
+    _size = size;
+    _count = count;
+    update_peak(size, count);
+  }
+
   inline void allocate(size_t sz) {
     size_t cnt = Atomic::add(&_count, size_t(1), memory_order_relaxed);
     if (sz > 0) {
@@ -176,16 +182,7 @@ class MallocMemorySnapshot {
   // Total malloc'd memory used by arenas
   size_t total_arena() const;
 
-  void copy_to(MallocMemorySnapshot* s) {
-     // Need to make sure that mtChunks don't get deallocated while the
-     // copy is going on, because their size is adjusted using this
-     // buffer in make_adjustment().
-     ThreadCritical tc;
-     s->_all_mallocs = _all_mallocs;
-     for (int index = 0; index < mt_number_of_types; index ++) {
-       s->_malloc[index] = _malloc[index];
-     }
-   }
+  void copy_to(MallocMemorySnapshot* s);
 
   // Make adjustment by subtracting chunks used by arenas
   // from total chunks to get total free chunk size

src/hotspot/share/nmt/virtualMemoryTracker.cpp

@@ -399,12 +399,16 @@ bool VirtualMemoryTracker::add_reserved_region(address base_addr, size_t size,
       }
 
       // Print some more details. Don't use UL here to avoid circularities.
-#ifdef ASSERT
       tty->print_cr("Error: existing region: [" INTPTR_FORMAT "-" INTPTR_FORMAT "), flag %u.\n"
                     "       new region: [" INTPTR_FORMAT "-" INTPTR_FORMAT "), flag %u.",
                     p2i(reserved_rgn->base()), p2i(reserved_rgn->end()), (unsigned)reserved_rgn->flag(),
                     p2i(base_addr), p2i(base_addr + size), (unsigned)flag);
-#endif
+      if (MemTracker::tracking_level() == NMT_detail) {
+        tty->print_cr("Existing region allocated from:");
+        reserved_rgn->call_stack()->print_on(tty);
+        tty->print_cr("New region allocated from:");
+        stack.print_on(tty);
+      }
       ShouldNotReachHere();
       return false;
     }
@@ -479,7 +483,7 @@ bool VirtualMemoryTracker::remove_released_region(ReservedMemoryRegion* rgn) {
 
   VirtualMemorySummary::record_released_memory(rgn->size(), rgn->flag());
   result =  _reserved_regions->remove(*rgn);
-  log_debug(nmt)("Removed region \'%s\' (" INTPTR_FORMAT ", " SIZE_FORMAT ") from _resvered_regions %s" ,
+  log_debug(nmt)("Removed region \'%s\' (" INTPTR_FORMAT ", " SIZE_FORMAT ") from _reserved_regions %s" ,
                 backup.flag_name(), p2i(backup.base()), backup.size(), (result ? "Succeeded" : "Failed"));
   return result;
 }

src/hotspot/share/opto/c2compiler.cpp

@@ -822,6 +822,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
   case vmIntrinsics::_notifyJvmtiVThreadMount:
   case vmIntrinsics::_notifyJvmtiVThreadUnmount:
   case vmIntrinsics::_notifyJvmtiVThreadHideFrames:
+  case vmIntrinsics::_notifyJvmtiVThreadDisableSuspend:
 #endif
     break;
 

src/hotspot/share/opto/compile.cpp

@@ -740,7 +740,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
     TracePhase tp("parse", &timers[_t_parser]);
 
     // Put top into the hash table ASAP.
-    initial_gvn()->transform_no_reclaim(top());
+    initial_gvn()->transform(top());
 
     // Set up tf(), start(), and find a CallGenerator.
     CallGenerator* cg = nullptr;
@@ -978,7 +978,7 @@ Compile::Compile( ciEnv* ci_env,
   {
     PhaseGVN gvn;
     set_initial_gvn(&gvn);    // not significant, but GraphKit guys use it pervasively
-    gvn.transform_no_reclaim(top());
+    gvn.transform(top());
 
     GraphKit kit;
     kit.gen_stub(stub_function, stub_name, is_fancy_jump, pass_tls, return_pc);

src/hotspot/share/opto/generateOptoStub.cpp

@@ -193,7 +193,7 @@ void GraphKit::gen_stub(address C_function,
     call->init_req(cnt++, returnadr());
   }
 
-  _gvn.transform_no_reclaim(call);
+  _gvn.transform(call);
 
   //-----------------------------
   // Now set up the return results

src/hotspot/share/opto/library_call.cpp

@@ -492,7 +492,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
                                                                                          "notifyJvmtiMount", false, false);
   case vmIntrinsics::_notifyJvmtiVThreadUnmount: return inline_native_notify_jvmti_funcs(CAST_FROM_FN_PTR(address, OptoRuntime::notify_jvmti_vthread_unmount()),
                                                                                          "notifyJvmtiUnmount", false, false);
-  case vmIntrinsics::_notifyJvmtiVThreadHideFrames: return inline_native_notify_jvmti_hide();
+  case vmIntrinsics::_notifyJvmtiVThreadHideFrames:     return inline_native_notify_jvmti_hide();
+  case vmIntrinsics::_notifyJvmtiVThreadDisableSuspend: return inline_native_notify_jvmti_sync();
 #endif
 
 #ifdef JFR_HAVE_INTRINSICS
@@ -2950,6 +2951,29 @@ bool LibraryCallKit::inline_native_notify_jvmti_hide() {
   return true;
 }
 
+// Always update the is_disable_suspend bit.
+bool LibraryCallKit::inline_native_notify_jvmti_sync() {
+  if (!DoJVMTIVirtualThreadTransitions) {
+    return true;
+  }
+  IdealKit ideal(this);
+
+  {
+    // unconditionally update the is_disable_suspend bit in current JavaThread
+    Node* thread = ideal.thread();
+    Node* arg = _gvn.transform(argument(1)); // argument for notification
+    Node* addr = basic_plus_adr(thread, in_bytes(JavaThread::is_disable_suspend_offset()));
+    const TypePtr *addr_type = _gvn.type(addr)->isa_ptr();
+
+    sync_kit(ideal);
+    access_store_at(nullptr, addr, addr_type, arg, _gvn.type(arg), T_BOOLEAN, IN_NATIVE | MO_UNORDERED);
+    ideal.sync_kit(this);
+  }
+  final_sync(ideal);
+
+  return true;
+}
+
 #endif // INCLUDE_JVMTI
 
 #ifdef JFR_HAVE_INTRINSICS

src/hotspot/share/opto/library_call.hpp

@@ -245,6 +245,7 @@ class LibraryCallKit : public GraphKit {
 #if INCLUDE_JVMTI
   bool inline_native_notify_jvmti_funcs(address funcAddr, const char* funcName, bool is_start, bool is_end);
   bool inline_native_notify_jvmti_hide();
+  bool inline_native_notify_jvmti_sync();
 #endif
 
 #ifdef JFR_HAVE_INTRINSICS

src/hotspot/share/opto/loopopts.cpp

@@ -786,25 +786,18 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) {
   // Avoid duplicated float compare.
   if (phis > 1 && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) return nullptr;
 
-  float infrequent_prob = PROB_UNLIKELY_MAG(3);
-  // Ignore cost and blocks frequency if CMOVE can be moved outside the loop.
-  if (used_inside_loop) {
-    if (cost >= ConditionalMoveLimit) return nullptr; // Too much goo
-
-    // BlockLayoutByFrequency optimization moves infrequent branch
-    // from hot path. No point in CMOV'ing in such case (110 is used
-    // instead of 100 to take into account not exactness of float value).
-    if (BlockLayoutByFrequency) {
-      infrequent_prob = MAX2(infrequent_prob, (float)BlockLayoutMinDiamondPercentage/110.0f);
-    }
+  // Ignore cost if CMOVE can be moved outside the loop.
+  if (used_inside_loop && cost >= ConditionalMoveLimit) {
+    return nullptr;
   }
   // Check for highly predictable branch.  No point in CMOV'ing if
   // we are going to predict accurately all the time.
+  constexpr float infrequent_prob = PROB_UNLIKELY_MAG(2);
   if (C->use_cmove() && (cmp_op == Op_CmpF || cmp_op == Op_CmpD)) {
     //keep going
-  } else if (iff->_prob < infrequent_prob ||
-      iff->_prob > (1.0f - infrequent_prob))
+  } else if (iff->_prob < infrequent_prob || iff->_prob > (1.0f - infrequent_prob)) {
     return nullptr;
+  }
 
   // --------------
   // Now replace all Phis with CMOV's