8305403: Shenandoah evacuation workers may deadlock

Reviewed-by: rkennke, ysr

src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp

@@ -40,7 +40,6 @@ void ShenandoahIUMode::initialize_flags() const {
   FLAG_SET_DEFAULT(ClassUnloadingWithConcurrentMark, false);
 
   if (ClassUnloading) {
-    FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true);
     FLAG_SET_DEFAULT(VerifyBeforeExit, false);
   }
 

src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp

@@ -35,7 +35,6 @@
 
 void ShenandoahSATBMode::initialize_flags() const {
   if (ClassUnloading) {
-    FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true);
     FLAG_SET_DEFAULT(VerifyBeforeExit, false);
   }
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1740,20 +1740,8 @@ size_t ShenandoahHeap::tlab_used(Thread* thread) const {
 }
 
 bool ShenandoahHeap::try_cancel_gc() {
-  while (true) {
-    jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
-    if (prev == CANCELLABLE) return true;
-    else if (prev == CANCELLED) return false;
-    assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
-    assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
-    Thread* thread = Thread::current();
-    if (thread->is_Java_thread()) {
-      // We need to provide a safepoint here, otherwise we might
-      // spin forever if a SP is pending.
-      ThreadBlockInVM sp(JavaThread::cast(thread));
-      SpinPause();
-    }
-  }
+  jbyte prev = _cancelled_gc.cmpxchg(CANCELLED, CANCELLABLE);
+  return prev == CANCELLABLE;
 }
 
 void ShenandoahHeap::cancel_gc(GCCause::Cause cause) {

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -327,12 +327,7 @@ private:
 
     // GC has been cancelled. Worker threads can not suspend for
     // safepoint but must finish their work as soon as possible.
-    CANCELLED,
-
-    // GC has not been cancelled and must not be cancelled. At least
-    // one worker thread checks for pending safepoint and may suspend
-    // if a safepoint is pending.
-    NOT_CANCELLED
+    CANCELLED
   };
 
   ShenandoahSharedEnumFlag<CancelState> _cancelled_gc;

src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp

@@ -244,25 +244,12 @@ inline bool ShenandoahHeap::cancelled_gc() const {
 }
 
 inline bool ShenandoahHeap::check_cancelled_gc_and_yield(bool sts_active) {
-  if (! (sts_active && ShenandoahSuspendibleWorkers)) {
-    return cancelled_gc();
-  }
-
-  jbyte prev = _cancelled_gc.cmpxchg(NOT_CANCELLED, CANCELLABLE);
-  if (prev == CANCELLABLE || prev == NOT_CANCELLED) {
+  if (sts_active && ShenandoahSuspendibleWorkers && !cancelled_gc()) {
     if (SuspendibleThreadSet::should_yield()) {
       SuspendibleThreadSet::yield();
     }
-
-    // Back to CANCELLABLE. The thread that poked NOT_CANCELLED first gets
-    // to restore to CANCELLABLE.
-    if (prev == CANCELLABLE) {
-      _cancelled_gc.set(CANCELLABLE);
-    }
-    return false;
-  } else {
-    return true;
   }
+  return cancelled_gc();
 }
 
 inline void ShenandoahHeap::clear_cancelled_gc() {

src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp

@@ -334,7 +334,7 @@
           "How many times to maximum attempt to flush SATB buffers at the " \
           "end of concurrent marking.")                                     \
                                                                             \
-  product(bool, ShenandoahSuspendibleWorkers, false, EXPERIMENTAL,          \
+  product(bool, ShenandoahSuspendibleWorkers, true, EXPERIMENTAL,           \
           "Suspend concurrent GC worker threads at safepoints")             \
                                                                             \
   product(bool, ShenandoahSATBBarrier, true, DIAGNOSTIC,                    \