[Wisp] Port 8.7.12_GA changes to JDK17.

src/hotspot/cpu/x86/c1_Runtime1_x86.cpp

@@ -92,7 +92,7 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre
     call_offset = offset();
   }
 
-  if (EnableSteal) {
+  if (EnableCoroutine) {
     // only if the entry_point is equal to `Runtime1::monitorenter()`, we will do this amendment.
     if (entry == monitorenter_address_C1) {
       WISP_V2v_UPDATE;
@@ -1425,7 +1425,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         oop_maps->add_gc_map(call_offset, map);
         restore_live_registers(sasm, save_fpu_registers);
 
-        if (EnableSteal) {
+        if (EnableCoroutine) {
           // r15 has been forcely restored in restore_live_registers so we need fix it.
           WISP_COMPILER_RESTORE_FORCE_UPDATE;
         }

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -1741,7 +1741,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
   // do the call, remove parameters
   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
 
-  if (EnableSteal) {
+  if (EnableCoroutine) {
     // only if the entry_point is equal to `Interpreter::monitorenter()`, we will do this amendment.
     if (entry_point == monitorenter_address_interp) {
       WISP_V2v_UPDATE;

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -1539,7 +1539,7 @@ void create_switchTo_contents(MacroAssembler *masm, int start,
 
 void generate_thread_fix(MacroAssembler *masm, Method *method) {
   // we can have a check here at the codegen time, so no cost in runtime.
-  if (EnableSteal) {
+  if (EnableCoroutine) {
     if (WispStealCandidate(method->method_holder()->name(), method->name(), method->signature()).is_steal_candidate()) {
       // as X86 calling conventions, the thread register r15 is one of the callee saved registers.
       // see: https://www.cs.cmu.edu/~aplatzer/course/Compilers11/calling_conventions.pdf, callee saved registers
@@ -2399,7 +2399,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
     restore_args(masm, total_c_args, c_arg, out_regs);
 
-    if (EnableSteal) {
+    if (EnableCoroutine) {
       // the r15 has been restored in restore_args so we need fix it.
       WISP_COMPILER_RESTORE_FORCE_UPDATE;
     }
@@ -4040,7 +4040,7 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma
   Register target_coroutine = rdx;
   // check that we're dealing with sane objects...
   DEBUG_ONLY(stop_if_null(masm, target_coroutine, "null new_coroutine"));
-  __ movptr(target_coroutine, Address(target_coroutine, java_dyn_CoroutineBase::get_data_offset()));
+  __ movptr(target_coroutine, Address(target_coroutine, java_dyn_CoroutineBase::get_native_coroutine_offset()));
   DEBUG_ONLY(stop_if_null(masm, target_coroutine, "new_coroutine without data"));
 
 /*
@@ -4062,7 +4062,7 @@ void create_switchTo_contents(MacroAssembler *masm, int start, OopMapSet* oop_ma
 
     // check that we're dealing with sane objects...
     DEBUG_ONLY(stop_if_null(masm, old_coroutine_obj, "null old_coroutine"));
-    __ movptr(old_coroutine, Address(old_coroutine_obj, java_dyn_CoroutineBase::get_data_offset()));
+    __ movptr(old_coroutine, Address(old_coroutine_obj, java_dyn_CoroutineBase::get_native_coroutine_offset()));
     DEBUG_ONLY(stop_if_null(masm, old_coroutine, "old_coroutine without data"));
 
 #if defined(_WINDOWS)

src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

@@ -364,7 +364,7 @@ class StubGenerator: public StubCodeGenerator {
     // pop parameters
     __ lea(rsp, rsp_after_call);
 
-    if (EnableSteal) {
+    if (EnableCoroutine) {
       Label L;    // the steal thread patch
       __ cmpptr(r15_thread, thread);
       __ jcc(Assembler::equal, L);
@@ -462,7 +462,7 @@ class StubGenerator: public StubCodeGenerator {
     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
     const Address thread        (rbp, thread_off         * wordSize);
 
-    if (EnableSteal) {
+    if (EnableCoroutine) {
       // to pass the assertion fail
       WISP_j2v_UPDATE;
     }

src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp

@@ -1004,7 +1004,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   // 64: result potentially in rax or xmm0
 
   // when we get back from native c++ code, our coroutine may be stolen by another thread.
-  if (EnableSteal) {
+  if (EnableCoroutine) {
     // it is a little hard to present method signature check here: because interpreter
     // will directly jump into this entry, which is in runtime.
     WISP_X86_CONVENTION_V2j_UPDATE;

src/hotspot/share/classfile/javaClasses.cpp

@@ -4735,21 +4735,21 @@ void java_nio_Buffer::compute_offsets() {
 
 /* stack manipulation */
 
-int java_dyn_CoroutineBase::_data_offset = 0;
+int java_dyn_CoroutineBase::_native_coroutine_offset = 0;
 
 void java_dyn_CoroutineBase::compute_offsets() {
   InstanceKlass* ik = vmClasses::java_dyn_CoroutineBase_klass();
   if (ik != NULL) {
-    compute_offset(_data_offset, ik, vmSymbols::data_name(), vmSymbols::long_signature());
+    compute_offset(_native_coroutine_offset, ik, vmSymbols::nativeCoroutine_name(), vmSymbols::long_signature());
   }
 }
 
-jlong java_dyn_CoroutineBase::data(oop obj) {
-  return obj->long_field(_data_offset);
+jlong java_dyn_CoroutineBase::native_coroutine(oop obj) {
+  return obj->long_field(_native_coroutine_offset);
 }
 
-void java_dyn_CoroutineBase::set_data(oop obj, jlong value) {
-  obj->long_field_put(_data_offset, value);
+void java_dyn_CoroutineBase::set_native_coroutine(oop obj, jlong value) {
+  obj->long_field_put(_native_coroutine_offset, value);
 }
 
 int com_alibaba_wisp_engine_WispEngine::_isInCritical_offset = 0;

src/hotspot/share/classfile/javaClasses.hpp

@@ -1794,16 +1794,16 @@ class InjectedField {
 class java_dyn_CoroutineBase: AllStatic {
 private:
   // Note that to reduce dependencies on the JDK we compute these offsets at run-time.
-  static int _data_offset;
+  static int _native_coroutine_offset;
 
   static void compute_offsets();
 
 public:
   // Accessors
-  static jlong data(oop obj);
-  static void set_data(oop obj, jlong value);
+  static jlong native_coroutine(oop obj);
+  static void set_native_coroutine(oop obj, jlong value);
 
-  static int get_data_offset()    { return _data_offset; }
+  static int get_native_coroutine_offset()    { return _native_coroutine_offset; }
 
   // Debugging
   friend class JavaClasses;

src/hotspot/share/classfile/vmSymbols.hpp

@@ -704,7 +704,7 @@
   template(interruptById_name,                         "interruptById")                                           \
   template(interrupted_name,                           "interrupted")                                             \
   template(yield_name,                                 "yield")                                                   \
-  template(data_name,                                  "data")                                                    \
+  template(nativeCoroutine_name,                       "nativeCoroutine")                                         \
   template(stack_name,                                 "stack")                                                   \
   template(current_name,                               "current")                                                 \
   template(java_dyn_CoroutineBase_signature,           "Ljava/dyn/CoroutineBase;")                                \

src/hotspot/share/opto/generateOptoStub.cpp

@@ -105,7 +105,7 @@ void GraphKit::gen_stub(address C_function,
   store_to_memory(control(), adr_sp, last_sp, T_ADDRESS, NoAlias, MemNode::unordered);
 
   Node *coro_task = NULL;
-  if (EnableSteal && C_function == (address)SharedRuntime::complete_monitor_locking_C) {
+  if (EnableCoroutine && C_function == (address)SharedRuntime::complete_monitor_locking_C) {
     Node *coro_adr = basic_plus_adr(top(), thread, in_bytes(JavaThread::current_coroutine_offset()));
     coro_task = make_load(NULL, coro_adr, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
   }
@@ -228,7 +228,7 @@ void GraphKit::gen_stub(address C_function,
   //-----------------------------
 
   Node *refetch = NULL;
-  if (EnableSteal && C_function == (address)SharedRuntime::complete_monitor_locking_C) {
+  if (EnableCoroutine && C_function == (address)SharedRuntime::complete_monitor_locking_C) {
     // we add ThreadRefetchNode so we can refetch r15 when monitorenter() ends, and
     // after refetching thread, every node based on ThreadLocalNode should be changed to use ThreadRefetchNode as thread.
     refetch = _gvn.transform(new ThreadRefetchNode(control(), coro_task));

src/hotspot/share/opto/machnode.cpp

@@ -360,7 +360,7 @@ const class TypePtr *MachNode::adr_type() const {
   // tlsLoad is special handled in adlc to return bottom type,
   // we do not want to hack for tlsRefetch because it is only
   // used for complete_monitor_locking_C stub, just check here
-  if (EnableSteal && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadRefetch) {
+  if (EnableCoroutine && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_ThreadRefetch) {
     return TypeRawPtr::BOTTOM;
   }
 

src/hotspot/share/prims/unsafe.cpp

@@ -872,8 +872,9 @@ UNSAFE_ENTRY(jint, Unsafe_GetLoadAverage0(JNIEnv *env, jobject unsafe, jdoubleAr
   return ret;
 } UNSAFE_END
 
-JVM_ENTRY(jlong, CoroutineSupport_getThreadCoroutine(JNIEnv* env, jclass klass))
-  DEBUG_CORO_PRINT("CoroutineSupport_getThreadCoroutine\n");
+JVM_ENTRY(jlong, CoroutineSupport_getNativeThreadCoroutine(JNIEnv* env, jclass klass))
+  DEBUG_CORO_PRINT("CoroutineSupport_getNativeThreadCoroutine\n");
+  assert(EnableCoroutine, "pre-condition");
 
   Coroutine* list = thread->coroutine_list();
   assert(list != NULL, "thread isn't initialized for coroutines");
@@ -891,10 +892,10 @@ JVM_ENTRY(void, CoroutineSupport_switchToAndTerminate(JNIEnv* env, jclass klass,
   assert(target_coroutine == NULL, "expecting NULL");
 
   oop old_oop = JNIHandles::resolve(old_coroutine);
-  Coroutine* coro = (Coroutine*)java_dyn_CoroutineBase::data(old_oop);
+  Coroutine* coro = (Coroutine*)java_dyn_CoroutineBase::native_coroutine(old_oop);
   assert(coro != NULL, "NULL old coroutine in switchToAndTerminate");
 
-  java_dyn_CoroutineBase::set_data(old_oop, 0);
+  java_dyn_CoroutineBase::set_native_coroutine(old_oop, 0);
 
   CoroutineStack* stack = coro->stack();
   stack->remove_from_list(thread->coroutine_stack_list());
@@ -972,6 +973,29 @@ JVM_ENTRY(void, CoroutineSupport_setWispBooted(JNIEnv* env, jclass klass))
   WispThread::set_wisp_booted(thread);
 JVM_END
 
+JVM_ENTRY (jobject, CoroutineSupport_getNextCoroutine(JNIEnv* env, jclass klass, jlong coroPtr))
+  assert(EnableCoroutine, "pre-condition");
+  Coroutine* coro = (Coroutine*)coroPtr;
+  assert (coro->next()->coroutine() != NULL, "coroutine oop can't be null");
+  return JNIHandles::make_local(THREAD, coro->next()->coroutine());
+JVM_END
+
+JVM_ENTRY (void, CoroutineSupport_moveCoroutine(JNIEnv* env, jclass klass, jlong coroPtr, jlong targetPtr))
+  assert(EnableCoroutine, "pre-condition");
+  Coroutine* coro = (Coroutine*)coroPtr;
+  Coroutine* target = (Coroutine*)targetPtr;
+  Coroutine::move(coro, target);
+JVM_END
+
+JVM_ENTRY (void, CoroutineSupport_markThreadCoroutine(JNIEnv* env, jclass klass, jlong coroPtr, jobject coroObj))
+  assert(EnableCoroutine, "pre-condition");
+  Coroutine* coro = (Coroutine*)coroPtr;
+  oop x = JNIHandles::resolve_non_null(coroObj);
+  coro->set_coroutine(x);
+  assert (coro->is_thread_coroutine() == true, "should be called by thread coro");
+  assert (coro->coroutine() != NULL, "coroutine oop can't be null");
+JVM_END
+
 JVM_ENTRY(jboolean, CoroutineSupport_stealCoroutine(JNIEnv* env, jclass klass, jlong coroPtr))
   // We've already locked the target's thread
   // and source's thread. target_thread->coroutine_list()s have
@@ -980,7 +1004,7 @@ JVM_ENTRY(jboolean, CoroutineSupport_stealCoroutine(JNIEnv* env, jclass klass, j
   // The lock will also block coroutine switch operation,
   // so we must finish the steal operation as soon as possible.
   Coroutine* coro = (Coroutine*) coroPtr;
-  if (!EnableSteal || coro == NULL || coro->enable_steal_count() != coro->java_call_counter()) {
+  if (coro == NULL || coro->enable_steal_count() != coro->java_call_counter()) {
       return false;       // an Exception throws and the coroutine being stealed is exited
   }
   assert(coro->thread() != thread, "steal from self");
@@ -1085,18 +1109,22 @@ static JNINativeMethod jdk_internal_misc_Unsafe_methods[] = {
 };
 
 #define COBA "Ljava/dyn/CoroutineBase;"
+#define COR "Ljava/dyn/Coroutine;"
 
 JNINativeMethod coroutine_support_methods[] = {
     {CC"switchTo",                CC "(" COBA COBA ")V", FN_PTR(CoroutineSupport_switchTo)},
     {CC"switchToAndTerminate",    CC "(" COBA COBA ")V", FN_PTR(CoroutineSupport_switchToAndTerminate)},
     {CC"switchToAndExit",         CC "(" COBA COBA ")V", FN_PTR(CoroutineSupport_switchToAndExit)},
-    {CC"getThreadCoroutine",      CC "()J",              FN_PTR(CoroutineSupport_getThreadCoroutine)},
+    {CC"getNativeThreadCoroutine",CC "()J",              FN_PTR(CoroutineSupport_getNativeThreadCoroutine)},
     {CC"createCoroutine",         CC "(" COBA "J)J",     FN_PTR(CoroutineSupport_createCoroutine)},
     {CC"testDisposableAndTryReleaseStack", 
                                   CC "(J)Z",             FN_PTR(CoroutineSupport_testDisposableAndTryReleaseStack)},
     {CC"cleanupCoroutine",        CC "()" COBA,          FN_PTR(CoroutineSupport_cleanupCoroutine)},
     {CC"setWispBooted",           CC "()V",              FN_PTR(CoroutineSupport_setWispBooted)},
     {CC"stealCoroutine",          CC "(J)Z",             FN_PTR(CoroutineSupport_stealCoroutine)},
+    {CC"getNextCoroutine",        CC "(J)" COR,          FN_PTR(CoroutineSupport_getNextCoroutine)},
+    {CC"moveCoroutine",           CC "(JJ)V",            FN_PTR(CoroutineSupport_moveCoroutine)},
+    {CC"markThreadCoroutine",     CC "(J" COBA ")V",     FN_PTR(CoroutineSupport_markThreadCoroutine)},
 };
 
 #define COMPILE_CORO_METHODS_BEFORE (3)

src/hotspot/share/runtime/arguments.cpp

@@ -3987,9 +3987,6 @@ jint Arguments::parse(const JavaVMInitArgs* initial_cmd_args) {
       UseBiasedLocking = false;
     }
   }
-  if (!EnableCoroutine) {
-    EnableSteal = false;
-  }
 
   // Set object alignment values.
   set_object_alignment();

src/hotspot/share/runtime/coroutine.cpp

@@ -133,6 +133,7 @@ Coroutine* Coroutine::create_thread_coroutine(JavaThread* thread, CoroutineStack
   coro->_wisp_engine  = NULL;
   coro->_wisp_task    = NULL;
   coro->_wisp_task_id = WISP_ID_NOT_SET;
+  coro->_coroutine    = NULL;
   coro->_enable_steal_count = 1;
   coro->_wisp_post_steal_resource_area = NULL;
   coro->_is_yielding  = false;
@@ -184,6 +185,7 @@ Coroutine* Coroutine::create_coroutine(JavaThread* thread, CoroutineStack* stack
   coro->_wisp_engine  = NULL;
   coro->_wisp_task    = NULL;
   coro->_wisp_task_id = WISP_ID_NOT_SET;
+  coro->_coroutine = coroutineObj;
   // if coro->_enable_steal_count == coro->_java_call_counter is true, we can do work steal.
   // when a coroutine starts, the `_java_call_counter` is 0,
   // then it will call java method `Coroutine.startInternal()` and then `_java_call_counter` is 1.
@@ -254,6 +256,9 @@ void Coroutine::oops_do(OopClosure* f, CodeBlobClosure* cf) {
     f->do_oop((oop*) &_wisp_engine);
     f->do_oop((oop*) &_wisp_task);
   }
+  if (_coroutine != NULL) {
+    f->do_oop((oop*) &_coroutine);
+  }
 }
 
 class nmethods_do_Closure: public FrameClosure {
@@ -793,7 +798,7 @@ void Coroutine::after_safepoint(JavaThread* thread) {
 
 EnableStealMark::EnableStealMark(Thread* thread) {
   assert(JavaThread::current() == thread, "current thread check");
-  if (EnableSteal) {
+  if (EnableCoroutine) {
     if (thread->is_Java_thread()) {
       _thread = (JavaThread*) thread;
       _coroutine = _thread->current_coroutine();
@@ -805,7 +810,7 @@ EnableStealMark::EnableStealMark(Thread* thread) {
 }
 
 EnableStealMark::~EnableStealMark() {
-  if (EnableSteal && _coroutine != NULL) {
+  if (EnableCoroutine && _coroutine != NULL) {
     assert(_coroutine->thread() == JavaThread::current(), "must be");
     bool eq = _enable_steal_count == _coroutine->dec_enable_steal_count();
     assert(eq, "enable_steal_count not balanced");
@@ -977,7 +982,7 @@ WispPostStealHandleUpdateMark::WispPostStealHandleUpdateMark(JavaThread *&th)
 bool WispPostStealHandleUpdateMark::check(JavaThread *current, bool sp) {
   assert(current == JavaThread::current(), "must be");
   Coroutine *coroutine = current->current_coroutine();
-  if (!EnableSteal ||
+  if (!EnableCoroutine ||
       coroutine == current->coroutine_list() ||
       // if we won't steal it, then no need to allocate handles.
       coroutine->enable_steal_count() != (sp ? current->java_call_counter()+1 : current->java_call_counter())) {

src/hotspot/share/runtime/coroutine.hpp

@@ -71,6 +71,7 @@ class DoublyLinkedList {
 
   void remove_from_list(pointer& list);
   void insert_into_list(pointer& list);
+  static void move(pointer& coro, pointer& target);
 
   T* last() const   { return _last; }
   T* next() const   { return _next; }
@@ -109,6 +110,7 @@ class Coroutine: public CHeapObj<mtThread>, public DoublyLinkedList<Coroutine> {
   int             _wisp_task_id;
   oop             _wisp_engine;
   oop             _wisp_task;
+  oop             _coroutine;
   WispThread*     _wisp_thread;
 
   ResourceArea*   _resource_area;
@@ -165,6 +167,9 @@ class Coroutine: public CHeapObj<mtThread>, public DoublyLinkedList<Coroutine> {
   oop wisp_task() const             { return _wisp_task; }
   void set_wisp_task(oop x)         { _wisp_task = x;    }
 
+  oop coroutine() const             { return _coroutine; }
+  void set_coroutine(oop x)         { _coroutine = x;    }
+
   WispThread* wisp_thread() const   { return _wisp_thread; }
 
   ResourceArea* resource_area() const     { return _resource_area; }
@@ -312,6 +317,22 @@ template<class T> void DoublyLinkedList<T>::insert_into_list(pointer& list) {
   }
 }
 
+template<class T> void DoublyLinkedList<T>::move(DoublyLinkedList<T>::pointer &coro,
+                                                 DoublyLinkedList<T>::pointer &target) {
+  assert(coro != NULL, "coroutine can't be null");
+  assert(target != NULL, "target can't be null");
+
+  // remove current coro from the ring
+  coro->_last->_next = coro->_next;
+  coro->_next->_last = coro->_last;
+
+  // insert at new position
+  coro->_next = target->_next;
+  coro->_last = target;
+  coro->_next->_last = coro;
+  target->_next = coro;
+}
+
 class ObjectWaiter;
 
 class WispThread: public JavaThread {

src/hotspot/share/runtime/globals.hpp

@@ -2101,9 +2101,6 @@ const intx ObjectAlignmentInBytes = 8;
   product(bool, UseWispMonitor, false,                                      \
           "yields to next coroutine when ObjectMonitor is contended")       \
                                                                             \
-  product(bool, EnableSteal, true,                                          \
-          "Enable coroutine steal")                                         \
-                                                                            \
   product(bool, UseWisp2, false,                                            \
           "Enable Wisp2")                                                   \
                                                                             \

src/hotspot/share/runtime/synchronizer.cpp

@@ -683,7 +683,7 @@ ObjectLocker::ObjectLocker(Handle obj, JavaThread* thread) {
 
   if (_obj() != NULL) {
     ObjectSynchronizer::enter(_obj, &_lock, _thread);
-    assert(!EnableSteal || thread == Thread::current(), "must not be stealed");
+    assert(!EnableCoroutine || thread == Thread::current(), "must not be stealed");
   }
 }