From 30a47ed7b76c2a984a4c768334e0cdf5e0ceefb4 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Fri, 27 Sep 2024 15:15:44 -0400 Subject: [PATCH 1/4] Use baseObj for Unsafe.CAS CardMarking for OffHeap on P When evaluating Unsafe.CAS while running with Balanced GC and OffHeap enabled, if the object child is the dataAddrPointer load, pass the baseObj to VMCardCheckEvaluator for correct card marking. The dstReg and temp2Reg in VMCardCheckEvaluator can share a reg. Signed-off-by: Abdulrahman Alattas --- .../compiler/p/codegen/J9TreeEvaluator.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp index 5497a3edf1f..268716ce812 100644 --- a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp @@ -8217,6 +8217,12 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera thirdChild = node->getChild(2); fourthChild = node->getChild(3); fifthChild = node->getChild(4); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + bool isObjOffHeapDataAddr = (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && secondChild->isDataAddrPointer()); + TR::Register *baseObjReg = NULL; + if (isObjOffHeapDataAddr) + TR::TreeEvaluator::stopUsingCopyReg(secondChild->getFirstChild(), baseObjReg, cg); +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ objReg = cg->evaluate(secondChild); // VM helper chops off the value in 32bit, and we don't want the whole long value either @@ -8473,7 +8479,13 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera } else if (!doWrtBar && doCrdMrk) { - TR::Register *temp1Reg = cg->allocateRegister(), *temp2Reg = cg->allocateRegister(), *temp3Reg; + TR::Register *temp1Reg = cg->allocateRegister(), *temp2Reg, *temp3Reg; +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + temp2Reg = baseObjReg; // Use baseObjReg as temp2Reg + else +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + temp2Reg = cg->allocateRegister(); TR::addDependency(conditions, objReg, TR::RealRegister::NoReg, TR_GPR, cg); conditions->getPostConditions()->getRegisterDependency(0)->setExcludeGPR0(); TR::addDependency(conditions, temp1Reg, TR::RealRegister::NoReg, TR_GPR, cg); @@ -8494,7 +8506,13 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera TR::addDependency(conditions, temp3Reg, TR::RealRegister::NoReg, TR_GPR, cg); } - VMCardCheckEvaluator(node, objReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + // Given that baseObjReg is trash-able it can be used as the objReg and temp2Reg + VMCardCheckEvaluator(node, baseObjReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg); + else +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + VMCardCheckEvaluator(node, objReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg); cg->stopUsingRegister(temp1Reg); cg->stopUsingRegister(temp2Reg); From 5b929a96a18850ba97e79b37faba753ffa2ccb85 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Mon, 30 Sep 2024 12:16:58 -0400 Subject: [PATCH 2/4] Use baseObj in CardMarking for OffHeap on X When running with Balanced GC and OffHeap enabled, if the destOwningObject is a dataAddrPointer load, use the baseObj as the owningObjectReg. Signed-off-by: Abdulrahman Alattas --- runtime/compiler/x/codegen/J9TreeEvaluator.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp index d693313f514..e5a25749d35 100644 --- a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp @@ -10599,7 +10599,13 @@ void J9::X86::TreeEvaluator::VMwrtbarWithoutStoreEvaluator( TR::Register *owningObjectReg; TR::Register *tempReg = NULL; - owningObjectReg = cg->evaluate(destOwningObject); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + bool stopUsingCopyBaseReg; + if (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && destOwningObject->isDataAddrPointer()) + owningObjectReg = cg->evaluate(destOwningObject->getFirstChild()); + else +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + owningObjectReg = cg->evaluate(destOwningObject); if (doInternalControlFlow) { From 15acde95e64ceb464cffb3b1efdcf474ea26fa46 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Mon, 30 Sep 2024 13:07:30 -0400 Subject: [PATCH 3/4] Use baseObj for Unsafe.CAS CardMarking for OffHeap on AArch64 When evaluating Unsafe.CAS while running with Balanced GC and OffHeap enabled, if the object child is the dataAddrPointer load, pass the baseObj to VMCardCheckEvaluator for correct card marking. The dstReg and temp2Reg in VMCardCheckEvaluator can share a reg, adding the argument clobberDstReg to indicate if dstReg can be used as the temp2Reg. Not allocating temp2Reg and adding a dep for baseObjReg uses the same number of registers. Signed-off-by: Abdulrahman Alattas --- .../aarch64/codegen/J9TreeEvaluator.cpp | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp index 6f580e306ad..236790e6fda 100644 --- a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp @@ -987,7 +987,8 @@ VMCardCheckEvaluator( TR::Register *dstReg, TR_ARM64ScratchRegisterManager *srm, TR::LabelSymbol *doneLabel, - TR::CodeGenerator *cg) + TR::CodeGenerator *cg, + bool clobberDstReg=false) { TR::Compilation *comp = cg->comp(); @@ -1014,7 +1015,11 @@ VMCardCheckEvaluator( cg->generateDebugCounter(TR::DebugCounter::debugCounterName(comp, "wrtbarEvaluator:020VMCardCheckEvaluator:01markThreadActiveCheckDone"), *srm); } - TR::Register *temp2Reg = srm->findOrCreateScratchRegister(); + TR::Register *temp2Reg; + if (clobberDstReg) + temp2Reg = dstReg; + else + temp2Reg = srm->findOrCreateScratchRegister(); /* * Generating code checking whether an object is in heap * @@ -5037,7 +5042,13 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera fourthChild = node->getChild(3); fifthChild = node->getChild(4); - objReg = cg->evaluate(secondChild); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + bool isObjOffHeapDataAddr = (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && secondChild->isDataAddrPointer()); + TR::Register *baseObjReg = NULL; + if (isObjOffHeapDataAddr) + TR::TreeEvaluator::stopUsingCopyReg(secondChild->getFirstChild(), baseObjReg, cg); +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + objReg = cg->evaluate(secondChild); if (thirdChild->getOpCode().isLoadConst() && thirdChild->getRegister() == NULL) { @@ -5200,11 +5211,20 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera generateCompareBranchInstruction(cg, TR::InstOpCode::cbzx, node, wrtBarSrcReg, doneLabel); cg->generateDebugCounter(TR::DebugCounter::debugCounterName(comp, "wrtbarEvaluator:000srcNullChk:NonNull"), *srm); } - VMCardCheckEvaluator(node, objReg, srm, doneLabel, cg); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + VMCardCheckEvaluator(node, baseObjReg, srm, doneLabel, cg, true); + else +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + VMCardCheckEvaluator(node, objReg, srm, doneLabel, cg); } TR_ARM64ScratchRegisterDependencyConditions scratchDeps; scratchDeps.addDependency(cg, objReg, doWrtBar ? TR::RealRegister::x0 : TR::RealRegister::NoReg); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + scratchDeps.addDependency(cg, baseObjReg, TR::RealRegister::NoReg); +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ scratchDeps.addDependency(cg, wrtBarSrcReg, doWrtBar ? TR::RealRegister::x1 : TR::RealRegister::NoReg); if (offsetInReg) { From 19953fd56d8f03963cc273c2386ee78a94f91665 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Mon, 30 Sep 2024 14:06:38 -0400 Subject: [PATCH 4/4] Use baseObj for Unsafe.CAS CardMarking for OffHeap on Z When evaluating Unsafe.CAS while running with Balanced GC and OffHeap enabled, if the object child is the dataAddrPointer load, pass the baseObj to VMCardCheckEvaluator for correct card marking. Using the VMCardCheckEvaluator ability to clobberDstReg, we use baseObjReg instead of the temp epReg. Signed-off-by: Abdulrahman Alattas --- .../compiler/z/codegen/J9TreeEvaluator.cpp | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp index 482c8c0ed4e..279164c6b82 100644 --- a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp @@ -12013,6 +12013,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * // Eval old and new vals // +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + bool isObjOffHeapDataAddr = (TR::Compiler->om.writeBarrierType() == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer()); + TR::Register *baseObjReg = NULL; + if (isObjOffHeapDataAddr) + baseObjReg = cg->gprClobberEvaluate(objNode->getFirstChild()); +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ objReg = cg->evaluate(objNode); oldVReg = cg->gprClobberEvaluate(oldVNode); // CS oldReg, newReg, OFF(objReg) newVReg = cg->evaluate(newVNode); // oldReg is clobbered @@ -12110,7 +12116,11 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * if (isObj && (doWrtBar || doCrdMrk)) { TR::LabelSymbol *doneLabelWrtBar = generateLabelSymbol(cg); - TR::Register *epReg = cg->allocateRegister(); + TR::Register *epReg; +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (!isObjOffHeapDataAddr) +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + epReg = cg->allocateRegister(); TR::Register *raReg = cg->allocateRegister(); TR::RegisterDependencyConditions* condWrtBar = new (cg->trHeapMemory()) TR::RegisterDependencyConditions(0, 5, cg); condWrtBar->addPostCondition(objReg, TR::RealRegister::GPR1); @@ -12118,7 +12128,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * condWrtBar->addPostCondition(newVReg, TR::RealRegister::AssignAny); //defect 92001 if (compressedValueRegister != objReg) // add this because I got conflicting dependencies on GPR1 and GPR2! condWrtBar->addPostCondition(compressedValueRegister, TR::RealRegister::GPR2); //defect 92001 - condWrtBar->addPostCondition(epReg, cg->getEntryPointRegister()); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + condWrtBar->addPostCondition(baseObjReg, cg->getEntryPointRegister()); + else +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + condWrtBar->addPostCondition(epReg, cg->getEntryPointRegister()); condWrtBar->addPostCondition(raReg, cg->getReturnAddressRegister()); // Cardmarking is not inlined for gencon. Consider doing so when perf issue arises. if (doWrtBar) @@ -12135,7 +12150,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * else if (doCrdMrk) { - VMCardCheckEvaluator(node, objReg, epReg, condWrtBar, cg, false, doneLabelWrtBar, false); +#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + VMCardCheckEvaluator(node, baseObjReg, NULL, condWrtBar, cg, true, doneLabelWrtBar, false); + else +#endif /* defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION) */ + VMCardCheckEvaluator(node, objReg, epReg, condWrtBar, cg, false, doneLabelWrtBar, false); // true #1 -> copy of objReg just happened, it's safe to clobber tempReg // false #2 -> Don't do compile time check for heap obj }