From 8467772217c27d650530bde7617e39478e620411 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Aug 2024 17:11:07 +0000 Subject: [PATCH] codegen: take gc roots (and alloca alignment) more seriously Due to limitations in the LLVM implementation, we are forced to emit fairly bad code here. But we need to make sure it is still correct with respect to GC rooting. The PR 50833c84d454ef989797e035294ba27b3cca79b7 also changed the meaning of haspadding without changing all of the existing users to use the new equivalent flag (haspadding || !isbitsegal), incurring additional breakage here as well and needing more tests for that. Fixes #54720 (cherry picked from commit 8bfef8f1b60155020abc9bdab8aef9788eb458ba) --- src/cgutils.cpp | 97 +++++++++++++++++++++++++++++++------------------ test/atomics.jl | 6 +++ 2 files changed, 67 insertions(+), 36 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 8f4571b85d10e..d049327c2bf36 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2092,12 +2092,15 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, FailOrder = AtomicOrdering::Monotonic; unsigned nb = isboxed ? sizeof(void*) : jl_datatype_size(jltype); AllocaInst *intcast = nullptr; + Type *intcast_eltyp = nullptr; + bool tracked_pointers = isboxed || CountTrackedPointers(elty).count > 0; if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) { + intcast_eltyp = elty; + elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb); if (!issetfield) { intcast = emit_static_alloca(ctx, elty); setName(ctx.emission_context, intcast, "atomic_store_box"); } - elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb); } Type *realelty = elty; if (Order != AtomicOrdering::NotAtomic && isa(elty)) { @@ -2106,14 +2109,20 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb2); } Value *r = nullptr; - if (issetfield || isswapfield || isreplacefield || issetfieldonce) { - if (isboxed) + if (issetfield || isswapfield || isreplacefield || issetfieldonce) { // e.g. !ismodifyfield + assert(isboxed || rhs.typ == jltype); + if (isboxed) { r = boxed(ctx, rhs); - else if (aliasscope || Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { + } + else if (intcast) { + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + r = ctx.builder.CreateLoad(realelty, intcast); + } + else if (aliasscope || Order != AtomicOrdering::NotAtomic || tracked_pointers) { r = emit_unbox(ctx, realelty, rhs, jltype); - if (realelty != elty) - r = ctx.builder.CreateZExt(r, elty); } + if (realelty != elty) + r = ctx.builder.CreateZExt(r, elty); } Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); if (ptr->getType() != ptrty) @@ -2197,7 +2206,14 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Current->addIncoming(instr, SkipBB); ctx.builder.SetInsertPoint(BB); } - Compare = emit_unbox(ctx, realelty, cmp, jltype); + cmp = update_julia_type(ctx, cmp, jltype); + if (intcast) { + emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + Compare = ctx.builder.CreateLoad(realelty, intcast); + } + else { + Compare = emit_unbox(ctx, realelty, cmp, jltype); + } if (realelty != elty) Compare = ctx.builder.CreateZExt(Compare, elty); } @@ -2244,16 +2260,17 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); if (intcast) { + assert(!isboxed); ctx.builder.CreateStore(realCompare, ctx.builder.CreateBitCast(intcast, realCompare->getType()->getPointerTo())); - if (maybe_null_if_boxed) - realCompare = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + if (tracked_pointers) + realCompare = ctx.builder.CreateLoad(intcast_eltyp, intcast); } - if (maybe_null_if_boxed) { - Value *first_ptr = isboxed ? Compare : extract_first_ptr(ctx, Compare); - if (first_ptr) - null_load_check(ctx, first_ptr, mod, var); + if (maybe_null_if_boxed && tracked_pointers) { + Value *first_ptr = isboxed ? realCompare : extract_first_ptr(ctx, realCompare); + assert(first_ptr); + null_load_check(ctx, first_ptr, mod, var); } - if (intcast) + if (intcast && !tracked_pointers) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); else oldval = mark_julia_type(ctx, realCompare, isboxed, jltype); @@ -2261,11 +2278,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (isboxed) { r = boxed(ctx, rhs); } - else if (Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { + else if (intcast) { + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + r = ctx.builder.CreateLoad(realelty, intcast); + if (!tracked_pointers) // oldval is a slot, so put the oldval back + ctx.builder.CreateStore(realCompare, intcast); + } + else if (Order != AtomicOrdering::NotAtomic) { + assert(!tracked_pointers); r = emit_unbox(ctx, realelty, rhs, jltype); - if (realelty != elty) - r = ctx.builder.CreateZExt(r, elty); } + if (realelty != elty) + r = ctx.builder.CreateZExt(r, elty); if (needlock) emit_lockstate_value(ctx, needlock, true); cmp = oldval; @@ -2331,9 +2355,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, realinstr = ctx.builder.CreateTrunc(realinstr, realelty); if (intcast) { ctx.builder.CreateStore(realinstr, ctx.builder.CreateBitCast(intcast, realinstr->getType()->getPointerTo())); + // n.b. this oldval is only used for emit_f_is in this branch, so we know a priori that it does not need a gc-root oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); if (maybe_null_if_boxed) - realinstr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + realinstr = ctx.builder.CreateLoad(intcast_eltyp, intcast); } else { oldval = mark_julia_type(ctx, realinstr, isboxed, jltype); @@ -2373,20 +2398,23 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.SetInsertPoint(DoneBB); if (needlock) emit_lockstate_value(ctx, needlock, false); - if (parent != NULL) { + if (parent != NULL && r && tracked_pointers && (!isboxed || !type_is_permalloc(rhs.typ))) { if (isreplacefield || issetfieldonce) { - // TODO: avoid this branch if we aren't making a write barrier BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "xchg_wb", ctx.f); DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg_wb", ctx.f); ctx.builder.CreateCondBr(Success, BB, DoneBB); ctx.builder.SetInsertPoint(BB); } - if (r) { - if (!isboxed) - emit_write_multibarrier(ctx, parent, r, rhs.typ); - else if (!type_is_permalloc(rhs.typ)) - emit_write_barrier(ctx, parent, r); + if (realelty != elty) + r = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, r, realelty)); + if (intcast) { + ctx.builder.CreateStore(r, intcast); + r = ctx.builder.CreateLoad(intcast_eltyp, intcast); } + if (!isboxed) + emit_write_multibarrier(ctx, parent, r, rhs.typ); + else if (!type_is_permalloc(rhs.typ)) + emit_write_barrier(ctx, parent, r); if (isreplacefield || issetfieldonce) { ctx.builder.CreateBr(DoneBB); ctx.builder.SetInsertPoint(DoneBB); @@ -2405,21 +2433,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, instr = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, instr, realelty)); if (intcast) { ctx.builder.CreateStore(instr, ctx.builder.CreateBitCast(intcast, instr->getType()->getPointerTo())); - instr = nullptr; + if (tracked_pointers) + instr = ctx.builder.CreateLoad(intcast_eltyp, intcast); } - if (maybe_null_if_boxed) { - if (intcast) - instr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + if (maybe_null_if_boxed && tracked_pointers) { Value *first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr); - if (first_ptr) - null_load_check(ctx, first_ptr, mod, var); - if (intcast && !first_ptr) - instr = nullptr; + assert(first_ptr); + null_load_check(ctx, first_ptr, mod, var); } - if (instr) - oldval = mark_julia_type(ctx, instr, isboxed, jltype); - else + if (intcast && !tracked_pointers) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); + else + oldval = mark_julia_type(ctx, instr, isboxed, jltype); if (isreplacefield) { Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); const jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; diff --git a/test/atomics.jl b/test/atomics.jl index 0d6c841073ad5..b2fc0e13c948d 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -128,6 +128,7 @@ test_field_operators(ARefxy{Any}(123_10, 123_20)) test_field_operators(ARefxy{Union{Nothing,Int}}(123_10, nothing)) test_field_operators(ARefxy{Complex{Int32}}(123_10, 123_20)) test_field_operators(ARefxy{Complex{Int128}}(123_10, 123_20)) +test_field_operators(ARefxy{Complex{Real}}(123_10, 123_20)) test_field_operators(ARefxy{PadIntA}(123_10, 123_20)) test_field_operators(ARefxy{PadIntB}(123_10, 123_20)) #FIXME: test_field_operators(ARefxy{Int24}(123_10, 123_20)) @@ -316,6 +317,7 @@ test_field_orderings(ARefxy{Any}(true, false), true, false) test_field_orderings(ARefxy{Union{Nothing,Missing}}(nothing, missing), nothing, missing) test_field_orderings(ARefxy{Union{Nothing,Int}}(nothing, 123_1), nothing, 123_1) test_field_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_field_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) test_field_orderings(10.0, 20.0) test_field_orderings(NaN, Inf) @@ -481,6 +483,7 @@ test_global_operators(Any) test_global_operators(Union{Nothing,Int}) test_global_operators(Complex{Int32}) test_global_operators(Complex{Int128}) +test_global_operators(Complex{Real}) test_global_operators(PadIntA) test_global_operators(PadIntB) #FIXME: test_global_operators(Int24) @@ -604,6 +607,7 @@ test_global_orderings(Any, true, false) test_global_orderings(Union{Nothing,Missing}, nothing, missing) test_global_orderings(Union{Nothing,Int}, nothing, 123_1) test_global_orderings(Complex{Int128}, Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_global_orderings(Complex{Real}, Complex{Real}(10, 30), Complex{Real}(20, 40)) test_global_orderings(Float64, 10.0, 20.0) test_global_orderings(Float64, NaN, Inf) @@ -757,6 +761,7 @@ test_memory_operators(Any) test_memory_operators(Union{Nothing,Int}) test_memory_operators(Complex{Int32}) test_memory_operators(Complex{Int128}) +test_memory_operators(Complex{Real}) test_memory_operators(PadIntA) test_memory_operators(PadIntB) #FIXME: test_memory_operators(Int24) @@ -944,6 +949,7 @@ test_memory_orderings(Any, true, false) test_memory_orderings(Union{Nothing,Missing}, nothing, missing) test_memory_orderings(Union{Nothing,Int}, nothing, 123_1) test_memory_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_memory_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) test_memory_orderings(10.0, 20.0) test_memory_orderings(NaN, Inf)