Skip to content

Commit

Permalink
add back old code
Browse files Browse the repository at this point in the history
  • Loading branch information
vchuravy committed Dec 5, 2024
1 parent 43d9bd4 commit 3ff51d2
Showing 1 changed file with 194 additions and 0 deletions.
194 changes: 194 additions & 0 deletions ext/UnsafeAtomicsLLVM/atomics.jl
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,143 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...}
end
end

@generated function llvm_atomic_op(
binop::AtomicRMWBinOpVal,
ptr::LLVMPtr{T,A},
val::T,
order::LLVMOrderingVal,
syncscope::Val{S},
) where {T,A,S}
@dispose ctx = Context() begin
T_val = convert(LLVMType, T)
T_ptr = convert(LLVMType, ptr)

T_typed_ptr = LLVM.PointerType(T_val, A)
llvm_f, _ = create_function(T_val, [T_ptr, T_val])

@dispose builder = IRBuilder() begin
entry = BasicBlock(llvm_f, "entry")
position!(builder, entry)

typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr)
rv = atomic_rmw!(
builder,
_valueof(binop()),
typed_ptr,
parameters(llvm_f)[2],
_valueof(order()),
syncscope_to_string(syncscope),
)

ret!(builder, rv)
end
call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val)
end
end

@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP}
@dynamic_order(order) do order
atomic_pointermodify(pointer, op, x, order)
end
end

@inline function atomic_pointermodify(
ptr::LLVMPtr{T},
op,
x::T,
::Val{:not_atomic},
) where {T}
old = atomic_pointerref(ptr, Val(:not_atomic))
new = op(old, x)
atomic_pointerset(ptr, new, Val(:not_atomic))
return old => new
end

@inline function atomic_pointermodify(
ptr::LLVMPtr{T},
::typeof(right),
x::T,
order::AtomicOrdering,
) where {T}
old = llvm_atomic_op(
Val(LLVM.API.LLVMAtomicRMWBinOpXchg),
ptr,
x,
llvm_from_julia_ordering(order),
)
return old => x
end

const atomictypes = Any[
Int8,
Int16,
Int32,
Int64,
Int128,
UInt8,
UInt16,
UInt32,
UInt64,
UInt128,
Float16,
Float32,
Float64,
]

for (opname, op, llvmop) in binoptable
opname === :xchg && continue
types = if opname in (:min, :max)
filter(t -> t <: Signed, atomictypes)
elseif opname in (:umin, :umax)
filter(t -> t <: Unsigned, atomictypes)
elseif opname in (:fadd, :fsub, :fmin, :fmax)
filter(t -> t <: AbstractFloat, atomictypes)
else
filter(t -> t <: Integer, atomictypes)
end
for T in types
@eval @inline function atomic_pointermodify(
ptr::LLVMPtr{$T},
::$(typeof(op)),
x::$T,
order::AtomicOrdering,
syncscope::Val{S} = Val{:system}(),
) where {S}
old =
syncscope isa Val{:system} ?
llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) :
llvm_atomic_op(
$(Val(llvmop)),
ptr,
x,
llvm_from_julia_ordering(order),
syncscope,
)
return old => $op(old, x)
end
end
end

@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new))
@inline atomic_pointerswap(pointer, new, order) =
first(atomic_pointermodify(pointer, right, new, order))

@inline function atomic_pointermodify(
ptr::LLVMPtr{T},
op,
x::T,
order::AllOrdering,
) where {T}
# Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256
fail_order = Val(:monotonic)
old = atomic_pointerref(ptr, fail_order)
while true
new = op(old, x)
(old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order)
success && return old => new
end
end

@generated function llvm_atomic_cas(
ptr::LLVMPtr{T,A},
cmp::T,
Expand Down Expand Up @@ -323,3 +460,60 @@ end
end
end
end

@inline function atomic_pointerreplace(
pointer,
expected,
desired,
success_order::Symbol,
fail_order::Symbol,
)
# This avoids abstract dispatch at run-time but probably too much codegen?
#=
@dynamic_order(success_order) do success_order
@dynamic_order(fail_order) do fail_order
atomic_pointerreplace(pointer, expected, desired, success_order, fail_order)
end
end
=#

# This avoids excessive codegen while hopefully imposes no cost when const-prop works:
so = @dynamic_order(success_order) do success_order
success_order
end
fo = @dynamic_order(fail_order) do fail_order
fail_order
end
return atomic_pointerreplace(pointer, expected, desired, so, fo)
end

@inline function atomic_pointerreplace(
ptr::LLVMPtr{T},
expected::T,
desired::T,
::Val{:not_atomic},
::Val{:not_atomic},
) where {T}
old = atomic_pointerref(ptr, Val(:not_atomic))
if old === expected
atomic_pointerset(ptr, desired, Val(:not_atomic))
success = true
else
success = false
end
return (; old, success)
end

@inline atomic_pointerreplace(
ptr::LLVMPtr{T},
expected::T,
desired::T,
success_order::_julia_ordering(((:not_atomic, :unordered))),
fail_order::_julia_ordering(((:not_atomic, :unordered, :release, :acquire_release))),
) where {T} = llvm_atomic_cas(
ptr,
expected,
desired,
llvm_from_julia_ordering(success_order),
llvm_from_julia_ordering(fail_order),
)

0 comments on commit 3ff51d2

Please sign in to comment.