Skip to content

Commit

Permalink
implement syncscopes everywhere
Browse files Browse the repository at this point in the history
  • Loading branch information
@vchuravy
vchuravy committed Dec 4, 2024
1 parent 6d6192f commit 0217a90
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 125 deletions.
2 changes: 1 addition & 1 deletion src/UnsafeAtomics.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ module Internal
using Base.Sys: WORD_SIZE
using Base: bitcast, llvmcall

using ..UnsafeAtomics: UnsafeAtomics, Ordering, right
using ..UnsafeAtomics: UnsafeAtomics, Ordering, SyncScope, right

include("utils.jl")
include("orderings.jl")
Expand Down
273 changes: 151 additions & 122 deletions src/core.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
@inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none)
@inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none)
@inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none)
@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord., none)
@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none)

#! format: off
# https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl#L23-L30
Expand Down Expand Up @@ -74,47 +74,51 @@ for typ in (inttypes..., floattypes...)
for ord in orderings
ord in (release, acq_rel) && continue

if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
end
else
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ}},
x,
)
for sync in syncscopes
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync)))
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
end
else
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ}},
x,
)
end
end
end
end

for ord in orderings
ord in (acquire, acq_rel) && continue

if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
return nothing
end
else
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
ret void
"""),
Cvoid,
Tuple{Ptr{$typ},$typ},
x,
v,
)

for sync in syncscopes
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync)))
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
return nothing
end
else
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
ret void
"""),
Cvoid,
Tuple{Ptr{$typ},$typ},
x,
v,
)
end
end
end
end
Expand All @@ -124,54 +128,58 @@ for typ in (inttypes..., floattypes...)

typ <: AbstractFloat && break

if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
return Core.Intrinsics.atomic_pointerreplace(
x,
cmp,
new,
base_ordering($success_ordering),
base_ordering($failure_ordering)
for sync in syncscopes
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
::$(typeof(sync)),
)
end
else
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
success = Ref{Int8}()
GC.@preserve success begin
old = llvmcall(
$(
"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
%rv = extractvalue { $lt, i1 } %rs, 0
%s1 = extractvalue { $lt, i1 } %rs, 1
%s8 = zext i1 %s1 to i8
%sptr = inttoptr i$WORD_SIZE %3 to i8*
store i8 %s8, i8* %sptr
ret $lt %rv
"""
),
$typ,
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
return Core.Intrinsics.atomic_pointerreplace(
x,
cmp,
new,
Ptr{Int8}(pointer_from_objref(success)),
base_ordering($success_ordering),
base_ordering($failure_ordering)
)
end
return (old = old, success = !iszero(success[]))
else
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
::$(typeof(sync)),
)
success = Ref{Int8}()
GC.@preserve success begin
old = llvmcall(
$(
"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
%rv = extractvalue { $lt, i1 } %rs, 0
%s1 = extractvalue { $lt, i1 } %rs, 1
%s8 = zext i1 %s1 to i8
%sptr = inttoptr i$WORD_SIZE %3 to i8*
store i8 %s8, i8* %sptr
ret $lt %rv
"""
),
$typ,
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
x,
cmp,
new,
Ptr{Int8}(pointer_from_objref(success)),
)
end
return (old = old, success = !iszero(success[]))
end
end
end
end
Expand All @@ -193,60 +201,81 @@ for typ in (inttypes..., floattypes...)
end
end
for ord in orderings
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.modify!(
for sync in syncscopes
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
op::typeof($op),
v::$typ,
::$(typeof(ord)),
::$(typeof(sync)),
)
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
end
else
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
op::typeof($op),
::typeof($op),
v::$typ,
::$(typeof(ord)),
::$(typeof(sync)),
)
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
end
else
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
::typeof($op),
v::$typ,
::$(typeof(ord)),
)
old = llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ},$typ},
x,
v,
)
return old => $op(old, v)
old = llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ},$typ},
x,
v,
)
return old => $op(old, v)
end
end
end
end
end
end

# Core.Intrinsics.atomic_fence was introduced in 1.10
function UnsafeAtomics.fence(ord::Ordering)
Core.Intrinsics.atomic_fence(base_ordering(ord))
return nothing
end
if Sys.ARCH == :x86_64
# FIXME: Disable this once on LLVM 19
# This is unfortunatly required for good-performance on AMD
# https://github.com/llvm/llvm-project/pull/106555
function UnsafeAtomics.fence(::typeof(seq_cst))
Base.llvmcall(
(raw"""
define void @fence() #0 {
entry:
tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory}
ret void
}
attributes #0 = { alwaysinline }
""", "fence"), Nothing, Tuple{})
for sync in syncscopes
if sync == none
# Core.Intrinsics.atomic_fence was introduced in 1.10
@eval function UnsafeAtomics.fence(ord::Ordering, ::$(typeof(sync)))
Core.Intrinsics.atomic_fence(base_ordering(ord))
return nothing
end
if Sys.ARCH == :x86_64
# FIXME: Disable this once on LLVM 19
# This is unfortunatly required for good-performance on AMD
# https://github.com/llvm/llvm-project/pull/106555
@eval function UnsafeAtomics.fence(::typeof(seq_cst), ::$(typeof(sync)))
Base.llvmcall(
(raw"""
define void @fence() #0 {
entry:
tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory}
ret void
}
attributes #0 = { alwaysinline }
""", "fence"), Nothing, Tuple{})
end
end
else
for ord in orderings
@eval function UnsafeAtomics.fence(::$(typeof(ord)), ::$(typeof(sync)))
return llvmcall(
$("""
fence $sync $ord
ret void
"""),
Cvoid,
Tuple{},
)
end
end
end
end

Expand Down
11 changes: 9 additions & 2 deletions src/syncscopes.jl
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
struct LLVMSyncScope{name} <: SyncScope end

const none = LLVMSyncScope{Symbol("")}()
const none = LLVMSyncScope{:none}()
const singlethread = LLVMSyncScope{:singlethread}()

const syncscopes = (none, singlethread)
const ConcreteSyncScopes = Union{map(typeof, syncscopes)...}

llvm_syncscope(::LLVMSyncScope{name}) where {name} = name

Base.string(s::LLVMSyncScope) = String(llvm_syncscope(s))
Base.string(s::LLVMSyncScope) = string("syncscope(\"", llvm_syncscope(s), "\")")
Base.string(s::typeof(none)) = ""

Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s))

Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_ordering(o))

0 comments on commit 0217a90

Please sign in to comment.