Skip to content

[SYCL][CUDA] libclc atomics update #2100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 8 commits into from
Closed

[SYCL][CUDA] libclc atomics update #2100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
eacbcd8
Remove data layout requirements from libclc atomics
Jul 13, 2020
849f145
Update SPIRVBuiltins.td for libclc OpenCL atomics
Jul 13, 2020
32e78b0
Use an include list for libclc test generation
Jul 13, 2020
4cd7963
Add libclc binding tests for atomics
Jul 13, 2020
fefb629
Update generated spirv_builtins.h
Jul 13, 2020
8d772ac
Remove manual mangling of libspirv atomics
Jul 13, 2020
f04e389
Update sycl use of libclc atomics for CUDA
Jul 13, 2020
8908673
Add comment for spirv type conversions in spirv.hpp
Jul 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions clang/lib/Sema/SPIRVBuiltins.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,52 @@ foreach name = ["MemoryBarrier"] in {
def : SPVBuiltin<name, [Void, UInt, UInt]>;
}

// 3.32.18. Atomic Instructions

// OpenCL validation rules:
// The Pointer operand must be a pointer to
// the Function, Workgroup, or CrossWorkGroup Storage Classes.
// Note that an Atomic Instruction on a pointer to the Function Storage Class is valid,
// but does not have defined behavior.
foreach AS = [GlobalAS, LocalAS, PrivateAS] in {
foreach Type = [Int, UInt, Long, ULong] in {
foreach name = ["AtomicLoad", "AtomicIIncrement", "AtomicIDecrement"] in {
def : SPVBuiltin<name, [Type, PointerType<ConstType<VolatileType<Type>>, AS>, UInt, UInt]>;
def : SPVBuiltin<name, [Type, PointerType<ConstType<Type>, AS>, UInt, UInt]>;
}

foreach name = ["AtomicStore"] in {
def : SPVBuiltin<name, [Void, PointerType<VolatileType<Type>, AS>, UInt, UInt, Type]>;
def : SPVBuiltin<name, [Void, PointerType<Type, AS>, UInt, UInt, Type]>;
}

foreach name = ["AtomicExchange", "AtomicIAdd", "AtomicISub",
"AtomicAnd", "AtomicOr", "AtomicXor"] in {
def : SPVBuiltin<name, [Type, PointerType<VolatileType<Type>, AS>, UInt, UInt, Type]>;
def : SPVBuiltin<name, [Type, PointerType<Type, AS>, UInt, UInt, Type]>;
}

foreach name = ["AtomicCompareExchange"] in {
def : SPVBuiltin<name, [Type, PointerType<VolatileType<Type>, AS>, UInt, UInt, UInt, Type, Type]>;
def : SPVBuiltin<name, [Type, PointerType<Type, AS>, UInt, UInt, UInt, Type, Type]>;
}
}

foreach Type = [Int, Long] in {
foreach name = ["AtomicSMin", "AtomicSMax"] in {
def : SPVBuiltin<name, [Type, PointerType<VolatileType<Type>, AS>, UInt, UInt, Type]>;
def : SPVBuiltin<name, [Type, PointerType<Type, AS>, UInt, UInt, Type]>;
}
}

foreach Type = [UInt, ULong] in {
foreach name = ["AtomicUMin", "AtomicUMax"] in {
def : SPVBuiltin<name, [Type, PointerType<VolatileType<Type>, AS>, UInt, UInt, Type]>;
def : SPVBuiltin<name, [Type, PointerType<Type, AS>, UInt, UInt, Type]>;
}
}
}

// 3.32.21. Group and Subgroup Instructions

foreach name = ["GroupAsyncCopy"] in {
Expand Down
543 changes: 543 additions & 0 deletions libclc/generic/include/spirv/spirv_builtins.h
View file
Open in desktop

Large diffs are not rendered by default.

119 changes: 97 additions & 22 deletions libclc/generic/libspirv/atomic/atomic_add.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,104 @@

#include <spirv/spirv.h>

// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicIAdd(volatile global int *p,
unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_add(p, val);
}

#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, FN_NAME) \
_CLC_DEF TYPE \
_Z18__spirv_AtomicIAddPU3##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
volatile AS TYPE *p, enum Scope scope, \
enum MemorySemanticsMask semantics, TYPE val) { \
return FN_NAME(p, val); \
}
_CLC_OVERLOAD _CLC_DEF unsigned int
__spirv_AtomicIAdd(volatile global unsigned int *p, unsigned int scope,
unsigned int semantics, unsigned int val) {
return __sync_fetch_and_add(p, val);
}

IMPL(int, i, global, AS1, __sync_fetch_and_add)
IMPL(unsigned int, j, global, AS1, __sync_fetch_and_add)
IMPL(int, i, local, AS3, __sync_fetch_and_add)
IMPL(unsigned int, j, local, AS3, __sync_fetch_and_add)
_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicIAdd(volatile local int *p,
unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_add(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned int
__spirv_AtomicIAdd(volatile local unsigned int *p, unsigned int scope,
unsigned int semantics, unsigned int val) {
return __sync_fetch_and_add(p, val);
}

_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicIAdd(global int *p, unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_add(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned int __spirv_AtomicIAdd(global unsigned int *p,
unsigned int scope,
unsigned int semantics,
unsigned int val) {
return __sync_fetch_and_add(p, val);
}

_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicIAdd(local int *p, unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_add(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned int __spirv_AtomicIAdd(local unsigned int *p,
unsigned int scope,
unsigned int semantics,
unsigned int val) {
return __sync_fetch_and_add(p, val);
}

#ifdef cl_khr_int64_base_atomics
IMPL(long, l, global, AS1, __sync_fetch_and_add_8)
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_add_8)
IMPL(long, l, local, AS3, __sync_fetch_and_add_8)
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_add_8)
IMPL(long, x, global, AS1, __sync_fetch_and_add_8)
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_add_8)
IMPL(long, x, local, AS3, __sync_fetch_and_add_8)
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_add_8)
#endif
#undef IMPL
_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicIAdd(volatile global long int *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long
__spirv_AtomicIAdd(volatile global unsigned long int *p, unsigned int scope,
unsigned int semantics, unsigned long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicIAdd(volatile local long int *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long
__spirv_AtomicIAdd(volatile local unsigned long int *p, unsigned int scope,
unsigned int semantics, unsigned long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicIAdd(global long int *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long
__spirv_AtomicIAdd(global unsigned long int *p, unsigned int scope,
unsigned int semantics, unsigned long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicIAdd(local long int *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_add_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long
__spirv_AtomicIAdd(local unsigned long int *p, unsigned int scope,
unsigned int semantics, unsigned long val) {
return __sync_fetch_and_add_8(p, val);
}
#endif // cl_khr_int64_base_atomics
120 changes: 98 additions & 22 deletions libclc/generic/libspirv/atomic/atomic_and.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,105 @@

#include <spirv/spirv.h>

// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicAnd(volatile global int *p,
unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_and(p, val);
}

#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, FN_NAME) \
_CLC_DEF TYPE \
_Z17__spirv_AtomicAndPU3##AS_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagE##TYPE_MANGLED( \
volatile AS TYPE *p, enum Scope scope, \
enum MemorySemanticsMask semantics, TYPE val) { \
return FN_NAME(p, val); \
}
_CLC_OVERLOAD _CLC_DEF unsigned int
__spirv_AtomicAnd(volatile global unsigned int *p, unsigned int scope,
unsigned int semantics, unsigned int val) {
return __sync_fetch_and_and(p, val);
}

IMPL(int, i, global, AS1, __sync_fetch_and_and)
IMPL(unsigned int, j, global, AS1, __sync_fetch_and_and)
IMPL(int, i, local, AS3, __sync_fetch_and_and)
IMPL(unsigned int, j, local, AS3, __sync_fetch_and_and)
_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicAnd(volatile local int *p,
unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_and(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned int
__spirv_AtomicAnd(volatile local unsigned int *p, unsigned int scope,
unsigned int semantics, unsigned int val) {
return __sync_fetch_and_and(p, val);
}

_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicAnd(global int *p, unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_and(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned int __spirv_AtomicAnd(global unsigned int *p,
unsigned int scope,
unsigned int semantics,
unsigned int val) {
return __sync_fetch_and_and(p, val);
}

_CLC_OVERLOAD _CLC_DEF int __spirv_AtomicAnd(local int *p, unsigned int scope,
unsigned int semantics, int val) {
return __sync_fetch_and_and(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned int __spirv_AtomicAnd(local unsigned int *p,
unsigned int scope,
unsigned int semantics,
unsigned int val) {
return __sync_fetch_and_and(p, val);
}

#ifdef cl_khr_int64_extended_atomics
IMPL(long, l, global, AS1, __sync_fetch_and_and_8)
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_and_8)
IMPL(long, l, local, AS3, __sync_fetch_and_and_8)
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_and_8)
IMPL(long, x, global, AS1, __sync_fetch_and_and_8)
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_and_8)
IMPL(long, x, local, AS3, __sync_fetch_and_and_8)
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_and_8)
#endif
#undef IMPL
_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicAnd(volatile global long *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long
__spirv_AtomicAnd(volatile global unsigned long *p, unsigned int scope,
unsigned int semantics, unsigned long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicAnd(volatile local long *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long
__spirv_AtomicAnd(volatile local unsigned long *p, unsigned int scope,
unsigned int semantics, unsigned long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicAnd(global long *p,
unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long __spirv_AtomicAnd(global unsigned long *p,
unsigned int scope,
unsigned int semantics,
unsigned long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF long __spirv_AtomicAnd(local long *p, unsigned int scope,
unsigned int semantics,
long val) {
return __sync_fetch_and_and_8(p, val);
}

_CLC_OVERLOAD _CLC_DEF unsigned long __spirv_AtomicAnd(local unsigned long *p,
unsigned int scope,
unsigned int semantics,
unsigned long val) {
return __sync_fetch_and_and_8(p, val);
}
#endif // cl_khr_int64_base_atomics
Loading