Skip to content

Commit a540d01

Browse files
steffenlarsensteffenlarsen
authored
[SYCL][PTX] Implements missing atomic builtins (#1173)
- Builtins for atomic load and store (`int`, `unsigned int`, `long`, `unsigned long`, `long long`, and `unsigned long long`) - Added `long`, `unsigned long`, `long long`, and `unsigned long long` variants of the following existing builtins: - Atomic add - Atomic sub - Atomic and - Atomic or - Atomic xor - Atomic exhange - Atomic compare-and-exchange - Atomic max - Atomic min Notice: `long` is currently used in place of `long long` due to a problem with the width of `long long` being defined as 128-bit. Signed-off-by: Steffen Larsen <[email protected]>
1 parent 03326f7 commit a540d01

File tree

19 files changed

+571
-0
lines changed

19 files changed

+571
-0
lines changed

‎libclc/generic/include/spirv/atomic/atomic_decl.inc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ __CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, m, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_
2828
#ifdef cl_khr_int64_base_atomics
2929
__CLC_DECLARE_ATOMIC_ADDRSPACE(long, l, __SPIRV_FUNCTION_S, __SPIRV_FUNCTION_S_LEN)
3030
__CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, m, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_LEN)
31+
__CLC_DECLARE_ATOMIC_ADDRSPACE(long, x, __SPIRV_FUNCTION_S, __SPIRV_FUNCTION_S_LEN)
32+
__CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, y, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_LEN)
3133
#endif
3234
#endif
3335

‎libclc/generic/include/spirv/atomic/atomic_load.h

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
10+
#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
11+
_CLC_DECL TYPE _Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE( \
12+
volatile AS const TYPE *, enum Scope, enum MemorySemanticsMask);
13+
14+
#define DECL_AS(TYPE, TYPE_MANGLED) \
15+
DECL(TYPE, TYPE_MANGLED, global, AS1) \
16+
DECL(TYPE, TYPE_MANGLED, local, AS3)
17+
18+
DECL_AS(int, i)
19+
DECL_AS(unsigned int, j)
20+
21+
#ifdef cl_khr_int64_base_atomics
22+
DECL_AS(long, l)
23+
DECL_AS(unsigned long, m)
24+
DECL_AS(long, x)
25+
DECL_AS(unsigned long, y)
26+
#endif
27+
28+
#undef DECL_AS
29+
#undef DECL

‎libclc/generic/include/spirv/atomic/atomic_store.h

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
10+
#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
11+
_CLC_DECL void _Z19__spirv_AtomicStorePU3##AS_MANGLED##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE##TYPE_MANGLED( \
12+
volatile AS TYPE *, enum Scope, enum MemorySemanticsMask, TYPE);
13+
14+
#define DECL_AS(TYPE, TYPE_MANGLED) \
15+
DECL(TYPE, TYPE_MANGLED, global, AS1) \
16+
DECL(TYPE, TYPE_MANGLED, local, AS3)
17+
18+
DECL_AS(int, i)
19+
DECL_AS(unsigned int, j)
20+
21+
#ifdef cl_khr_int64_base_atomics
22+
DECL_AS(long, l)
23+
DECL_AS(unsigned long, m)
24+
DECL_AS(long, x)
25+
DECL_AS(unsigned long, y)
26+
#endif
27+
28+
#undef DECL_AS
29+
#undef DECL

‎libclc/generic/include/spirv/spirv.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,8 @@
235235
#include <spirv/atomic/atomic_sub.h>
236236
#include <spirv/atomic/atomic_xchg.h>
237237
#include <spirv/atomic/atomic_xor.h>
238+
#include <spirv/atomic/atomic_load.h>
239+
#include <spirv/atomic/atomic_store.h>
238240

239241
/* cl_khr extension atomics are omitted from __spirv */
240242

‎libclc/generic/libspirv/SOURCES

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ atomic/atomic_or.cl
1212
atomic/atomic_sub.cl
1313
atomic/atomic_xchg.cl
1414
atomic/atomic_xor.cl
15+
atomic/atomic_load.cl
16+
atomic/atomic_store.cl
1517
common/degrees.cl
1618
common/mix.cl
1719
common/radians.cl

‎libclc/generic/libspirv/atomic/atomic_add.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_add_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_add_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_add_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_add_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_add_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_add_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_add_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_add_8)
2933
#endif
3034
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_and.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_and_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_and_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_and_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_and_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_and_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_and_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_and_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_and_8)
2933
#endif
3034
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_cmpxchg.cl

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,24 @@ _CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS1mN5__spv5ScopeENS1_19Memor
5050
volatile global ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
5151
return __sync_val_compare_and_swap_8(p, cmp, val);
5252
}
53+
54+
_CLC_DEF long _Z29__spirv_AtomicCompareExchangePU3AS3xN5__spv5ScopeENS1_19MemorySemanticsMaskES3_xx(
55+
volatile local long *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, long val, long cmp) {
56+
return __sync_val_compare_and_swap_8(p, cmp, val);
57+
}
58+
59+
_CLC_DEF long _Z29__spirv_AtomicCompareExchangePU3AS1xN5__spv5ScopeENS1_19MemorySemanticsMaskES3_xx(
60+
volatile global long *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, long val, long cmp) {
61+
return __sync_val_compare_and_swap_8(p, cmp, val);
62+
}
63+
64+
_CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS3yN5__spv5ScopeENS1_19MemorySemanticsMaskES3_yy(
65+
volatile local ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
66+
return __sync_val_compare_and_swap_8(p, cmp, val);
67+
}
68+
69+
_CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS1yN5__spv5ScopeENS1_19MemorySemanticsMaskES3_yy(
70+
volatile global ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
71+
return __sync_val_compare_and_swap_8(p, cmp, val);
72+
}
5373
#endif

‎libclc/generic/libspirv/atomic/atomic_load.cl

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include <spirv/spirv.h>
10+
11+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
12+
13+
#define FDECL(TYPE, PREFIX, AS, BYTE_SIZE, MEM_ORDER) \
14+
TYPE __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *);
15+
16+
#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, PREFIX, BYTE_SIZE) \
17+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, unordered) \
18+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, acquire) \
19+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, seq_cst) \
20+
_CLC_DEF TYPE _Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE( \
21+
volatile AS const TYPE *p, enum Scope scope, enum MemorySemanticsMask semantics) { \
22+
if (semantics == Acquire) { \
23+
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_acquire(p); \
24+
} \
25+
if (semantics == SequentiallyConsistent) { \
26+
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_seq_cst(p); \
27+
} \
28+
return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_unordered(p); \
29+
}
30+
31+
#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
32+
IMPL(TYPE, TYPE_MANGLED, global, AS1, PREFIX, BYTE_SIZE) \
33+
IMPL(TYPE, TYPE_MANGLED, local, AS3, PREFIX, BYTE_SIZE)
34+
35+
IMPL_AS(int, i, , 4)
36+
IMPL_AS(unsigned int, j, u, 4)
37+
38+
#ifdef cl_khr_int64_base_atomics
39+
IMPL_AS(long, l, , 8)
40+
IMPL_AS(unsigned long, m, u, 8)
41+
IMPL_AS(long, x, , 8)
42+
IMPL_AS(unsigned long, y, u, 8)
43+
#endif
44+
45+
#undef FDECL
46+
#undef IMPL_AS
47+
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_max.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,9 @@ IMPL(long, l, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
3131
IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
3232
IMPL(long, l, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
3333
IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
34+
IMPL(long, x, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
35+
IMPL(unsigned long, y, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
36+
IMPL(long, x, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
37+
IMPL(unsigned long, y, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
3438
#endif
3539
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_min.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,9 @@ IMPL(long, l, global, AS1, __spirv_AtomicSMin, __clc, min_global_8)
3131
IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMin, __clc, umin_global_8)
3232
IMPL(long, l, local, AS3, __spirv_AtomicSMin, __clc, min_local_8)
3333
IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMin, __clc, umin_local_8)
34+
IMPL(long, x, global, AS1, __spirv_AtomicSMin, __clc, min_global_8)
35+
IMPL(unsigned long, y, global, AS1, __spirv_AtomicUMin, __clc, umin_global_8)
36+
IMPL(long, x, local, AS3, __spirv_AtomicSMin, __clc, min_local_8)
37+
IMPL(unsigned long, y, local, AS3, __spirv_AtomicUMin, __clc, umin_local_8)
3438
#endif
3539
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_or.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_or_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_or_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_or_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_or_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_or_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_or_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_or_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_or_8)
2933
#endif
3034
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_store.cl

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
#include <spirv/spirv.h>
10+
11+
// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
12+
13+
_CLC_DEF void _Z19__spirv_AtomicStorePU3AS1fN5__spv5ScopeENS1_19MemorySemanticsMaskEf(
14+
volatile global float *p, enum Scope scope, enum MemorySemanticsMask semantics, float val) {
15+
_Z19__spirv_AtomicStorePU3AS1jN5__spv5ScopeENS1_19MemorySemanticsMaskEj(
16+
(volatile global uint *)p, scope, semantics, as_uint(val));
17+
}
18+
19+
_CLC_DEF void _Z19__spirv_AtomicStorePU3AS3fN5__spv5ScopeENS1_19MemorySemanticsMaskEf(
20+
volatile local float *p, enum Scope scope, enum MemorySemanticsMask semantics, float val) {
21+
_Z19__spirv_AtomicStorePU3AS3jN5__spv5ScopeENS1_19MemorySemanticsMaskEj(
22+
(volatile local uint *)p, scope, semantics, as_uint(val));
23+
}
24+
25+
#define FDECL(TYPE, PREFIX, AS, BYTE_SIZE, MEM_ORDER) \
26+
TYPE __clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *, TYPE);
27+
28+
#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, PREFIX, BYTE_SIZE) \
29+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, unordered) \
30+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, release) \
31+
FDECL(TYPE, PREFIX, AS, BYTE_SIZE, seq_cst) \
32+
_CLC_DEF void _Z19__spirv_AtomicStorePU3##AS_MANGLED##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE##TYPE_MANGLED( \
33+
volatile AS TYPE *p, enum Scope scope, enum MemorySemanticsMask semantics, TYPE val) { \
34+
if (semantics == Release) { \
35+
__clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_release(p, val); \
36+
} else if (semantics == SequentiallyConsistent) { \
37+
__clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_seq_cst(p, val); \
38+
} else { \
39+
__clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_unordered(p, val); \
40+
} \
41+
}
42+
43+
#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
44+
IMPL(TYPE, TYPE_MANGLED, global, AS1, PREFIX, BYTE_SIZE) \
45+
IMPL(TYPE, TYPE_MANGLED, local, AS3, PREFIX, BYTE_SIZE)
46+
47+
IMPL_AS(int, i, , 4)
48+
IMPL_AS(unsigned int, j, u, 4)
49+
50+
#ifdef cl_khr_int64_base_atomics
51+
IMPL_AS(long, l, , 8)
52+
IMPL_AS(unsigned long, m, u, 8)
53+
IMPL_AS(long, x, , 8)
54+
IMPL_AS(unsigned long, y, u, 8)
55+
#endif
56+
57+
#undef FDECL
58+
#undef IMPL_AS
59+
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_sub.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_sub_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_sub_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_sub_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_sub_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_sub_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_sub_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_sub_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_sub_8)
2933
#endif
3034
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_xchg.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,9 @@ IMPL(long, l, global, AS1, __sync_swap_8)
3838
IMPL(unsigned long, m, global, AS1, __sync_swap_8)
3939
IMPL(long, l, local, AS3, __sync_swap_8)
4040
IMPL(unsigned long, m, local, AS3, __sync_swap_8)
41+
IMPL(long, x, global, AS1, __sync_swap_8)
42+
IMPL(unsigned long, y, global, AS1, __sync_swap_8)
43+
IMPL(long, x, local, AS3, __sync_swap_8)
44+
IMPL(unsigned long, y, local, AS3, __sync_swap_8)
4145
#endif
4246
#undef IMPL

‎libclc/generic/libspirv/atomic/atomic_xor.cl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_xor_8)
2626
IMPL(unsigned long, m, global, AS1, __sync_fetch_and_xor_8)
2727
IMPL(long, l, local, AS3, __sync_fetch_and_xor_8)
2828
IMPL(unsigned long, m, local, AS3, __sync_fetch_and_xor_8)
29+
IMPL(long, x, global, AS1, __sync_fetch_and_xor_8)
30+
IMPL(unsigned long, y, global, AS1, __sync_fetch_and_xor_8)
31+
IMPL(long, x, local, AS3, __sync_fetch_and_xor_8)
32+
IMPL(unsigned long, y, local, AS3, __sync_fetch_and_xor_8)
2933
#endif
3034
#undef IMPL

‎libclc/ptx-nvidiacl/libspirv/SOURCES

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
atomic/loadstore_helpers.ll
2+
cl_khr_int64_extended_atomics/minmax_helpers.ll
13
synchronization/barrier.cl
24
workitem/get_global_id.cl
35
workitem/get_global_offset.cl

0 commit comments

Comments
 (0)