Skip to content

Commit f04e389

Browse files
author
Alexander Johnston
committed
Update sycl use of libclc atomics for CUDA
Authored-by: Stuart Adams <[email protected]> Co-authored-by: Alexander Johnston <[email protected]> Signed-off-by: Alexander Johnston <[email protected]>
1 parent 8d772ac commit f04e389

File tree

4 files changed

+97
-110
lines changed

4 files changed

+97
-110
lines changed

‎sycl/include/CL/__spirv/spirv_ops.hpp

Lines changed: 0 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -54,84 +54,6 @@ extern SYCL_EXTERNAL TempRetT __spirv_ImageSampleExplicitLod(SampledType,
5454
#define OpGroupAsyncCopyGlobalToLocal __spirv_GroupAsyncCopy
5555
#define OpGroupAsyncCopyLocalToGlobal __spirv_GroupAsyncCopy
5656

57-
// Atomic SPIR-V builtins
58-
#define __SPIRV_ATOMIC_LOAD(AS, Type) \
59-
extern SYCL_EXTERNAL Type __spirv_AtomicLoad( \
60-
AS const Type *P, __spv::Scope::Flag S, \
61-
__spv::MemorySemanticsMask::Flag O);
62-
#define __SPIRV_ATOMIC_STORE(AS, Type) \
63-
extern SYCL_EXTERNAL void __spirv_AtomicStore( \
64-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
65-
Type V);
66-
#define __SPIRV_ATOMIC_EXCHANGE(AS, Type) \
67-
extern SYCL_EXTERNAL Type __spirv_AtomicExchange( \
68-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
69-
Type V);
70-
#define __SPIRV_ATOMIC_CMP_EXCHANGE(AS, Type) \
71-
extern SYCL_EXTERNAL Type __spirv_AtomicCompareExchange( \
72-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag E, \
73-
__spv::MemorySemanticsMask::Flag U, Type V, Type C);
74-
#define __SPIRV_ATOMIC_IADD(AS, Type) \
75-
extern SYCL_EXTERNAL Type __spirv_AtomicIAdd( \
76-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
77-
Type V);
78-
#define __SPIRV_ATOMIC_ISUB(AS, Type) \
79-
extern SYCL_EXTERNAL Type __spirv_AtomicISub( \
80-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
81-
Type V);
82-
#define __SPIRV_ATOMIC_SMIN(AS, Type) \
83-
extern SYCL_EXTERNAL Type __spirv_AtomicSMin( \
84-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
85-
Type V);
86-
#define __SPIRV_ATOMIC_UMIN(AS, Type) \
87-
extern SYCL_EXTERNAL Type __spirv_AtomicUMin( \
88-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
89-
Type V);
90-
#define __SPIRV_ATOMIC_SMAX(AS, Type) \
91-
extern SYCL_EXTERNAL Type __spirv_AtomicSMax( \
92-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
93-
Type V);
94-
#define __SPIRV_ATOMIC_UMAX(AS, Type) \
95-
extern SYCL_EXTERNAL Type __spirv_AtomicUMax( \
96-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
97-
Type V);
98-
#define __SPIRV_ATOMIC_AND(AS, Type) \
99-
extern SYCL_EXTERNAL Type __spirv_AtomicAnd( \
100-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
101-
Type V);
102-
#define __SPIRV_ATOMIC_OR(AS, Type) \
103-
extern SYCL_EXTERNAL Type __spirv_AtomicOr( \
104-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
105-
Type V);
106-
#define __SPIRV_ATOMIC_XOR(AS, Type) \
107-
extern SYCL_EXTERNAL Type __spirv_AtomicXor( \
108-
AS Type *P, __spv::Scope::Flag S, __spv::MemorySemanticsMask::Flag O, \
109-
Type V);
110-
111-
#define __SPIRV_ATOMIC_FLOAT(AS, Type) \
112-
__SPIRV_ATOMIC_LOAD(AS, Type) \
113-
__SPIRV_ATOMIC_STORE(AS, Type) \
114-
__SPIRV_ATOMIC_EXCHANGE(AS, Type)
115-
116-
#define __SPIRV_ATOMIC_BASE(AS, Type) \
117-
__SPIRV_ATOMIC_FLOAT(AS, Type) \
118-
__SPIRV_ATOMIC_CMP_EXCHANGE(AS, Type) \
119-
__SPIRV_ATOMIC_IADD(AS, Type) \
120-
__SPIRV_ATOMIC_ISUB(AS, Type) \
121-
__SPIRV_ATOMIC_AND(AS, Type) \
122-
__SPIRV_ATOMIC_OR(AS, Type) \
123-
__SPIRV_ATOMIC_XOR(AS, Type)
124-
125-
#define __SPIRV_ATOMIC_SIGNED(AS, Type) \
126-
__SPIRV_ATOMIC_BASE(AS, Type) \
127-
__SPIRV_ATOMIC_SMIN(AS, Type) \
128-
__SPIRV_ATOMIC_SMAX(AS, Type)
129-
130-
#define __SPIRV_ATOMIC_UNSIGNED(AS, Type) \
131-
__SPIRV_ATOMIC_BASE(AS, Type) \
132-
__SPIRV_ATOMIC_UMIN(AS, Type) \
133-
__SPIRV_ATOMIC_UMAX(AS, Type)
134-
13557
// Helper atomic operations which select correct signed/unsigned version
13658
// of atomic min/max based on the signed-ness of the type
13759
#define __SPIRV_ATOMIC_MINMAX(AS, Op) \
@@ -154,14 +76,6 @@ extern SYCL_EXTERNAL TempRetT __spirv_ImageSampleExplicitLod(SampledType,
15476
macro(__attribute__((opencl_global)), Arg) \
15577
macro(__attribute__((opencl_local)), Arg)
15678

157-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_FLOAT, float)
158-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_FLOAT, double)
159-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_SIGNED, int)
160-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_SIGNED, long)
161-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_SIGNED, long long)
162-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_UNSIGNED, unsigned int)
163-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_UNSIGNED, unsigned long)
164-
__SPIRV_ATOMICS(__SPIRV_ATOMIC_UNSIGNED, unsigned long long)
16579
__SPIRV_ATOMICS(__SPIRV_ATOMIC_MINMAX, Min)
16680
__SPIRV_ATOMICS(__SPIRV_ATOMIC_MINMAX, Max)
16781

‎sycl/include/CL/sycl/atomic.hpp

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
#include <CL/__spirv/spirv_ops.hpp>
1212
#include <CL/sycl/access/access.hpp>
13+
#include <CL/sycl/detail/generic_type_traits.hpp>
1314
#include <CL/sycl/detail/helpers.hpp>
1415

1516
#ifndef __SYCL_DEVICE_ONLY__
@@ -215,16 +216,42 @@ class atomic {
215216
Ptr = RHS.Ptr;
216217
}
217218

219+
#ifdef __SYCL_DEVICE_ONLY__
220+
template <typename T2 = T>
221+
detail::enable_if_t<!std::is_same<cl_float, T2>::value, T>
222+
store(T2 Operand, memory_order Order = memory_order::relaxed) {
223+
using I = detail::atomic_integer_arg_t<T>;
224+
auto *TempPtr =
225+
reinterpret_cast<typename multi_ptr<I, addressSpace>::pointer_t>(Ptr);
226+
auto SPIRVOrder = detail::getSPIRVMemorySemanticsMask(Order);
227+
__spirv_AtomicStore(TempPtr, SpirvScope, SPIRVOrder, (I)Operand);
228+
}
229+
230+
template <typename T2 = T>
231+
detail::enable_if_t<std::is_same<cl_float, T2>::value, T>
232+
store(T2 Operand, memory_order Order = memory_order::relaxed) {
233+
using I = detail::make_unsinged_integer_t<T>;
234+
auto *TempPtr =
235+
reinterpret_cast<typename multi_ptr<I, addressSpace>::pointer_t>(Ptr);
236+
auto SPIRVOrder = detail::getSPIRVMemorySemanticsMask(Order);
237+
I ValueInt = detail::bit_cast<I>(Operand);
238+
__spirv_AtomicStore(TempPtr, SpirvScope, SPIRVOrder, ValueInt);
239+
}
240+
#else
218241
void store(T Operand, memory_order Order = memory_order::relaxed) {
219242
__spirv_AtomicStore(
220243
Ptr, SpirvScope, detail::getSPIRVMemorySemanticsMask(Order), Operand);
221244
}
245+
#endif
222246

223247
#ifdef __SYCL_DEVICE_ONLY__
224248
template <typename T2 = T>
225249
detail::enable_if_t<!std::is_same<cl_float, T2>::value, T>
226250
load(memory_order Order = memory_order::relaxed) const {
227-
return __spirv_AtomicLoad(Ptr, SpirvScope,
251+
using I = detail::make_unsinged_integer_t<T>;
252+
auto *TempPtr =
253+
reinterpret_cast<typename multi_ptr<I, addressSpace>::pointer_t>(Ptr);
254+
return __spirv_AtomicLoad(TempPtr, SpirvScope,
228255
detail::getSPIRVMemorySemanticsMask(Order));
229256
}
230257
template <typename T2 = T>

‎sycl/include/CL/sycl/detail/generic_type_traits.hpp

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -625,6 +625,29 @@ template <typename... Args> inline void check_vector_size() {
625625
"with the same number of elements.");
626626
}
627627

628+
template <std::size_t Size, bool IsSigned> struct atomic_integer_arg_by_size {};
629+
630+
template <> struct atomic_integer_arg_by_size<4, true> {
631+
using type = int32_t;
632+
};
633+
634+
template <> struct atomic_integer_arg_by_size<8, true> {
635+
using type = int64_t;
636+
};
637+
638+
template <> struct atomic_integer_arg_by_size<4, false> {
639+
using type = uint32_t;
640+
};
641+
642+
template <> struct atomic_integer_arg_by_size<8, false> {
643+
using type = uint64_t;
644+
};
645+
646+
template <typename T>
647+
using atomic_integer_arg_t =
648+
typename atomic_integer_arg_by_size<sizeof(T),
649+
std::is_signed<T>::value>::type;
650+
628651
} // namespace detail
629652
} // namespace sycl
630653
} // __SYCL_INLINE_NAMESPACE(cl)

‎sycl/include/CL/sycl/detail/spirv.hpp

Lines changed: 46 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,12 @@ AtomicCompareExchange(multi_ptr<T, AddressSpace> MPtr,
122122
auto SPIRVSuccess = getMemorySemanticsMask(Success);
123123
auto SPIRVFailure = getMemorySemanticsMask(Failure);
124124
auto SPIRVScope = getScope(Scope);
125-
auto *Ptr = MPtr.get();
126-
return __spirv_AtomicCompareExchange(Ptr, SPIRVScope, SPIRVSuccess,
127-
SPIRVFailure, Desired, Expected);
125+
using I = detail::atomic_integer_arg_t<T>;
126+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
127+
MPtr.get());
128+
129+
return (T)__spirv_AtomicCompareExchange(
130+
Ptr, SPIRVScope, SPIRVSuccess, SPIRVFailure, (I)Desired, (I)Expected);
128131
}
129132

130133
template <typename T, access::address_space AddressSpace>
@@ -150,10 +153,12 @@ template <typename T, access::address_space AddressSpace>
150153
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
151154
AtomicLoad(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
152155
intel::memory_order Order) {
153-
auto *Ptr = MPtr.get();
156+
using I = detail::atomic_integer_arg_t<T>;
157+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
158+
MPtr.get());
154159
auto SPIRVOrder = getMemorySemanticsMask(Order);
155160
auto SPIRVScope = getScope(Scope);
156-
return __spirv_AtomicLoad(Ptr, SPIRVScope, SPIRVOrder);
161+
return (T)__spirv_AtomicLoad(Ptr, SPIRVScope, SPIRVOrder);
157162
}
158163

159164
template <typename T, access::address_space AddressSpace>
@@ -174,10 +179,12 @@ template <typename T, access::address_space AddressSpace>
174179
inline typename detail::enable_if_t<std::is_integral<T>::value>
175180
AtomicStore(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
176181
intel::memory_order Order, T Value) {
177-
auto *Ptr = MPtr.get();
182+
using I = detail::atomic_integer_arg_t<T>;
183+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
184+
MPtr.get());
178185
auto SPIRVOrder = getMemorySemanticsMask(Order);
179186
auto SPIRVScope = getScope(Scope);
180-
__spirv_AtomicStore(Ptr, SPIRVScope, SPIRVOrder, Value);
187+
__spirv_AtomicStore(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
181188
}
182189

183190
template <typename T, access::address_space AddressSpace>
@@ -198,10 +205,12 @@ template <typename T, access::address_space AddressSpace>
198205
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
199206
AtomicExchange(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
200207
intel::memory_order Order, T Value) {
201-
auto *Ptr = MPtr.get();
208+
using I = detail::atomic_integer_arg_t<T>;
209+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
210+
MPtr.get());
202211
auto SPIRVOrder = getMemorySemanticsMask(Order);
203212
auto SPIRVScope = getScope(Scope);
204-
return __spirv_AtomicExchange(Ptr, SPIRVScope, SPIRVOrder, Value);
213+
return (T)__spirv_AtomicExchange(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
205214
}
206215

207216
template <typename T, access::address_space AddressSpace>
@@ -224,70 +233,84 @@ template <typename T, access::address_space AddressSpace>
224233
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
225234
AtomicIAdd(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
226235
intel::memory_order Order, T Value) {
227-
auto *Ptr = MPtr.get();
236+
using I = detail::atomic_integer_arg_t<T>;
237+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
238+
MPtr.get());
228239
auto SPIRVOrder = getMemorySemanticsMask(Order);
229240
auto SPIRVScope = getScope(Scope);
230-
return __spirv_AtomicIAdd(Ptr, SPIRVScope, SPIRVOrder, Value);
241+
return (T)__spirv_AtomicIAdd(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
231242
}
232243

233244
template <typename T, access::address_space AddressSpace>
234245
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
235246
AtomicISub(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
236247
intel::memory_order Order, T Value) {
237-
auto *Ptr = MPtr.get();
248+
using I = detail::atomic_integer_arg_t<T>;
249+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
250+
MPtr.get());
238251
auto SPIRVOrder = getMemorySemanticsMask(Order);
239252
auto SPIRVScope = getScope(Scope);
240-
return __spirv_AtomicISub(Ptr, SPIRVScope, SPIRVOrder, Value);
253+
return (T)__spirv_AtomicISub(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
241254
}
242255

243256
template <typename T, access::address_space AddressSpace>
244257
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
245258
AtomicAnd(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
246259
intel::memory_order Order, T Value) {
247-
auto *Ptr = MPtr.get();
260+
using I = detail::atomic_integer_arg_t<T>;
261+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
262+
MPtr.get());
248263
auto SPIRVOrder = getMemorySemanticsMask(Order);
249264
auto SPIRVScope = getScope(Scope);
250-
return __spirv_AtomicAnd(Ptr, SPIRVScope, SPIRVOrder, Value);
265+
return (T)__spirv_AtomicAnd(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
251266
}
252267

253268
template <typename T, access::address_space AddressSpace>
254269
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
255270
AtomicOr(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
256271
intel::memory_order Order, T Value) {
257-
auto *Ptr = MPtr.get();
272+
using I = detail::atomic_integer_arg_t<T>;
273+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
274+
MPtr.get());
258275
auto SPIRVOrder = getMemorySemanticsMask(Order);
259276
auto SPIRVScope = getScope(Scope);
260-
return __spirv_AtomicOr(Ptr, SPIRVScope, SPIRVOrder, Value);
277+
return (T)__spirv_AtomicOr(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
261278
}
262279

263280
template <typename T, access::address_space AddressSpace>
264281
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
265282
AtomicXor(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
266283
intel::memory_order Order, T Value) {
267-
auto *Ptr = MPtr.get();
284+
using I = detail::atomic_integer_arg_t<T>;
285+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
286+
MPtr.get());
268287
auto SPIRVOrder = getMemorySemanticsMask(Order);
269288
auto SPIRVScope = getScope(Scope);
270-
return __spirv_AtomicXor(Ptr, SPIRVScope, SPIRVOrder, Value);
289+
return (T)__spirv_AtomicXor(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
271290
}
272291

273292
template <typename T, access::address_space AddressSpace>
274293
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
275294
AtomicMin(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
276295
intel::memory_order Order, T Value) {
277-
auto *Ptr = MPtr.get();
296+
using I = detail::atomic_integer_arg_t<T>;
297+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
298+
MPtr.get());
278299
auto SPIRVOrder = getMemorySemanticsMask(Order);
279300
auto SPIRVScope = getScope(Scope);
280-
return __spirv_AtomicMin(Ptr, SPIRVScope, SPIRVOrder, Value);
301+
return (T)__spirv_AtomicMin(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
281302
}
282303

283304
template <typename T, access::address_space AddressSpace>
284305
inline typename detail::enable_if_t<std::is_integral<T>::value, T>
285306
AtomicMax(multi_ptr<T, AddressSpace> MPtr, intel::memory_scope Scope,
286307
intel::memory_order Order, T Value) {
287-
auto *Ptr = MPtr.get();
308+
using I = detail::atomic_integer_arg_t<T>;
309+
auto *Ptr = reinterpret_cast<typename multi_ptr<I, AddressSpace>::pointer_t>(
310+
MPtr.get());
288311
auto SPIRVOrder = getMemorySemanticsMask(Order);
289312
auto SPIRVScope = getScope(Scope);
290-
return __spirv_AtomicMax(Ptr, SPIRVScope, SPIRVOrder, Value);
313+
return (T)__spirv_AtomicMax(Ptr, SPIRVScope, SPIRVOrder, (I)Value);
291314
}
292315

293316
// Native shuffles map directly to a SPIR-V SubgroupShuffle intrinsic

0 commit comments

Comments
 (0)