[SYCL][CUDA][libclc] Added atomics with scopes and memory orders

libclc/ptx-nvidiacl/libspirv/SOURCES

@@ -85,5 +85,13 @@ images/image_helpers.ll
 images/image.cl
 group/collectives_helpers.ll
 group/collectives.cl
-SPV_EXT_shader_atomic_float_add/atomicfaddext.cl
-SPV_EXT_shader_atomic_float_add/faddext_helpers.ll
+atomic/atomic_add.cl
+atomic/atomic_and.cl
+atomic/atomic_cmpxchg.cl
+atomic/atomic_dec.cl
+atomic/atomic_inc.cl
+atomic/atomic_max.cl
+atomic/atomic_min.cl
+atomic/atomic_xchg.cl
+atomic/atomic_or.cl
+atomic/atomic_xor.cl

libclc/ptx-nvidiacl/libspirv/atomic/atomic_add.cl

@@ -0,0 +1,25 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <spirv/spirv.h>
+#include <spirv/spirv_types.h>
+#include <atomic_helpers.h>
+
+__CLC_NVVM_ATOMIC(int, i, int, i, add, _Z18__spirv_AtomicIAdd)
+__CLC_NVVM_ATOMIC(uint, j, int, i, add, _Z18__spirv_AtomicIAdd)
+__CLC_NVVM_ATOMIC(long, l, long, l, add, _Z18__spirv_AtomicIAdd)
+__CLC_NVVM_ATOMIC(ulong, m, long, l, add, _Z18__spirv_AtomicIAdd)
+
+__CLC_NVVM_ATOMIC(float, f, float, f, add, _Z21__spirv_AtomicFAddEXT)
+#ifdef cl_khr_int64_base_atomics
+__CLC_NVVM_ATOMIC(double, d, double, d, add, _Z21__spirv_AtomicFAddEXT)
+#endif
+
+#undef __CLC_NVVM_ATOMIC_TYPES
+#undef __CLC_NVVM_ATOMIC
+#undef __CLC_NVVM_ATOMIC_IMPL

libclc/ptx-nvidiacl/libspirv/atomic/atomic_and.cl

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <spirv/spirv.h>
+#include <spirv/spirv_types.h>
+#include <atomic_helpers.h>
+
+__CLC_NVVM_ATOMIC(int, i, int, i, and, _Z17__spirv_AtomicAnd)
+__CLC_NVVM_ATOMIC(long, l, long, l, and, _Z17__spirv_AtomicAnd)
+__CLC_NVVM_ATOMIC(unsigned int, j, int, i, and, _Z17__spirv_AtomicAnd)
+__CLC_NVVM_ATOMIC(unsigned long, m, long, l, and, _Z17__spirv_AtomicAnd)
+
+#undef __CLC_NVVM_ATOMIC_TYPES
+#undef __CLC_NVVM_ATOMIC
+#undef __CLC_NVVM_ATOMIC_IMPL

libclc/ptx-nvidiacl/libspirv/atomic/atomic_cmpxchg.cl

@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <spirv/spirv.h>
+#include <spirv/spirv_types.h>
+
+#define __CLC_NVVM_ATOMIC_CAS_IMPL_ORDER(TYPE, TYPE_NV, TYPE_MANGLED_NV, OP, ADDR_SPACE, ADDR_SPACE_NV, ORDER) \
+switch(scope){ \
+	case Subgroup: \
+	case Workgroup: { \
+		  TYPE_NV res = __nvvm_atom_cta##ORDER##_##OP##ADDR_SPACE_NV##TYPE_MANGLED_NV((ADDR_SPACE TYPE_NV*)pointer, *(TYPE_NV*)&value, cmp);  \
+		  return *(TYPE*)&res; \
+	} \
+	case Device: { \
+		  TYPE_NV res = __nvvm_atom##ORDER##_##OP##ADDR_SPACE_NV##TYPE_MANGLED_NV((ADDR_SPACE TYPE_NV*)pointer, *(TYPE_NV*)&value, cmp);  \
+		  return *(TYPE*)&res; \
+	} \
+	case CrossDevice: \
+	default: { \
+		  TYPE_NV res = __nvvm_atom_sys##ORDER##_##OP##ADDR_SPACE_NV##TYPE_MANGLED_NV((ADDR_SPACE TYPE_NV*)pointer, *(TYPE_NV*)&value, cmp);  \
+		  return *(TYPE*)&res; \
+	} \
+}
+
+#define __CLC_NVVM_ATOMIC_CAS_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, OP_MANGLED, ADDR_SPACE, ADDR_SPACE_MANGLED, ADDR_SPACE_NV) \
+_CLC_DECL TYPE _Z29__spirv_Atomic##OP_MANGLED##PU3##ADDR_SPACE_MANGLED##TYPE_MANGLED##N5__spv5Scope4FlagENS1_19MemorySemanticsMask4FlagES5_##TYPE_MANGLED##TYPE_MANGLED( \
+    volatile ADDR_SPACE TYPE *pointer, enum Scope scope, enum MemorySemanticsMask semantics1, enum MemorySemanticsMask semantics2, \
+    TYPE cmp, TYPE value) { \
+		/* Semantics mask may include memory order, storage class and other info   \
+          Memory order is stored in the lowest 5 bits */                          \
+        unsigned int order = (semantics1 | semantics2) & 0x1F; \
+		switch (order) {                                                           \
+			case None:                                                                 \
+			  __CLC_NVVM_ATOMIC_CAS_IMPL_ORDER(TYPE, TYPE_NV, TYPE_MANGLED_NV, OP, ADDR_SPACE, ADDR_SPACE_NV, )  \
+			case Acquire:                                                              \
+			  __CLC_NVVM_ATOMIC_CAS_IMPL_ORDER(TYPE, TYPE_NV, TYPE_MANGLED_NV, OP, ADDR_SPACE, ADDR_SPACE_NV, _acquire)  \
+			case Release:                                                              \
+			  __CLC_NVVM_ATOMIC_CAS_IMPL_ORDER(TYPE, TYPE_NV, TYPE_MANGLED_NV, OP, ADDR_SPACE, ADDR_SPACE_NV, _release)  \
+			default: \
+			case AcquireRelease:                                                       \
+			  __CLC_NVVM_ATOMIC_CAS_IMPL_ORDER(TYPE, TYPE_NV, TYPE_MANGLED_NV, OP, ADDR_SPACE, ADDR_SPACE_NV, _acq_rel)  \
+			}                                                                          \
+}
+
+#define __CLC_NVVM_ATOMIC_CAS(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, OP_MANGLED) \
+__CLC_NVVM_ATOMIC_CAS_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, OP_MANGLED, __global, AS1, _global_) \
+__CLC_NVVM_ATOMIC_CAS_IMPL(TYPE, TYPE_MANGLED, TYPE_NV, TYPE_MANGLED_NV, OP, OP_MANGLED, __local, AS3, _shared_)
+
+__CLC_NVVM_ATOMIC_CAS(int, i, int, i, cas, CompareExchange)
+__CLC_NVVM_ATOMIC_CAS(long, l, long, l, cas, CompareExchange)
+__CLC_NVVM_ATOMIC_CAS(unsigned int, j, int, i, cas, CompareExchange)
+__CLC_NVVM_ATOMIC_CAS(unsigned long, m, long, l, cas, CompareExchange)
+
+#undef __CLC_NVVM_ATOMIC_CAS_IMPL_ORDER
+#undef __CLC_NVVM_ATOMIC_CAS
+#undef __CLC_NVVM_ATOMIC_CAS_IMPL