[SYCL][PTX] Implements missing atomic builtins

libclc/generic/include/spirv/atomic/atomic_decl.inc

@@ -28,6 +28,8 @@ __CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, m, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_
 #ifdef cl_khr_int64_base_atomics
 __CLC_DECLARE_ATOMIC_ADDRSPACE(long, l, __SPIRV_FUNCTION_S, __SPIRV_FUNCTION_S_LEN)
 __CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, m, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_LEN)
+__CLC_DECLARE_ATOMIC_ADDRSPACE(long, x, __SPIRV_FUNCTION_S, __SPIRV_FUNCTION_S_LEN)
+__CLC_DECLARE_ATOMIC_ADDRSPACE(ulong, y, __SPIRV_FUNCTION_U, __SPIRV_FUNCTION_U_LEN)
 #endif
 #endif
 

libclc/generic/include/spirv/atomic/atomic_load.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
+#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
+_CLC_DECL TYPE _Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE( \
+    volatile AS const TYPE *, enum Scope, enum MemorySemanticsMask);
+
+#define DECL_AS(TYPE, TYPE_MANGLED) \
+DECL(TYPE, TYPE_MANGLED, global, AS1) \
+DECL(TYPE, TYPE_MANGLED, local, AS3)
+
+DECL_AS(int, i)
+DECL_AS(unsigned int, j)
+
+#ifdef cl_khr_int64_base_atomics
+DECL_AS(long, l)
+DECL_AS(unsigned long, m)
+DECL_AS(long, x)
+DECL_AS(unsigned long, y)
+#endif
+
+#undef DECL_AS
+#undef DECL

libclc/generic/include/spirv/atomic/atomic_store.h

@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
+#define DECL(TYPE, TYPE_MANGLED, AS, AS_MANGLED) \
+_CLC_DECL void _Z19__spirv_AtomicStorePU3##AS_MANGLED##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE##TYPE_MANGLED( \
+    volatile AS TYPE *, enum Scope, enum MemorySemanticsMask, TYPE);
+
+#define DECL_AS(TYPE, TYPE_MANGLED) \
+DECL(TYPE, TYPE_MANGLED, global, AS1) \
+DECL(TYPE, TYPE_MANGLED, local, AS3)
+
+DECL_AS(int, i)
+DECL_AS(unsigned int, j)
+
+#ifdef cl_khr_int64_base_atomics
+DECL_AS(long, l)
+DECL_AS(unsigned long, m)
+DECL_AS(long, x)
+DECL_AS(unsigned long, y)
+#endif
+
+#undef DECL_AS
+#undef DECL

libclc/generic/include/spirv/spirv.h

@@ -235,6 +235,8 @@
 #include <spirv/atomic/atomic_sub.h>
 #include <spirv/atomic/atomic_xchg.h>
 #include <spirv/atomic/atomic_xor.h>
+#include <spirv/atomic/atomic_load.h>
+#include <spirv/atomic/atomic_store.h>
 
 /* cl_khr extension atomics are omitted from __spirv */
 

libclc/generic/libspirv/SOURCES

@@ -12,6 +12,8 @@ atomic/atomic_or.cl
 atomic/atomic_sub.cl
 atomic/atomic_xchg.cl
 atomic/atomic_xor.cl
+atomic/atomic_load.cl
+atomic/atomic_store.cl
 common/degrees.cl
 common/mix.cl
 common/radians.cl

libclc/generic/libspirv/atomic/atomic_add.cl

@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_add_8)
 IMPL(unsigned long, m, global, AS1, __sync_fetch_and_add_8)
 IMPL(long, l, local, AS3, __sync_fetch_and_add_8)
 IMPL(unsigned long, m, local, AS3, __sync_fetch_and_add_8)
+IMPL(long, x, global, AS1, __sync_fetch_and_add_8)
+IMPL(unsigned long, y, global, AS1, __sync_fetch_and_add_8)
+IMPL(long, x, local, AS3, __sync_fetch_and_add_8)
+IMPL(unsigned long, y, local, AS3, __sync_fetch_and_add_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_and.cl

@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_and_8)
 IMPL(unsigned long, m, global, AS1, __sync_fetch_and_and_8)
 IMPL(long, l, local, AS3, __sync_fetch_and_and_8)
 IMPL(unsigned long, m, local, AS3, __sync_fetch_and_and_8)
+IMPL(long, x, global, AS1, __sync_fetch_and_and_8)
+IMPL(unsigned long, y, global, AS1, __sync_fetch_and_and_8)
+IMPL(long, x, local, AS3, __sync_fetch_and_and_8)
+IMPL(unsigned long, y, local, AS3, __sync_fetch_and_and_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_cmpxchg.cl

@@ -50,4 +50,24 @@ _CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS1mN5__spv5ScopeENS1_19Memor
     volatile global ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
   return __sync_val_compare_and_swap_8(p, cmp, val);
 }
+
+_CLC_DEF long _Z29__spirv_AtomicCompareExchangePU3AS3xN5__spv5ScopeENS1_19MemorySemanticsMaskES3_xx(
+    volatile local long *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, long val, long cmp) {
+  return __sync_val_compare_and_swap_8(p, cmp, val);
+}
+
+_CLC_DEF long _Z29__spirv_AtomicCompareExchangePU3AS1xN5__spv5ScopeENS1_19MemorySemanticsMaskES3_xx(
+    volatile global long *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, long val, long cmp) {
+  return __sync_val_compare_and_swap_8(p, cmp, val);
+}
+
+_CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS3yN5__spv5ScopeENS1_19MemorySemanticsMaskES3_yy(
+    volatile local ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
+  return __sync_val_compare_and_swap_8(p, cmp, val);
+}
+
+_CLC_DEF ulong _Z29__spirv_AtomicCompareExchangePU3AS1yN5__spv5ScopeENS1_19MemorySemanticsMaskES3_yy(
+    volatile global ulong *p, enum Scope scope, enum MemorySemanticsMask eq, enum MemorySemanticsMask neq, ulong val, ulong cmp) {
+  return __sync_val_compare_and_swap_8(p, cmp, val);
+}
 #endif

libclc/generic/libspirv/atomic/atomic_load.cl

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <spirv/spirv.h>
+
+// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
+
+#define FDECL(TYPE, PREFIX, AS, BYTE_SIZE, MEM_ORDER) \
+TYPE __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *);
+
+#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, PREFIX, BYTE_SIZE) \
+FDECL(TYPE, PREFIX, AS, BYTE_SIZE, unordered) \
+FDECL(TYPE, PREFIX, AS, BYTE_SIZE, acquire) \
+FDECL(TYPE, PREFIX, AS, BYTE_SIZE, seq_cst) \
+_CLC_DEF TYPE _Z18__spirv_AtomicLoadPU3##AS_MANGLED##K##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE( \
+    volatile AS const TYPE *p, enum Scope scope, enum MemorySemanticsMask semantics) { \
+  if (semantics == Acquire) { \
+    return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_acquire(p); \
+  } \
+  if (semantics == SequentiallyConsistent) { \
+      return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_seq_cst(p); \
+  } \
+  return __clc__atomic_##PREFIX##load_##AS##_##BYTE_SIZE##_unordered(p); \
+}
+
+#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
+IMPL(TYPE, TYPE_MANGLED, global, AS1, PREFIX, BYTE_SIZE) \
+IMPL(TYPE, TYPE_MANGLED, local, AS3, PREFIX, BYTE_SIZE)
+
+IMPL_AS(int, i, , 4)
+IMPL_AS(unsigned int, j, u, 4)
+
+#ifdef cl_khr_int64_base_atomics
+IMPL_AS(long, l, , 8)
+IMPL_AS(unsigned long, m, u, 8)
+IMPL_AS(long, x, , 8)
+IMPL_AS(unsigned long, y, u, 8)
+#endif
+
+#undef FDECL
+#undef IMPL_AS
+#undef IMPL

libclc/generic/libspirv/atomic/atomic_max.cl

@@ -31,5 +31,9 @@ IMPL(long, l, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
 IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
 IMPL(long, l, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
 IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
+IMPL(long, x, global, AS1, __spirv_AtomicSMax, __clc, max_global_8)
+IMPL(unsigned long, y, global, AS1, __spirv_AtomicUMax, __clc, umax_global_8)
+IMPL(long, x, local, AS3, __spirv_AtomicSMax, __clc, max_local_8)
+IMPL(unsigned long, y, local, AS3, __spirv_AtomicUMax, __clc, umax_local_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_min.cl

@@ -31,5 +31,9 @@ IMPL(long, l, global, AS1, __spirv_AtomicSMin, __clc, min_global_8)
 IMPL(unsigned long, m, global, AS1, __spirv_AtomicUMin, __clc, umin_global_8)
 IMPL(long, l, local, AS3, __spirv_AtomicSMin, __clc, min_local_8)
 IMPL(unsigned long, m, local, AS3, __spirv_AtomicUMin, __clc, umin_local_8)
+IMPL(long, x, global, AS1, __spirv_AtomicSMin, __clc, min_global_8)
+IMPL(unsigned long, y, global, AS1, __spirv_AtomicUMin, __clc, umin_global_8)
+IMPL(long, x, local, AS3, __spirv_AtomicSMin, __clc, min_local_8)
+IMPL(unsigned long, y, local, AS3, __spirv_AtomicUMin, __clc, umin_local_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_or.cl

@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_or_8)
 IMPL(unsigned long, m, global, AS1, __sync_fetch_and_or_8)
 IMPL(long, l, local, AS3, __sync_fetch_and_or_8)
 IMPL(unsigned long, m, local, AS3, __sync_fetch_and_or_8)
+IMPL(long, x, global, AS1, __sync_fetch_and_or_8)
+IMPL(unsigned long, y, global, AS1, __sync_fetch_and_or_8)
+IMPL(long, x, local, AS3, __sync_fetch_and_or_8)
+IMPL(unsigned long, y, local, AS3, __sync_fetch_and_or_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_store.cl

@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <spirv/spirv.h>
+
+// TODO: Stop manually mangling this name. Need C++ namespaces to get the exact mangling.
+
+_CLC_DEF void _Z19__spirv_AtomicStorePU3AS1fN5__spv5ScopeENS1_19MemorySemanticsMaskEf(
+    volatile global float *p, enum Scope scope, enum MemorySemanticsMask semantics, float val) {
+  _Z19__spirv_AtomicStorePU3AS1jN5__spv5ScopeENS1_19MemorySemanticsMaskEj(
+    (volatile global uint *)p, scope, semantics, as_uint(val));
+}
+
+_CLC_DEF void _Z19__spirv_AtomicStorePU3AS3fN5__spv5ScopeENS1_19MemorySemanticsMaskEf(
+    volatile local float *p, enum Scope scope, enum MemorySemanticsMask semantics, float val) {
+  _Z19__spirv_AtomicStorePU3AS3jN5__spv5ScopeENS1_19MemorySemanticsMaskEj(
+    (volatile local uint *)p, scope, semantics, as_uint(val));
+}
+
+#define FDECL(TYPE, PREFIX, AS, BYTE_SIZE, MEM_ORDER) \
+TYPE __clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_##MEM_ORDER(volatile AS const TYPE *, TYPE);
+
+#define IMPL(TYPE, TYPE_MANGLED, AS, AS_MANGLED, PREFIX, BYTE_SIZE) \
+FDECL(TYPE, PREFIX, AS, BYTE_SIZE, unordered) \
+FDECL(TYPE, PREFIX, AS, BYTE_SIZE, release) \
+FDECL(TYPE, PREFIX, AS, BYTE_SIZE, seq_cst) \
+_CLC_DEF void _Z19__spirv_AtomicStorePU3##AS_MANGLED##TYPE_MANGLED##N5__spv5ScopeENS1_19MemorySemanticsMaskE##TYPE_MANGLED( \
+    volatile AS TYPE *p, enum Scope scope, enum MemorySemanticsMask semantics, TYPE val) { \
+  if (semantics == Release) { \
+    __clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_release(p, val); \
+  } else if (semantics == SequentiallyConsistent) { \
+    __clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_seq_cst(p, val); \
+  } else { \
+    __clc__atomic_##PREFIX##store_##AS##_##BYTE_SIZE##_unordered(p, val); \
+  } \
+}
+
+#define IMPL_AS(TYPE, TYPE_MANGLED, PREFIX, BYTE_SIZE) \
+IMPL(TYPE, TYPE_MANGLED, global, AS1, PREFIX, BYTE_SIZE) \
+IMPL(TYPE, TYPE_MANGLED, local, AS3, PREFIX, BYTE_SIZE)
+
+IMPL_AS(int, i, , 4)
+IMPL_AS(unsigned int, j, u, 4)
+
+#ifdef cl_khr_int64_base_atomics
+IMPL_AS(long, l, , 8)
+IMPL_AS(unsigned long, m, u, 8)
+IMPL_AS(long, x, , 8)
+IMPL_AS(unsigned long, y, u, 8)
+#endif
+
+#undef FDECL
+#undef IMPL_AS
+#undef IMPL

libclc/generic/libspirv/atomic/atomic_sub.cl

@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_sub_8)
 IMPL(unsigned long, m, global, AS1, __sync_fetch_and_sub_8)
 IMPL(long, l, local, AS3, __sync_fetch_and_sub_8)
 IMPL(unsigned long, m, local, AS3, __sync_fetch_and_sub_8)
+IMPL(long, x, global, AS1, __sync_fetch_and_sub_8)
+IMPL(unsigned long, y, global, AS1, __sync_fetch_and_sub_8)
+IMPL(long, x, local, AS3, __sync_fetch_and_sub_8)
+IMPL(unsigned long, y, local, AS3, __sync_fetch_and_sub_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_xchg.cl

@@ -38,5 +38,9 @@ IMPL(long, l, global, AS1, __sync_swap_8)
 IMPL(unsigned long, m, global, AS1, __sync_swap_8)
 IMPL(long, l, local, AS3, __sync_swap_8)
 IMPL(unsigned long, m, local, AS3, __sync_swap_8)
+IMPL(long, x, global, AS1, __sync_swap_8)
+IMPL(unsigned long, y, global, AS1, __sync_swap_8)
+IMPL(long, x, local, AS3, __sync_swap_8)
+IMPL(unsigned long, y, local, AS3, __sync_swap_8)
 #endif
 #undef IMPL

libclc/generic/libspirv/atomic/atomic_xor.cl

@@ -26,5 +26,9 @@ IMPL(long, l, global, AS1, __sync_fetch_and_xor_8)
 IMPL(unsigned long, m, global, AS1, __sync_fetch_and_xor_8)
 IMPL(long, l, local, AS3, __sync_fetch_and_xor_8)
 IMPL(unsigned long, m, local, AS3, __sync_fetch_and_xor_8)
+IMPL(long, x, global, AS1, __sync_fetch_and_xor_8)
+IMPL(unsigned long, y, global, AS1, __sync_fetch_and_xor_8)
+IMPL(long, x, local, AS3, __sync_fetch_and_xor_8)
+IMPL(unsigned long, y, local, AS3, __sync_fetch_and_xor_8)
 #endif
 #undef IMPL

libclc/ptx-nvidiacl/libspirv/SOURCES

@@ -1,3 +1,5 @@
+atomic/loadstore_helpers.ll
+cl_khr_int64_extended_atomics/minmax_helpers.ll
 synchronization/barrier.cl
 workitem/get_global_id.cl
 workitem/get_global_offset.cl