[SYCL] Implementation of fallback assert (#3767)

Signed-off-by: Sergey Kanaev <[email protected]>

Author: sergei

libdevice/atomic.hpp

@@ -0,0 +1,93 @@
+//==-------------- atomic.hpp - support of atomic operations ---------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include <cstdint>
+
+#include "device.h"
+
+#ifdef __SPIR__
+
+#define SPIR_GLOBAL __attribute__((opencl_global))
+
+namespace __spv {
+struct Scope {
+
+  enum Flag : uint32_t {
+    CrossDevice = 0,
+    Device = 1,
+    Workgroup = 2,
+    Subgroup = 3,
+    Invocation = 4,
+  };
+
+  constexpr Scope(Flag flag) : flag_value(flag) {}
+
+  constexpr operator uint32_t() const { return flag_value; }
+
+  Flag flag_value;
+};
+
+struct MemorySemanticsMask {
+
+  enum Flag : uint32_t {
+    None = 0x0,
+    Acquire = 0x2,
+    Release = 0x4,
+    AcquireRelease = 0x8,
+    SequentiallyConsistent = 0x10,
+    UniformMemory = 0x40,
+    SubgroupMemory = 0x80,
+    WorkgroupMemory = 0x100,
+    CrossWorkgroupMemory = 0x200,
+    AtomicCounterMemory = 0x400,
+    ImageMemory = 0x800,
+  };
+
+  constexpr MemorySemanticsMask(Flag flag) : flag_value(flag) {}
+
+  constexpr operator uint32_t() const { return flag_value; }
+
+  Flag flag_value;
+};
+} // namespace __spv
+
+extern DEVICE_EXTERNAL int
+__spirv_AtomicCompareExchange(int SPIR_GLOBAL *, __spv::Scope::Flag,
+                              __spv::MemorySemanticsMask::Flag,
+                              __spv::MemorySemanticsMask::Flag, int, int);
+
+extern DEVICE_EXTERNAL int __spirv_AtomicLoad(const int SPIR_GLOBAL *,
+                                              __spv::Scope::Flag,
+                                              __spv::MemorySemanticsMask::Flag);
+
+extern DEVICE_EXTERNAL void
+__spirv_AtomicStore(int SPIR_GLOBAL *, __spv::Scope::Flag,
+                    __spv::MemorySemanticsMask::Flag, int);
+
+/// Atomically set the value in *Ptr with Desired if and only if it is Expected
+/// Return the value which already was in *Ptr
+static inline int atomicCompareAndSet(SPIR_GLOBAL int *Ptr, int Desired,
+                                      int Expected) {
+  return __spirv_AtomicCompareExchange(
+      Ptr, __spv::Scope::Device,
+      __spv::MemorySemanticsMask::SequentiallyConsistent,
+      __spv::MemorySemanticsMask::SequentiallyConsistent, Desired, Expected);
+}
+
+static inline int atomicLoad(SPIR_GLOBAL int *Ptr) {
+  return __spirv_AtomicLoad(Ptr, __spv::Scope::Device,
+                            __spv::MemorySemanticsMask::SequentiallyConsistent);
+}
+
+static inline void atomicStore(SPIR_GLOBAL int *Ptr, int V) {
+  __spirv_AtomicStore(Ptr, __spv::Scope::Device,
+                      __spv::MemorySemanticsMask::SequentiallyConsistent, V);
+}
+
+#endif // __SPIR__

libdevice/fallback-cassert.cpp

@@ -6,26 +6,93 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "atomic.hpp"
+#include "include/assert-happened.hpp"
 #include "wrapper.h"
 
 #ifdef __SPIR__
-static const __attribute__((opencl_constant)) char assert_fmt[] =
-    "%s:%d: %s: global id: [%lu,%lu,%lu], local id: [%lu,%lu,%lu] "
-    "Assertion `%s` failed.\n";
+
+#define ASSERT_NONE 0
+#define ASSERT_START 1
+#define ASSERT_FINISH 2
+
+// definition
+SPIR_GLOBAL AssertHappened SPIR_AssertHappenedMem;
+
+DEVICE_EXTERN_C void __devicelib_assert_read(void *_Dst) {
+  AssertHappened *Dst = (AssertHappened *)_Dst;
+  int Flag = atomicLoad(&SPIR_AssertHappenedMem.Flag);
+
+  if (ASSERT_NONE == Flag) {
+    Dst->Flag = Flag;
+    return;
+  }
+
+  if (Flag != ASSERT_FINISH)
+    while (ASSERT_START == atomicLoad(&SPIR_AssertHappenedMem.Flag))
+      ;
+
+  *Dst = SPIR_AssertHappenedMem;
+}
 
 DEVICE_EXTERN_C void __devicelib_assert_fail(const char *expr, const char *file,
                                              int32_t line, const char *func,
                                              uint64_t gid0, uint64_t gid1,
                                              uint64_t gid2, uint64_t lid0,
                                              uint64_t lid1, uint64_t lid2) {
-  // intX_t types are used instead of `int' and `long' because the format string
-  // is defined in terms of *device* types (OpenCL types): %d matches a 32 bit
-  // integer, %lu matches a 64 bit unsigned integer. Host `int' and
-  // `long' types may be different, so we cannot use them.
-  __spirv_ocl_printf(assert_fmt, file, (int32_t)line,
-                     // WORKAROUND: IGC does not handle this well
-                     // (func) ? func : "<unknown function>",
-                     func, gid0, gid1, gid2, lid0, lid1, lid2, expr);
+  int Expected = ASSERT_NONE;
+  int Desired = ASSERT_START;
+
+  if (atomicCompareAndSet(&SPIR_AssertHappenedMem.Flag, Desired, Expected) ==
+      Expected) {
+    SPIR_AssertHappenedMem.Line = line;
+    SPIR_AssertHappenedMem.GID0 = gid0;
+    SPIR_AssertHappenedMem.GID1 = gid1;
+    SPIR_AssertHappenedMem.GID2 = gid2;
+    SPIR_AssertHappenedMem.LID0 = lid0;
+    SPIR_AssertHappenedMem.LID1 = lid1;
+    SPIR_AssertHappenedMem.LID2 = lid2;
+
+    int ExprLength = 0;
+    int FileLength = 0;
+    int FuncLength = 0;
+
+    if (expr)
+      for (const char *C = expr; *C != '\0'; ++C, ++ExprLength)
+        ;
+    if (file)
+      for (const char *C = file; *C != '\0'; ++C, ++FileLength)
+        ;
+    if (func)
+      for (const char *C = func; *C != '\0'; ++C, ++FuncLength)
+        ;
+
+    int MaxExprIdx = sizeof(SPIR_AssertHappenedMem.Expr) - 1;
+    int MaxFileIdx = sizeof(SPIR_AssertHappenedMem.File) - 1;
+    int MaxFuncIdx = sizeof(SPIR_AssertHappenedMem.Func) - 1;
+
+    if (ExprLength < MaxExprIdx)
+      MaxExprIdx = ExprLength;
+    if (FileLength < MaxFileIdx)
+      MaxFileIdx = FileLength;
+    if (FuncLength < MaxFuncIdx)
+      MaxFuncIdx = FuncLength;
+
+    for (int Idx = 0; Idx < MaxExprIdx; ++Idx)
+      SPIR_AssertHappenedMem.Expr[Idx] = expr[Idx];
+    SPIR_AssertHappenedMem.Expr[MaxExprIdx] = '\0';
+
+    for (int Idx = 0; Idx < MaxFileIdx; ++Idx)
+      SPIR_AssertHappenedMem.File[Idx] = file[Idx];
+    SPIR_AssertHappenedMem.File[MaxFileIdx] = '\0';
+
+    for (int Idx = 0; Idx < MaxFuncIdx; ++Idx)
+      SPIR_AssertHappenedMem.Func[Idx] = func[Idx];
+    SPIR_AssertHappenedMem.Func[MaxFuncIdx] = '\0';
+
+    // Show we've done copying
+    atomicStore(&SPIR_AssertHappenedMem.Flag, ASSERT_FINISH);
+  }
 
   // FIXME: call SPIR-V unreachable instead
   // volatile int *die = (int *)0x0;

libdevice/include/assert-happened.hpp

@@ -0,0 +1,48 @@
+//==-- assert-happened.hpp - Structure and declaration for assert support --==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+// Treat this header as system one to workaround frontend's restriction
+#pragma clang system_header
+
+#ifdef __SPIR__
+
+// NOTE Layout of this structure should be aligned with the one in
+// sycl/include/CL/sycl/detail/assert_happened.hpp
+struct AssertHappened {
+  int Flag = 0;
+  char Expr[256 + 1] = "";
+  char File[256 + 1] = "";
+  char Func[128 + 1] = "";
+
+  int32_t Line = 0;
+
+  uint64_t GID0 = 0;
+  uint64_t GID1 = 0;
+  uint64_t GID2 = 0;
+
+  uint64_t LID0 = 0;
+  uint64_t LID1 = 0;
+  uint64_t LID2 = 0;
+};
+
+#ifndef SPIR_GLOBAL_VAR
+#ifdef __SYCL_DEVICE_ONLY__
+#define SPIR_GLOBAL_VAR __attribute__((sycl_global_var))
+#else
+#warning "SPIR_GLOBAL_VAR not defined in host mode. Defining as empty macro."
+#define SPIR_GLOBAL_VAR
+#endif
+#endif
+
+#define __SYCL_GLOBAL__ __attribute__((opencl_global))
+
+// declaration
+extern SPIR_GLOBAL_VAR __SYCL_GLOBAL__ AssertHappened SPIR_AssertHappenedMem;
+
+#endif

sycl/cmake/modules/AddSYCLUnitTest.cmake

@@ -21,7 +21,8 @@ macro(add_sycl_unittest test_dirname link_variant)
   else()
     add_unittest(SYCLUnitTests ${test_dirname}
                 $<TARGET_OBJECTS:${sycl_obj_target}> ${ARGN})
-    target_compile_definitions(${test_dirname} PRIVATE __SYCL_BUILD_SYCL_DLL)
+    target_compile_definitions(${test_dirname}
+                               PRIVATE __SYCL_BUILD_SYCL_DLL)
 
     get_target_property(SYCL_LINK_LIBS ${sycl_so_target} LINK_LIBRARIES)
   endif()

sycl/doc/Assert.md

@@ -149,10 +149,9 @@ The following sequence of events describes how user code gets notified:
    2. A host-task is enqueued to check value of assert failure flag.
    3. The host task calls abort whenever assert failure flag is set.
 
-DPCPP Runtime will automatically check if assertions are enabled in the kernel
+DPCPP Runtime will automatically check if assertions are used in the kernel
 being run, and won't enqueue the auxiliary kernels if assertions are not
-enabled. So there is no host-side runtime overhead when assertion are not
-enabled.
+used. So there is no host-side runtime overhead when assertion are not used.
 
 Illustrating this with an example, lets assume the user enqueues three kernels:
  - `Kernel #1`, uses assert
@@ -172,18 +171,25 @@ same binary image where fallback `__devicelib_assert_fail` resides.
 declaration:</a>
 
 ```c++
-namespace cl {
-namespace sycl {
-namespace detail {
-struct AssertHappened {
+struct __SYCL_AssertHappened {
   int Flag = 0;
+  char Expr[256 + 1] = "";
+  char File[256 + 1] = "";
+  char Func[128 + 1] = "";
+
+  int32_t Line = 0;
+
+  uint64_t GID0 = 0;
+  uint64_t GID1 = 0;
+  uint64_t GID2 = 0;
+
+  uint64_t LID0 = 0;
+  uint64_t LID1 = 0;
+  uint64_t LID2 = 0;
 };
-}
-}
-}
 
 #ifdef __SYCL_DEVICE_ONLY__
-extern SYCL_GLOBAL_VAR AssertHappened AssertHappenedMem;
+extern SYCL_GLOBAL_VAR __SYCL_AssertHappened __SYCL_AssertHappenedMem;
 #endif
 ```
 
@@ -193,6 +199,28 @@ mutable program-scope variable.
 The reference to extern variable is resolved within online-linking against
 fallback devicelib.
 
+#### Description of fields
+
+The value stored here denotes if assert happened at all. There are two valid
+values at host:
+
+| Value | Meaning |
+| ----- | ------- |
+| 0     | No assert failure detected |
+| 2     | Assert failure detected and reported within this instance of struct |
+
+At device-side, there's another valid value: 1, which means that assert failure
+is detected and the structure is filling up at the moment. This value is for
+device-side only and should never be reported to host. Otherwise, it means, that
+atomic operation malfunctioned.
+
+`Expr`, `File`, `Func`, `Line` are to describe the assert message itself and
+contain the expression, file name, function name, line in the file where assert
+failure had happened respectively.
+
+`GID*` and `LID*` fields describe the global and local ID respectively of a
+work-item in which assert had failed.
+
 ### Online-linking fallback `__devicelib_assert_fail`
 
 Online linking against fallback implementation of `__devicelib_assert_fail` is

sycl/doc/PreprocessorMacros.md

@@ -33,6 +33,17 @@ This file describes macros that have effect on SYCL compiler and run-time.
 
   Disables all deprecation warnings in SYCL runtime headers, including SYCL 1.2.1 deprecations.
 
+- **SYCL_DISABLE_FALLBACK_ASSERT**
+
+  Defining this macro eliminates some overhead that is associated with
+  submitting kernels that call `assert()`. When this macro is defined, the logic
+  for detecting assertion failures in kernels is disabled, so a failed assert
+  will not cause a message to be printed and will not cause the program to
+  abort. However, this macro only affects kernels that are submitted to devices
+  that do **not** have native support for `assert()` because devices with native
+  support do not impose any extra overhead. One can check to see if a device has
+  native support for `assert()` via `aspect::ext_oneapi_native_assert`.
+
 ## Version macros
 
 - `__LIBSYCL_MAJOR_VERSION` is set to SYCL runtime library major version.

sycl/doc/extensions/C-CXX-StandardLibrary/DeviceLibExtensions.rst

@@ -14,13 +14,17 @@ cl_intel_devicelib_cassert
                                 __generic const char *func,
                                 size_t gid0, size_t gid1, size_t gid2,
                                 size_t lid0, size_t lid1, size_t lid2);
+
 Semantic:
 the function is called when an assertion expression `expr` is false,
 and it indicates that a program does not execute as expected.
 The function should print a message containing the information
 provided in the arguments. In addition to that, the function is free
 to terminate the current kernel invocation.
 
+Fallback implementation of the function raises a flag to be read later by `__devicelib_assert_read`.
+The flag remains raised until the program finishes.
+
 Arguments:
 
   - `expr` is a string representation of the assertion condition
@@ -33,6 +37,16 @@ Example of a message:
 .. code:
    foo.cpp:42: void foo(int): global id: [0,0,0], local id: [0,0,0] Assertion `buf[wiID] == 0 && "Invalid value"` failed.
 
+.. code:
+   int __devicelib_assert_read();
+
+Semantic:
+the function is called to read assert failure flag raised by
+`__devicelib_assert_fail`.
+The function is only used in fallback implementation.
+Invoking `__devicelib_assert_read` after a kernel doesn't imply the kernel has
+assertion failed.
+
 See also: assert_extension_.
 .. _assert_extension: ../Assert/SYCL_ONEAPI_ASSERT.asciidoc)
 

sycl/include/CL/sycl/aspects.hpp

@@ -46,6 +46,7 @@ enum class aspect {
   atomic64 = 28,
   ext_intel_device_info_uuid = 29,
   ext_oneapi_srgb = 30,
+  ext_oneapi_native_assert = 31,
 };
 
 } // namespace sycl