Avoid unnecessary calls to cuFuncSetAttribute

Calling cuFuncSetAttribute to set
CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES is required to
launch kernels using more than 48 kB of local memory[1] (CUDA
dynamic shared memory). Without this, cuLaunchKernel fails with
CUDA_ERROR_INVALID_VALUE. However, calling cuFuncSetAttribute
introduces synchronisation in the CUDA runtime which blocks its
execution until all H2D/D2H memory copies are finished (don't know
why), therefore effectively blocking kernel launches from
overlapping with memory copies. This introduces significant
performance degradation in some workflows, specifically in
applications launching overlapping memory copies and kernels from
multiple host threads into multiple CUDA streams to the same GPU.

Avoid the CUDA runtime synchronisation causing poor performance by
removing the cuFuncSetAttribute call unless it's strictly
necessary. Call it only when a specific carveout is requested by
user (using env variables) or when the kernel launch would fail
without it (local memory size >48kB). Good performance is recovered
for default settings with kernels using little or no local memory.

No performance effects were observed for kernel execution time
after removing the attribute across a wide range of tested kernels
using various amounts of local memory.

[1] Related to the 48 kB static shared memory limit, see the
footnote for "Maximum amount of shared memory per thread block" in
https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#features-and-technical-specifications-technical-specifications-per-compute-capability

Author: rafbiels

sycl/cmake/modules/FetchUnifiedRuntime.cmake

@@ -75,7 +75,19 @@ cmake_path(NORMAL_PATH UR_INTREE_SOURCE_DIR OUTPUT_VARIABLE UR_INTREE_SOURCE_DIR
 
 if(IS_DIRECTORY "${UR_INTREE_SOURCE_DIR}")
   set(UR_INTREE_BINARY_DIR ${LLVM_BINARY_DIR}/unified-runtime)
-  add_subdirectory(${UR_INTREE_SOURCE_DIR} ${UR_INTREE_BINARY_DIR})
+  set(UNIFIED_RUNTIME_SOURCE_DIR
+    "${UR_INTREE_SOURCE_DIR}" CACHE PATH
+    "Path to Unified Runtime Headers" FORCE)
+  set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "EXAMPLES")
+  # Due to the use of dependentloadflag and no installer for UMF and hwloc we need
+  # to link statically on windows
+  if(WIN32)
+    set(UMF_BUILD_SHARED_LIBRARY OFF CACHE INTERNAL "Build UMF shared library")
+    set(UMF_LINK_HWLOC_STATICALLY ON CACHE INTERNAL "static HWLOC")
+  else()
+    set(UMF_DISABLE_HWLOC ${SYCL_UMF_DISABLE_HWLOC} CACHE INTERNAL "Disable hwloc for UMF")
+  endif()
+  add_subdirectory(${UNIFIED_RUNTIME_SOURCE_DIR} ${UR_INTREE_BINARY_DIR})
 elseif(SYCL_UR_USE_FETCH_CONTENT)
   include(FetchContent)
 
@@ -122,7 +134,7 @@ elseif(SYCL_UR_USE_FETCH_CONTENT)
       CACHE PATH "Path to external '${name}' adapter source dir" FORCE)
   endfunction()
 
-  set(UNIFIED_RUNTIME_REPO "https://github.com/rafbiels/unified-runtime.git")
+  set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git")
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/UnifiedRuntimeTag.cmake)
 
   set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "EXAMPLES")

sycl/cmake/modules/UnifiedRuntimeTag.cmake

@@ -1,5 +1,7 @@
-# commit 4ae15957e0c7575cde45d9d3ae01bd334e052e95
-# Author: Rafal Bielski <[email protected]>
-# Date:   Fri Feb 7 19:22:22 2025 +0000
-#    Avoid unnecessary calls to cuFuncSetAttribute
-set(UNIFIED_RUNTIME_TAG cuda-avoid-cuFuncSetAttribute)
+# commit d03f19a88e42cb98be9604ff24b61190d1e48727
+# Merge: 3ce6fcc9 84454b0e
+# Author: Kenneth Benzie (Benie) <[email protected]>
+# Date:   Thu Feb 13 11:43:34 2025 +0000
+#     Merge pull request #2680 from ldorau/Set_UMF_CUDA_INCLUDE_DIR_to_not_fetch_cudart_from_gitlab
+#     Do not fetch cudart from gitlab for UMF
+set(UNIFIED_RUNTIME_TAG d03f19a88e42cb98be9604ff24b61190d1e48727)

unified-runtime/source/adapters/cuda/enqueue.cpp

@@ -290,7 +290,9 @@ setKernelParams([[maybe_unused]] const ur_context_handle_t Context,
           CuFunc, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES,
           Device->getMaxChosenLocalMem()));
 
-    } else {
+    } else if (LocalSize > 48 * 1024) {
+      // CUDA requires explicit carveout of dynamic shared memory size if larger
+      // than 48 kB, otherwise cuLaunchKernel fails.
       UR_CHECK_ERROR(cuFuncSetAttribute(
           CuFunc, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, LocalSize));
     }