enable building L0 prov in CI on Linux

Author: bratpiorka

.github/workflows/basic.yml

@@ -21,6 +21,7 @@ jobs:
         pool_tracking: ['ON', 'OFF']
         shared_library: ['OFF']
         os_provider: ['ON']
+        level_zero_provider: ['ON']
     runs-on: ubuntu-22.04
     container:
       image: intel/oneapi:latest
@@ -46,15 +47,15 @@ jobs:
         -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
         -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
         -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
-        -DUMF_BUILD_OS_MEMORY_PROVIDER=${{matrix.os_provider}}
+        -DUMF_BUILD_OS_MEMORY_PROVIDER=${{matrix.os_provider}}        
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=${{matrix.level_zero_provider}}      
         -DUMF_ENABLE_POOL_TRACKING=${{matrix.pool_tracking}}
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
         -DUMF_BUILD_EXAMPLES=ON
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
     - name: Build UMF
       run: cmake --build build -j $(nproc)
@@ -86,6 +87,7 @@ jobs:
         pool_tracking: ['ON', 'OFF']
         shared_library: ['OFF']
         os_provider: ['ON']
+        level_zero_provider: ['ON']
         include:
           - os: 'ubuntu-20.04'
             build_type: Release
@@ -119,6 +121,13 @@ jobs:
             compiler: {c: gcc, cxx: g++}
             shared_library: 'OFF'
             os_provider: 'OFF'
+          # test level_zero_provider='OFF'
+          - os: 'ubuntu-22.04'
+            build_type: Release
+            compiler: {c: gcc, cxx: g++}
+            shared_library: 'OFF'
+            os_provider: 'ON'
+            level_zero_provider: 'OFF'
     runs-on: ${{matrix.os}}
 
     steps:
@@ -145,13 +154,13 @@ jobs:
         -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
         -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
         -DUMF_BUILD_OS_MEMORY_PROVIDER=${{matrix.os_provider}}
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=${{matrix.level_zero_provider}}
         -DUMF_ENABLE_POOL_TRACKING=${{matrix.pool_tracking}}
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -187,6 +196,7 @@ jobs:
         pool_tracking: ['ON', 'OFF']
         shared_library: ['OFF']
         os_provider: ['ON']
+        # TODO add level zero provider
         include:
           - os: 'windows-2022'
             build_type: Release
@@ -268,8 +278,7 @@ jobs:
         -DCMAKE_BUILD_TYPE=Release
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
-        -DUMF_ENABLE_POOL_TRACKING=ON        
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
+        -DUMF_ENABLE_POOL_TRACKING=ON
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(sysctl -n hw.logicalcpu)

.github/workflows/benchmarks.yml

@@ -26,6 +26,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y cmake libhwloc-dev libnuma-dev libjemalloc-dev libtbb-dev
 
+      # TODO add GPU benchmarks
       - name: Configure build
         run: >
           cmake
@@ -40,11 +41,11 @@ jobs:
           -DUMF_FORMAT_CODE_STYLE=OFF
           -DUMF_DEVELOPER_MODE=OFF
           -DUMF_BUILD_OS_MEMORY_PROVIDER=ON
+          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
           -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
           -DUMF_ENABLE_POOL_TRACKING=OFF
-          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
       - name: Build UMF
         run: cmake --build ${{github.workspace}}/build -j $(nproc)

.github/workflows/codeql.yml

@@ -75,7 +75,6 @@ jobs:
         -DUMF_DEVELOPER_MODE=ON
         -DUMF_ENABLE_POOL_TRACKING=ON
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=${{matrix.pool_scalable}}
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build --config Release -j

.github/workflows/coverity.yml

@@ -41,7 +41,6 @@ jobs:
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=OFF
         -DUMF_ENABLE_POOL_TRACKING=ON
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
         -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
         -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON

.github/workflows/gpu.yml

@@ -35,9 +35,9 @@ jobs:
           -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
           -DUMF_BUILD_BENCHMARKS=ON
           -DUMF_BUILD_TESTS=ON
+          -DUMF_BUILD_GPU_TESTS=ON
           -DUMF_FORMAT_CODE_STYLE=ON
           -DUMF_DEVELOPER_MODE=ON
-          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_ENABLE_POOL_TRACKING=ON
 

.github/workflows/pr_push.yml

@@ -22,19 +22,22 @@ jobs:
             pool_scalable: 'OFF'
             disjoint: 'OFF'
             jemalloc: 'OFF'
+            level_zero_provider: 'OFF' # TODO enable
           # pure C build (Windows)
           - os: windows-latest
             pool_scalable: 'OFF'
             disjoint: 'OFF'
             jemalloc: 'OFF'
             # Tests' building is off for a pure C build
             extra_build_options: '-DUMF_BUILD_TESTS=OFF'
+            level_zero_provider: 'OFF' # TODO enable
           - os: ubuntu-latest
             pool_scalable: 'ON'
             disjoint: 'ON'
             jemalloc: 'ON'
             # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command
             extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON'
+            level_zero_provider: 'ON'
           # pure C build (Linux)
           - os: ubuntu-latest
             pool_scalable: 'ON'
@@ -43,6 +46,7 @@ jobs:
             # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command
             # Tests' building is off for a pure C build
             extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_TESTS=OFF'
+            level_zero_provider: 'ON'
     runs-on: ${{matrix.os}}
 
     steps:
@@ -81,7 +85,7 @@ jobs:
         -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=${{matrix.jemalloc}}
         -DUMF_BUILD_TESTS=ON
         -DUMF_BUILD_EXAMPLES=ON
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=${{matrix.level_zero_provider}}
         ${{matrix.extra_build_options}}
 
     - name: Build
@@ -111,7 +115,7 @@ jobs:
         -DUMF_FORMAT_CODE_STYLE=ON
         -DUMF_BUILD_OS_MEMORY_PROVIDER=OFF
         -DUMF_BUILD_TESTS=OFF
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=${{matrix.level_zero_provider}}
 
     - name: Check clang-format
       run: cmake --build build --target clang-format-check

.github/workflows/proxy_lib.yml

@@ -42,7 +42,6 @@ jobs:
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
           -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
           -DUMF_ENABLE_POOL_TRACKING=ON
-          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
       - name: Build UMF
         run: cmake --build ${{github.workspace}}/build -j $(nproc)

.github/workflows/sanitizers.yml

@@ -43,6 +43,7 @@ jobs:
         -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
         -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
         -DUMF_BUILD_OS_MEMORY_PROVIDER=ON
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
         -DUMF_ENABLE_POOL_TRACKING=ON
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
@@ -53,7 +54,6 @@ jobs:
         -DUSE_UBSAN=${{matrix.sanitizers.ubsan}}
         -DUSE_TSAN=${{matrix.sanitizers.tsan}}
         -DUMF_BUILD_EXAMPLES=ON
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -89,6 +89,7 @@ jobs:
         -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
         -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
         -DUMF_BUILD_OS_MEMORY_PROVIDER=ON
+        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
         -DUMF_ENABLE_POOL_TRACKING=ON
         -DUMF_FORMAT_CODE_STYLE=OFF
         -DUMF_DEVELOPER_MODE=ON
@@ -99,7 +100,6 @@ jobs:
         -DUSE_UBSAN=${{matrix.sanitizers.ubsan}}
         -DUSE_TSAN=${{matrix.sanitizers.tsan}}
         -DUMF_BUILD_EXAMPLES=ON
-        -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF
 
     - name: Build UMF
       run: cmake --build ${{env.BUILD_DIR}} -j $(nproc)
@@ -146,6 +146,7 @@ jobs:
       run: vcpkg install
       shell: pwsh # Specifies PowerShell as the shell for running the script.
 
+    # TODO enable level zero provider
     - name: Configure build
       run: >
         cmake

CMakeLists.txt

@@ -13,6 +13,7 @@ option(UMF_BUILD_LIBUMF_POOL_DISJOINT "Build the libumf_pool_disjoint static lib
 option(UMF_BUILD_LIBUMF_POOL_JEMALLOC "Build the libumf_pool_jemalloc static library" OFF)
 option(UMF_BUILD_LIBUMF_POOL_SCALABLE "Build the libumf_pool_scalable static library" OFF)
 option(UMF_BUILD_TESTS "Build UMF tests" ON)
+option(UMF_BUILD_GPU_TESTS "Build UMF GPU tests" OFF)
 option(UMF_BUILD_BENCHMARKS "Build UMF benchmarks" OFF)
 option(UMF_BUILD_BENCHMARKS_MT "Build UMF multithreaded benchmarks" OFF)
 option(UMF_BUILD_EXAMPLES "Build UMF examples" ON)

README.md

@@ -34,7 +34,7 @@ For development and contributions:
 For building tests, multithreaded benchmarks and Disjoint Pool:
 - C++ compiler with C++17 support
 
-For Level Zero memory provider and its tests:
+For Level Zero memory provider tests:
 - Level Zero headers and libraries
 - compatible GPU with installed driver
 
@@ -96,6 +96,7 @@ List of options provided by CMake:
 | UMF_BUILD_LIBUMF_POOL_JEMALLOC | Build the libumf_pool_jemalloc static library | ON/OFF | OFF |
 | UMF_BUILD_LIBUMF_POOL_SCALABLE | Build the libumf_pool_scalable static library | ON/OFF | OFF |
 | UMF_BUILD_TESTS | Build UMF tests | ON/OFF | ON |
+| UMF_BUILD_GPU_TESTS | Build UMF GPU tests | ON/OFF | OFF |
 | UMF_BUILD_BENCHMARKS | Build UMF benchmarks | ON/OFF | OFF |
 | UMF_BUILD_EXAMPLES | Build UMF examples | ON/OFF | ON |
 | UMF_ENABLE_POOL_TRACKING | Build UMF with pool tracking | ON/OFF | ON |
@@ -138,9 +139,10 @@ A memory provider that provides memory from L0 device.
 
 1) System with Level Zero compatible GPU
 2) Linux OS
-3) The `UMF_BUILD_LIBUMF_PROVIDER_LEVEL_ZERO` option turned `ON`
-4) Required packages:
-   - level-zero-dev (TODO: check this)
+3) The `UMF_BUILD_LIBUMF_PROVIDER_LEVEL_ZERO` option turned `ON` (by default)
+4) Required for tests (Linux-only yet): `UMF_BUILD_GPU_TESTS` option turned `ON`
+4) Required packages for tests (Linux-only yet):
+   - liblevel-zero-dev
 
 ### Memory pool managers
 

src/CMakeLists.txt

@@ -3,6 +3,32 @@
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake)
+        
+if(UMF_BUILD_LEVEL_ZERO_PROVIDER)
+    include(FetchContent)
+
+    set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
+    set(LEVEL_ZERO_LOADER_TAG v1.16.1)
+
+    message(STATUS "Will fetch Level Zero Loader from ${LEVEL_ZERO_LOADER_REPO}")
+    include(FetchContent)
+    FetchContent_Declare(level-zero-loader
+        GIT_REPOSITORY    ${LEVEL_ZERO_LOADER_REPO}
+        GIT_TAG           ${LEVEL_ZERO_LOADER_TAG}
+        EXCLUDE_FROM_ALL
+    )
+    if(MSVC)
+        set(USE_Z7 ON)
+    endif()
+
+    FetchContent_GetProperties(level-zero-loader)
+    if(NOT level-zero-loader_POPULATED)
+        FetchContent_Populate(level-zero-loader)
+    endif()
+
+    set(LEVEL_ZERO_INCLUDE_DIRS
+        ${level-zero-loader_SOURCE_DIR}/include CACHE PATH "Path to Level Zero Headers")    
+endif()
 
 # set UMF_PROXY_LIB_BASED_ON_POOL to one of:
 # - SCALABLE
@@ -75,18 +101,6 @@ if(UMF_BUILD_OS_MEMORY_PROVIDER)
     endif()
 endif()
 
-if(UMF_BUILD_LEVEL_ZERO_PROVIDER)
-    # Currently, Level Zero provider is only built for Linux
-    # TODO add fetch content or PkgConfig/find_package for Level Zero
-    set(UMF_SOURCES_LINUX ${UMF_SOURCES_LINUX}
-        provider/provider_level_zero.c)
-
-    if(LINUX)
-        set(UMF_COMPILE_DEFINITIONS ${UMF_COMPILE_DEFINITIONS} 
-            "UMF_BUILD_LEVEL_ZERO_PROVIDER=1")
-    endif()
-endif()
-
 if(LINUX)
     set(UMF_SOURCES ${UMF_SOURCES} ${UMF_SOURCES_LINUX})
 elseif(WINDOWS)
@@ -129,12 +143,35 @@ else()
     target_sources(umf PRIVATE memory_pool_default.c)
 endif()
 
+# Currently, Level Zero provider is only built for Linux
+if(LINUX AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
+
+    # WA for error ze_api.h:14234:20: no newline at end of file [-Werror,-Wnewline-eof]
+    #set(CMAKE_C_FLAGS_BAK "${CMAKE_C_FLAGS}")
+    #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-newline-eof")
+
+    target_sources(umf PRIVATE
+        provider/provider_level_zero.c)
+    set_source_files_properties(provider/provider_level_zero.c 
+        PROPERTIES COMPILE_FLAGS -Wno-newline-eof)
+
+    set(UMF_COMPILE_DEFINITIONS ${UMF_COMPILE_DEFINITIONS} 
+        "UMF_BUILD_LEVEL_ZERO_PROVIDER=1")          
+    
+    # restore original C flags
+    #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_BAK}")
+endif()
+
 add_library(${PROJECT_NAME}::umf ALIAS umf)
 
 if(LIBHWLOC_INCLUDE_DIRS)
     target_include_directories(umf PRIVATE ${LIBHWLOC_INCLUDE_DIRS})
 endif()
 
+if (LEVEL_ZERO_INCLUDE_DIRS)
+    target_include_directories(umf PRIVATE ${LEVEL_ZERO_INCLUDE_DIRS})
+endif()
+
 target_include_directories(umf PUBLIC
     $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>

src/provider/provider_level_zero.c

@@ -12,7 +12,8 @@
 #include <stddef.h>
 #include <unistd.h>
 
-#include <level_zero/ze_api.h>
+// Level Zero API
+#include <ze_api.h>
 
 #include <umf.h>
 #include <umf/memory_provider_ops.h>
@@ -84,7 +85,7 @@ enum umf_result_t ze_memory_provider_initialize(void *params, void **provider) {
 
     ze_provider->context = ze_params->level_zero_context_handle;
     ze_provider->device = ze_params->level_zero_device_handle;
-    ze_provider->memory_type = ze_params->memory_type;
+    ze_provider->memory_type = (ze_memory_type_t)ze_params->memory_type;
 
     *provider = ze_provider;
 

test/CMakeLists.txt

@@ -139,7 +139,7 @@ if(UMF_BUILD_OS_MEMORY_PROVIDER AND LINUX) # OS-specific functions are implement
 endif()
 
 # TODO add support for Windows
-if(UMF_BUILD_LEVEL_ZERO_PROVIDER AND LINUX)
+if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER AND LINUX)
     # we have two test binaries here that use the same sources, but differ in 
     # the way they are linked to the Level Zero (statically or at runtime using 
     # dlopen)