Adapt project to work on windows with L0 and GPU

Author: szadam

.github/workflows/codeql.yml

@@ -63,7 +63,6 @@ jobs:
     - name: Install pip packages
       run: python3 -m pip install -r third_party/requirements.txt
 
-    # TODO enable Level Zero provider
     - name: Configure CMake
       run: >
         cmake

.github/workflows/gpu.yml

@@ -9,51 +9,82 @@ permissions:
   contents: read
 
 jobs:
-  # TODO: add support for tests on Windows
   gpu:
     name: Build
+    env:
+      BUILD_TYPE: Release
+      VCPKG_PATH: "${{github.workspace}}/../../../../vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/../../../../vcpkg/packages/tbb_x64-windows;${{github.workspace}}/../../../../vcpkg/packages/jemalloc_x64-windows"
     # run only on upstream; forks will not have the HW
     if: github.repository == 'oneapi-src/unified-memory-framework'
-
     strategy:
       matrix:
-        build_type: [Release]
-        compiler: [{c: gcc, cxx: g++}]
         shared_library: ['ON', 'OFF']
-    # TODO enable testing on DSS-WINDOWS
-    runs-on:  [ "DSS-LEVEL_ZERO", "DSS-UBUNTU" ]
+        os: ['Ubuntu', 'Windows']
+        include:
+          - os: 'Ubuntu'
+            compiler: {c: gcc, cxx: g++}
+            number_of_processors: '$(nproc)'
+          - os: 'Windows'
+            compiler: {c: cl, cxx: cl}
+            number_of_processors: '$Env:NUMBER_OF_PROCESSORS'
 
+    runs-on: ["DSS-LEVEL_ZERO", "DSS-${{matrix.os}}"]
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Get information about platform
+        if: matrix.os == 'Ubuntu'
         run: .github/scripts/get_system_info.sh
 
-      - name: Configure build
+      - name: Configure build for Win
+        if: matrix.os == 'Windows'
+        run: >
+          cmake
+          -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
+          -B ${{github.workspace}}/build
+          -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+          -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
+          -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
+          -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
+          -DUMF_BUILD_BENCHMARKS=ON
+          -DUMF_BUILD_TESTS=ON
+          -DUMF_BUILD_GPU_TESTS=ON
+          -DUMF_BUILD_GPU_EXAMPLES=ON
+          -DUMF_FORMAT_CODE_STYLE=OFF
+          -DUMF_DEVELOPER_MODE=ON
+          -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
+          -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
+          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
+          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
+      
+      - name: Configure build for Ubuntu
+        if: matrix.os == 'Ubuntu'
         run: >
           cmake
           -B ${{github.workspace}}/build
-          -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
+          -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
           -DCMAKE_C_COMPILER=${{matrix.compiler.c}}
           -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}}
           -DUMF_BUILD_SHARED_LIBRARY=${{matrix.shared_library}}
           -DUMF_BUILD_BENCHMARKS=ON
           -DUMF_BUILD_TESTS=ON
           -DUMF_BUILD_GPU_TESTS=ON
           -DUMF_BUILD_GPU_EXAMPLES=ON
-          -DUMF_FORMAT_CODE_STYLE=ON
+          -DUMF_FORMAT_CODE_STYLE=OFF
           -DUMF_DEVELOPER_MODE=ON
           -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON
+          -DUMF_BUILD_LIBUMF_POOL_SCALABLE=ON
+          -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
+          -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
 
       - name: Build UMF
-        run: cmake --build ${{github.workspace}}/build -j $(nproc)
+        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j ${{matrix.number_of_processors}}
 
       - name: Run tests
         working-directory: ${{github.workspace}}/build
-        run: ctest --output-on-failure --test-dir test
+        run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --test-dir test
 
       - name: Run examples
         working-directory: ${{github.workspace}}/build
-        run: ctest --output-on-failure --test-dir examples -C ${{matrix.build_type}}
-
+        run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}}

.github/workflows/pr_push.yml

@@ -168,4 +168,4 @@ jobs:
     uses: ./.github/workflows/valgrind.yml
   MultiNuma:
     needs: [Build]
-    uses: ./.github/workflows/multi_numa.yml
+    uses: ./.github/workflows/multi_numa.yml

CMakeLists.txt

@@ -224,7 +224,7 @@ if(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL SCALABLE)
     if(UMF_BUILD_LIBUMF_POOL_SCALABLE)
         set(UMF_PROXY_LIB_ENABLED ON)
         set(PROXY_LIB_USES_SCALABLE_POOL ON)
-        set(PROXY_LIBS umf scalable_pool)
+        set(PROXY_LIBS scalable_pool umf)
     else()
         message(
             STATUS
@@ -235,7 +235,7 @@ elseif(UMF_PROXY_LIB_BASED_ON_POOL STREQUAL JEMALLOC)
     if(UMF_BUILD_LIBUMF_POOL_JEMALLOC)
         set(UMF_PROXY_LIB_ENABLED ON)
         set(PROXY_LIB_USES_JEMALLOC_POOL ON)
-        set(PROXY_LIBS umf jemalloc_pool)
+        set(PROXY_LIBS jemalloc_pool umf)
     else()
         message(
             STATUS
@@ -263,6 +263,17 @@ if(UMF_BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
+# Conditional configuration for Level Zero provider
+if(UMF_BUILD_LEVEL_ZERO_PROVIDER)
+    set(OPTIONAL_SYMBOLS "umfLevelZeroMemoryProviderOps")
+else()
+    set(OPTIONAL_SYMBOLS "")
+endif()
+
+# Configure the DEF file based on whether Level Zero provider is built
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/libumf.def.in"
+               "${CMAKE_CURRENT_BINARY_DIR}/src/libumf.def" @ONLY)
+
 if(UMF_FORMAT_CODE_STYLE)
     find_program(CLANG_FORMAT NAMES clang-format-15 clang-format-15.0
                                     clang-format)

README.md

@@ -151,7 +151,7 @@ If the shared memory mapping is used then an anonymous file descriptor for memor
 Required packages for tests (Linux-only yet):
    - libnuma-dev
 
-#### Level Zero memory provider (Linux-only yet)
+#### Level Zero memory provider
 
 A memory provider that provides memory from L0 device.
 
@@ -162,11 +162,10 @@ A memory provider that provides memory from L0 device.
 
 Additionally, required for tests:
 
-3) Linux OS
-4) The `UMF_BUILD_GPU_TESTS` option turned `ON`
-5) System with Level Zero compatible GPU
-6) Required packages:
-   - liblevel-zero-dev
+3) The `UMF_BUILD_GPU_TESTS` option turned `ON`
+4) System with Level Zero compatible GPU
+5) Required packages:
+   - liblevel-zero-dev (Linux) or level-zero-sdk (Windows)
 
 ### Memory pool managers
 

benchmark/CMakeLists.txt

@@ -34,7 +34,7 @@ function(add_umf_benchmark)
 
     set(BENCH_NAME umf-bench-${ARG_NAME})
 
-    set(BENCH_LIBS umf ${ARG_LIBS})
+    set(BENCH_LIBS ${ARG_LIBS} umf)
 
     add_umf_executable(
         NAME ${BENCH_NAME}

examples/CMakeLists.txt

@@ -8,7 +8,7 @@ if(UMF_BUILD_LIBUMF_POOL_SCALABLE)
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS basic/basic.c
-        LIBS umf scalable_pool)
+        LIBS scalable_pool umf)
 
     target_include_directories(
         ${EXAMPLE_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils
@@ -37,14 +37,13 @@ endif()
 
 if(UMF_BUILD_GPU_EXAMPLES
    AND UMF_BUILD_LIBUMF_POOL_DISJOINT
-   AND UMF_BUILD_LEVEL_ZERO_PROVIDER
-   AND LINUX)
+   AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
     set(EXAMPLE_NAME umf_example_gpu_shared_memory)
 
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS basic/gpu_shared_memory.c
-        LIBS umf disjoint_pool ze_loader)
+        LIBS disjoint_pool ze_loader umf)
 
     target_include_directories(
         ${EXAMPLE_NAME}
@@ -73,15 +72,17 @@ else()
             "to be turned ON - skipping")
 endif()
 
+# TODO: it looks like there is some problem with IPC implementation in Level
+# Zero on windows
 if(UMF_BUILD_GPU_EXAMPLES
    AND UMF_BUILD_LIBUMF_POOL_DISJOINT
-   AND UMF_BUILD_LEVEL_ZERO_PROVIDER)
+   AND LINUX)
     set(EXAMPLE_NAME umf_example_ipc_level_zero)
 
     add_umf_executable(
         NAME ${EXAMPLE_NAME}
         SRCS basic/ipc_level_zero.c
-        LIBS umf disjoint_pool ze_loader)
+        LIBS disjoint_pool ze_loader umf)
 
     target_include_directories(
         ${EXAMPLE_NAME}
@@ -118,7 +119,7 @@ if(LINUX AND UMF_BUILD_LIBUMF_POOL_SCALABLE)
         add_umf_executable(
             NAME ${EX_NAME}
             SRCS basic/${BASE_NAME}_${loop_var}.c
-            LIBS umf scalable_pool)
+            LIBS scalable_pool umf)
 
         target_include_directories(
             ${EX_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils

examples/basic/utils_level_zero.h

@@ -10,7 +10,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#ifdef _WIN32
+#include <ze_api.h>
+#else
 #include <level_zero/ze_api.h>
+#endif
 
 static int init_level_zero() {
     ze_init_flag_t flags = ZE_INIT_FLAG_GPU_ONLY;

src/CMakeLists.txt

@@ -111,7 +111,7 @@ if(UMF_BUILD_SHARED_LIBRARY)
         SRCS ${UMF_SOURCES}
         LIBS ${UMF_LIBS}
         LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.map
-        WINDOWS_DEF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.def)
+        WINDOWS_DEF_FILE ${CMAKE_CURRENT_BINARY_DIR}/libumf.def)
     set(UMF_PRIVATE_COMPILE_DEFINITIONS ${UMF_PRIVATE_COMPILE_DEFINITIONS}
                                         "UMF_SHARED_LIBRARY")
     set_target_properties(

src/libumf.def renamed to src/libumf.def.in

@@ -50,3 +50,4 @@ EXPORTS
     umfProxyPoolOps
     umfPutIPCHandle
     umfOsMemoryProviderOps
+    @OPTIONAL_SYMBOLS@

src/pool/pool_scalable.c

@@ -108,28 +108,29 @@ static int init_tbb_callbacks(tbb_callbacks_t *tbb_callbacks) {
     assert(tbb_callbacks);
 
     const char *lib_name = tbb_symbol[TBB_LIB_NAME];
-    tbb_callbacks->lib_handle = util_open_library(lib_name);
+    tbb_callbacks->lib_handle = util_open_library(lib_name, 0);
     if (!tbb_callbacks->lib_handle) {
         LOG_ERR("%s not found.", lib_name);
         return -1;
     }
 
     *(void **)&tbb_callbacks->pool_malloc = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_MALLOC]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_MALLOC], lib_name);
     *(void **)&tbb_callbacks->pool_realloc = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_REALLOC]);
-    *(void **)&tbb_callbacks->pool_aligned_malloc = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_ALIGNED_MALLOC]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_REALLOC], lib_name);
+    *(void **)&tbb_callbacks->pool_aligned_malloc =
+        util_get_symbol_addr(tbb_callbacks->lib_handle,
+                             tbb_symbol[TBB_POOL_ALIGNED_MALLOC], lib_name);
     *(void **)&tbb_callbacks->pool_free = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_FREE]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_FREE], lib_name);
     *(void **)&tbb_callbacks->pool_create_v1 = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_CREATE_V1]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_CREATE_V1], lib_name);
     *(void **)&tbb_callbacks->pool_destroy = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_DESTROY]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_DESTROY], lib_name);
     *(void **)&tbb_callbacks->pool_identify = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_IDENTIFY]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_IDENTIFY], lib_name);
     *(void **)&tbb_callbacks->pool_msize = util_get_symbol_addr(
-        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_MSIZE]);
+        tbb_callbacks->lib_handle, tbb_symbol[TBB_POOL_MSIZE], lib_name);
 
     if (!tbb_callbacks->pool_malloc || !tbb_callbacks->pool_realloc ||
         !tbb_callbacks->pool_aligned_malloc || !tbb_callbacks->pool_free ||

src/provider/provider_level_zero.c

@@ -57,23 +57,29 @@ static UTIL_ONCE_FLAG ze_is_initialized = UTIL_ONCE_FLAG_INIT;
 static bool Init_ze_global_state_failed;
 
 static void init_ze_global_state(void) {
+#ifdef _WIN32
+    const char *lib_name = "ze_loader.dll";
+#else
+    const char *lib_name = "libze_loader.so";
+#endif
     // check if Level Zero shared library is already loaded
     // we pass 0 as a handle to search the global symbol table
     *(void **)&g_ze_ops.zeMemAllocHost =
-        util_get_symbol_addr(0, "zeMemAllocHost");
+        util_get_symbol_addr(0, "zeMemAllocHost", lib_name);
     *(void **)&g_ze_ops.zeMemAllocDevice =
-        util_get_symbol_addr(0, "zeMemAllocDevice");
+        util_get_symbol_addr(0, "zeMemAllocDevice", lib_name);
     *(void **)&g_ze_ops.zeMemAllocShared =
-        util_get_symbol_addr(0, "zeMemAllocShared");
-    *(void **)&g_ze_ops.zeMemFree = util_get_symbol_addr(0, "zeMemFree");
+        util_get_symbol_addr(0, "zeMemAllocShared", lib_name);
+    *(void **)&g_ze_ops.zeMemFree =
+        util_get_symbol_addr(0, "zeMemFree", lib_name);
     *(void **)&g_ze_ops.zeMemGetIpcHandle =
-        util_get_symbol_addr(0, "zeMemGetIpcHandle");
+        util_get_symbol_addr(0, "zeMemGetIpcHandle", lib_name);
     *(void **)&g_ze_ops.zeMemPutIpcHandle =
-        util_get_symbol_addr(0, "zeMemPutIpcHandle");
+        util_get_symbol_addr(0, "zeMemPutIpcHandle", lib_name);
     *(void **)&g_ze_ops.zeMemOpenIpcHandle =
-        util_get_symbol_addr(0, "zeMemOpenIpcHandle");
+        util_get_symbol_addr(0, "zeMemOpenIpcHandle", lib_name);
     *(void **)&g_ze_ops.zeMemCloseIpcHandle =
-        util_get_symbol_addr(0, "zeMemCloseIpcHandle");
+        util_get_symbol_addr(0, "zeMemCloseIpcHandle", lib_name);
 
     if (!g_ze_ops.zeMemAllocHost || !g_ze_ops.zeMemAllocDevice ||
         !g_ze_ops.zeMemAllocShared || !g_ze_ops.zeMemFree ||

src/utils/CMakeLists.txt

@@ -5,7 +5,7 @@
 include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake)
 include(FindThreads)
 
-set(UMF_UTILS_SOURCES_COMMON utils_common.c utils_log.c)
+set(UMF_UTILS_SOURCES_COMMON utils_common.c utils_log.c utils_load_library.c)
 
 set(UMF_UTILS_SOURCES_POSIX utils_posix_common.c utils_posix_concurrency.c
                             utils_posix_math.c)