[L0 v2] implement USM allocation functions using UMF

cmake/FetchLevelZero.cmake

@@ -0,0 +1,94 @@
+# Copyright (C) 2024 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+set(UR_LEVEL_ZERO_LOADER_LIBRARY "" CACHE FILEPATH "Path of the Level Zero Loader library")
+set(UR_LEVEL_ZERO_INCLUDE_DIR "" CACHE FILEPATH "Directory containing the Level Zero Headers")
+set(UR_LEVEL_ZERO_LOADER_REPO "" CACHE STRING "Github repo to get the Level Zero loader sources from")
+set(UR_LEVEL_ZERO_LOADER_TAG "" CACHE STRING " GIT tag of the Level Loader taken from github repo")
+
+# Copy Level Zero loader/headers locally to the build to avoid leaking their path.
+set(LEVEL_ZERO_COPY_DIR ${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader)
+if (NOT UR_LEVEL_ZERO_LOADER_LIBRARY STREQUAL "")
+    get_filename_component(LEVEL_ZERO_LIB_NAME "${UR_LEVEL_ZERO_LOADER_LIBRARY}" NAME)
+    set(LEVEL_ZERO_LIBRARY ${LEVEL_ZERO_COPY_DIR}/${LEVEL_ZERO_LIB_NAME})
+    message(STATUS "Level Zero Adapter: Copying Level Zero loader to local build tree")
+    file(COPY ${UR_LEVEL_ZERO_LOADER_LIBRARY} DESTINATION ${LEVEL_ZERO_COPY_DIR} FOLLOW_SYMLINK_CHAIN)
+endif()
+if (NOT UR_LEVEL_ZERO_INCLUDE_DIR STREQUAL "")
+    set(LEVEL_ZERO_INCLUDE_DIR ${LEVEL_ZERO_COPY_DIR})
+    message(STATUS "Level Zero Adapter: Copying Level Zero headers to local build tree")
+    file(COPY ${UR_LEVEL_ZERO_INCLUDE_DIR}/ DESTINATION ${LEVEL_ZERO_COPY_DIR})
+endif()
+
+if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
+    message(STATUS "Level Zero Adapter: Download Level Zero loader and headers from github.com")
+
+    # Workaround warnings/errors for Level Zero build
+    set(CMAKE_CXX_FLAGS_BAK "${CMAKE_CXX_FLAGS}")
+    if (UNIX)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-pedantic")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-stringop-truncation")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++98-compat-extra-semi")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option")
+    endif()
+
+    if (UR_LEVEL_ZERO_LOADER_REPO STREQUAL "")
+        set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
+    endif()
+    if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "")
+        set(UR_LEVEL_ZERO_LOADER_TAG v1.17.39)
+    endif()
+
+    # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104
+    set(CMAKE_INCLUDE_CURRENT_DIR OFF)
+    # Prevent L0 loader from exporting extra symbols
+    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS OFF)
+
+    message(STATUS "Level Zero Adapter: Will fetch Level Zero Loader from ${UR_LEVEL_ZERO_LOADER_REPO}")
+    include(FetchContent)
+    FetchContent_Declare(level-zero-loader
+        GIT_REPOSITORY    ${UR_LEVEL_ZERO_LOADER_REPO}
+        GIT_TAG           ${UR_LEVEL_ZERO_LOADER_TAG}
+    )
+    if(MSVC)
+        set(USE_Z7 ON)
+    endif()
+    FetchContent_MakeAvailable(level-zero-loader)
+    FetchContent_GetProperties(level-zero-loader)
+
+    # Restore original flags
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_BAK}")
+
+    target_compile_options(ze_loader PRIVATE
+        $<$<IN_LIST:$<CXX_COMPILER_ID>,GNU;Clang;Intel;IntelLLVM>:-Wno-error>
+        $<$<CXX_COMPILER_ID:MSVC>:/WX- /UUNICODE>
+    )
+
+    set(LEVEL_ZERO_LIBRARY ze_loader)
+    set(LEVEL_ZERO_INCLUDE_DIR
+        ${level-zero-loader_SOURCE_DIR}/include CACHE PATH "Path to Level Zero Headers")
+endif()
+
+add_library(LevelZeroLoader INTERFACE)
+# The MSVC linker does not like / at the start of a path, so to work around this
+# we split it into a link library and a library path, where the path is allowed
+# to have leading /.
+get_filename_component(LEVEL_ZERO_LIBRARY_SRC "${LEVEL_ZERO_LIBRARY}" DIRECTORY)
+get_filename_component(LEVEL_ZERO_LIB_NAME "${LEVEL_ZERO_LIBRARY}" NAME)
+target_link_directories(LevelZeroLoader
+    INTERFACE "$<BUILD_INTERFACE:${LEVEL_ZERO_LIBRARY_SRC}>"
+              "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+)
+target_link_libraries(LevelZeroLoader
+    INTERFACE "${LEVEL_ZERO_LIB_NAME}"
+)
+
+add_library(LevelZeroLoader-Headers INTERFACE)
+target_include_directories(LevelZeroLoader-Headers
+    INTERFACE "$<BUILD_INTERFACE:${LEVEL_ZERO_INCLUDE_DIR}>"
+              "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+)

source/adapters/level_zero/CMakeLists.txt

@@ -3,96 +3,6 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-set(UR_LEVEL_ZERO_LOADER_LIBRARY "" CACHE FILEPATH "Path of the Level Zero Loader library")
-set(UR_LEVEL_ZERO_INCLUDE_DIR "" CACHE FILEPATH "Directory containing the Level Zero Headers")
-set(UR_LEVEL_ZERO_LOADER_REPO "" CACHE STRING "Github repo to get the Level Zero loader sources from")
-set(UR_LEVEL_ZERO_LOADER_TAG "" CACHE STRING " GIT tag of the Level Loader taken from github repo")
-
-# Copy Level Zero loader/headers locally to the build to avoid leaking their path.
-set(LEVEL_ZERO_COPY_DIR ${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader)
-if (NOT UR_LEVEL_ZERO_LOADER_LIBRARY STREQUAL "")
-    get_filename_component(LEVEL_ZERO_LIB_NAME "${UR_LEVEL_ZERO_LOADER_LIBRARY}" NAME)
-    set(LEVEL_ZERO_LIBRARY ${LEVEL_ZERO_COPY_DIR}/${LEVEL_ZERO_LIB_NAME})
-    message(STATUS "Level Zero Adapter: Copying Level Zero loader to local build tree")
-    file(COPY ${UR_LEVEL_ZERO_LOADER_LIBRARY} DESTINATION ${LEVEL_ZERO_COPY_DIR} FOLLOW_SYMLINK_CHAIN)
-endif()
-if (NOT UR_LEVEL_ZERO_INCLUDE_DIR STREQUAL "")
-    set(LEVEL_ZERO_INCLUDE_DIR ${LEVEL_ZERO_COPY_DIR})
-    message(STATUS "Level Zero Adapter: Copying Level Zero headers to local build tree")
-    file(COPY ${UR_LEVEL_ZERO_INCLUDE_DIR}/ DESTINATION ${LEVEL_ZERO_COPY_DIR})
-endif()
-
-if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
-    message(STATUS "Level Zero Adapter: Download Level Zero loader and headers from github.com")
-
-    # Workaround warnings/errors for Level Zero build
-    set(CMAKE_CXX_FLAGS_BAK "${CMAKE_CXX_FLAGS}")
-    if (UNIX)
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-but-set-variable")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-pedantic")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-stringop-truncation")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++98-compat-extra-semi")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option")
-    endif()
-
-    if (UR_LEVEL_ZERO_LOADER_REPO STREQUAL "")
-        set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
-    endif()
-    if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "")
-        set(UR_LEVEL_ZERO_LOADER_TAG v1.17.39)
-    endif()
-
-    # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104
-    set(CMAKE_INCLUDE_CURRENT_DIR OFF)
-    # Prevent L0 loader from exporting extra symbols
-    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS OFF)
-
-    message(STATUS "Level Zero Adapter: Will fetch Level Zero Loader from ${UR_LEVEL_ZERO_LOADER_REPO}")
-    include(FetchContent)
-    FetchContent_Declare(level-zero-loader
-        GIT_REPOSITORY    ${UR_LEVEL_ZERO_LOADER_REPO}
-        GIT_TAG           ${UR_LEVEL_ZERO_LOADER_TAG}
-    )
-    if(MSVC)
-        set(USE_Z7 ON)
-    endif()
-    FetchContent_MakeAvailable(level-zero-loader)
-    FetchContent_GetProperties(level-zero-loader)
-
-    # Restore original flags
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_BAK}")
-
-    target_compile_options(ze_loader PRIVATE
-        $<$<IN_LIST:$<CXX_COMPILER_ID>,GNU;Clang;Intel;IntelLLVM>:-Wno-error>
-        $<$<CXX_COMPILER_ID:MSVC>:/WX- /UUNICODE>
-    )
-
-    set(LEVEL_ZERO_LIBRARY ze_loader)
-    set(LEVEL_ZERO_INCLUDE_DIR
-        ${level-zero-loader_SOURCE_DIR}/include CACHE PATH "Path to Level Zero Headers")
-endif()
-
-add_library(LevelZeroLoader INTERFACE)
-# The MSVC linker does not like / at the start of a path, so to work around this
-# we split it into a link library and a library path, where the path is allowed
-# to have leading /.
-get_filename_component(LEVEL_ZERO_LIBRARY_SRC "${LEVEL_ZERO_LIBRARY}" DIRECTORY)
-get_filename_component(LEVEL_ZERO_LIB_NAME "${LEVEL_ZERO_LIBRARY}" NAME)
-target_link_directories(LevelZeroLoader
-    INTERFACE "$<BUILD_INTERFACE:${LEVEL_ZERO_LIBRARY_SRC}>"
-              "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
-)
-target_link_libraries(LevelZeroLoader
-    INTERFACE "${LEVEL_ZERO_LIB_NAME}"
-)
-
-add_library(LevelZeroLoader-Headers INTERFACE)
-target_include_directories(LevelZeroLoader-Headers
-    INTERFACE "$<BUILD_INTERFACE:${LEVEL_ZERO_INCLUDE_DIR}>"
-              "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
-)
-
 if(UR_BUILD_ADAPTER_L0)
     set(ADAPTER_LIB_TYPE SHARED)
     if(UR_STATIC_ADAPTER_L0)

source/adapters/level_zero/v2/context.cpp

@@ -27,7 +27,8 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext,
                        return std::make_unique<v2::provider_normal>(
                            context, device, v2::EVENT_COUNTER,
                            v2::QUEUE_IMMEDIATE);
-                     }) {}
+                     }),
+      defaultUSMPool(this, nullptr) {}
 
 ur_result_t ur_context_handle_t_::retain() {
   RefCount.increment();
@@ -60,6 +61,10 @@ bool ur_context_handle_t_::isValidDevice(ur_device_handle_t hDevice) const {
   return false;
 }
 
+ur_usm_pool_handle_t ur_context_handle_t_::getDefaultUSMPool() {
+  return &defaultUSMPool;
+}
+
 namespace ur::level_zero {
 ur_result_t urContextCreate(uint32_t deviceCount,
                             const ur_device_handle_t *phDevices,

source/adapters/level_zero/v2/context.hpp

@@ -15,6 +15,7 @@
 #include "command_list_cache.hpp"
 #include "common.hpp"
 #include "event_pool_cache.hpp"
+#include "usm.hpp"
 
 struct ur_context_handle_t_ : _ur_object {
   ur_context_handle_t_(ze_context_handle_t hContext, uint32_t numDevices,
@@ -26,6 +27,7 @@ struct ur_context_handle_t_ : _ur_object {
   inline ze_context_handle_t getZeHandle() const { return hContext.get(); }
   ur_platform_handle_t getPlatform() const;
   const std::vector<ur_device_handle_t> &getDevices() const;
+  ur_usm_pool_handle_t getDefaultUSMPool();
 
   // Checks if Device is covered by this context.
   // For that the Device or its root devices need to be in the context.
@@ -35,4 +37,5 @@ struct ur_context_handle_t_ : _ur_object {
   const std::vector<ur_device_handle_t> hDevices;
   v2::command_list_cache_t commandListCache;
   v2::event_pool_cache eventPoolCache;
+  ur_usm_pool_handle_t_ defaultUSMPool;
 };

source/adapters/level_zero/v2/queue_immediate_in_order.cpp

@@ -545,24 +545,61 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueUSMPrefetch(
     const void *pMem, size_t size, ur_usm_migration_flags_t flags,
     uint32_t numEventsInWaitList, const ur_event_handle_t *phEventWaitList,
     ur_event_handle_t *phEvent) {
-  std::ignore = pMem;
-  std::ignore = size;
   std::ignore = flags;
-  std::ignore = numEventsInWaitList;
-  std::ignore = phEventWaitList;
-  std::ignore = phEvent;
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+
+  std::scoped_lock<ur_shared_mutex> Lock(this->Mutex);
+
+  auto handler = getCommandListHandlerForCompute();
+  auto signalEvent = getSignalEvent(handler, phEvent);
+
+  auto [pWaitEvents, numWaitEvents] =
+      getWaitListView(phEventWaitList, numEventsInWaitList, handler);
+
+  if (pWaitEvents) {
+    ZE2UR_CALL(zeCommandListAppendBarrier, (handler->commandList.get(), nullptr,
+                                            numWaitEvents, pWaitEvents));
+  }
+  // TODO: figure out how to translate "flags"
+  ZE2UR_CALL(zeCommandListAppendMemoryPrefetch,
+             (handler->commandList.get(), pMem, size));
+  ZE2UR_CALL(zeCommandListAppendSignalEvent,
+             (handler->commandList.get(), signalEvent));
+
+  lastHandler = handler;
+
+  return UR_RESULT_SUCCESS;
 }
 
 ur_result_t
 ur_queue_immediate_in_order_t::enqueueUSMAdvise(const void *pMem, size_t size,
                                                 ur_usm_advice_flags_t advice,
                                                 ur_event_handle_t *phEvent) {
-  std::ignore = pMem;
-  std::ignore = size;
-  std::ignore = advice;
-  std::ignore = phEvent;
-  return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;
+  std::ignore = flags;
+
+  auto zeAdvice = ur_cast<ze_memory_advice_t>(advice);
+
+  std::scoped_lock<ur_shared_mutex> Lock(this->Mutex);
+
+  auto handler = getCommandListHandlerForCompute();
+  auto signalEvent = getSignalEvent(handler, phEvent);
+
+  auto [pWaitEvents, numWaitEvents] = getWaitListView(nullptr, 0, handler);
+
+  if (pWaitEvents) {
+    ZE2UR_CALL(zeCommandListAppendBarrier, (handler->commandList.get(), nullptr,
+                                            numWaitEvents, pWaitEvents));
+  }
+
+  // TODO: figure out how to translate "flags"
+  ZE2UR_CALL(zeCommandListAppendMemAdvise,
+             (handler->commandList.get(), this->hDevice->ZeDevice, pMem, size,
+              zeAdvice));
+  ZE2UR_CALL(zeCommandListAppendSignalEvent,
+             (handler->commandList.get(), signalEvent));
+
+  lastHandler = handler;
+
+  return UR_RESULT_SUCCESS;
 }
 
 ur_result_t ur_queue_immediate_in_order_t::enqueueUSMFill2D(