[SYCL][UR][L0 v2] implement OOO immediate queue

by using multiple in-order queues and round-robin
strategy to dispatch work.

With this approach we don't need to worry about events'
lifetime. Since we are still using counter-based events,
we don't need any special logic to handle cases where
event release is called right after being passed as signal
event or as part of a wait list.

Author: igchor

unified-runtime/source/adapters/level_zero/CMakeLists.txt

@@ -171,6 +171,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/lockable.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.hpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.hpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/api.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/command_buffer.cpp
@@ -187,6 +188,7 @@ if(UR_BUILD_ADAPTER_L0_V2)
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_create.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_in_order.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_immediate_out_of_order.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/v2/usm.cpp
     )
     install_ur_library(ur_adapter_level_zero_v2)

unified-runtime/source/adapters/level_zero/v2/event_pool.hpp

@@ -61,4 +61,34 @@ class event_pool {
   std::unique_ptr<std::mutex> mutex;
 };
 
+// Only create an event when requested by the user.
+static inline ur_event_handle_t
+createEventIfRequested(event_pool *eventPool, ur_event_handle_t *phEvent,
+                       ur_queue_t_ *queue) {
+  if (phEvent == nullptr) {
+    return nullptr;
+  }
+
+  (*phEvent) = eventPool->allocate();
+  (*phEvent)->setQueue(queue);
+  return (*phEvent);
+}
+
+// Always creates an event (used in functions that need to store the event
+// internally). If event was requested by the user, also increase ref count of
+// that event to avoid pre-mature release.
+static inline ur_event_handle_t createEventAndRetain(event_pool *eventPool,
+                                                     ur_event_handle_t *phEvent,
+                                                     ur_queue_t_ *queue) {
+  auto hEvent = eventPool->allocate();
+  hEvent->setQueue(queue);
+
+  if (phEvent) {
+    (*phEvent) = hEvent;
+    hEvent->retain();
+  }
+
+  return hEvent;
+}
+
 } // namespace v2

unified-runtime/source/adapters/level_zero/v2/lockable.hpp

@@ -18,6 +18,7 @@ template <typename T> struct locked {
     object_ = object;
   }
   T *operator->() { return object_; }
+  auto &operator[](size_t index) { return (*object_)[index]; }
 
 private:
   std::unique_lock<std::mutex> lock_;

unified-runtime/source/adapters/level_zero/v2/queue_create.cpp

@@ -69,9 +69,18 @@ ur_result_t urQueueCreate(ur_context_handle_t hContext,
 
   auto zeIndex = v2::getZeIndex(pProperties);
 
-  *phQueue = ur_queue_handle_t_::create<v2::ur_queue_immediate_in_order_t>(
-      hContext, hDevice, v2::getZeOrdinal(hDevice), v2::getZePriority(flags),
-      zeIndex, v2::eventFlagsFromQueueFlags(flags), flags);
+  if ((flags & UR_QUEUE_FLAG_OUT_OF_ORDER_EXEC_MODE_ENABLE) != 0 &&
+      !zeIndex.has_value()) {
+    *phQueue =
+        ur_queue_handle_t_::create<v2::ur_queue_immediate_out_of_order_t>(
+            hContext, hDevice, v2::getZeOrdinal(hDevice),
+            v2::getZePriority(flags), v2::eventFlagsFromQueueFlags(flags),
+            flags);
+  } else {
+    *phQueue = ur_queue_handle_t_::create<v2::ur_queue_immediate_in_order_t>(
+        hContext, hDevice, v2::getZeOrdinal(hDevice), v2::getZePriority(flags),
+        zeIndex, v2::eventFlagsFromQueueFlags(flags), flags);
+  }
 
   return UR_RESULT_SUCCESS;
 } catch (...) {

unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp

@@ -15,11 +15,13 @@
 
 #include "../common.hpp"
 #include "queue_immediate_in_order.hpp"
+#include "queue_immediate_out_of_order.hpp"
 #include <ur_api.h>
 #include <variant>
 
 struct ur_queue_handle_t_ : ur::handle_base<ur::level_zero::ddi_getter> {
-  using data_variant = std::variant<v2::ur_queue_immediate_in_order_t>;
+  using data_variant = std::variant<v2::ur_queue_immediate_in_order_t,
+                                    v2::ur_queue_immediate_out_of_order_t>;
   data_variant queue_data;
 
   static constexpr uintptr_t queue_offset =

unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp

@@ -140,10 +140,12 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueEventsWaitWithBarrier(
   // zeCommandListAppendWaitOnEvents
   if ((flags & UR_QUEUE_FLAG_PROFILING_ENABLE) != 0) {
     return commandListManager.lock()->appendEventsWaitWithBarrier(
-        numEventsInWaitList, phEventWaitList, createEventIfRequested(phEvent));
+        numEventsInWaitList, phEventWaitList,
+        createEventIfRequested(eventPool.get(), phEvent, this));
   } else {
     return commandListManager.lock()->appendEventsWait(
-        numEventsInWaitList, phEventWaitList, createEventIfRequested(phEvent));
+        numEventsInWaitList, phEventWaitList,
+        createEventIfRequested(eventPool.get(), phEvent, this));
   }
 }