Merge pull request #1623 from igchor/legacy_path_queue

[L0] Create initial structure for supporting queue dispatcher

Author: kbenzie

source/adapters/level_zero/command_buffer.cpp

@@ -1175,9 +1175,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferAppendUSMFillExp(
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urCommandBufferEnqueueExp(
-    ur_exp_command_buffer_handle_t CommandBuffer, ur_queue_handle_t Queue,
+    ur_exp_command_buffer_handle_t CommandBuffer, ur_queue_handle_t UrQueue,
     uint32_t NumEventsInWaitList, const ur_event_handle_t *EventWaitList,
     ur_event_handle_t *Event) {
+  auto Queue = Legacy(UrQueue);
   std::scoped_lock<ur_shared_mutex> lock(Queue->Mutex);
   // Use compute engine rather than copy engine
   const auto UseCopyEngine = false;

source/adapters/level_zero/context.cpp

@@ -576,8 +576,8 @@ void ur_context_handle_t_::addEventToContextCache(ur_event_handle_t Event) {
   std::scoped_lock<ur_mutex> Lock(EventCacheMutex);
   ur_device_handle_t Device = nullptr;
 
-  if (!Event->IsMultiDevice && Event->UrQueue) {
-    Device = Event->UrQueue->Device;
+  if (!Event->IsMultiDevice && Legacy(Event->UrQueue)) {
+    Device = Legacy(Event->UrQueue)->Device;
   }
 
   auto Cache = getEventCache(Event->isHostVisible(),
@@ -598,10 +598,10 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
 
   ze_device_handle_t ZeDevice = nullptr;
   bool UsingImmediateCommandlists =
-      !Event->UrQueue || Event->UrQueue->UsingImmCmdLists;
+      !Legacy(Event->UrQueue) || Legacy(Event->UrQueue)->UsingImmCmdLists;
 
-  if (!Event->IsMultiDevice && Event->UrQueue) {
-    ZeDevice = Event->UrQueue->Device->ZeDevice;
+  if (!Event->IsMultiDevice && Legacy(Event->UrQueue)) {
+    ZeDevice = Legacy(Event->UrQueue)->Device->ZeDevice;
   }
 
   std::list<ze_event_pool_handle_t> *ZePoolCache = getZeEventPoolCache(
@@ -644,7 +644,7 @@ static const size_t CmdListsCleanupThreshold = [] {
 
 // Retrieve an available command list to be used in a PI call.
 ur_result_t ur_context_handle_t_::getAvailableCommandList(
-    ur_queue_handle_t Queue, ur_command_list_ptr_t &CommandList,
+    ur_queue_handle_legacy_t Queue, ur_command_list_ptr_t &CommandList,
     bool UseCopyEngine, uint32_t NumEventsInWaitList,
     const ur_event_handle_t *EventWaitList, bool AllowBatching,
     ze_command_queue_handle_t *ForcedCmdQueue) {

source/adapters/level_zero/context.hpp

@@ -293,7 +293,7 @@ struct ur_context_handle_t_ : _ur_object {
   // for executing on this device. Immediate commandlists are created only
   // once for each SYCL Queue and after that they are reused.
   ur_result_t getAvailableCommandList(
-      ur_queue_handle_t Queue, ur_command_list_ptr_t &CommandList,
+      ur_queue_handle_legacy_t Queue, ur_command_list_ptr_t &CommandList,
       bool UseCopyEngine, uint32_t NumEventsInWaitList,
       const ur_event_handle_t *EventWaitList, bool AllowBatching = false,
       ze_command_queue_handle_t *ForcedCmdQueue = nullptr);

source/adapters/level_zero/event.cpp

@@ -46,21 +46,21 @@ static const bool UseMultipleCmdlistBarriers = [] {
 }();
 
 bool WaitListEmptyOrAllEventsFromSameQueue(
-    ur_queue_handle_t Queue, uint32_t NumEventsInWaitList,
+    ur_queue_handle_legacy_t Queue, uint32_t NumEventsInWaitList,
     const ur_event_handle_t *EventWaitList) {
   if (!NumEventsInWaitList)
     return true;
 
   for (uint32_t i = 0; i < NumEventsInWaitList; ++i) {
-    if (Queue != EventWaitList[i]->UrQueue)
+    if (Queue != Legacy(EventWaitList[i]->UrQueue))
       return false;
   }
 
   return true;
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWait(
-    ur_queue_handle_t Queue,      ///< [in] handle of the queue object
+    ur_queue_handle_t UrQueue,    ///< [in] handle of the queue object
     uint32_t NumEventsInWaitList, ///< [in] size of the event wait list
     const ur_event_handle_t
         *EventWaitList, ///< [in][optional][range(0, numEventsInWaitList)]
@@ -72,6 +72,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWait(
         *OutEvent ///< [in,out][optional] return an event object that identifies
                   ///< this particular command instance.
 ) {
+  auto Queue = Legacy(UrQueue);
   if (EventWaitList) {
     bool UseCopyEngine = false;
 
@@ -152,7 +153,7 @@ static const bool InOrderBarrierBySignal = [] {
 }();
 
 UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWaitWithBarrier(
-    ur_queue_handle_t Queue,      ///< [in] handle of the queue object
+    ur_queue_handle_t UrQueue,    ///< [in] handle of the queue object
     uint32_t NumEventsInWaitList, ///< [in] size of the event wait list
     const ur_event_handle_t
         *EventWaitList, ///< [in][optional][range(0, numEventsInWaitList)]
@@ -164,6 +165,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWaitWithBarrier(
         *OutEvent ///< [in,out][optional] return an event object that identifies
                   ///< this particular command instance.
 ) {
+  auto Queue = Legacy(UrQueue);
 
   // Lock automatically releases when this goes out of scope.
   std::scoped_lock<ur_shared_mutex> lock(Queue->Mutex);
@@ -299,8 +301,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWaitWithBarrier(
   for (auto &QueueMap :
        {Queue->ComputeQueueGroupsByTID, Queue->CopyQueueGroupsByTID})
     for (auto &QueueGroup : QueueMap) {
-      bool UseCopyEngine =
-          QueueGroup.second.Type != ur_queue_handle_t_::queue_type::Compute;
+      bool UseCopyEngine = QueueGroup.second.Type !=
+                           ur_queue_handle_legacy_t_::queue_type::Compute;
       if (Queue->UsingImmCmdLists) {
         // If immediate command lists are being used, each will act as their own
         // queue, so we must insert a barrier into each.
@@ -369,8 +371,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueEventsWaitWithBarrier(
 
   // Execute each command list so the barriers can be encountered.
   for (ur_command_list_ptr_t &CmdList : CmdLists) {
-    bool IsCopy =
-        CmdList->second.isCopy(reinterpret_cast<ur_queue_handle_t>(Queue));
+    bool IsCopy = CmdList->second.isCopy(
+        reinterpret_cast<ur_queue_handle_legacy_t>(Queue));
     const auto &CommandBatch =
         (IsCopy) ? Queue->CopyCommandBatch : Queue->ComputeCommandBatch;
     // Only batch if the matching CmdList is already open.
@@ -414,7 +416,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetInfo(
     // possible that this is trying to query some event's status that
     // is part of the batch.  This isn't strictly required, but it seems
     // like a reasonable thing to do.
-    auto UrQueue = Event->UrQueue;
+    auto UrQueue = Legacy(Event->UrQueue);
     if (UrQueue) {
       // Lock automatically releases when this goes out of scope.
       std::unique_lock<ur_shared_mutex> Lock(UrQueue->Mutex, std::try_to_lock);
@@ -486,8 +488,9 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo(
     return UR_RESULT_ERROR_PROFILING_INFO_NOT_AVAILABLE;
   }
 
-  ur_device_handle_t Device =
-      Event->UrQueue ? Event->UrQueue->Device : Event->Context->Devices[0];
+  ur_device_handle_t Device = Legacy(Event->UrQueue)
+                                  ? Legacy(Event->UrQueue)->Device
+                                  : Event->Context->Devices[0];
 
   uint64_t ZeTimerResolution = Device->ZeDeviceProperties->timerResolution;
   const uint64_t TimestampMaxValue =
@@ -512,10 +515,10 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo(
         return ReturnValue(Event->RecordEventEndTimestamp);
 
       // Otherwise we need to collect it from the queue.
-      auto Entry = Event->UrQueue->EndTimeRecordings.find(Event);
+      auto Entry = Legacy(Event->UrQueue)->EndTimeRecordings.find(Event);
 
       // Unexpected state if there is no end-time record.
-      if (Entry == Event->UrQueue->EndTimeRecordings.end())
+      if (Entry == Legacy(Event->UrQueue)->EndTimeRecordings.end())
         return UR_RESULT_ERROR_UNKNOWN;
       auto &EndTimeRecording = Entry->second;
 
@@ -540,7 +543,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo(
       // anymore, so we cache it on the event and evict the record from the
       // queue.
       Event->RecordEventEndTimestamp = ContextEndTime;
-      Event->UrQueue->EndTimeRecordings.erase(Entry);
+      Legacy(Event->UrQueue)->EndTimeRecordings.erase(Entry);
 
       return ReturnValue(ContextEndTime);
     }
@@ -659,7 +662,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetProfilingInfo(
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp(
-    ur_queue_handle_t Queue,      ///< [in] handle of the queue object
+    ur_queue_handle_t UrQueue,    ///< [in] handle of the queue object
     bool Blocking,                ///< [in] blocking or non-blocking enqueue
     uint32_t NumEventsInWaitList, ///< [in] size of the event wait list
     const ur_event_handle_t
@@ -673,6 +676,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp(
         *OutEvent ///< [in,out] return an event object that identifies
                   ///< this particular command instance.
 ) {
+  auto Queue = Legacy(UrQueue);
   // Lock automatically releases when this goes out of scope.
   std::scoped_lock<ur_shared_mutex> lock(Queue->Mutex);
 
@@ -701,7 +705,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp(
 
   // Create a new entry in the queue's recordings.
   Queue->EndTimeRecordings[*OutEvent] =
-      ur_queue_handle_t_::end_time_recording{};
+      ur_queue_handle_legacy_t_::end_time_recording{};
 
   ZE2UR_CALL(zeCommandListAppendWriteGlobalTimestamp,
              (CommandList->first,
@@ -717,6 +721,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueTimestampRecordingExp(
 
 ur_result_t ur_event_handle_t_::getOrCreateHostVisibleEvent(
     ze_event_handle_t &ZeHostVisibleEvent) {
+  auto UrQueue = Legacy(this->UrQueue);
 
   std::scoped_lock<ur_shared_mutex, ur_shared_mutex> Lock(UrQueue->Mutex,
                                                           this->Mutex);
@@ -771,7 +776,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventWait(
 ) {
   for (uint32_t I = 0; I < NumEvents; I++) {
     auto e = EventWaitList[I];
-    if (e->UrQueue && e->UrQueue->ZeEventsScope == OnDemandHostVisibleProxy) {
+    auto UrQueue = Legacy(e->UrQueue);
+    if (UrQueue && UrQueue->ZeEventsScope == OnDemandHostVisibleProxy) {
       // Make sure to add all host-visible "proxy" event signals if needed.
       // This ensures that all signalling commands are submitted below and
       // thus proxy events can be waited without a deadlock.
@@ -788,15 +794,15 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventWait(
   // Submit dependent open command lists for execution, if any
   for (uint32_t I = 0; I < NumEvents; I++) {
     ur_event_handle_t_ *Event = ur_cast<ur_event_handle_t_ *>(EventWaitList[I]);
-    auto UrQueue = Event->UrQueue;
+    auto UrQueue = Legacy(Event->UrQueue);
     if (UrQueue) {
       // Lock automatically releases when this goes out of scope.
       std::scoped_lock<ur_shared_mutex> lock(UrQueue->Mutex);
 
       UR_CALL(UrQueue->executeAllOpenCommandLists());
     }
   }
-  std::unordered_set<ur_queue_handle_t> Queues;
+  std::unordered_set<ur_queue_handle_legacy_t> Queues;
   for (uint32_t I = 0; I < NumEvents; I++) {
     {
       ur_event_handle_t_ *Event =
@@ -823,12 +829,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventWait(
           Event->Completed = true;
         }
       }
-      if (auto Q = Event->UrQueue) {
+      if (auto Q = Legacy(Event->UrQueue)) {
         if (Q->UsingImmCmdLists && Q->isInOrderQueue())
           // Use information about waited event to cleanup completed events in
           // the in-order queue.
           CleanupEventsInImmCmdLists(
-              Event->UrQueue, false /* QueueLocked */, false /* QueueSynced */,
+              Legacy(Event->UrQueue), false /* QueueLocked */,
+              false /* QueueSynced */,
               reinterpret_cast<ur_event_handle_t>(Event));
         else {
           // NOTE: we are cleaning up after the event here to free resources
@@ -884,7 +891,7 @@ UR_APIEXPORT ur_result_t UR_APICALL urEventGetNativeHandle(
   // Event can potentially be in an open command-list, make sure that
   // it is submitted for execution to avoid potential deadlock if
   // interop app is going to wait for it.
-  auto Queue = Event->UrQueue;
+  auto Queue = Legacy(Event->UrQueue);
   if (Queue) {
     std::scoped_lock<ur_shared_mutex> lock(Queue->Mutex);
     const auto &OpenCommandList = Queue->eventOpenCommandList(Event);
@@ -1014,7 +1021,7 @@ ur_result_t urEventReleaseInternal(ur_event_handle_t Event) {
   }
 
   // Save pointer to the queue before deleting/resetting event.
-  auto Queue = Event->UrQueue;
+  auto Queue = Legacy(Event->UrQueue);
 
   // If the event was a timestamp recording, we try to evict its entry in the
   // queue.
@@ -1028,7 +1035,7 @@ ur_result_t urEventReleaseInternal(ur_event_handle_t Event) {
         EndTimeRecording.EventHasDied = true;
       } else {
         // Otherwise we evict the entry.
-        Event->UrQueue->EndTimeRecordings.erase(Entry);
+        Legacy(Event->UrQueue)->EndTimeRecordings.erase(Entry);
       }
     }
   }
@@ -1046,8 +1053,8 @@ ur_result_t urEventReleaseInternal(ur_event_handle_t Event) {
   // created so that we can avoid ur_queue_handle_t is released before the
   // associated ur_event_handle_t is released. Here we have to decrement it so
   // ur_queue_handle_t can be released successfully.
-  if (Queue) {
-    UR_CALL(urQueueReleaseInternal(Queue));
+  if (Event->UrQueue) {
+    UR_CALL(urQueueReleaseInternal(Event->UrQueue));
   }
 
   return UR_RESULT_SUCCESS;
@@ -1091,7 +1098,7 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
   ur_kernel_handle_t AssociatedKernel = nullptr;
   // List of dependent events.
   std::list<ur_event_handle_t> EventsToBeReleased;
-  ur_queue_handle_t AssociatedQueue = nullptr;
+  ur_queue_handle_legacy_t AssociatedQueue = nullptr;
   {
     // If the Event is already locked, then continue with the cleanup, otherwise
     // block on locking the event.
@@ -1105,7 +1112,7 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
     if (Event->CleanedUp)
       return UR_RESULT_SUCCESS;
 
-    AssociatedQueue = Event->UrQueue;
+    AssociatedQueue = Legacy(Event->UrQueue);
 
     // Remember the kernel associated with this event if there is one. We are
     // going to release it later.
@@ -1222,9 +1229,9 @@ ur_result_t CleanupCompletedEvent(ur_event_handle_t Event, bool QueueLocked,
 // The "HostVisible" argument specifies if event needs to be allocated from
 // a host-visible pool.
 //
-ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
-                        bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent,
+ur_result_t EventCreate(ur_context_handle_t Context,
+                        ur_queue_handle_legacy_t Queue, bool IsMultiDevice,
+                        bool HostVisible, ur_event_handle_t *RetEvent,
                         bool CounterBasedEventEnabled,
                         bool ForceDisableProfiling) {
   bool ProfilingEnabled =
@@ -1311,7 +1318,7 @@ ur_result_t ur_event_handle_t_::reset() {
 
 ur_result_t _ur_ze_event_list_t::createAndRetainUrZeEventList(
     uint32_t EventListLength, const ur_event_handle_t *EventList,
-    ur_queue_handle_t CurQueue, bool UseCopyEngine) {
+    ur_queue_handle_legacy_t CurQueue, bool UseCopyEngine) {
   this->Length = 0;
   this->ZeEventList = nullptr;
   this->UrEventList = nullptr;
@@ -1427,7 +1434,7 @@ ur_result_t _ur_ze_event_list_t::createAndRetainUrZeEventList(
           }
         }
 
-        auto Queue = EventList[I]->UrQueue;
+        auto Queue = Legacy(EventList[I]->UrQueue);
 
         auto CurQueueDevice = CurQueue->Device;
         std::optional<std::unique_lock<ur_shared_mutex>> QueueLock =
@@ -1628,7 +1635,7 @@ ur_result_t _ur_ze_event_list_t::collectEventsForReleaseAndDestroyUrZeEventList(
 // Tells if this event is with profiling capabilities.
 bool ur_event_handle_t_::isProfilingEnabled() const {
   return !UrQueue || // tentatively assume user events are profiling enabled
-         (UrQueue->Properties & UR_QUEUE_FLAG_PROFILING_ENABLE) != 0;
+         (Legacy(UrQueue)->Properties & UR_QUEUE_FLAG_PROFILING_ENABLE) != 0;
 }
 
 // Tells if this event was created as a timestamp event, allowing profiling

source/adapters/level_zero/event.hpp

@@ -29,9 +29,9 @@
 
 extern "C" {
 ur_result_t urEventReleaseInternal(ur_event_handle_t Event);
-ur_result_t EventCreate(ur_context_handle_t Context, ur_queue_handle_t Queue,
-                        bool IsMultiDevice, bool HostVisible,
-                        ur_event_handle_t *RetEvent,
+ur_result_t EventCreate(ur_context_handle_t Context,
+                        ur_queue_handle_legacy_t Queue, bool IsMultiDevice,
+                        bool HostVisible, ur_event_handle_t *RetEvent,
                         bool CounterBasedEventEnabled = false,
                         bool ForceDisableProfiling = false);
 } // extern "C"
@@ -89,7 +89,7 @@ struct _ur_ze_event_list_t {
   // command-lists.
   ur_result_t createAndRetainUrZeEventList(uint32_t EventListLength,
                                            const ur_event_handle_t *EventList,
-                                           ur_queue_handle_t CurQueue,
+                                           ur_queue_handle_legacy_t CurQueue,
                                            bool UseCopyEngine);
 
   // Add all the events in this object's UrEventList to the end

source/adapters/level_zero/image.cpp

@@ -753,12 +753,13 @@ UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesSampledImageCreateExp(
 }
 
 UR_APIEXPORT ur_result_t UR_APICALL urBindlessImagesImageCopyExp(
-    ur_queue_handle_t hQueue, void *pDst, void *pSrc,
+    ur_queue_handle_t hUrQueue, void *pDst, void *pSrc,
     const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc,
     ur_exp_image_copy_flags_t imageCopyFlags, ur_rect_offset_t srcOffset,
     ur_rect_offset_t dstOffset, ur_rect_region_t copyExtent,
     ur_rect_region_t hostExtent, uint32_t numEventsInWaitList,
     const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) {
+  auto hQueue = Legacy(hUrQueue);
   std::scoped_lock<ur_shared_mutex> Lock(hQueue->Mutex);
 
   UR_ASSERT(hQueue, UR_RESULT_ERROR_INVALID_NULL_HANDLE);