Merge pull request #1298 from pbalcer/immcmd-out-of-order-completion-batching

[L0] optimize # of event status queries through batching

Author: aarongreig

source/adapters/level_zero/context.cpp

@@ -13,6 +13,7 @@
 #include <mutex>
 #include <string.h>
 
+#include "adapters/level_zero/queue.hpp"
 #include "context.hpp"
 #include "ur_level_zero.hpp"
 
@@ -596,29 +597,6 @@ ur_context_handle_t_::decrementUnreleasedEventsInPool(ur_event_handle_t Event) {
   return UR_RESULT_SUCCESS;
 }
 
-// Get value of the threshold for number of events in immediate command lists.
-// If number of events in the immediate command list exceeds this threshold then
-// cleanup process for those events is executed.
-static const size_t ImmCmdListsEventCleanupThreshold = [] {
-  const char *UrRet =
-      std::getenv("UR_L0_IMMEDIATE_COMMANDLISTS_EVENT_CLEANUP_THRESHOLD");
-  const char *PiRet = std::getenv(
-      "SYCL_PI_LEVEL_ZERO_IMMEDIATE_COMMANDLISTS_EVENT_CLEANUP_THRESHOLD");
-  const char *ImmCmdListsEventCleanupThresholdStr =
-      UrRet ? UrRet : (PiRet ? PiRet : nullptr);
-  static constexpr int Default = 1000;
-  if (!ImmCmdListsEventCleanupThresholdStr)
-    return Default;
-
-  int Threshold = std::atoi(ImmCmdListsEventCleanupThresholdStr);
-
-  // Basically disable threshold if negative value is provided.
-  if (Threshold < 0)
-    return INT_MAX;
-
-  return Threshold;
-}();
-
 // Get value of the threshold for number of active command lists allowed before
 // we start heuristically cleaning them up.
 static const size_t CmdListsCleanupThreshold = [] {
@@ -648,8 +626,8 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList(
   // Immediate commandlists have been pre-allocated and are always available.
   if (Queue->UsingImmCmdLists) {
     CommandList = Queue->getQueueGroup(UseCopyEngine).getImmCmdList();
-    if (CommandList->second.EventList.size() >
-        ImmCmdListsEventCleanupThreshold) {
+    if (CommandList->second.EventList.size() >=
+        Queue->getImmdCmmdListsEventCleanupThreshold()) {
       std::vector<ur_event_handle_t> EventListToCleanup;
       Queue->resetCommandList(CommandList, false, EventListToCleanup);
       CleanupEventListFromResetCmdList(EventListToCleanup, true);
@@ -743,11 +721,13 @@ ur_result_t ur_context_handle_t_::getAvailableCommandList(
         ZE2UR_CALL(zeFenceCreate, (ZeCommandQueue, &ZeFenceDesc, &ZeFence));
         ZeStruct<ze_command_queue_desc_t> ZeQueueDesc;
         ZeQueueDesc.ordinal = QueueGroupOrdinal;
+
         CommandList =
             Queue->CommandListMap
                 .emplace(ZeCommandList,
-                         ur_command_list_info_t{ZeFence, true, false,
-                                                ZeCommandQueue, ZeQueueDesc})
+                         ur_command_list_info_t(ZeFence, true, false,
+                                                ZeCommandQueue, ZeQueueDesc,
+                                                Queue->useCompletionBatching()))
                 .first;
       }
       ZeCommandListCache.erase(ZeCommandListIt);

source/adapters/level_zero/event.cpp

@@ -11,6 +11,7 @@
 #include <algorithm>
 #include <climits>
 #include <mutex>
+#include <optional>
 #include <string.h>
 
 #include "command_buffer.hpp"
@@ -1129,6 +1130,7 @@ ur_result_t ur_event_handle_t_::reset() {
   RefCountExternal = 0;
   RefCount.reset();
   CommandList = std::nullopt;
+  completionBatch = std::nullopt;
 
   if (!isHostVisible())
     HostVisibleEvent = nullptr;

source/adapters/level_zero/event.hpp

@@ -222,6 +222,10 @@ struct ur_event_handle_t_ : _ur_object {
 
   // Get the host-visible event or create one and enqueue its signal.
   ur_result_t getOrCreateHostVisibleEvent(ze_event_handle_t &HostVisibleEvent);
+
+  // completion batch for this event. Only used for out-of-order immediate
+  // command lists.
+  std::optional<ur_completion_batch_it> completionBatch;
 };
 
 // Helper function to implement zeHostSynchronize.