[SYCL] Add a leaf limit to the execution graph

This patch adds a leaf limit (per memory object) for the command
execution graph in order to avoid leaf bloat in applications that have
an overwhelming number of command groups that can be executed in
parallel. Limiting the number of leaves is necessary for reducing
performance overhead of regular cleanup of finished command nodes.

Whenever the limit is exceeded, the oldest leaf is added as a
dependency of the new one instead.

Signed-off-by: Sergey Semenov <[email protected]>

Author: sergey-semenov

sycl/include/CL/sycl/detail/circular_buffer.hpp

@@ -0,0 +1,94 @@
+//==---------------- circular_buffer.hpp - Circular buffer -----------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <CL/sycl/detail/defines.hpp>
+
+#include <deque>
+#include <utility>
+
+__SYCL_INLINE namespace cl {
+namespace sycl {
+namespace detail {
+
+// A partial implementation of a circular buffer: once its capacity is full,
+// new data overwrites the old.
+template <typename T> class CircularBuffer {
+public:
+  explicit CircularBuffer(size_t Capacity) : MCapacity{Capacity} {};
+
+  using value_type = T;
+  using pointer = T *;
+  using const_pointer = const T *;
+  using reference = T &;
+  using const_reference = const T &;
+
+  using iterator = typename std::deque<T>::iterator;
+  using const_iterator = typename std::deque<T>::const_iterator;
+
+  iterator begin() { return MValues.begin(); }
+
+  const_iterator begin() const { return MValues.begin(); }
+
+  iterator end() { return MValues.end(); }
+
+  const_iterator end() const { return MValues.end(); }
+
+  reference front() { return MValues.front(); }
+
+  const_reference front() const { return MValues.front(); }
+
+  reference back() { return MValues.back(); }
+
+  const_reference back() const { return MValues.back(); }
+
+  reference operator[](size_t Idx) { return MValues[Idx]; }
+
+  const_reference operator[](size_t Idx) const { return MValues[Idx]; }
+
+  size_t size() const { return MValues.size(); }
+
+  size_t capacity() const { return MCapacity; }
+
+  bool empty() const { return MValues.empty(); };
+
+  bool full() const { return MValues.size() == MCapacity; };
+
+  void push_back(T Val) {
+    if (MValues.size() == MCapacity)
+      MValues.pop_front();
+    MValues.push_back(std::move(Val));
+  }
+
+  void push_front(T Val) {
+    if (MValues.size() == MCapacity)
+      MValues.pop_back();
+    MValues.push_front(std::move(Val));
+  }
+
+  void pop_back() { MValues.pop_back(); }
+
+  void pop_front() { MValues.pop_front(); }
+
+  void erase(const_iterator Pos) { MValues.erase(Pos); }
+
+  void erase(const_iterator First, const_iterator Last) {
+    MValues.erase(First, Last);
+  }
+
+  void clear() { MValues.clear(); }
+
+private:
+  std::deque<T> MValues;
+  const size_t MCapacity;
+};
+
+} // namespace detail
+} // namespace sycl
+} // namespace cl

sycl/include/CL/sycl/detail/scheduler/scheduler.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include <CL/sycl/detail/cg.hpp>
+#include <CL/sycl/detail/circular_buffer.hpp>
 #include <CL/sycl/detail/scheduler/commands.hpp>
 #include <CL/sycl/detail/sycl_mem_obj_i.hpp>
 
@@ -32,21 +33,25 @@ using ContextImplPtr = std::shared_ptr<detail::context_impl>;
 // The MemObjRecord is created for each memory object used in command
 // groups. There should be only one MemObjRecord for SYCL memory object.
 struct MemObjRecord {
+  MemObjRecord(ContextImplPtr CurContext, size_t LeafLimit)
+      : MReadLeaves{LeafLimit}, MWriteLeaves{LeafLimit}, MCurContext{
+                                                             CurContext} {}
+
   // Contains all allocation commands for the memory object.
   std::vector<AllocaCommandBase *> MAllocaCommands;
 
   // Contains latest read only commands working with memory object.
-  std::vector<Command *> MReadLeaves;
+  CircularBuffer<Command *> MReadLeaves;
 
   // Contains latest write commands working with memory object.
-  std::vector<Command *> MWriteLeaves;
+  CircularBuffer<Command *> MWriteLeaves;
 
   // The context which has the latest state of the memory object.
   ContextImplPtr MCurContext;
 
   // The flag indicates that the content of the memory object was/will be
   // modified. Used while deciding if copy back needed.
-  bool MMemModified;
+  bool MMemModified = false;
 };
 
 class Scheduler {
@@ -165,6 +170,9 @@ class Scheduler {
     std::set<Command *> findDepsForReq(MemObjRecord *Record, Requirement *Req,
                                        const ContextImplPtr &Context);
 
+    // Finds a command dependency corresponding to the record
+    DepDesc findDepForRecord(Command *Cmd, MemObjRecord *Record);
+
     // Searches for suitable alloca in memory record.
     AllocaCommandBase *findAllocaForReq(MemObjRecord *Record, Requirement *Req,
                                         const ContextImplPtr &Context);

sycl/source/detail/scheduler/graph_builder.cpp

@@ -120,11 +120,9 @@ Scheduler::GraphBuilder::getOrInsertMemObjRecord(const QueueImplPtr &Queue,
   if (nullptr != Record)
     return Record;
 
-  MemObject->MRecord.reset(new MemObjRecord{/*MAllocaCommands*/ {},
-                                            /*MReadLeaves*/ {},
-                                            /*MWriteLeaves*/ {},
-                                            Queue->getContextImplPtr(),
-                                            /*MMemModified*/ false});
+  const size_t LeafLimit = 8;
+  MemObject->MRecord.reset(
+      new MemObjRecord{Queue->getContextImplPtr(), LeafLimit});
 
   MMemObjs.push_back(MemObject);
   return MemObject->MRecord.get();
@@ -153,10 +151,21 @@ void Scheduler::GraphBuilder::UpdateLeaves(const std::set<Command *> &Cmds,
 void Scheduler::GraphBuilder::AddNodeToLeaves(MemObjRecord *Record,
                                               Command *Cmd,
                                               access::mode AccessMode) {
-  if (AccessMode == access::mode::read)
-    Record->MReadLeaves.push_back(Cmd);
-  else
-    Record->MWriteLeaves.push_back(Cmd);
+  CircularBuffer<Command *> &Leaves{AccessMode == access::mode::read
+                                        ? Record->MReadLeaves
+                                        : Record->MWriteLeaves};
+  if (Leaves.full()) {
+    Command *OldLeaf = Leaves.front();
+    if (OldLeaf == Cmd)
+      return;
+    // Add the old leaf as a dependency for the new one by duplicating one of
+    // the requirements for the current record
+    DepDesc Dep = findDepForRecord(Cmd, Record);
+    Dep.MDepCommand = OldLeaf;
+    Cmd->addDep(Dep);
+    OldLeaf->addUser(Cmd);
+  }
+  Leaves.push_back(Cmd);
 }
 
 UpdateHostRequirementCommand *Scheduler::GraphBuilder::insertUpdateHostReqCmd(
@@ -389,9 +398,8 @@ Scheduler::GraphBuilder::findDepsForReq(MemObjRecord *Record, Requirement *Req,
   std::set<Command *> Visited;
   const bool ReadOnlyReq = Req->MAccessMode == access::mode::read;
 
-  std::vector<Command *> ToAnalyze;
-
-  ToAnalyze = Record->MWriteLeaves;
+  std::vector<Command *> ToAnalyze{Record->MWriteLeaves.begin(),
+                                   Record->MWriteLeaves.end()};
 
   if (!ReadOnlyReq)
     ToAnalyze.insert(ToAnalyze.begin(), Record->MReadLeaves.begin(),
@@ -436,6 +444,19 @@ Scheduler::GraphBuilder::findDepsForReq(MemObjRecord *Record, Requirement *Req,
   return RetDeps;
 }
 
+// A helper function for finding a command dependency on a specific memory
+// object
+DepDesc Scheduler::GraphBuilder::findDepForRecord(Command *Cmd,
+                                                  MemObjRecord *Record) {
+  for (const DepDesc &DD : Cmd->MDeps) {
+    if (getMemObjRecord(DD.MDepRequirement->MSYCLMemObj) == Record) {
+      return DD;
+    }
+  }
+  assert(false && "No dependency found for a leaf of the record");
+  return {nullptr, nullptr, nullptr};
+}
+
 // The function searches for the alloca command matching context and
 // requirement.
 AllocaCommandBase *Scheduler::GraphBuilder::findAllocaForReq(

sycl/source/detail/scheduler/scheduler.cpp

@@ -146,7 +146,7 @@ EventImplPtr Scheduler::addHostAccessor(Requirement *Req) {
 void Scheduler::releaseHostAccessor(Requirement *Req) {
   Req->MBlockedCmd->MCanEnqueue = true;
   MemObjRecord* Record = Req->MSYCLMemObj->MRecord.get();
-  auto EnqueueLeaves = [](std::vector<Command *> &Leaves) {
+  auto EnqueueLeaves = [](CircularBuffer<Command *> &Leaves) {
     for (Command *Cmd : Leaves) {
       EnqueueResultT Res;
       bool Enqueued = GraphProcessor::enqueueCommand(Cmd, Res);