[SYCL] reset enqueue status so that buffer can be used after failure is addressed (#17871)

CMPLRLLVM-65738 - If we run out of memory, an exception is thrown and
can be caught by the user application. They can address this in several
ways: free memory, .wait() on outstanding operations, etc. But despite
addressing the problem, SYCL has a bug wherein the buffer is not usable.
This is because the EnqueueStatus isn't reset, the Command/Requirement
cannot be re-enqueued.
The fix here is straightforward. We also improve the FailedDependency
unit test.

Author: cperkinsintel

sycl/source/detail/scheduler/graph_processor.cpp

@@ -84,6 +84,13 @@ bool Scheduler::GraphProcessor::enqueueCommand(
     return false;
   }
 
+  // Reset enqueue status if reattempting
+
+  if (!Cmd->isHostTask() &&
+      Cmd->MEnqueueStatus == EnqueueResultT::SyclEnqueueFailed) {
+    Cmd->MEnqueueStatus = EnqueueResultT::SyclEnqueueReady;
+  }
+
   // Recursively enqueue all the implicit + explicit backend level dependencies
   // first and exit immediately if any of the commands cannot be enqueued.
   for (const EventImplPtr &Event : Cmd->getPreparedDepsEvents()) {

sycl/unittests/scheduler/FailedCommands.cpp

@@ -20,13 +20,14 @@ TEST_F(SchedulerTest, FailedDependency) {
   queue Queue(context(Plt), default_selector_v);
 
   detail::Requirement MockReq = getMockRequirement();
-  MockCommand MDep(detail::getSyclObjImpl(Queue));
+  MockCommand MDepFail(
+      false, detail::getSyclObjImpl(Queue)); // <-- will fail to enqueue
   MockCommand MUser(detail::getSyclObjImpl(Queue));
-  MDep.addUser(&MUser);
+  MDepFail.addUser(&MUser);
   std::vector<detail::Command *> ToCleanUp;
-  (void)MUser.addDep(detail::DepDesc{&MDep, &MockReq, nullptr}, ToCleanUp);
+  (void)MUser.addDep(detail::DepDesc{&MDepFail, &MockReq, nullptr}, ToCleanUp);
   MUser.MEnqueueStatus = detail::EnqueueResultT::SyclEnqueueReady;
-  MDep.MEnqueueStatus = detail::EnqueueResultT::SyclEnqueueFailed;
+  MDepFail.MEnqueueStatus = detail::EnqueueResultT::SyclEnqueueReady;
 
   MockScheduler MS;
   auto Lock = MS.acquireGraphReadLock();
@@ -35,13 +36,13 @@ TEST_F(SchedulerTest, FailedDependency) {
       MockScheduler::enqueueCommand(&MUser, Res, detail::NON_BLOCKING);
 
   ASSERT_FALSE(Enqueued) << "Enqueue process must fail\n";
-  ASSERT_EQ(Res.MCmd, &MDep) << "Wrong failed command\n";
+  ASSERT_EQ(Res.MCmd, &MDepFail) << "Wrong failed command\n";
   ASSERT_EQ(Res.MResult, detail::EnqueueResultT::SyclEnqueueFailed)
       << "Enqueue process must fail\n";
   ASSERT_EQ(MUser.MEnqueueStatus, detail::EnqueueResultT::SyclEnqueueReady)
       << "MUser shouldn't be marked as failed\n";
-  ASSERT_EQ(MDep.MEnqueueStatus, detail::EnqueueResultT::SyclEnqueueFailed)
-      << "MDep should be marked as failed\n";
+  ASSERT_EQ(MDepFail.MEnqueueStatus, detail::EnqueueResultT::SyclEnqueueFailed)
+      << "MDepFail should be marked as failed\n";
 }
 
 void RunWithFailedCommandsAndCheck(bool SyncExceptionExpected,

sycl/unittests/scheduler/SchedulerTestUtils.hpp

@@ -53,6 +53,17 @@ class MockCommand : public sycl::detail::Command {
     EXPECT_CALL(*this, enqueue).Times(AnyNumber());
   }
 
+  // This Mock will fail to enqueue.
+  MockCommand(
+      bool, sycl::detail::QueueImplPtr Queue,
+      sycl::detail::Command::CommandType Type = sycl::detail::Command::RUN_CG)
+      : Command{Type, Queue}, MRequirement{std::move(getMockRequirement())} {
+    using namespace testing;
+    ON_CALL(*this, enqueue)
+        .WillByDefault(Invoke(this, &MockCommand::enqueueFail));
+    EXPECT_CALL(*this, enqueue).Times(AnyNumber());
+  }
+
   void printDot(std::ostream &) const override {}
   void emitInstrumentationData() override {}
 
@@ -70,6 +81,14 @@ class MockCommand : public sycl::detail::Command {
                      std::vector<sycl::detail::Command *> &ToCleanUp) {
     return sycl::detail::Command::enqueue(EnqueueResult, Blocking, ToCleanUp);
   }
+  bool enqueueFail(sycl::detail::EnqueueResultT &EnqueueResult,
+                   sycl::detail::BlockingT Blocking,
+                   std::vector<sycl::detail::Command *> &ToCleanUp) {
+    this->MEnqueueStatus = sycl::detail::EnqueueResultT::SyclEnqueueFailed;
+    EnqueueResult = {sycl::detail::EnqueueResultT::SyclEnqueueFailed, this};
+    ToCleanUp.push_back(this);
+    return false;
+  }
 
   ur_result_t MRetVal = UR_RESULT_SUCCESS;