[SYCL][CUDA] Use cuEventQuery to check event completion status (#3544)

Previously the completion status of a PI CUDA event (returned by a
`get_info` query for `info::event::command_execution_status`) would only
be changed to `command_execution_status::complete` once the event had
been waited upon.

This changes the mechanism to query the underlying event's execution
status using `cuEventQuery` if the event has not been waited upon.

Author: psalz

sycl/plugins/cuda/pi_cuda.cpp

@@ -322,9 +322,9 @@ pi_result cuda_piEventRetain(pi_event event);
 /// \endcond
 
 _pi_event::_pi_event(pi_command_type type, pi_context context, pi_queue queue)
-    : commandType_{type}, refCount_{1}, isCompleted_{false}, isRecorded_{false},
-      isStarted_{false}, evEnd_{nullptr}, evStart_{nullptr}, evQueued_{nullptr},
-      queue_{queue}, context_{context} {
+    : commandType_{type}, refCount_{1}, hasBeenWaitedOn_{false},
+      isRecorded_{false}, isStarted_{false}, evEnd_{nullptr}, evStart_{nullptr},
+      evQueued_{nullptr}, queue_{queue}, context_{context} {
 
   bool profilingEnabled = queue_->properties_ & PI_QUEUE_PROFILING_ENABLE;
 
@@ -367,6 +367,23 @@ pi_result _pi_event::start() {
   return result;
 }
 
+bool _pi_event::is_completed() const noexcept {
+  if (!isRecorded_) {
+    return false;
+  }
+  if (!hasBeenWaitedOn_) {
+    const CUresult ret = cuEventQuery(evEnd_);
+    if (ret != CUDA_SUCCESS && ret != CUDA_ERROR_NOT_READY) {
+      PI_CHECK_ERROR(ret);
+      return false;
+    }
+    if (ret == CUDA_ERROR_NOT_READY) {
+      return false;
+    }
+  }
+  return true;
+}
+
 pi_uint64 _pi_event::get_queued_time() const {
   float miliSeconds = 0.0f;
   assert(is_started());
@@ -428,7 +445,7 @@ pi_result _pi_event::wait() {
   pi_result retErr;
   try {
     retErr = PI_CHECK_ERROR(cuEventSynchronize(evEnd_));
-    isCompleted_ = true;
+    hasBeenWaitedOn_ = true;
   } catch (pi_result error) {
     retErr = error;
   }

sycl/plugins/cuda/pi_cuda.hpp

@@ -410,7 +410,7 @@ class _pi_event {
 
   bool is_started() const noexcept { return isStarted_; }
 
-  bool is_completed() const noexcept { return isCompleted_; };
+  bool is_completed() const noexcept;
 
   pi_int32 get_execution_status() const noexcept {
 
@@ -462,8 +462,9 @@ class _pi_event {
 
   std::atomic_uint32_t refCount_; // Event reference count.
 
-  bool isCompleted_; // Signifies whether the operations have completed
-                     //
+  bool hasBeenWaitedOn_; // Signifies whether the event has been waited
+                         // on through a call to wait(), which implies
+                         // that it has completed.
 
   bool isRecorded_; // Signifies wether a native CUDA event has been recorded
                     // yet.