[SYCL][XPTI] Add unique queue ID to trace

sycl/source/detail/queue_impl.cpp

@@ -26,6 +26,9 @@
 namespace sycl {
 inline namespace _V1 {
 namespace detail {
+
+std::atomic<unsigned long long> queue_impl::MNextAvailableQueueID = 0;
+
 template <>
 uint32_t queue_impl::get_info<info::queue::reference_count>() const {
   sycl::detail::pi::PiResult result = PI_SUCCESS;
@@ -75,6 +78,7 @@ event queue_impl::memset(const std::shared_ptr<detail::queue_impl> &Self,
     xpti::addMetadata(TEvent, "memory_ptr", reinterpret_cast<size_t>(Ptr));
     xpti::addMetadata(TEvent, "value_set", Value);
     xpti::addMetadata(TEvent, "memory_size", Count);
+    xpti::addMetadata(TEvent, "queue_id", MQueueID);
   });
   // Notify XPTI about the memset submission
   PrepareNotify.notify();
@@ -152,6 +156,7 @@ event queue_impl::memcpy(const std::shared_ptr<detail::queue_impl> &Self,
     xpti::addMetadata(TEvent, "dest_memory_ptr",
                       reinterpret_cast<size_t>(Dest));
     xpti::addMetadata(TEvent, "memory_size", Count);
+    xpti::addMetadata(TEvent, "queue_id", MQueueID);
   });
   // Notify XPTI about the memset submission
   PrepareNotify.notify();
@@ -448,7 +453,7 @@ void *queue_impl::instrumentationProlog(const detail::code_location &CodeLoc,
       DevStr = "ACCELERATOR";
     else
       DevStr = "UNKNOWN";
-    xpti::addMetadata(WaitEvent, "sycl_device", DevStr);
+    xpti::addMetadata(WaitEvent, "sycl_device_type", DevStr);
     if (HasSourceInfo) {
       xpti::addMetadata(WaitEvent, "sym_function_name", CodeLoc.functionName());
       xpti::addMetadata(WaitEvent, "sym_source_file_name", CodeLoc.fileName());

sycl/source/detail/queue_impl.hpp

@@ -113,39 +113,9 @@ class queue_impl {
             has_property<ext::oneapi::property::queue::discard_events>()),
         MIsProfilingEnabled(has_property<property::queue::enable_profiling>()),
         MHasDiscardEventsSupport(MDiscardEvents &&
-                                 (MHostQueue ? true : MIsInorder)) {
-    // We enable XPTI tracing events using the TLS mechanism; if the code
-    // location data is available, then the tracing data will be rich.
-#if XPTI_ENABLE_INSTRUMENTATION
-    /// This section of code is relying on scoped objects, so they cannot be
-    /// encapsulated in a function
-    constexpr uint16_t NotificationTraceType =
-        static_cast<uint16_t>(xpti::trace_point_type_t::queue_create);
-    XPTIScope PrepareNotify((void *)this, NotificationTraceType,
-                            SYCL_STREAM_NAME, "queue_create");
-    // Cache the trace event, stream id and instance IDs for the destructor
-    if (xptiCheckTraceEnabled(PrepareNotify.streamID(),
-                              NotificationTraceType)) {
-      MTraceEvent = (void *)PrepareNotify.traceEvent();
-      MStreamID = PrepareNotify.streamID();
-      MInstanceID = PrepareNotify.instanceID();
-      // Add the function to capture meta data for the XPTI trace event
-      PrepareNotify.addMetadata([&](auto TEvent) {
-        xpti::addMetadata(TEvent, "sycl_context",
-                          reinterpret_cast<size_t>(MContext->getHandleRef()));
-        if (MDevice) {
-          xpti::addMetadata(TEvent, "sycl_device_name",
-                            MDevice->getDeviceName());
-          xpti::addMetadata(
-              TEvent, "sycl_device",
-              reinterpret_cast<size_t>(
-                  MDevice->is_host() ? 0 : MDevice->getHandleRef()));
-        }
-        xpti::addMetadata(TEvent, "is_inorder", MIsInorder);
-      });
-      PrepareNotify.notify();
-    }
-#endif
+                                 (MHostQueue ? true : MIsInorder)),
+        MQueueID{
+            MNextAvailableQueueID.fetch_add(1, std::memory_order_relaxed)} {
     if (has_property<property::queue::enable_profiling>()) {
       if (has_property<ext::oneapi::property::queue::discard_events>())
         throw sycl::exception(make_error_code(errc::invalid),
@@ -196,28 +166,19 @@ class queue_impl {
       // This section is the second part of the instrumentation that uses the
       // tracepoint information and notifies
     }
-  }
-
-private:
-  void queue_impl_interop(sycl::detail::pi::PiQueue PiQueue) {
-    // The following commented section provides a guideline on how to use the
-    // TLS enabled mechanism to create a tracepoint and notify using XPTI. This
-    // is the prolog section and the epilog section will initiate the
-    // notification.
+    // We enable XPTI tracing events using the TLS mechanism; if the code
+    // location data is available, then the tracing data will be rich.
 #if XPTI_ENABLE_INSTRUMENTATION
-    /// This section of code is relying on scoped objects, so they cannot be
-    /// encapsulated in a function
     constexpr uint16_t NotificationTraceType =
         static_cast<uint16_t>(xpti::trace_point_type_t::queue_create);
     XPTIScope PrepareNotify((void *)this, NotificationTraceType,
                             SYCL_STREAM_NAME, "queue_create");
+    // Cache the trace event, stream id and instance IDs for the destructor
     if (xptiCheckTraceEnabled(PrepareNotify.streamID(),
                               NotificationTraceType)) {
-      // Cache the trace event, stream id and instance IDs for the destructor
       MTraceEvent = (void *)PrepareNotify.traceEvent();
       MStreamID = PrepareNotify.streamID();
       MInstanceID = PrepareNotify.instanceID();
-
       // Add the function to capture meta data for the XPTI trace event
       PrepareNotify.addMetadata([&](auto TEvent) {
         xpti::addMetadata(TEvent, "sycl_context",
@@ -231,10 +192,18 @@ class queue_impl {
                   MDevice->is_host() ? 0 : MDevice->getHandleRef()));
         }
         xpti::addMetadata(TEvent, "is_inorder", MIsInorder);
+        xpti::addMetadata(TEvent, "queue_id", MQueueID);
+        if (!MHostQueue)
+          xpti::addMetadata(TEvent, "queue_handle",
+                            reinterpret_cast<size_t>(getHandleRef()));
       });
       PrepareNotify.notify();
     }
 #endif
+  }
+
+private:
+  void queue_impl_interop(sycl::detail::pi::PiQueue PiQueue) {
     if (has_property<ext::oneapi::property::queue::discard_events>() &&
         has_property<property::queue::enable_profiling>()) {
       throw sycl::exception(make_error_code(errc::invalid),
@@ -255,6 +224,42 @@ class queue_impl {
           make_error_code(errc::invalid),
           "Device provided by native Queue not found in Context.");
     }
+    // The following commented section provides a guideline on how to use the
+    // TLS enabled mechanism to create a tracepoint and notify using XPTI. This
+    // is the prolog section and the epilog section will initiate the
+    // notification.
+#if XPTI_ENABLE_INSTRUMENTATION
+    constexpr uint16_t NotificationTraceType =
+        static_cast<uint16_t>(xpti::trace_point_type_t::queue_create);
+    XPTIScope PrepareNotify((void *)this, NotificationTraceType,
+                            SYCL_STREAM_NAME, "queue_create");
+    if (xptiCheckTraceEnabled(PrepareNotify.streamID(),
+                              NotificationTraceType)) {
+      // Cache the trace event, stream id and instance IDs for the destructor
+      MTraceEvent = (void *)PrepareNotify.traceEvent();
+      MStreamID = PrepareNotify.streamID();
+      MInstanceID = PrepareNotify.instanceID();
+
+      // Add the function to capture meta data for the XPTI trace event
+      PrepareNotify.addMetadata([&](auto TEvent) {
+        xpti::addMetadata(TEvent, "sycl_context",
+                          reinterpret_cast<size_t>(MContext->getHandleRef()));
+        if (MDevice) {
+          xpti::addMetadata(TEvent, "sycl_device_name",
+                            MDevice->getDeviceName());
+          xpti::addMetadata(
+              TEvent, "sycl_device",
+              reinterpret_cast<size_t>(
+                  MDevice->is_host() ? 0 : MDevice->getHandleRef()));
+        }
+        xpti::addMetadata(TEvent, "is_inorder", MIsInorder);
+        xpti::addMetadata(TEvent, "queue_id", MQueueID);
+        if (!MHostQueue)
+          xpti::addMetadata(TEvent, "queue_handle", getHandleRef());
+      });
+      PrepareNotify.notify();
+    }
+#endif
   }
 
 public:
@@ -273,7 +278,9 @@ class queue_impl {
             has_property<ext::oneapi::property::queue::discard_events>()),
         MIsProfilingEnabled(has_property<property::queue::enable_profiling>()),
         MHasDiscardEventsSupport(MDiscardEvents &&
-                                 (MHostQueue ? true : MIsInorder)) {
+                                 (MHostQueue ? true : MIsInorder)),
+        MQueueID{
+            MNextAvailableQueueID.fetch_add(1, std::memory_order_relaxed)} {
     queue_impl_interop(PiQueue);
   }
 
@@ -310,6 +317,7 @@ class queue_impl {
                             (xpti::trace_event_data_t *)MTraceEvent,
                             MInstanceID,
                             static_cast<const void *>("queue_destroy"));
+      xptiReleaseEvent((xpti::trace_event_data_t *)MTraceEvent);
     }
 #endif
     throw_asynchronous();
@@ -695,6 +703,8 @@ class queue_impl {
     return MGraph.lock();
   }
 
+  unsigned long long getQueueID() { return MQueueID; }
+
 protected:
   // Hook to the scheduler to clean up any fusion command held on destruction.
   void cleanup_fusion_cmd();
@@ -873,6 +883,9 @@ class queue_impl {
   // recording commands to it.
   std::weak_ptr<ext::oneapi::experimental::detail::graph_impl> MGraph{};
 
+  unsigned long long MQueueID;
+  static std::atomic<unsigned long long> MNextAvailableQueueID;
+
   friend class sycl::ext::oneapi::experimental::detail::node_impl;
 };