[SYCL][XPTI] Revisit resource management strategy (#4494)

Global objects are typically destroyed on process tear down, but the order of the destructor calls is undefined. Since SYCL applications can potentially call SYCL APIs from global context, XPTI and all of its resources have to outlive user application. This patch refactors XPTI proxy library and framework to allocate global objects on heap and manages their lifetime based on communications from traced application.

Each user of XPTI (e.g. SYCL runtime) has to call xptiFrameworkInitialize() once prior to any other XPTI API. When application is done collecting trace information, it must close streams and then call xptiFrameworkFinalize().

The XPTI framework will maintain a reference counter, and will only free resources and unload libraries, when the counter hits 0. This will allow the subscribers to survive past DllMain call or global shared library destructor.

Author: Alexander Batashev

sycl/source/detail/pi.cpp

@@ -435,6 +435,8 @@ static void initializePlugins(std::vector<plugin> &Plugins) {
   }
 
 #ifdef XPTI_ENABLE_INSTRUMENTATION
+  GlobalHandler::instance().getXPTIRegistry().initializeFrameworkOnce();
+
   if (!(xptiTraceEnabled() && !XPTIInitDone))
     return;
   // Not sure this is the best place to initialize the framework; SYCL runtime

sycl/source/detail/xpti_registry.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include <mutex>
 #include <string>
 #include <unordered_set>
 
@@ -31,6 +32,12 @@ inline constexpr const char *SYCL_PIDEBUGCALL_STREAM_NAME = "sycl.pi.debug";
 
 class XPTIRegistry {
 public:
+  void initializeFrameworkOnce() {
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+    std::call_once(MInitialized, [] { xptiFrameworkInitialize(); });
+#endif
+  }
+
   /// Notifies XPTI subscribers about new stream.
   ///
   /// \param StreamName is a name of newly initialized stream.
@@ -50,11 +57,13 @@ class XPTIRegistry {
     for (const auto &StreamName : MActiveStreams) {
       xptiFinalize(StreamName.c_str());
     }
+    xptiFrameworkFinalize();
 #endif // XPTI_ENABLE_INSTRUMENTATION
   }
 
 private:
   std::unordered_set<std::string> MActiveStreams;
+  std::once_flag MInitialized;
 };
 } // namespace detail
 } // namespace sycl

xpti/include/xpti/xpti_trace_framework.h

@@ -35,6 +35,20 @@
 
 extern "C" {
 
+/// @brief Initializes XPTI framework.
+/// @details Initialize XPTI framework resources. Each user of XPTI must call
+/// this function prior to any other XPTI API call. It is framework's
+/// responsibility to ensure that resources are initialized once. Each call to
+/// this function must have corresponding call to xptiFrameworkFinalize() to
+/// ensure resources are freed.
+XPTI_EXPORT_API void xptiFrameworkInitialize();
+
+/// @brief Deinitializes XPTI framework.
+/// @details Call to this function decrements framework's internal reference
+/// counter. Once its value is equal to zero, XPTI framework can release
+/// resources and unload subscribers.
+XPTI_EXPORT_API void xptiFrameworkFinalize();
+
 /// @brief Initialization function that is called when a new stream is generated
 /// @details When a runtime or application that uses XPTI instrumentation API
 /// starts to generate a new stream, a call to xptiInitialize() must be made to
@@ -414,6 +428,8 @@ XPTI_EXPORT_API void xptiReset();
 /// The proxy/stub library does not implement this function.
 XPTI_EXPORT_API void xptiForceSetTraceEnabled(bool yesOrNo);
 
+typedef xpti::result_t (*xpti_framework_initialize_t)();
+typedef xpti::result_t (*xpti_framework_finalize_t)();
 typedef xpti::result_t (*xpti_initialize_t)(const char *, uint32_t, uint32_t,
                                             const char *);
 typedef void (*xpti_finalize_t)(const char *);

xpti/include/xpti/xpti_trace_framework.hpp

@@ -7,6 +7,7 @@
 //
 #pragma once
 
+#include <atomic>
 #include <cstdint>
 #include <memory>
 #include <sstream>
@@ -273,6 +274,20 @@ class PlatformHelper {
     return false;
   }
 };
+
+/// This is an implementation of a SpinLock synchronization primitive, that has
+/// trivial constructor and destructor.
+class SpinLock {
+public:
+  void lock() {
+    while (MLock.test_and_set(std::memory_order_acquire))
+      std::this_thread::yield();
+  }
+  void unlock() { MLock.clear(std::memory_order_release); }
+
+private:
+  std::atomic_flag MLock = ATOMIC_FLAG_INIT;
+};
 } // namespace utils
 
 namespace framework {