[SYCL][PI][XPTI] Instrumenting PI Layer API - Part I (#1655)

+ Added new trace point type 'function_begin/function_end'
  to support pure API tracing without having to create a
  trace event.
+ PI Layer API tracing enabled using function_begin/end
+ xptiNotifySubscribers() behaves differently when the
  tracepoint type is function_begin/end and requires the
  per_instance_user_data field to be valid with a function
  name.
+ Updated XPTI API documentation to reflect this change
  and added additional tests to test xptiNotifySubscribers

Signed-off-by: Vasanth Tovinkere <[email protected]>

Author: tovinkere

.github/CODEOWNERS

@@ -85,3 +85,4 @@ sycl/doc/ @pvchupin @kbobrovs
 sycl/doc/extensions/ @mkinsner @jbrodman
 
 xpti/ @tovinkere @andykaylor
+xptifw/ @tovinkere  @andykaylor

sycl/include/CL/sycl/detail/common.hpp

@@ -27,12 +27,13 @@
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 namespace detail {
-// We define a sycl stream name and this will
-// be used by the instrumentation framework
+// We define a sycl stream name and this will be used by the instrumentation
+// framework
 constexpr const char *SYCL_STREAM_NAME = "sycl";
-// Data structure that captures the user code
-// location information using the builtin capabilities
-// of the compiler
+// Stream name being used for traces generated from the SYCL plugin layer
+constexpr const char *SYCL_PICALL_STREAM_NAME = "sycl.pi";
+// Data structure that captures the user code location information using the
+// builtin capabilities of the compiler
 struct code_location {
 #ifdef _MSC_VER
   // Since MSVC does not support the required builtins, we

sycl/include/CL/sycl/detail/pi.hpp

@@ -150,6 +150,18 @@ template <PiApiKind PiApiOffset> struct PiFuncInfo {};
   };
 #include <CL/sycl/detail/pi.def>
 
+/// Emits an XPTI trace before a PI API call is made
+/// \param FName The name of the PI API call
+/// \return The correlation ID for the API call that is to be used by the
+/// emitFunctionEndTrace() call
+uint64_t emitFunctionBeginTrace(const char *FName);
+
+/// Emits an XPTI trace after the PI API call has been made
+/// \param CorrelationID The correlation ID for the API call generated by the
+/// emitFunctionBeginTrace() call.
+/// \param FName The name of the PI API call
+void emitFunctionEndTrace(uint64_t CorrelationID, const char *FName);
+
 // Helper utilities for PI Tracing
 // The run-time tracing of PI calls.
 // Print functions used by Trace class.

sycl/source/detail/pi.cpp

@@ -37,25 +37,84 @@ __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 namespace detail {
 #ifdef XPTI_ENABLE_INSTRUMENTATION
-// Stream name being used for traces generated from the SYCL runtime
-constexpr const char *PICALL_STREAM_NAME = "sycl.pi";
 // Global (to the SYCL runtime) graph handle that all command groups are a
 // child of
-///< Event to be used by graph related activities
+/// Event to be used by graph related activities
 xpti_td *GSYCLGraphEvent = nullptr;
-///< Event to be used by PI layer related activities
+/// Event to be used by PI layer related activities
 xpti_td *GPICallEvent = nullptr;
-///< Constansts being used as placeholder until one is able to reliably get the
-///< version of the SYCL runtime
+/// Constants being used as placeholder until one is able to reliably get the
+/// version of the SYCL runtime
 constexpr uint32_t GMajVer = 1;
 constexpr uint32_t GMinVer = 0;
 constexpr const char *GVerStr = "sycl 1.0";
-#endif
+#endif // XPTI_ENABLE_INSTRUMENTATION
 
 namespace pi {
 
 bool XPTIInitDone = false;
 
+// Implementation of the SYCL PI API call tracing methods that use XPTI
+// framework to emit these traces that will be used by tools.
+uint64_t emitFunctionBeginTrace(const char *FName) {
+  uint64_t CorrelationID = 0;
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+  // The function_begin and function_end trace point types are defined to
+  // trace library API calls and they are currently enabled here for support
+  // tools that need the API scope. The methods emitFunctionBeginTrace() and
+  // emitFunctionEndTrace() can be extended to also trace the arguments of the
+  // PI API call using a trace point type the extends the predefined trace
+  // point types.
+  //
+  // You can use the sample collector in llvm/xptifw/samples/syclpi_collector
+  // to print the API traces and also extend them to support  arguments that
+  // may be traced later.
+  //
+  /// Example Usage:
+  /// \code{cpp}
+  /// // Two diagnostic trace types defined for function begin and function end
+  /// // with different semantics than the one in the default trace type list.
+  /// typedef enum {
+  ///   diagnostic_func_begin = XPTI_TRACE_POINT_BEGIN(0),
+  ///   diagnostic_func_end = XPTI_TRACE_POINT_END(0),
+  /// }syclpi_extension_t;
+  /// ...
+  /// uint16_t pi_func_begin =
+  ///     xptiRegisterUserDefinedTracePoint("sycl.pi", func_begin);
+  /// uint16_t pi_func_end =
+  ///     xptiRegisterUserDefinedTracePoint("sycl.pi", func_end);
+  /// ...
+  /// // Setup argument data for the function being traced
+  /// ...
+  /// xptiNotifySubscribers(stream_id, pi_func_begin, parent, event, instance,
+  ///                       (void *)argument_data);
+  /// \endcode
+  if (xptiTraceEnabled()) {
+    uint8_t StreamID = xptiRegisterStream(SYCL_PICALL_STREAM_NAME);
+    CorrelationID = xptiGetUniqueId();
+    xptiNotifySubscribers(
+        StreamID, (uint16_t)xpti::trace_point_type_t::function_begin,
+        GPICallEvent, nullptr, CorrelationID, static_cast<const void *>(FName));
+  }
+#endif // XPTI_ENABLE_INSTRUMENTATION
+  return CorrelationID;
+}
+
+void emitFunctionEndTrace(uint64_t CorrelationID, const char *FName) {
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+  if (xptiTraceEnabled()) {
+    // CorrelationID is the unique ID that ties together a function_begin and
+    // function_end pair of trace calls. The splitting of a scoped_notify into
+    // two function calls incurs an additional overhead as the StreamID must
+    // be looked up twice.
+    uint8_t StreamID = xptiRegisterStream(SYCL_PICALL_STREAM_NAME);
+    xptiNotifySubscribers(
+        StreamID, (uint16_t)xpti::trace_point_type_t::function_end,
+        GPICallEvent, nullptr, CorrelationID, static_cast<const void *>(FName));
+  }
+#endif // XPTI_ENABLE_INSTRUMENTATION
+}
+
 void contextSetExtendedDeleter(const cl::sycl::context &context,
                                pi_context_extended_deleter func,
                                void *user_data) {
@@ -285,6 +344,8 @@ vector_class<plugin> initialize() {
                           GSYCLGraphEvent, GraphInstanceNo, nullptr);
   }
 
+  // Let subscribers know a new stream is being initialized
+  xptiInitialize(SYCL_PICALL_STREAM_NAME, GMajVer, GMinVer, GVerStr);
   xpti::payload_t PIPayload("Plugin Interface Layer");
   uint64_t PiInstanceNo;
   GPICallEvent =

sycl/source/detail/plugin.hpp

@@ -12,10 +12,17 @@
 #include <CL/sycl/detail/pi.hpp>
 #include <CL/sycl/stl.hpp>
 
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+// Include the headers necessary for emitting traces using the trace framework
+#include "xpti_trace_framework.h"
+#endif
+
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 namespace detail {
-
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+extern xpti::trace_event_data_t *GPICallEvent;
+#endif
 /// The plugin class provides a unified interface to the underlying low-level
 /// runtimes for the device-agnostic SYCL runtime.
 ///
@@ -53,6 +60,13 @@ class plugin {
   template <PiApiKind PiApiOffset, typename... ArgsT>
   RT::PiResult call_nocheck(ArgsT... Args) const {
     RT::PiFuncInfo<PiApiOffset> PiCallInfo;
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+    // Emit a function_begin trace for the PI API before the call is executed.
+    // If arguments need to be captured, then a data structure can be sent in
+    // the per_instance_user_data field.
+    std::string PIFnName = PiCallInfo.getFuncName();
+    uint64_t CorrelationID = pi::emitFunctionBeginTrace(PIFnName.c_str());
+#endif
     if (pi::trace(pi::TraceLevel::PI_TRACE_CALLS)) {
       std::string FnName = PiCallInfo.getFuncName();
       std::cout << "---> " << FnName << "(" << std::endl;
@@ -63,6 +77,10 @@ class plugin {
       std::cout << ") ---> ";
       RT::printArgs(R);
     }
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+    // Close the function begin with a call to function end
+    pi::emitFunctionEndTrace(CorrelationID, PIFnName.c_str());
+#endif
     return R;
   }
 

xpti/include/xpti_data_types.h

@@ -246,6 +246,15 @@ enum class trace_point_type_t : uint16_t {
   wait_begin = XPTI_TRACE_POINT_BEGIN(11),
   /// Models the explicit barrier end in SYCL
   wait_end = XPTI_TRACE_POINT_END(11),
+  /// Used to trace function call begin, from libraries, for example. This trace
+  /// point type does not require an event object for the parent or the event of
+  /// interest, but information about the function being traced needs to be sent
+  /// using the user_data parameter in the xptiNotifySubscribers() call.
+  function_begin = XPTI_TRACE_POINT_BEGIN(12),
+  /// Used to trace function call end
+  function_end = XPTI_TRACE_POINT_END(12),
+  /// Use to notify that a new metadata entry is available for a given event
+  metadata = XPTI_TRACE_POINT_BEGIN(13),
   /// Indicates that the trace point is user defined and only the tool defined
   /// for a stream will be able to handle it
   user_defined = 1 << 7
@@ -363,7 +372,7 @@ struct trace_event_data_t {
   reserved_data_t reserved;
   /// User defined data, if required; owned by the user shared object and will
   /// not be deleted when event data is destroyed
-  void *user_data = nullptr;
+  void *global_user_data = nullptr;
 };
 
 ///

xpti/include/xpti_trace_framework.h

@@ -288,7 +288,7 @@ xptiRegisterCallback(uint8_t stream_id, uint16_t trace_type,
 ///            1. XPTI_RESULT_SUCCESS when the unregistration is successful
 ///            2. XPTI_RESULT_DUPLICATE when the callback function has already
 ///               been disabled for the stream and trace point type
-///            3. XPTI_RESULT_NOTFOUND if the callbackhas not been previously
+///            3. XPTI_RESULT_NOTFOUND if the callback has not been previously
 ///               registered.
 XPTI_EXPORT_API xpti::result_t
 xptiUnregisterCallback(uint8_t stream_id, uint16_t trace_type,
@@ -308,17 +308,27 @@ xptiUnregisterCallback(uint8_t stream_id, uint16_t trace_type,
 /// @param object The event object for which the notification must be sent out.
 /// @param instance The instance number of the current event and this value is
 /// guaranteed to be static for the duration of the callback handler.
-/// @param temporal_user_data This is the field where each tool can send in some
-/// state information and the handshake of the type of this data type must be
-/// handled by extending tracepoint types that handle diffent types od user
-/// data.
+/// @param per_instance_user_data This is the field where each tool can send in
+/// some state information and the handshake of the type of this data type must
+/// be handled by extending tracepoint types that handle diffent types of user
+/// data. If the trace type is function_begin/function_end, then the parent and
+/// object parameters can be null, but the per_instance_user_data must contain
+/// information about the function being traced (preferably the function name).
 /// @return The result code which can be one of:
 ///            1. XPTI_RESULT_SUCCESS when the notification is successful
+///            2. XPTI_RESULT_FALSE when tracing is turned off
+///            3. XPTI_RESULT_INVALIDARG when one or more input parameters are
+///            invalid. For example, for all trace types except function_begin
+///            and function_end, the event 'object' cannot be NULL. If a NULL
+///            value is provided for this parameter, you will see an
+///            XPTI_RESULT_INVALIDARG return value. Similarly, for
+///            function_begin and function_end, the per_instance_user_data value
+///            must be populated to not get this return value.
 XPTI_EXPORT_API xpti::result_t
 xptiNotifySubscribers(uint8_t stream_id, uint16_t trace_type,
                       xpti::trace_event_data_t *parent,
                       xpti::trace_event_data_t *object, uint64_t instance,
-                      const void *temporal_user_data);
+                      const void *per_instance_user_data);
 
 /// @brief Associates <key-value> pairs with an event
 /// @details If the instrumentation embedded in applications need to send

xpti/include/xpti_trace_framework.hpp

@@ -285,7 +285,7 @@ class scoped_notify {
                 const void *user_data = nullptr)
       : m_object(object), m_parent(parent), m_stream_id(0),
         m_trace_type(trace_type), m_user_data(user_data), m_instance(instance) {
-    if (xptiTraceEnabled() && object) {
+    if (xptiTraceEnabled()) {
       uint16_t open = m_trace_type & 0xfffe;
       m_stream_id = xptiRegisterStream(stream);
       xptiNotifySubscribers(m_stream_id, open, parent, object, instance,
@@ -294,7 +294,7 @@ class scoped_notify {
   }
 
   ~scoped_notify() {
-    if (xptiTraceEnabled() && m_object) {
+    if (xptiTraceEnabled()) {
       switch (m_trace_type) {
       case signal:
       case graph_create:

xptifw/CMakeLists.txt

@@ -24,6 +24,7 @@ include_directories(${CMAKE_SOURCE_DIR}/include ${XPTI_DIR}/include)
 add_subdirectory(src)
 add_subdirectory(unit_test)
 add_subdirectory(samples/basic_collector)
+add_subdirectory(samples/syclpi_collector)
 # The tests in basic_test are written using TBB, so these tests are enabled
 # only if TBB has been enabled.
 if (XPTI_ENABLE_TBB)

xptifw/doc/XPTI_Framework.md

@@ -269,7 +269,12 @@ below.
    ```
 
    You can now run a SYCL application that has been linked with a runtime that
-   supports the XPTI instrumentation and inspect the resulting stream.
+   supports the XPTI instrumentation and inspect the resulting stream. An
+   example collector that subscribes to a specific stream is also provided under
+   `xptifw/samples/syclpi_collector`.  This example demonstrates how a tool can
+   selectively subscribe to a known stream and ignore all other traces. All
+   trace notifications for the streams that have no callbacks registered will
+   return immediately.
 
 3. **Running the unit tests:** The unit tests included cover the exported API
    and incorporate some correctness tests.

xptifw/samples/syclpi_collector/CMakeLists.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 2.8.9)
+project (syclpi_collector)
+
+file(GLOB SOURCES *.cpp)
+include_directories(${XPTIFW_DIR}/include)
+include_directories(${XPTI_DIR}/include)
+include_directories(${XPTIFW_DIR}/samples/include)
+
+remove_definitions(-DXPTI_STATIC_LIBRARY)
+add_definitions(-DXPTI_API_EXPORTS)
+add_library(syclpi_collector SHARED ${SOURCES})
+add_dependencies(syclpi_collector xptifw)
+target_link_libraries(syclpi_collector PRIVATE xptifw)
+if(UNIX)
+  target_link_libraries(syclpi_collector PRIVATE dl)
+endif()
+
+if (XPTI_ENABLE_TBB)
+  target_link_libraries(syclpi_collector PRIVATE tbb)
+endif()
+# Set the location of the library installation
+install(TARGETS syclpi_collector DESTINATION ${CMAKE_BINARY_DIR})

xptifw/samples/syclpi_collector/README.md

@@ -0,0 +1,27 @@
+# Example SYCL PI Layer Collector
+
+The SYCL PI layer collector demonstrates the creation of a subscriber and prints
+of the data received from SYCL PI layer stream. In order to obtain the data from
+an application instrumented with XPTI, the following steps must be performed.
+
+1. Set the environment variable that indicates that tracing has been enabled.
+
+   This is defined by the variable `XPTI_TRACE_ENABLE`. The possible
+   values taken by this environment variable are:
+
+   To enable: `XPTI_TRACE_ENABLE=1` or `XPTI_TRACE_ENABLE=true`
+
+   To disable: `XPTI_TRACE_ENABLE=0` or `XPTI_TRACE_ENABLE=false`
+
+2. Set the environment variable that points to the XPTI framework dispatcher so
+   the stub library can dynamically load it and dispatch the calls to the
+   dispatcher.
+   `XPTI_FRAMEWORK_DISPATCHER=/path/to/libxptifw.[so,dll,dylib]`
+
+3. Set the environment variable that points to the subscriber, which in this
+  case is `libsyclpi_collector.[so,dll,dylib]`.
+
+     `XPTI_SUBSCRIBERS=/path/to/libsyclpi_collector.[so,dll,dylib]`
+
+For more detail on the framework, the tests that are provided and their usage,
+please consult the [XPTI Framework library documentation](doc/XPTI_Framework.md).