[SYCL][XPTI] Revisit resource management strategy

sycl/source/detail/pi.cpp

@@ -447,6 +447,8 @@ static void initializePlugins(std::vector<plugin> *Plugins) {
   }
 
 #ifdef XPTI_ENABLE_INSTRUMENTATION
+  GlobalHandler::instance().getXPTIRegistry().initializeFrameworkOnce();
+
   if (!(xptiTraceEnabled() && !XPTIInitDone))
     return;
   // Not sure this is the best place to initialize the framework; SYCL runtime

sycl/source/detail/xpti_registry.hpp

@@ -31,6 +31,15 @@ inline constexpr const char *SYCL_PIDEBUGCALL_STREAM_NAME = "sycl.pi.debug";
 
 class XPTIRegistry {
 public:
+  void initializeFrameworkOnce() {
+#ifdef XPTI_ENABLE_INSTRUMENTATION
+    if (!MInitialized) {
+      xptiFrameworkInitialize();
+      MInitialized = true;
+    }
+#endif
+  }
+
   /// Notifies XPTI subscribers about new stream.
   ///
   /// \param StreamName is a name of newly initialized stream.
@@ -50,11 +59,13 @@ class XPTIRegistry {
     for (const auto &StreamName : MActiveStreams) {
       xptiFinalize(StreamName.c_str());
     }
+    xptiFrameworkFinalize();
 #endif // XPTI_ENABLE_INSTRUMENTATION
   }
 
 private:
   std::unordered_set<std::string> MActiveStreams;
+  bool MInitialized = false;
 };
 } // namespace detail
 } // namespace sycl

xpti/include/xpti_trace_framework.h

@@ -34,6 +34,20 @@
 
 extern "C" {
 
+/// @brief Initializes XPTI framework.
+/// @details Initialize XPTI framework resources. Each user of XPTI must call
+/// this function prior to any other XPTI API call. It is framework's
+/// responsibility to ensure that resources are initialized once. Each call to
+/// this function must have corresponding call to xptiFrameworkFinalize() to
+/// ensure resources are freed.
+XPTI_EXPORT_API void xptiFrameworkInitialize();
+
+/// @brief Deinitializes XPTI framework.
+/// @details Call to this function decrements framework's internal reference
+/// counter. Once its value is equal to zero, XPTI framework can release
+/// resources and unload subscribers.
+XPTI_EXPORT_API void xptiFrameworkFinalize();
+
 /// @brief Initialization function that is called when a new stream is generated
 /// @details When a runtime or application that uses XPTI instrumentation API
 /// starts to generate a new stream, a call to xptiInitialize() must be made to
@@ -413,6 +427,8 @@ XPTI_EXPORT_API void xptiReset();
 /// The proxy/stub library does not implement this function.
 XPTI_EXPORT_API void xptiForceSetTraceEnabled(bool yesOrNo);
 
+typedef xpti::result_t (*xpti_framework_initialize_t)();
+typedef xpti::result_t (*xpti_framework_finalize_t)();
 typedef xpti::result_t (*xpti_initialize_t)(const char *, uint32_t, uint32_t,
                                             const char *);
 typedef void (*xpti_finalize_t)(const char *);

xpti/include/xpti_trace_framework.hpp

@@ -6,6 +6,8 @@
 //
 //
 #pragma once
+
+#include <atomic>
 #include <cstdint>
 #include <memory>
 #include <sstream>
@@ -268,6 +270,20 @@ class PlatformHelper {
     return false;
   }
 };
+
+/// This is an implementation of a SpinLock synchronization primitive, that has
+/// trivial constructor and destructor.
+class SpinLock {
+public:
+  void lock() {
+    while (MLock.test_and_set(std::memory_order_acquire))
+      std::this_thread::yield();
+  }
+  void unlock() { MLock.clear(std::memory_order_release); }
+
+private:
+  std::atomic_flag MLock = ATOMIC_FLAG_INIT;
+};
 } // namespace utils
 
 namespace framework {

xpti/src/xpti_proxy.cpp

@@ -7,12 +7,15 @@
 //
 #include "xpti_trace_framework.hpp"
 
+#include <atomic>
 #include <iostream>
 #include <memory>
 #include <unordered_map>
 #include <vector>
 
 enum functions_t {
+  XPTI_FRAMEWORK_INITIALIZE,
+  XPTI_FRAMEWORK_FINALIZE,
   XPTI_INITIALIZE,
   XPTI_FINALIZE,
   XPTI_GET_UNIQUE_ID,
@@ -42,6 +45,8 @@ enum functions_t {
 namespace xpti {
 class ProxyLoader {
   std::unordered_map<int, const char *> m_function_names = {
+      {XPTI_FRAMEWORK_INITIALIZE, "xptiFrameworkInitialize"},
+      {XPTI_FRAMEWORK_FINALIZE, "xptiFrameworkFinalize"},
       {XPTI_INITIALIZE, "xptiInitialize"},
       {XPTI_FINALIZE, "xptiFinalize"},
       {XPTI_GET_UNIQUE_ID, "xptiGetUniqueId"},
@@ -116,26 +121,52 @@ class ProxyLoader {
   inline bool noErrors() { return m_loaded; }
 
   void *functionByIndex(int index) {
-    if (index >= XPTI_INITIALIZE && index < XPTI_FW_API_COUNT) {
+    if (index >= XPTI_FRAMEWORK_INITIALIZE && index < XPTI_FW_API_COUNT) {
       return reinterpret_cast<void *>(m_dispatch_table[index]);
     }
     return nullptr;
   }
 
+  static ProxyLoader &instance() {
+    static ProxyLoader *loader = new ProxyLoader();
+    return *loader;
+  }
+
 private:
   bool m_loaded;
   xpti_plugin_handle_t m_fw_plugin_handle;
   dispatch_table_t m_dispatch_table;
   xpti::utils::PlatformHelper m_loader;
 };
-
-static ProxyLoader g_loader;
 } // namespace xpti
 
+XPTI_EXPORT_API void xptiFrameworkInitialize() {
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    void *f = xpti::ProxyLoader::instance().functionByIndex(
+        XPTI_FRAMEWORK_INITIALIZE);
+    if (f) {
+      (*reinterpret_cast<xpti_framework_initialize_t>(f))();
+    }
+  }
+}
+
+XPTI_EXPORT_API void xptiFrameworkFinalize() {
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    void *f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_FRAMEWORK_FINALIZE);
+    if (f) {
+      (*reinterpret_cast<xpti_framework_finalize_t>(f))();
+    }
+  }
+
+  delete &xpti::ProxyLoader::instance();
+}
+
 XPTI_EXPORT_API uint16_t xptiRegisterUserDefinedTracePoint(
     const char *tool_name, uint8_t user_defined_tp) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_REGISTER_USER_DEFINED_TP);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(
+        XPTI_REGISTER_USER_DEFINED_TP);
     if (f) {
       return (*(xpti_register_user_defined_tp_t)f)(tool_name, user_defined_tp);
     }
@@ -145,8 +176,9 @@ XPTI_EXPORT_API uint16_t xptiRegisterUserDefinedTracePoint(
 
 XPTI_EXPORT_API uint16_t xptiRegisterUserDefinedEventType(
     const char *tool_name, uint8_t user_defined_event) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_REGISTER_USER_DEFINED_ET);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(
+        XPTI_REGISTER_USER_DEFINED_ET);
     if (f) {
       return (*(xpti_register_user_defined_et_t)f)(tool_name,
                                                    user_defined_event);
@@ -158,8 +190,8 @@ XPTI_EXPORT_API uint16_t xptiRegisterUserDefinedEventType(
 XPTI_EXPORT_API xpti::result_t xptiInitialize(const char *stream, uint32_t maj,
                                               uint32_t min,
                                               const char *version) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_INITIALIZE);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_INITIALIZE);
     if (f) {
       return (*(xpti_initialize_t)f)(stream, maj, min, version);
     }
@@ -168,17 +200,17 @@ XPTI_EXPORT_API xpti::result_t xptiInitialize(const char *stream, uint32_t maj,
 }
 
 XPTI_EXPORT_API void xptiFinalize(const char *stream) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_FINALIZE);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_FINALIZE);
     if (f) {
       (*(xpti_finalize_t)f)(stream);
     }
   }
 }
 
 XPTI_EXPORT_API uint64_t xptiGetUniqueId() {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_GET_UNIQUE_ID);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_GET_UNIQUE_ID);
     if (f) {
       return (*(xpti_get_unique_id_t)f)();
     }
@@ -188,8 +220,9 @@ XPTI_EXPORT_API uint64_t xptiGetUniqueId() {
 
 XPTI_EXPORT_API xpti::string_id_t xptiRegisterString(const char *string,
                                                      char **table_string) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_REGISTER_STRING);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_REGISTER_STRING);
     if (f) {
       return (*(xpti_register_string_t)f)(string, table_string);
     }
@@ -198,8 +231,8 @@ XPTI_EXPORT_API xpti::string_id_t xptiRegisterString(const char *string,
 }
 
 XPTI_EXPORT_API const char *xptiLookupString(xpti::string_id_t id) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_LOOKUP_STRING);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_LOOKUP_STRING);
     if (f) {
       return (*(xpti_lookup_string_t)f)(id);
     }
@@ -208,8 +241,9 @@ XPTI_EXPORT_API const char *xptiLookupString(xpti::string_id_t id) {
 }
 
 XPTI_EXPORT_API uint64_t xptiRegisterPayload(xpti::payload_t *payload) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_REGISTER_PAYLOAD);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_REGISTER_PAYLOAD);
     if (f) {
       return (*(xpti_register_payload_t)f)(payload);
     }
@@ -218,8 +252,9 @@ XPTI_EXPORT_API uint64_t xptiRegisterPayload(xpti::payload_t *payload) {
 }
 
 XPTI_EXPORT_API uint8_t xptiRegisterStream(const char *stream_name) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_REGISTER_STREAM);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_REGISTER_STREAM);
     if (f) {
       return (*(xpti_register_stream_t)f)(stream_name);
     }
@@ -228,8 +263,9 @@ XPTI_EXPORT_API uint8_t xptiRegisterStream(const char *stream_name) {
 }
 
 XPTI_EXPORT_API xpti::result_t xptiUnregisterStream(const char *stream_name) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_UNREGISTER_STREAM);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_UNREGISTER_STREAM);
     if (f) {
       return (*(xpti_unregister_stream_t)f)(stream_name);
     }
@@ -239,8 +275,8 @@ XPTI_EXPORT_API xpti::result_t xptiUnregisterStream(const char *stream_name) {
 XPTI_EXPORT_API xpti::trace_event_data_t *
 xptiMakeEvent(const char *name, xpti::payload_t *payload, uint16_t event,
               xpti::trace_activity_type_t activity, uint64_t *instance_no) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_MAKE_EVENT);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_MAKE_EVENT);
     if (f) {
       return (*(xpti_make_event_t)f)(name, payload, event, activity,
                                      instance_no);
@@ -250,8 +286,8 @@ xptiMakeEvent(const char *name, xpti::payload_t *payload, uint16_t event,
 }
 
 XPTI_EXPORT_API const xpti::trace_event_data_t *xptiFindEvent(uint64_t uid) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_FIND_EVENT);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_FIND_EVENT);
     if (f) {
       return (*(xpti_find_event_t)f)(uid);
     }
@@ -261,8 +297,8 @@ XPTI_EXPORT_API const xpti::trace_event_data_t *xptiFindEvent(uint64_t uid) {
 
 XPTI_EXPORT_API const xpti::payload_t *
 xptiQueryPayload(xpti::trace_event_data_t *lookup_object) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_QUERY_PAYLOAD);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_QUERY_PAYLOAD);
     if (f) {
       return (*(xpti_query_payload_t)f)(lookup_object);
     }
@@ -271,8 +307,9 @@ xptiQueryPayload(xpti::trace_event_data_t *lookup_object) {
 }
 
 XPTI_EXPORT_API const xpti::payload_t *xptiQueryPayloadByUID(uint64_t uid) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_QUERY_PAYLOAD_BY_UID);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(
+        XPTI_QUERY_PAYLOAD_BY_UID);
     if (f) {
       return (*(xpti_query_payload_by_uid_t)f)(uid);
     }
@@ -283,8 +320,9 @@ XPTI_EXPORT_API const xpti::payload_t *xptiQueryPayloadByUID(uint64_t uid) {
 XPTI_EXPORT_API xpti::result_t
 xptiRegisterCallback(uint8_t stream_id, uint16_t trace_type,
                      xpti::tracepoint_callback_api_t cb) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_REGISTER_CALLBACK);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_REGISTER_CALLBACK);
     if (f) {
       return (*(xpti_register_cb_t)f)(stream_id, trace_type, cb);
     }
@@ -295,8 +333,9 @@ xptiRegisterCallback(uint8_t stream_id, uint16_t trace_type,
 XPTI_EXPORT_API xpti::result_t
 xptiUnregisterCallback(uint8_t stream_id, uint16_t trace_type,
                        xpti::tracepoint_callback_api_t cb) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_UNREGISTER_CALLBACK);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_UNREGISTER_CALLBACK);
     if (f) {
       return (*(xpti_unregister_cb_t)f)(stream_id, trace_type, cb);
     }
@@ -309,8 +348,9 @@ xptiNotifySubscribers(uint8_t stream_id, uint16_t trace_type,
                       xpti::trace_event_data_t *parent,
                       xpti::trace_event_data_t *object, uint64_t instance,
                       const void *user_data) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_NOTIFY_SUBSCRIBERS);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f =
+        xpti::ProxyLoader::instance().functionByIndex(XPTI_NOTIFY_SUBSCRIBERS);
     if (f) {
       return (*(xpti_notify_subscribers_t)f)(stream_id, trace_type, parent,
                                              object, instance, user_data);
@@ -320,8 +360,8 @@ xptiNotifySubscribers(uint8_t stream_id, uint16_t trace_type,
 }
 
 XPTI_EXPORT_API bool xptiTraceEnabled() {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_TRACE_ENABLED);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_TRACE_ENABLED);
     if (f) {
       return (*(xpti_trace_enabled_t)f)();
     }
@@ -332,8 +372,8 @@ XPTI_EXPORT_API bool xptiTraceEnabled() {
 XPTI_EXPORT_API xpti::result_t xptiAddMetadata(xpti::trace_event_data_t *e,
                                                const char *key,
                                                const char *value) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_ADD_METADATA);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_ADD_METADATA);
     if (f) {
       return (*(xpti_add_metadata_t)f)(e, key, value);
     }
@@ -343,8 +383,8 @@ XPTI_EXPORT_API xpti::result_t xptiAddMetadata(xpti::trace_event_data_t *e,
 
 XPTI_EXPORT_API xpti::metadata_t *
 xptiQueryMetadata(xpti::trace_event_data_t *lookup_object) {
-  if (xpti::g_loader.noErrors()) {
-    auto f = xpti::g_loader.functionByIndex(XPTI_QUERY_METADATA);
+  if (xpti::ProxyLoader::instance().noErrors()) {
+    auto f = xpti::ProxyLoader::instance().functionByIndex(XPTI_QUERY_METADATA);
     if (f) {
       return (*(xpti_query_metadata_t)f)(lookup_object);
     }