[SYCL][L0] Implement Plugin Lifetime Management (#2942)

The patch introduces new PI API - piTearDown which is called by the SYCL RT
prior to plugin library unloading. Global variables are now released in the
level zero plugin.

Signed-off-by: Byoungro So <[email protected]>

Author: bso-intel

sycl/doc/GlobalObjectsInRuntime.md

@@ -88,7 +88,14 @@ constructor and destructor.
 
 ## Plugins
 
-TBD
+Plugin lifetime is managed by utilizing piPluginInit() and piTearDown().
+GlobalHandler::shutdown() will tear down all registered globals before SYCL RT
+library is unloaded. It will invoke piTearDown() and unload() for each
+plugin. piTearDown() is going to perform any necessary tear-down process at the
+plugin PI level. These two APIs allow on-demand plugin lifetime management. SYCL
+RT can control the beginning and the end of the plugin. 
+
+![](images/plugin-lifetime.jpg)
 
 ## Low-level runtimes
 

sycl/doc/PluginInterface.md

@@ -121,9 +121,22 @@ The trace shows the PI API calls made when using SYCL_PI_TRACE=-1.
 bound.)
 
 ### Plugin Unloading
-The plugins not chosen to be connected to should be unloaded.
-
-TBD - Unloading a bound plugin.
+The plugins not chosen to be connected to should be unloaded. piInitializePlugins()
+can be called to load and bound the necessary plugins. In addition, piTearDown()
+can be called when plugins are not needed any more. It notifies each
+plugin to start performing its own tear-down process such as global memory
+deallocation. In the future, piTearDown() can include any other jobs that need to
+be done before the plugin is unloaded from memory. Possibly, a
+notification of the plugin unloading to lower-level plugins can be added so that
+they can clean up their own memory [TBD].
+After piTearDown() is called, the plugin can be safely unloaded by calling unload(),
+which is going to invoke OS-specific system calls to remove the dynamic library
+from memory.
+
+Each plugin should not create global variables that require non-trivial
+destructor. Pointer variables with heap memory allocation is a good example
+to be created at the global scope. A std::vector object is not. piTearDown
+will take care of deallocation of these global variables safely.
 
 ## PI API Specification
 

sycl/doc/images/plugin-lifetime.jpg

@@ -126,5 +126,6 @@ _PI_API(piextUSMGetMemAllocInfo)
 
 _PI_API(piextKernelSetArgMemObj)
 _PI_API(piextKernelSetArgSampler)
+_PI_API(piTearDown)
 
 #undef _PI_API

sycl/include/CL/sycl/detail/pi.def

@@ -1600,6 +1600,11 @@ __SYCL_EXPORT pi_result piextUSMGetMemAllocInfo(
     pi_context context, const void *ptr, pi_mem_info param_name,
     size_t param_value_size, void *param_value, size_t *param_value_size_ret);
 
+/// API to notify that the plugin should clean up its resources.
+/// No PI calls should be made until the next piPluginInit call.
+/// \param PluginParameter placeholder for future use, currenly not used.
+__SYCL_EXPORT pi_result piTearDown(void *PluginParameter);
+
 struct _pi_plugin {
   // PI version supported by host passed to the plugin. The Plugin
   // checks and writes the appropriate Function Pointers in

sycl/include/CL/sycl/detail/pi.h

@@ -123,6 +123,13 @@ __SYCL_EXPORT void contextSetExtendedDeleter(const cl::sycl::context &constext,
 // Implementation is OS dependent.
 void *loadOsLibrary(const std::string &Library);
 
+// Function to unload the shared library
+// Implementation is OS dependent (see posix-pi.cpp and windows-pi.cpp)
+int unloadOsLibrary(void *Library);
+
+// OS agnostic function to unload the shared library
+int unloadPlugin(void *Library);
+
 // Function to get Address of a symbol defined in the shared
 // library, implementation is OS dependent.
 void *getOsLibraryFuncAddress(void *Library, const std::string &FunctionName);

sycl/include/CL/sycl/detail/pi.hpp

@@ -4475,6 +4475,11 @@ pi_result cuda_piextUSMGetMemAllocInfo(pi_context context, const void *ptr,
   return result;
 }
 
+// This API is called by Sycl RT to notify the end of the plugin lifetime.
+// TODO: add a global variable lifetime management code here (see
+// pi_level_zero.cpp for reference) Currently this is just a NOOP.
+pi_result cuda_piTearDown(void *PluginParameter) { return PI_SUCCESS; }
+
 const char SupportedVersion[] = _PI_H_VERSION_STRING;
 
 pi_result piPluginInit(pi_plugin *PluginInit) {
@@ -4610,6 +4615,7 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
 
   _PI_CL(piextKernelSetArgMemObj, cuda_piextKernelSetArgMemObj)
   _PI_CL(piextKernelSetArgSampler, cuda_piextKernelSetArgSampler)
+  _PI_CL(piTearDown, cuda_piTearDown)
 
 #undef _PI_CL
 

sycl/plugins/cuda/pi_cuda.cpp

@@ -12,6 +12,7 @@
 /// \ingroup sycl_pi_level_zero
 
 #include "pi_level_zero.hpp"
+#include <CL/sycl/detail/spinlock.hpp>
 #include <algorithm>
 #include <cstdarg>
 #include <cstdio>
@@ -172,6 +173,17 @@ class ReturnHelper {
 
 } // anonymous namespace
 
+// Global variables used in PI_Level_Zero
+// Note we only create a simple pointer variables such that C++ RT won't
+// deallocate them automatically at the end of the main program.
+// The heap memory allocated for these global variables reclaimed only when
+// Sycl RT calls piTearDown().
+static std::vector<pi_platform> *PiPlatformsCache =
+    new std::vector<pi_platform>;
+static sycl::detail::SpinLock *PiPlatformsCacheMutex =
+    new sycl::detail::SpinLock;
+static bool PiPlatformCachePopulated = false;
+
 // TODO:: In the following 4 methods we may want to distinguish read access vs.
 // write (as it is OK for multiple threads to read the map without locking it).
 
@@ -821,16 +833,8 @@ pi_result piPlatformsGet(pi_uint32 NumEntries, pi_platform *Platforms,
   // 1. sycl::platform equality issue; we always return the same pi_platform.
   // 2. performance; we can save time by immediately return from cache.
   //
-  // Note: The memory for "PiPlatformsCache" and "PiPlatformsCacheMutex" is
-  // intentionally leaked because the application may call into the SYCL
-  // runtime from a global destructor, and such a call could eventually
-  // access these variables. Therefore, there is no safe time when
-  // "PiPlatformsCache" and "PiPlatformsCacheMutex" could be deleted.
-  static auto PiPlatformsCache = new std::vector<pi_platform>;
-  static auto PiPlatformsCacheMutex = new std::mutex;
-  static bool PiPlatformCachePopulated = false;
-
-  std::lock_guard<std::mutex> Lock(*PiPlatformsCacheMutex);
+
+  const std::lock_guard<sycl::detail::SpinLock> Lock{*PiPlatformsCacheMutex};
   if (!PiPlatformCachePopulated) {
     const char *CommandListCacheSize =
         std::getenv("SYCL_PI_LEVEL_ZERO_MAX_COMMAND_LIST_CACHE");
@@ -5349,4 +5353,18 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
   return PI_SUCCESS;
 }
 
+// SYCL RT calls this api to notify the end of plugin lifetime.
+// It can include all the jobs to tear down resources before
+// the plugin is unloaded from memory.
+pi_result piTearDown(void *PluginParameter) {
+  // reclaim pi_platform objects here since we don't have piPlatformRelease.
+  for (pi_platform &Platform : *PiPlatformsCache) {
+    delete Platform;
+  }
+  delete PiPlatformsCache;
+  delete PiPlatformsCacheMutex;
+
+  return PI_SUCCESS;
+}
+
 } // extern "C"

sycl/plugins/level_zero/pi_level_zero.cpp

@@ -1170,6 +1170,11 @@ pi_result piextProgramGetNativeHandle(pi_program program,
   return piextGetNativeHandle(program, nativeHandle);
 }
 
+// This API is called by Sycl RT to notify the end of the plugin lifetime.
+// TODO: add a global variable lifetime management code here (see
+// pi_level_zero.cpp for reference) Currently this is just a NOOP.
+pi_result piTearDown(void *PluginParameter) { return PI_SUCCESS; }
+
 pi_result piPluginInit(pi_plugin *PluginInit) {
   int CompareVersions = strcmp(PluginInit->PiVersion, SupportedVersion);
   if (CompareVersions < 0) {
@@ -1297,6 +1302,7 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
 
   _PI_CL(piextKernelSetArgMemObj, piextKernelSetArgMemObj)
   _PI_CL(piextKernelSetArgSampler, piextKernelSetArgSampler)
+  _PI_CL(piTearDown, piTearDown)
 
 #undef _PI_CL
 

sycl/plugins/opencl/pi_opencl.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <CL/sycl/detail/device_filter.hpp>
+#include <CL/sycl/detail/pi.hpp>
 #include <CL/sycl/detail/spinlock.hpp>
 #include <detail/global_handler.hpp>
 #include <detail/platform_impl.hpp>
@@ -113,7 +114,18 @@ GlobalHandler::getDeviceFilterList(const std::string &InitValue) {
   return *MDeviceFilterList;
 }
 
-void shutdown() { delete &GlobalHandler::instance(); }
+void shutdown() {
+  for (plugin &Plugin : GlobalHandler::instance().getPlugins()) {
+    // PluginParameter is reserved for future use that can control
+    // some parameters in the plugin tear-down process.
+    // Currently, it is not used.
+    void *PluginParameter = nullptr;
+    Plugin.call_nocheck<PiApiKind::piTearDown>(PluginParameter);
+    Plugin.unload();
+  }
+
+  delete &GlobalHandler::instance();
+}
 
 #ifdef _WIN32
 BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved) {

sycl/source/detail/global_handler.cpp

@@ -256,6 +256,10 @@ void *loadPlugin(const std::string &PluginPath) {
   return loadOsLibrary(PluginPath);
 }
 
+// Unload the given plugin by calling teh OS-specific library unloading call.
+// \param Library OS-specific library handle created when loading.
+int unloadPlugin(void *Library) { return unloadOsLibrary(Library); }
+
 // Binds all the PI Interface APIs to Plugin Library Function Addresses.
 // TODO: Remove the 'OclPtr' extension to PI_API.
 // TODO: Change the functionality such that a single getOsLibraryFuncAddress
@@ -339,18 +343,20 @@ static void initializePlugins(vector_class<plugin> *Plugins) {
         PluginNames[I].first.find("opencl") != std::string::npos) {
       // Use the OpenCL plugin as the GlobalPlugin
       GlobalPlugin =
-          std::make_shared<plugin>(PluginInformation, backend::opencl);
+          std::make_shared<plugin>(PluginInformation, backend::opencl, Library);
     } else if (InteropBE == backend::cuda &&
                PluginNames[I].first.find("cuda") != std::string::npos) {
       // Use the CUDA plugin as the GlobalPlugin
-      GlobalPlugin = std::make_shared<plugin>(PluginInformation, backend::cuda);
+      GlobalPlugin =
+          std::make_shared<plugin>(PluginInformation, backend::cuda, Library);
     } else if (InteropBE == backend::level_zero &&
                PluginNames[I].first.find("level_zero") != std::string::npos) {
       // Use the LEVEL_ZERO plugin as the GlobalPlugin
-      GlobalPlugin =
-          std::make_shared<plugin>(PluginInformation, backend::level_zero);
+      GlobalPlugin = std::make_shared<plugin>(PluginInformation,
+                                              backend::level_zero, Library);
     }
-    Plugins->emplace_back(plugin(PluginInformation, PluginNames[I].second));
+    Plugins->emplace_back(
+        plugin(PluginInformation, PluginNames[I].second, Library));
     if (trace(TraceLevel::PI_TRACE_BASIC))
       std::cerr << "SYCL_PI_TRACE[basic]: "
                 << "Plugin found and successfully loaded: "

sycl/source/detail/pi.cpp

@@ -34,8 +34,8 @@ class plugin {
 public:
   plugin() = delete;
 
-  plugin(RT::PiPlugin Plugin, backend UseBackend)
-      : MPlugin(Plugin), MBackend(UseBackend),
+  plugin(RT::PiPlugin Plugin, backend UseBackend, void *LibraryHandle)
+      : MPlugin(Plugin), MBackend(UseBackend), MLibraryHandle(LibraryHandle),
         TracingMutex(std::make_shared<std::mutex>()) {}
 
   plugin &operator=(const plugin &) = default;
@@ -107,10 +107,14 @@ class plugin {
   }
 
   backend getBackend(void) const { return MBackend; }
+  void *getLibraryHandle() const { return MLibraryHandle; }
+  void *getLibraryHandle() { return MLibraryHandle; }
+  int unload() { return RT::unloadPlugin(MLibraryHandle); }
 
 private:
   RT::PiPlugin MPlugin;
   backend MBackend;
+  void *MLibraryHandle; // the handle returned from dlopen
   std::shared_ptr<std::mutex> TracingMutex;
 }; // class plugin
 } // namespace detail

sycl/source/detail/plugin.hpp

@@ -22,6 +22,8 @@ void *loadOsLibrary(const std::string &PluginPath) {
   return dlopen(PluginPath.c_str(), RTLD_NOW);
 }
 
+int unloadOsLibrary(void *Library) { return dlclose(Library); }
+
 void *getOsLibraryFuncAddress(void *Library, const std::string &FunctionName) {
   return dlsym(Library, FunctionName.c_str());
 }

sycl/source/detail/posix_pi.cpp

@@ -8,9 +8,9 @@
 
 #include <CL/sycl/detail/defines.hpp>
 
+#include <string>
 #include <windows.h>
 #include <winreg.h>
-#include <string>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
@@ -21,6 +21,10 @@ void *loadOsLibrary(const std::string &PluginPath) {
   return (void *)LoadLibraryA(PluginPath.c_str());
 }
 
+int unloadOsLibrary(void *Library) {
+  return (int)FreeLibrary((HMODULE)Library);
+}
+
 void *getOsLibraryFuncAddress(void *Library, const std::string &FunctionName) {
   return reinterpret_cast<void *>(
       GetProcAddress((HMODULE)Library, FunctionName.c_str()));

sycl/source/detail/windows_pi.cpp

@@ -76,6 +76,7 @@ piSamplerCreate
 piSamplerGetInfo
 piSamplerRelease
 piSamplerRetain
+piTearDown
 piclProgramCreateWithSource
 piextContextCreateWithNativeHandle
 piextContextGetNativeHandle

sycl/test/abi/pi_level_zero_symbol_check.dump

@@ -24,6 +24,7 @@ piProgramCreateWithBinary
 piProgramLink
 piQueueCreate
 piSamplerCreate
+piTearDown
 piclProgramCreateWithSource
 piextContextCreateWithNativeHandle
 piextContextGetNativeHandle

sycl/test/abi/pi_opencl_symbol_check.dump

@@ -121,7 +121,8 @@ class PiMock {
     // Copy the PiPlugin, thus untying our to-be mock platform from other
     // platforms within the context. Reset our platform to use the new plugin.
     auto NewPluginPtr = std::make_shared<detail::plugin>(
-        OriginalPiPlugin.getPiPlugin(), OriginalPiPlugin.getBackend());
+        OriginalPiPlugin.getPiPlugin(), OriginalPiPlugin.getBackend(),
+        OriginalPiPlugin.getLibraryHandle());
     ImplPtr->setPlugin(NewPluginPtr);
     // Extract the new PiPlugin instance by a non-const pointer,
     // explicitly allowing modification