[SYCL] Add OpenCL interop API following SYCL-2020 backend generalization (#1846)

The implementation is following https://github.com/KhronosGroup/SYCL-Shared/blob/master/proposals/sycl_generalization.md.

It is OpenCL centric, and a Level-Zero counterpart is coming here: #1723

An important "side-effect" is that plugins are initialized from these APIs and not only from get_platforms() in the current state.

Signed-off-by: Sergey V Maslov <[email protected]>

Author: smaslov-intel

sycl/include/CL/sycl.hpp

@@ -11,7 +11,6 @@
 #include <CL/sycl/accessor.hpp>
 #include <CL/sycl/atomic.hpp>
 #include <CL/sycl/backend.hpp>
-#include <CL/sycl/backend/opencl.hpp>
 #include <CL/sycl/buffer.hpp>
 #include <CL/sycl/builtins.hpp>
 #include <CL/sycl/context.hpp>

sycl/include/CL/sycl/backend/opencl.hpp

@@ -16,16 +16,83 @@
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
+template <> struct interop<backend::opencl, platform> {
+  using type = cl_platform_id;
+};
+
+template <> struct interop<backend::opencl, device> {
+  using type = cl_device_id;
+};
+
+template <> struct interop<backend::opencl, context> {
+  using type = cl_context;
+};
+
 template <> struct interop<backend::opencl, queue> {
   using type = cl_command_queue;
 };
 
+template <> struct interop<backend::opencl, program> {
+  using type = cl_program;
+};
+
 template <typename DataT, int Dimensions, access::mode AccessMode>
 struct interop<backend::opencl, accessor<DataT, Dimensions, AccessMode,
                                          access::target::global_buffer,
                                          access::placeholder::false_t>> {
   using type = cl_mem;
 };
 
+namespace opencl {
+
+// Implementation of various "make" functions resides in SYCL RT because
+// creating SYCL objects requires knowing details not acessible here.
+// Note that they take opaque pi_native_handle that real OpenCL handles
+// are casted to.
+//
+platform make_platform(pi_native_handle NativeHandle);
+device make_device(pi_native_handle NativeHandle);
+context make_context(pi_native_handle NativeHandle);
+program make_program(const context &Context, pi_native_handle NativeHandle);
+queue make_queue(const context &Context, pi_native_handle InteropHandle);
+
+// Construction of SYCL platform.
+template <typename T, typename std::enable_if<
+                          std::is_same<T, platform>::value>::type * = nullptr>
+T make(typename interop<backend::opencl, T>::type Interop) {
+  return make_platform(detail::pi::cast<pi_native_handle>(Interop));
+}
+
+// Construction of SYCL device.
+template <typename T, typename std::enable_if<
+                          std::is_same<T, device>::value>::type * = nullptr>
+T make(typename interop<backend::opencl, T>::type Interop) {
+  return make_device(detail::pi::cast<pi_native_handle>(Interop));
+}
+
+// Construction of SYCL context.
+template <typename T, typename std::enable_if<
+                          std::is_same<T, context>::value>::type * = nullptr>
+T make(typename interop<backend::opencl, T>::type Interop) {
+  return make_context(detail::pi::cast<pi_native_handle>(Interop));
+}
+
+// Construction of SYCL program.
+template <typename T, typename std::enable_if<
+                          std::is_same<T, program>::value>::type * = nullptr>
+T make(const context &Context,
+       typename interop<backend::opencl, T>::type Interop) {
+  return make_program(Context, detail::pi::cast<pi_native_handle>(Interop));
+}
+
+// Construction of SYCL queue.
+template <typename T, typename std::enable_if<
+                          std::is_same<T, queue>::value>::type * = nullptr>
+T make(const context &Context,
+       typename interop<backend::opencl, T>::type Interop) {
+  return make_queue(Context, detail::pi::cast<pi_native_handle>(Interop));
+}
+
+} // namespace opencl
 } // namespace sycl
 } // __SYCL_INLINE_NAMESPACE(cl)

sycl/include/CL/sycl/detail/pi.def

@@ -17,6 +17,8 @@
 // Platform
 _PI_API(piPlatformsGet)
 _PI_API(piPlatformGetInfo)
+_PI_API(piextPlatformGetNativeHandle)
+_PI_API(piextPlatformCreateWithNativeHandle)
 // Device
 _PI_API(piDevicesGet)
 _PI_API(piDeviceGetInfo)

sycl/include/CL/sycl/detail/pi.h

@@ -805,6 +805,21 @@ __SYCL_EXPORT pi_result piPlatformGetInfo(pi_platform platform,
                                           void *param_value,
                                           size_t *param_value_size_ret);
 
+/// Gets the native handle of a PI platform object.
+///
+/// \param platform is the PI platform to get the native handle of.
+/// \param nativeHandle is the native handle of platform.
+__SYCL_EXPORT pi_result piextPlatformGetNativeHandle(
+    pi_platform platform, pi_native_handle *nativeHandle);
+
+/// Creates PI platform object from a native handle.
+/// NOTE: The created PI object takes ownership of the native handle.
+///
+/// \param nativeHandle is the native handle to create PI device from.
+/// \param platform is the PI platform created from the native handle.
+__SYCL_EXPORT pi_result piextPlatformCreateWithNativeHandle(
+    pi_native_handle nativeHandle, pi_platform *platform);
+
 __SYCL_EXPORT pi_result piDevicesGet(pi_platform platform,
                                      pi_device_type device_type,
                                      pi_uint32 num_entries, pi_device *devices,

sycl/include/CL/sycl/detail/pi.hpp

@@ -139,6 +139,9 @@ extern std::shared_ptr<plugin> GlobalPlugin;
 // Performs PI one-time initialization.
 const vector_class<plugin> &initialize();
 
+// Get the plugin serving given backend.
+template <backend BE> const plugin &getPlugin();
+
 // Utility Functions to get Function Name for a PI Api.
 template <PiApiKind PiApiOffset> struct PiFuncInfo {};
 

sycl/include/CL/sycl/device.hpp

@@ -176,8 +176,7 @@ class __SYCL_EXPORT device {
   /// \return a native handle, the type of which defined by the backend.
   template <backend BackendName>
   auto get_native() const -> typename interop<BackendName, device>::type {
-    return static_cast<typename interop<BackendName, device>::type>(
-        getNative());
+    return (typename interop<BackendName, device>::type)getNative();
   }
 
 private:

sycl/include/CL/sycl/platform.hpp

@@ -102,7 +102,18 @@ class __SYCL_EXPORT platform {
   /// \return a vector of all available SYCL platforms.
   static vector_class<platform> get_platforms();
 
+  /// Gets the native handle of the SYCL platform.
+  ///
+  /// \return a native handle, the type of which defined by the backend.
+  template <backend BackendName>
+  auto get_native() const -> typename interop<BackendName, platform>::type {
+    return detail::pi::cast<typename interop<BackendName, platform>::type>(
+        getNative());
+  }
+
 private:
+  pi_native_handle getNative() const;
+
   shared_ptr_class<detail::platform_impl> impl;
   platform(shared_ptr_class<detail::platform_impl> impl) : impl(impl) {}
 

sycl/include/CL/sycl/queue.hpp

@@ -703,8 +703,12 @@ class __SYCL_EXPORT queue {
   pi_native_handle getNative() const;
 
   shared_ptr_class<detail::queue_impl> impl;
+  queue(shared_ptr_class<detail::queue_impl> impl) : impl(impl) {}
+
   template <class Obj>
   friend decltype(Obj::impl) detail::getSyclObjImpl(const Obj &SyclObject);
+  template <class T>
+  friend T detail::createSyclObjFromImpl(decltype(T::impl) ImplObj);
 
   /// A template-free version of submit.
   event submit_impl(function_class<void(handler &)> CGH,

sycl/plugins/level_zero/pi_level0.cpp

@@ -565,6 +565,28 @@ pi_result piPlatformGetInfo(pi_platform Platform, pi_platform_info ParamName,
   return PI_SUCCESS;
 }
 
+pi_result piextPlatformGetNativeHandle(pi_platform Platform,
+                                       pi_native_handle *NativeHandle) {
+  assert(Platform);
+  assert(NativeHandle);
+
+  auto ZeDriver = pi_cast<ze_driver_handle_t *>(NativeHandle);
+  // Extract the L0 driver handle from the given PI platform
+  *ZeDriver = Platform->ZeDriver;
+  return PI_SUCCESS;
+}
+
+pi_result piextPlatformCreateWithNativeHandle(pi_native_handle NativeHandle,
+                                              pi_platform *Platform) {
+  assert(NativeHandle);
+  assert(Platform);
+
+  // Create PI platform from the given L0 driver handle.
+  auto ZeDriver = pi_cast<ze_driver_handle_t>(NativeHandle);
+  *Platform = new _pi_platform(ZeDriver);
+  return PI_SUCCESS;
+}
+
 pi_result piDevicesGet(pi_platform Platform, pi_device_type DeviceType,
                        pi_uint32 NumEntries, pi_device *Devices,
                        pi_uint32 *NumDevices) {

sycl/plugins/opencl/pi_opencl.cpp

@@ -176,6 +176,14 @@ pi_result piPlatformsGet(pi_uint32 num_entries, pi_platform *platforms,
   return static_cast<pi_result>(result);
 }
 
+pi_result piextPlatformCreateWithNativeHandle(pi_native_handle nativeHandle,
+                                              pi_platform *platform) {
+  assert(platform);
+  assert(nativeHandle);
+  *platform = reinterpret_cast<pi_platform>(nativeHandle);
+  return PI_SUCCESS;
+}
+
 // Example of a PI interface that does not map exactly to an OpenCL one.
 pi_result piDevicesGet(pi_platform platform, pi_device_type device_type,
                        pi_uint32 num_entries, pi_device *devices,
@@ -1072,6 +1080,11 @@ static pi_result piextGetNativeHandle(void *piObj,
   return PI_SUCCESS;
 }
 
+pi_result piextPlatformGetNativeHandle(pi_platform platform,
+                                       pi_native_handle *nativeHandle) {
+  return piextGetNativeHandle(platform, nativeHandle);
+}
+
 pi_result piextDeviceGetNativeHandle(pi_device device,
                                      pi_native_handle *nativeHandle) {
   return piextGetNativeHandle(device, nativeHandle);
@@ -1113,6 +1126,9 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
   // Platform
   _PI_CL(piPlatformsGet, piPlatformsGet)
   _PI_CL(piPlatformGetInfo, clGetPlatformInfo)
+  _PI_CL(piextPlatformGetNativeHandle, piextPlatformGetNativeHandle)
+  _PI_CL(piextPlatformCreateWithNativeHandle,
+         piextPlatformCreateWithNativeHandle)
   // Device
   _PI_CL(piDevicesGet, piDevicesGet)
   _PI_CL(piDeviceGetInfo, clGetDeviceInfo)

sycl/source/CMakeLists.txt

@@ -94,6 +94,7 @@ endfunction(add_sycl_rt_library)
 
 set(SYCL_SOURCES
     "${sycl_inc_dir}/CL/sycl.hpp"
+    "backend/opencl.cpp"
     "detail/accessor_impl.cpp"
     "detail/buffer_impl.cpp"
     "detail/builtins_common.cpp"

sycl/source/backend/opencl.cpp

@@ -0,0 +1,88 @@
+//==------- opencl.cpp - SYCL OpenCL backend -------------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <CL/sycl.hpp>
+#include <detail/platform_impl.hpp>
+#include <detail/plugin.hpp>
+#include <detail/program_impl.hpp>
+#include <detail/queue_impl.hpp>
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+namespace opencl {
+using namespace detail;
+
+//----------------------------------------------------------------------------
+// Implementation of opencl::make<platform>
+__SYCL_EXPORT platform make_platform(pi_native_handle NativeHandle) {
+  const auto &Plugin = pi::getPlugin<backend::opencl>();
+  // Create PI platform first.
+  pi::PiPlatform PiPlatform;
+  Plugin.call<PiApiKind::piextPlatformCreateWithNativeHandle>(NativeHandle,
+                                                              &PiPlatform);
+
+  // Construct the SYCL platform from PI platfrom.
+  return detail::createSyclObjFromImpl<platform>(
+      std::make_shared<platform_impl>(PiPlatform, Plugin));
+}
+
+//----------------------------------------------------------------------------
+// Implementation of opencl::make<device>
+__SYCL_EXPORT device make_device(pi_native_handle NativeHandle) {
+  const auto &Plugin = pi::getPlugin<backend::opencl>();
+  // Create PI device first.
+  pi::PiDevice PiDevice;
+  Plugin.call<PiApiKind::piextDeviceCreateWithNativeHandle>(NativeHandle,
+                                                            &PiDevice);
+  // Construct the SYCL device from PI device.
+  return detail::createSyclObjFromImpl<device>(
+      std::make_shared<device_impl>(PiDevice, Plugin));
+}
+
+//----------------------------------------------------------------------------
+// Implementation of opencl::make<context>
+__SYCL_EXPORT context make_context(pi_native_handle NativeHandle) {
+  const auto &Plugin = pi::getPlugin<backend::opencl>();
+  // Create PI context first.
+  pi::PiContext PiContext;
+  Plugin.call<PiApiKind::piextContextCreateWithNativeHandle>(NativeHandle,
+                                                             &PiContext);
+  // Construct the SYCL context from PI context.
+  return detail::createSyclObjFromImpl<context>(
+      std::make_shared<context_impl>(PiContext, async_handler{}, Plugin));
+}
+
+//----------------------------------------------------------------------------
+// Implementation of opencl::make<program>
+__SYCL_EXPORT program make_program(const context &Context,
+                                   pi_native_handle NativeHandle) {
+  // Construct the SYCL program from native program.
+  // TODO: move here the code that creates PI program, and remove the
+  // native interop constructor.
+  return detail::createSyclObjFromImpl<program>(
+      std::make_shared<program_impl>(getSyclObjImpl(Context), NativeHandle));
+}
+
+//----------------------------------------------------------------------------
+// Implementation of opencl::make<queue>
+__SYCL_EXPORT queue make_queue(const context &Context,
+                               pi_native_handle NativeHandle) {
+  const auto &Plugin = pi::getPlugin<backend::opencl>();
+  const auto &ContextImpl = getSyclObjImpl(Context);
+  // Create PI queue first.
+  pi::PiQueue PiQueue;
+  Plugin.call<PiApiKind::piextQueueCreateWithNativeHandle>(NativeHandle,
+                                                           &PiQueue);
+  // Construct the SYCL queue from PI queue.
+  return detail::createSyclObjFromImpl<queue>(std::make_shared<queue_impl>(
+      PiQueue, ContextImpl, ContextImpl->get_async_handler()));
+}
+
+} // namespace opencl
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)

sycl/source/detail/pi.cpp

@@ -372,6 +372,25 @@ static void initializePlugins(vector_class<plugin> *Plugins) {
 #endif
 }
 
+// Get the plugin serving given backend.
+template <backend BE> const plugin &getPlugin() {
+  static const plugin *Plugin = nullptr;
+  if (Plugin)
+    return *Plugin;
+
+  const vector_class<plugin> &Plugins = pi::initialize();
+  for (const auto &P : Plugins)
+    if (P.getBackend() == BE) {
+      Plugin = &P;
+      return *Plugin;
+    }
+
+  throw runtime_error("pi::getPlugin couldn't find plugin",
+                      PI_INVALID_OPERATION);
+}
+
+template const plugin &getPlugin<backend::opencl>();
+
 // Report error and no return (keeps compiler from printing warnings).
 // TODO: Probably change that to throw a catchable exception,
 //       but for now it is useful to see every failure.

sycl/source/detail/platform_impl.cpp

@@ -276,6 +276,13 @@ bool platform_impl::has_extension(const string_class &ExtensionName) const {
   return (AllExtensionNames.find(ExtensionName) != std::string::npos);
 }
 
+pi_native_handle platform_impl::getNative() const {
+  const auto &Plugin = getPlugin();
+  pi_native_handle Handle;
+  Plugin.call<PiApiKind::piextPlatformGetNativeHandle>(getHandleRef(), &Handle);
+  return Handle;
+}
+
 template <info::platform param>
 typename info::param_traits<info::platform, param>::return_type
 platform_impl::get_info() const {

sycl/source/detail/platform_impl.hpp

@@ -120,6 +120,11 @@ class platform_impl {
     MPlugin = std::move(PluginPtr);
   }
 
+  /// Gets the native handle of the SYCL platform.
+  ///
+  /// \return a native handle.
+  pi_native_handle getNative() const;
+
 private:
   bool MHostPlatform = false;
   RT::PiPlatform MPlatform = 0;

sycl/source/platform.cpp

@@ -49,6 +49,8 @@ platform::get_info() const {
   return impl->get_info<param>();
 }
 
+pi_native_handle platform::getNative() const { return impl->getNative(); }
+
 #define PARAM_TRAITS_SPEC(param_type, param, ret_type)                         \
   template __SYCL_EXPORT ret_type                                              \
   platform::get_info<info::param_type::param>() const;