[SYCL] Added missing SYCL 2020 USM features

sycl/include/CL/sycl/aspects.hpp

@@ -38,7 +38,9 @@ enum class aspect {
   ext_intel_gpu_subslices_per_slice = 22,
   ext_intel_gpu_eu_count_per_subslice = 23,
   ext_intel_max_mem_bandwidth = 24,
-  ext_intel_mem_channel = 25
+  ext_intel_mem_channel = 25,
+  usm_atomic_host_allocations = 26,
+  usm_atomic_shared_allocations = 27,
 };
 
 } // namespace sycl

sycl/include/CL/sycl/detail/cg.hpp

@@ -93,6 +93,7 @@ namespace detail {
 // Used to represent a type of an extended member
 enum class ExtendedMembersType : unsigned int {
   HANDLER_KERNEL_BUNDLE = 0,
+  HANDLER_MEM_ADVICE,
 };
 
 // Holds a pointer to an object of an arbitrary type and an ID value which
@@ -164,6 +165,7 @@ class CG {
     PREFETCH_USM = 12,
     CODEPLAY_INTEROP_TASK = 13,
     CODEPLAY_HOST_TASK = 14,
+    ADVISE_USM = 15,
   };
 
   CG(CGTYPE Type, vector_class<vector_class<char>> ArgsStorage,
@@ -414,6 +416,37 @@ class CGPrefetchUSM : public CG {
   size_t getLength() { return MLength; }
 };
 
+/// "Advise USM" command group class.
+class CGAdviseUSM : public CG {
+  void *MDst;
+  size_t MLength;
+
+public:
+  CGAdviseUSM(void *DstPtr, size_t Length,
+              vector_class<vector_class<char>> ArgsStorage,
+              vector_class<detail::AccessorImplPtr> AccStorage,
+              vector_class<shared_ptr_class<const void>> SharedPtrStorage,
+              vector_class<Requirement *> Requirements,
+              vector_class<detail::EventImplPtr> Events,
+              detail::code_location loc = {})
+      : CG(ADVISE_USM, std::move(ArgsStorage), std::move(AccStorage),
+           std::move(SharedPtrStorage), std::move(Requirements),
+           std::move(Events), std::move(loc)),
+        MDst(DstPtr), MLength(Length) {}
+  void *getDst() { return MDst; }
+  size_t getLength() { return MLength; }
+
+  pi_mem_advice getAdvice() {
+    auto ExtendedMembers = getExtendedMembers();
+    if (!ExtendedMembers)
+      return PI_MEM_ADVISE_UNKNOWN;
+    for (const ExtendedMemberT &EM : *ExtendedMembers)
+      if ((ExtendedMembersType::HANDLER_MEM_ADVICE == EM.MType) && EM.MData)
+        return *std::static_pointer_cast<pi_mem_advice>(EM.MData);
+    return PI_MEM_ADVISE_UNKNOWN;
+  }
+};
+
 class CGInteropTask : public CG {
 public:
   std::unique_ptr<InteropTask> MInteropTask;

sycl/include/CL/sycl/detail/memory_manager.hpp

@@ -151,6 +151,11 @@ class __SYCL_EXPORT MemoryManager {
   static void prefetch_usm(void *Ptr, QueueImplPtr Queue, size_t Len,
                            std::vector<RT::PiEvent> DepEvents,
                            RT::PiEvent &OutEvent);
+
+  static void advise_usm(const void *Ptr, QueueImplPtr Queue, size_t Len,
+                         pi_mem_advice Advice,
+                         std::vector<RT::PiEvent> DepEvents,
+                         RT::PiEvent &OutEvent);
 };
 } // namespace detail
 } // namespace sycl

sycl/include/CL/sycl/handler.hpp

@@ -2183,6 +2183,19 @@ class __SYCL_EXPORT handler {
   /// \param Count is a number of bytes to copy.
   void memcpy(void *Dest, const void *Src, size_t Count);
 
+  /// Copies data from one memory region to another, both pointed by
+  /// USM pointers.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if either \param Dest or \param Src is nullptr. The behavior is undefined
+  /// if any of the pointer parameters is invalid.
+  ///
+  /// \param Dest is a USM pointer to the destination memory.
+  /// \param Src is a USM pointer to the source memory.
+  /// \param Count is a number of elements of type T to copy.
+  template <typename T> void copy(T *Dest, const T *Src, size_t Count) {
+    this->memcpy(Dest, Src, Count * sizeof(T));
+  }
+
   /// Fills the memory pointed by a USM pointer with the value specified.
   /// No operations is done if \param Count is zero. An exception is thrown
   /// if \param Dest is nullptr. The behavior is undefined if \param Dest
@@ -2201,6 +2214,14 @@ class __SYCL_EXPORT handler {
   /// \param Count is a number of bytes to be prefetched.
   void prefetch(const void *Ptr, size_t Count);
 
+  /// Provides additional information to the underlying runtime about how
+  /// different allocations are used.
+  ///
+  /// \param Ptr is a USM pointer to the allocation.
+  /// \param Length is a number of bytes in the allocation.
+  /// \param Advice is a device-defined advice for the specified allocation.
+  void mem_advise(const void *Ptr, size_t Length, pi_mem_advice Advice);
+
 private:
   shared_ptr_class<detail::queue_impl> MQueue;
   /// The storage for the arguments passed.

sycl/include/CL/sycl/queue.hpp

@@ -320,17 +320,52 @@ class __SYCL_EXPORT queue {
 
   /// Fills the specified memory with the specified pattern.
   ///
-  /// \param Ptr is the pointer to the memory to fill
+  /// \param Ptr is the pointer to the memory to fill.
   /// \param Pattern is the pattern to fill into the memory.  T should be
   /// trivially copyable.
   /// \param Count is the number of times to fill Pattern into Ptr.
+  /// \return an event representing fill operation.
   template <typename T> event fill(void *Ptr, const T &Pattern, size_t Count) {
     return submit([&](handler &CGH) { CGH.fill<T>(Ptr, Pattern, Count); });
   }
 
+  /// Fills the specified memory with the specified pattern.
+  ///
+  /// \param Ptr is the pointer to the memory to fill.
+  /// \param Pattern is the pattern to fill into the memory.  T should be
+  /// trivially copyable.
+  /// \param Count is the number of times to fill Pattern into Ptr.
+  /// \param DepEvent is an event that specifies the kernel dependencies.
+  /// \return an event representing fill operation.
+  template <typename T>
+  event fill(void *Ptr, const T &Pattern, size_t Count, event DepEvent) {
+    return submit([&](handler &CGH) {
+      CGH.depends_on(DepEvent);
+      CGH.fill<T>(Ptr, Pattern, Count);
+    });
+  }
+
+  /// Fills the specified memory with the specified pattern.
+  ///
+  /// \param Ptr is the pointer to the memory to fill.
+  /// \param Pattern is the pattern to fill into the memory.  T should be
+  /// trivially copyable.
+  /// \param Count is the number of times to fill Pattern into Ptr.
+  /// \param DepEvents is a vector of events that specifies the kernel
+  /// dependencies.
+  /// \return an event representing fill operation.
+  template <typename T>
+  event fill(void *Ptr, const T &Pattern, size_t Count,
+             const vector_class<event> &DepEvents) {
+    return submit([&](handler &CGH) {
+      CGH.depends_on(DepEvents);
+      CGH.fill<T>(Ptr, Pattern, Count);
+    });
+  }
+
   /// Fills the memory pointed by a USM pointer with the value specified.
   /// No operations is done if \param Count is zero. An exception is thrown
-  /// if \param Dest is nullptr. The behavior is undefined if \param Ptr
+  /// if \param Ptr is nullptr. The behavior is undefined if \param Ptr
   /// is invalid.
   ///
   /// \param Ptr is a USM pointer to the memory to fill.
@@ -339,6 +374,32 @@ class __SYCL_EXPORT queue {
   /// \return an event representing fill operation.
   event memset(void *Ptr, int Value, size_t Count);
 
+  /// Fills the memory pointed by a USM pointer with the value specified.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if \param Ptr is nullptr. The behavior is undefined if \param Ptr
+  /// is invalid.
+  ///
+  /// \param Ptr is a USM pointer to the memory to fill.
+  /// \param Value is a value to be set. Value is cast as an unsigned char.
+  /// \param Count is a number of bytes to fill.
+  /// \param DepEvent is an event that specifies the kernel dependencies.
+  /// \return an event representing fill operation.
+  event memset(void *Ptr, int Value, size_t Count, event DepEvent);
+
+  /// Fills the memory pointed by a USM pointer with the value specified.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if \param Ptr is nullptr. The behavior is undefined if \param Ptr
+  /// is invalid.
+  ///
+  /// \param Ptr is a USM pointer to the memory to fill.
+  /// \param Value is a value to be set. Value is cast as an unsigned char.
+  /// \param Count is a number of bytes to fill.
+  /// \param DepEvents is a vector of events that specifies the kernel
+  /// dependencies.
+  /// \return an event representing fill operation.
+  event memset(void *Ptr, int Value, size_t Count,
+               const vector_class<event> &DepEvents);
+
   /// Copies data from one memory region to another, both pointed by
   /// USM pointers.
   /// No operations is done if \param Count is zero. An exception is thrown
@@ -351,6 +412,81 @@ class __SYCL_EXPORT queue {
   /// \return an event representing copy operation.
   event memcpy(void *Dest, const void *Src, size_t Count);
 
+  /// Copies data from one memory region to another, both pointed by
+  /// USM pointers.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if either \param Dest or \param Src is nullptr. The behavior is undefined
+  /// if any of the pointer parameters is invalid.
+  ///
+  /// \param Dest is a USM pointer to the destination memory.
+  /// \param Src is a USM pointer to the source memory.
+  /// \param Count is a number of bytes to copy.
+  /// \param DepEvent is an event that specifies the kernel dependencies.
+  /// \return an event representing copy operation.
+  event memcpy(void *Dest, const void *Src, size_t Count, event DepEvent);
+
+  /// Copies data from one memory region to another, both pointed by
+  /// USM pointers.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if either \param Dest or \param Src is nullptr. The behavior is undefined
+  /// if any of the pointer parameters is invalid.
+  ///
+  /// \param Dest is a USM pointer to the destination memory.
+  /// \param Src is a USM pointer to the source memory.
+  /// \param Count is a number of bytes to copy.
+  /// \param DepEvents is a vector of events that specifies the kernel
+  /// dependencies.
+  /// \return an event representing copy operation.
+  event memcpy(void *Dest, const void *Src, size_t Count,
+               const vector_class<event> &DepEvents);
+
+  /// Copies data from one memory region to another, both pointed by
+  /// USM pointers.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if either \param Dest or \param Src is nullptr. The behavior is undefined
+  /// if any of the pointer parameters is invalid.
+  ///
+  /// \param Dest is a USM pointer to the destination memory.
+  /// \param Src is a USM pointer to the source memory.
+  /// \param Count is a number of elements of type T to copy.
+  /// \return an event representing copy operation.
+  template <typename T> event copy(T *Dest, const T *Src, size_t Count) {
+    return this->memcpy(Dest, Src, Count * sizeof(T));
+  }
+
+  /// Copies data from one memory region to another, both pointed by
+  /// USM pointers.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if either \param Dest or \param Src is nullptr. The behavior is undefined
+  /// if any of the pointer parameters is invalid.
+  ///
+  /// \param Dest is a USM pointer to the destination memory.
+  /// \param Src is a USM pointer to the source memory.
+  /// \param Count is a number of elements of type T to copy.
+  /// \param DepEvent is an event that specifies the kernel dependencies.
+  /// \return an event representing copy operation.
+  template <typename T>
+  event copy(T *Dest, const T *Src, size_t Count, event DepEvent) {
+    return this->memcpy(Dest, Src, Count * sizeof(T), DepEvent);
+  }
+
+  /// Copies data from one memory region to another, both pointed by
+  /// USM pointers.
+  /// No operations is done if \param Count is zero. An exception is thrown
+  /// if either \param Dest or \param Src is nullptr. The behavior is undefined
+  /// if any of the pointer parameters is invalid.
+  ///
+  /// \param Dest is a USM pointer to the destination memory.
+  /// \param Src is a USM pointer to the source memory.
+  /// \param Count is a number of elements of type T to copy.
+  /// \param DepEvents is a vector of events that specifies the kernel
+  /// \return an event representing copy operation.
+  template <typename T>
+  event copy(T *Dest, const T *Src, size_t Count,
+             const vector_class<event> &DepEvents) {
+    return this->memcpy(Dest, Src, Count * sizeof(T), DepEvents);
+  }
+
   /// Provides additional information to the underlying runtime about how
   /// different allocations are used.
   ///
@@ -360,16 +496,72 @@ class __SYCL_EXPORT queue {
   /// \return an event representing advice operation.
   event mem_advise(const void *Ptr, size_t Length, pi_mem_advice Advice);
 
+  /// Provides additional information to the underlying runtime about how
+  /// different allocations are used.
+  ///
+  /// \param Ptr is a USM pointer to the allocation.
+  /// \param Length is a number of bytes in the allocation.
+  /// \param Advice is a device-defined advice for the specified allocation.
+  /// \param DepEvent is an event that specifies the kernel dependencies.
+  /// \return an event representing advice operation.
+  event mem_advise(const void *Ptr, size_t Length, pi_mem_advice Advice,
+                   event DepEvent);
+
+  /// Provides additional information to the underlying runtime about how
+  /// different allocations are used.
+  ///
+  /// \param Ptr is a USM pointer to the allocation.
+  /// \param Length is a number of bytes in the allocation.
+  /// \param Advice is a device-defined advice for the specified allocation.
+  /// \param DepEvents is a vector of events that specifies the kernel
+  /// dependencies.
+  /// \return an event representing advice operation.
+  event mem_advise(const void *Ptr, size_t Length, pi_mem_advice Advice,
+                   const vector_class<event> &DepEvents);
+
   /// Provides hints to the runtime library that data should be made available
   /// on a device earlier than Unified Shared Memory would normally require it
   /// to be available.
   ///
   /// \param Ptr is a USM pointer to the memory to be prefetched to the device.
   /// \param Count is a number of bytes to be prefetched.
+  /// \return an event representing prefetch operation.
   event prefetch(const void *Ptr, size_t Count) {
     return submit([=](handler &CGH) { CGH.prefetch(Ptr, Count); });
   }
 
+  /// Provides hints to the runtime library that data should be made available
+  /// on a device earlier than Unified Shared Memory would normally require it
+  /// to be available.
+  ///
+  /// \param Ptr is a USM pointer to the memory to be prefetched to the device.
+  /// \param Count is a number of bytes to be prefetched.
+  /// \param DepEvent is an event that specifies the kernel dependencies.
+  /// \return an event representing prefetch operation.
+  event prefetch(const void *Ptr, size_t Count, event DepEvent) {
+    return submit([=](handler &CGH) {
+      CGH.depends_on(DepEvent);
+      CGH.prefetch(Ptr, Count);
+    });
+  }
+
+  /// Provides hints to the runtime library that data should be made available
+  /// on a device earlier than Unified Shared Memory would normally require it
+  /// to be available.
+  ///
+  /// \param Ptr is a USM pointer to the memory to be prefetched to the device.
+  /// \param Count is a number of bytes to be prefetched.
+  /// \param DepEvents is a vector of events that specifies the kernel
+  /// dependencies.
+  /// \return an event representing prefetch operation.
+  event prefetch(const void *Ptr, size_t Count,
+                 const vector_class<event> &DepEvents) {
+    return submit([=](handler &CGH) {
+      CGH.depends_on(DepEvents);
+      CGH.prefetch(Ptr, Count);
+    });
+  }
+
   /// single_task version with a kernel represented as a lambda.
   ///
   /// \param KernelFunc is the Kernel functor or lambda