[SYCL] Change in buffer management for integrated GPUs.

Signed-off-by: rdeodhar <[email protected]>

Author: rdeodhar

sycl/plugins/level_zero/pi_level_zero.cpp

@@ -1862,34 +1862,50 @@ pi_result piMemBufferCreate(pi_context Context, pi_mem_flags Flags, size_t Size,
   assert(RetMem);
 
   void *Ptr;
+  ze_device_handle_t ZeDevice = Context->Devices[0]->ZeDevice;
+
+  // We treat integrated devices (physical memory shared with the CPU)
+  // differently from discrete devices (those with distinct memories).
+  // For integrated devices, allocating the buffer in host shared memory
+  // enables automatic access from the device, and makes copying
+  // unnecessary in the map/unmap operations. This improves performance.
+  bool DeviceIsIntegrated = Context->Devices.size() == 1 &&
+                            Context->Devices[0]->ZeDeviceProperties.flags &
+                                ZE_DEVICE_PROPERTY_FLAG_INTEGRATED;
+
+  if (DeviceIsIntegrated) {
+    ze_host_mem_alloc_desc_t ZeDesc = {};
+    ZeDesc.flags = 0;
 
-  ze_device_mem_alloc_desc_t ZeDeviceMemDesc = {};
-  ZeDeviceMemDesc.flags = 0;
-  ZeDeviceMemDesc.ordinal = 0;
+    ZE_CALL(zeMemAllocHost(Context->ZeContext, &ZeDesc, Size, 4096, &Ptr));
 
-  if (Context->Devices.size() == 1) {
-    ZE_CALL(zeMemAllocDevice(Context->ZeContext, &ZeDeviceMemDesc, Size,
-                             1, // TODO: alignment
-                             Context->Devices[0]->ZeDevice, &Ptr));
   } else {
-    ze_host_mem_alloc_desc_t ZeHostMemDesc = {};
-    ZeHostMemDesc.flags = 0;
-    ZE_CALL(zeMemAllocShared(Context->ZeContext, &ZeDeviceMemDesc,
-                             &ZeHostMemDesc, Size,
-                             1,       // TODO: alignment
-                             nullptr, // not bound to any device
+    ze_device_mem_alloc_desc_t ZeDesc = {};
+    ZeDesc.flags = 0;
+    ZeDesc.ordinal = 0;
+
+    ZE_CALL(zeMemAllocDevice(Context->ZeContext, &ZeDesc, Size, 4096, ZeDevice,
                              &Ptr));
   }
+  if (HostPtr) {
+    if ((Flags & PI_MEM_FLAGS_HOST_PTR_USE) != 0 ||
+        (Flags & PI_MEM_FLAGS_HOST_PTR_COPY) != 0) {
+      // Initialize the buffer with user data
+      if (DeviceIsIntegrated) {
+        // Do a host to host copy
+        memcpy(Ptr, HostPtr, Size);
+      } else {
 
-  if ((Flags & PI_MEM_FLAGS_HOST_PTR_USE) != 0 ||
-      (Flags & PI_MEM_FLAGS_HOST_PTR_COPY) != 0) {
-    // Initialize the buffer synchronously with immediate offload
-    ZE_CALL(zeCommandListAppendMemoryCopy(Context->ZeCommandListInit, Ptr,
-                                          HostPtr, Size, nullptr, 0, nullptr));
-  } else if (Flags == 0 || (Flags == PI_MEM_FLAGS_ACCESS_RW)) {
-    // Nothing more to do.
-  } else {
-    die("piMemBufferCreate: not implemented");
+        // Initialize the buffer synchronously with immediate offload
+        ZE_CALL(zeCommandListAppendMemoryCopy(Context->ZeCommandListInit, Ptr,
+                                              HostPtr, Size, nullptr, 0,
+                                              nullptr));
+      }
+    } else if (Flags == 0 || (Flags == PI_MEM_FLAGS_ACCESS_RW)) {
+      // Nothing more to do.
+    } else {
+      die("piMemBufferCreate: not implemented");
+    }
   }
 
   auto HostPtrOrNull =
@@ -3305,7 +3321,26 @@ pi_result piEventsWait(pi_uint32 NumEvents, const pi_event *EventList) {
   for (uint32_t I = 0; I < NumEvents; I++) {
     ze_event_handle_t ZeEvent = EventList[I]->ZeEvent;
     zePrint("ZeEvent = %lx\n", pi_cast<std::uintptr_t>(ZeEvent));
-    ZE_CALL(zeEventHostSynchronize(ZeEvent, UINT32_MAX));
+
+    // If event comes from a Map/UnMap operation in integrated device, then do
+    // sync, memcpy, and signaling on the host
+    if (EventList[I]->HostSyncforMap) {
+#if 0
+      for (auto ZeWaitEvent : EventList[I]->waitEvents) {
+        zePrint("ZeWaitEvent = %lx\n", pi_cast<std::uintptr_t>(ZeWaitEvent));
+        if (ZeWaitEvent)
+          ZE_CALL(zeEventHostSynchronize(ZeWaitEvent, UINT32_MAX));
+      }
+      if (EventList[I]->CopyPending) {
+        memcpy(EventList[I]->DstBuffer, EventList[I]->SrcBuffer,
+               EventList[I]->RetMapSize);
+        EventList[I]->CopyPending = false;
+      }
+#endif
+      ZE_CALL(zeEventHostSignal(ZeEvent));
+    } else {
+      ZE_CALL(zeEventHostSynchronize(ZeEvent, UINT32_MAX));
+    }
 
     // NOTE: we are destroying associated command lists here to free
     // resources sooner in case RT is not calling piEventRelease soon enough.
@@ -3983,14 +4018,22 @@ piEnqueueMemBufferMap(pi_queue Queue, pi_mem Buffer, pi_bool BlockingMap,
   // Lock automatically releases when this goes out of scope.
   std::lock_guard<std::mutex> lock(Queue->PiQueueMutex);
 
-  // Get a new command list to be used on this call
+  // Query the buffer allocation to determine if host allocation
+  ze_memory_allocation_properties_t ZeMemoryAllocationProperties = {};
+  ze_device_handle_t ZeDeviceHandle;
+
+  ZE_CALL(
+      zeMemGetAllocProperties(Queue->Context->ZeContext, Buffer->getZeHandle(),
+                              &ZeMemoryAllocationProperties, &ZeDeviceHandle));
+
+  bool BufferUsesHostMem =
+      (ZeMemoryAllocationProperties.type == ZE_MEMORY_TYPE_HOST);
+
+  // For discrete devices we don't need a commandlist
   ze_command_list_handle_t ZeCommandList = nullptr;
   ze_fence_handle_t ZeFence = nullptr;
-  if (auto Res = Queue->Device->getAvailableCommandList(Queue, &ZeCommandList,
-                                                        &ZeFence))
-    return Res;
-
   ze_event_handle_t ZeEvent = nullptr;
+
   if (Event) {
     auto Res = piEventCreate(Queue->Context, Event);
     if (Res != PI_SUCCESS)
@@ -4003,38 +4046,84 @@ piEnqueueMemBufferMap(pi_queue Queue, pi_mem Buffer, pi_bool BlockingMap,
     ZeEvent = (*Event)->ZeEvent;
   }
 
+  // TODO: Level Zero is missing the memory "mapping" capabilities, so we are
+  // left to doing new memory allocation and a copy (read) on discrete devices.
+  // On integrated devices  we have allocated the buffer in host memory
+  // so no actions are needed here except for synchronizing on incoming events
+  // and doing a host-to-host copy if a host pointer had been supplied
+  // during buffer creation.
+  //
+  // TODO: for discrete, check if the input buffer is already allocated
+  // in shared memory and thus is accessible from the host as is.
+  // Can we get SYCL RT to predict/allocate in shared memory
+  // from the beginning?
+  if (BufferUsesHostMem) {
+    (*Event)->HostSyncforMap = true;
+#if 0
+    for (uint32_t i = 0; i < NumEventsInWaitList; i++) {
+      zePrint("Map added ZeWaitEvent = %lx\n",
+              pi_cast<std::uintptr_t>(EventWaitList[i]->ZeEvent));
+      (*Event)->waitEvents.push_back(EventWaitList[i]->ZeEvent);
+    }
+#else
+    for (uint32_t i = 0; i < NumEventsInWaitList; i++) {
+      auto Queue = EventWaitList[i]->Queue;
+      zePrint("Got Q\n");
+      if (Queue->RefCount > 0) {
+        zePrint("Executing commandlist\n");
+        if (auto Res = Queue->executeOpenCommandList())
+          return Res;
+      }
+    }
+    for (uint32_t i = 0; i < NumEventsInWaitList; i++) {
+      zePrint("Going to wait on ZeWaitEvent = %lx\n",
+              pi_cast<std::uintptr_t>(EventWaitList[i]->ZeEvent));
+      auto ZeWaitEvent = EventWaitList[i]->ZeEvent;
+      if (ZeWaitEvent)
+        ZE_CALL(zeEventHostSynchronize(ZeWaitEvent, UINT32_MAX));
+    }
+#endif
+    if (Buffer->MapHostPtr) {
+      *RetMap = Buffer->MapHostPtr + Offset;
+#if 0
+      (*Event)->SrcBuffer = pi_cast<char*>(Buffer->getZeHandle()) + Offset;
+      (*Event)->DstBuffer = *RetMap;
+      (*Event)->RetMapSize = Size;
+      (*Event)->CopyPending = true;
+#else
+      zePrint("Doing memcpy %p %p %zu\n", *RetMap,
+              pi_cast<char *>(Buffer->getZeHandle()) + Offset, Size);
+      memcpy(*RetMap, pi_cast<char *>(Buffer->getZeHandle()) + Offset, Size);
+      zePrint("DONE\n");
+#endif
+    } else {
+      *RetMap = pi_cast<char *>(Buffer->getZeHandle()) + Offset;
+    }
+
+    return Buffer->addMapping(*RetMap, Offset, Size);
+  }
+
+  // For discrete devices we need a command list
+  if (auto Res = Queue->Device->getAvailableCommandList(Queue, &ZeCommandList,
+                                                        &ZeFence))
+    return Res;
+
   ze_event_handle_t *ZeEventWaitList =
       _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
 
-  ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
-                                          ZeEventWaitList));
-
-  // TODO: Level Zero is missing the memory "mapping" capabilities, so we are
-  // left to doing new memory allocation and a copy (read).
-  //
-  // TODO: check if the input buffer is already allocated in shared
-  // memory and thus is accessible from the host as is. Can we get SYCL RT
-  // to predict/allocate in shared memory from the beginning?
   if (Buffer->MapHostPtr) {
-    // NOTE: borrowing below semantics from OpenCL as SYCL RT relies on it.
-    // It is also better for performance.
-    //
-    // "If the buffer object is created with CL_MEM_USE_HOST_PTR set in
-    // mem_flags, the following will be true:
-    // - The host_ptr specified in clCreateBuffer is guaranteed to contain the
-    //   latest bits in the region being mapped when the clEnqueueMapBuffer
-    //   command has completed.
-    // - The pointer value returned by clEnqueueMapBuffer will be derived from
-    //   the host_ptr specified when the buffer object is created."
     *RetMap = Buffer->MapHostPtr + Offset;
   } else {
     ze_host_mem_alloc_desc_t ZeDesc = {};
     ZeDesc.flags = 0;
-    ZE_CALL(zeMemAllocHost(Queue->Context->ZeContext, &ZeDesc, Size,
-                           1, // TODO: alignment
-                           RetMap));
+
+    ZE_CALL(
+        zeMemAllocHost(Queue->Context->ZeContext, &ZeDesc, Size, 4096, RetMap));
   }
 
+  ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
+                                          ZeEventWaitList));
+
   ZE_CALL(zeCommandListAppendMemoryCopy(
       ZeCommandList, *RetMap, pi_cast<char *>(Buffer->getZeHandle()) + Offset,
       Size, ZeEvent, 0, nullptr));
@@ -4055,12 +4144,21 @@ pi_result piEnqueueMemUnmap(pi_queue Queue, pi_mem MemObj, void *MappedPtr,
   // Lock automatically releases when this goes out of scope.
   std::lock_guard<std::mutex> lock(Queue->PiQueueMutex);
 
-  // Get a new command list to be used on this call
+  // Query the buffer allocation to determine if host allocation
+  ze_memory_allocation_properties_t ZeMemoryAllocationProperties = {};
+  ze_device_handle_t ZeDeviceHandle;
+
+  ZE_CALL(
+      zeMemGetAllocProperties(Queue->Context->ZeContext, MemObj->getZeHandle(),
+                              &ZeMemoryAllocationProperties, &ZeDeviceHandle));
+
+  bool BufferUsesHostMem =
+      (ZeMemoryAllocationProperties.type == ZE_MEMORY_TYPE_HOST);
+
+  // Integrated devices don't need a command list.
+  // If discrete we will get a commandlist later.
   ze_command_list_handle_t ZeCommandList = nullptr;
   ze_fence_handle_t ZeFence = nullptr;
-  if (auto Res = Queue->Device->getAvailableCommandList(Queue, &ZeCommandList,
-                                                        &ZeFence))
-    return Res;
 
   // TODO: handle the case when user does not care to follow the event
   // of unmap completion.
@@ -4079,38 +4177,74 @@ pi_result piEnqueueMemUnmap(pi_queue Queue, pi_mem MemObj, void *MappedPtr,
     ZeEvent = (*Event)->ZeEvent;
   }
 
-  ze_event_handle_t *ZeEventWaitList =
-      _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
-
-  ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
-                                          ZeEventWaitList));
-
-  // TODO: Level Zero is missing the memory "mapping" capabilities, so we are
-  // left to doing copy (write back to the device).
-  //
-  // NOTE: Keep this in sync with the implementation of
-  // piEnqueueMemBufferMap/piEnqueueMemImageMap.
   _pi_mem::Mapping MapInfo = {};
   if (pi_result Res = MemObj->removeMapping(MappedPtr, MapInfo))
     return Res;
 
-  ZE_CALL(zeCommandListAppendMemoryCopy(
-      ZeCommandList, pi_cast<char *>(MemObj->getZeHandle()) + MapInfo.Offset,
-      MappedPtr, MapInfo.Size, ZeEvent, 0, nullptr));
-
   // NOTE: we still have to free the host memory allocated/returned by
   // piEnqueueMemBufferMap, but can only do so after the above copy
   // is completed. Instead of waiting for It here (blocking), we shall
   // do so in piEventRelease called for the pi_event tracking the unmap.
+  // In the case of an integrated device, the map operation does not allocate
+  // any memory, so there is nothing to free. This is indicated by a nullptr.
   if (Event)
-    (*Event)->CommandData = MemObj->MapHostPtr ? nullptr : MappedPtr;
+    (*Event)->CommandData =
+        (BufferUsesHostMem ? nullptr
+                           : (MemObj->MapHostPtr ? nullptr : MappedPtr));
+
+  if (BufferUsesHostMem) {
+    (*Event)->HostSyncforMap = true;
+#if 0
+    for (uint32_t i = 0; i < NumEventsInWaitList; i++) {
+      zePrint("UnMap Added ZeWaitEvent = %lx\n",
+        pi_cast<std::uintptr_t>(EventWaitList[i]->ZeEvent));
+      (*Event)->waitEvents.push_back(EventWaitList[i]->ZeEvent);
+    }
+    (*Event)->SrcBuffer = MappedPtr;
+    (*Event)->DstBuffer =
+      pi_cast<char*>(MemObj->getZeHandle()) + MapInfo.Offset;
+    (*Event)->RetMapSize = MapInfo.Size;
+    (*Event)->CopyPending = true;
+#else
+    for (uint32_t i = 0; i < NumEventsInWaitList; i++) {
+      auto ZeWaitEvent = EventWaitList[i]->ZeEvent;
+      if (ZeWaitEvent)
+        ZE_CALL(zeEventHostSynchronize(ZeWaitEvent, UINT32_MAX));
+    }
+    memcpy(pi_cast<char *>(MemObj->getZeHandle()) + MapInfo.Offset, MappedPtr,
+           MapInfo.Size);
+#endif
+  } else {
 
-  // Execute command list asynchronously, as the event will be used
-  // to track down its completion.
-  if (auto Res = Queue->executeCommandList(ZeCommandList, ZeFence))
-    return Res;
+    if (auto Res = Queue->Device->getAvailableCommandList(Queue, &ZeCommandList,
+                                                          &ZeFence))
+      return Res;
 
-  _pi_event::deleteZeEventList(ZeEventWaitList);
+    ze_event_handle_t *ZeEventWaitList =
+        _pi_event::createZeEventList(NumEventsInWaitList, EventWaitList);
+
+    ZE_CALL(zeCommandListAppendWaitOnEvents(ZeCommandList, NumEventsInWaitList,
+                                            ZeEventWaitList));
+
+    // TODO: Level Zero is missing the memory "mapping" capabilities, so we are
+    // left to doing copy (write back to the device).
+    // See https://gitlab.devtools.intel.com/one-api/level_zero/issues/293. //
+    // INTEL
+    //
+    // NOTE: Keep this in sync with the implementation of
+    // piEnqueueMemBufferMap/piEnqueueMemImageMap.
+
+    ZE_CALL(zeCommandListAppendMemoryCopy(
+        ZeCommandList, pi_cast<char *>(MemObj->getZeHandle()) + MapInfo.Offset,
+        MappedPtr, MapInfo.Size, ZeEvent, 0, nullptr));
+
+    // Execute command list asynchronously, as the event will be used
+    // to track down its completion.
+    if (auto Res = Queue->executeCommandList(ZeCommandList, ZeFence))
+      return Res;
+
+    _pi_event::deleteZeEventList(ZeEventWaitList);
+  }
 
   return PI_SUCCESS;
 }

sycl/plugins/level_zero/pi_level_zero.hpp

@@ -451,6 +451,22 @@ struct _pi_event : _pi_object {
   // Level Zero event pool handle.
   ze_event_pool_handle_t ZeEventPool;
 
+  // The following are used by MemBufferMap/UnMap on integrated devices
+  // Flag to indicate that a copy is pending from a map/unmap operation
+  bool HostSyncforMap = false;
+#if 0
+  // List of incoming events to be satisfied before the copy can be done
+  std::vector<ze_event_handle_t> waitEvents;
+  // The destination of the map/unmap copy
+  void *DstBuffer = nullptr;
+  // The source of the map/unmap copy
+  void *SrcBuffer = nullptr;
+  // The size of the copy
+  size_t RetMapSize = 0;
+  // A flag that enables doing the copy only once, for a list of events
+  bool CopyPending = false;
+#endif
+
   // Level Zero command list where the command signaling this event was appended
   // to. This is currently used to remember/destroy the command list after all
   // commands in it are completed, i.e. this event signaled.