merge main into amd-staging

brings in : [OpenMP][NFC] Move mapping related code into OpenMP/Mapping.cpp (llvm#75239)

Change-Id: I55c8d32fd1e96d0b769ac276aa9b20eac0cf0f8b

Author: ronlieb

openmp/libomptarget/include/ExclusiveAccess.h

@@ -11,6 +11,7 @@
 #ifndef OMPTARGET_EXCLUSIVE_ACCESS
 #define OMPTARGET_EXCLUSIVE_ACCESS
 
+#include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <mutex>

openmp/libomptarget/include/OpenMP/Mapping.h

@@ -13,6 +13,7 @@
 #ifndef OMPTARGET_OPENMP_MAPPING_H
 #define OMPTARGET_OPENMP_MAPPING_H
 
+#include "ExclusiveAccess.h"
 #include "Shared/EnvironmentVar.h"
 #include "omptarget.h"
 
@@ -446,4 +447,94 @@ int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                      void **ArgMappers, AsyncInfoTy &AsyncInfo,
                      bool FromMapper = false);
 
+struct MappingInfoTy {
+  MappingInfoTy(DeviceTy &Device) : Device(Device) {}
+
+  /// Host data to device map type with a wrapper key indirection that allows
+  /// concurrent modification of the entries without invalidating the underlying
+  /// entries.
+  using HostDataToTargetListTy =
+      std::set<HostDataToTargetMapKeyTy, std::less<>>;
+
+  /// The HDTTMap is a protected object that can only be accessed by one thread
+  /// at a time.
+  ProtectedObj<HostDataToTargetListTy> HostDataToTargetMap;
+
+  /// The type used to access the HDTT map.
+  using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy;
+
+  /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures
+  /// exclusive access to the HDTT map.
+  LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
+                             int64_t Size,
+                             HostDataToTargetTy *OwnedTPR = nullptr);
+
+  /// Get the target pointer based on host pointer begin and base. If the
+  /// mapping already exists, the target pointer will be returned directly. In
+  /// addition, if required, the memory region pointed by \p HstPtrBegin of size
+  /// \p Size will also be transferred to the device. If the mapping doesn't
+  /// exist, and if unified shared memory is not enabled, a new mapping will be
+  /// created and the data will also be transferred accordingly. nullptr will be
+  /// returned because of any of following reasons:
+  /// - Data allocation failed;
+  /// - The user tried to do an illegal mapping;
+  /// - Data transfer issue fails.
+  TargetPointerResultTy getTargetPointer(
+      HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
+      int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
+      bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
+      bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+      AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
+      bool ReleaseHDTTMap = true);
+
+  /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
+  /// ensures exclusive access to the HDTT map.
+  void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
+                       int64_t Size);
+
+  /// Return the target pointer begin (where the data will be moved).
+  /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
+  /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry
+  /// reference counters will be decremented.
+  /// - \p MustContain enforces that the query must not extend beyond an already
+  /// mapped entry to be valid.
+  /// - \p ForceDelete deletes the entry regardless of its reference counting
+  /// (unless it is infinite).
+  /// - \p FromDataEnd tracks the number of threads referencing the entry at
+  /// targetDataEnd for delayed deletion purpose.
+  [[nodiscard]] TargetPointerResultTy
+  getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
+                 bool UseHoldRefCount, bool MustContain = false,
+                 bool ForceDelete = false, bool FromDataEnd = false);
+
+  /// Remove the \p Entry from the data map. Expect the entry's total reference
+  /// count to be zero and the caller thread to be the last one using it. \p
+  /// HDTTMap ensure the caller holds exclusive access and can modify the map.
+  /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c
+  /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling
+  /// this function if the map entry is not expected to exist because \p
+  /// HstPtrBegin uses shared memory.
+  [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
+                                  HostDataToTargetTy *Entry, int64_t Size);
+
+  /// Deallocate the \p Entry from the device memory and delete it. Return \c
+  /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and
+  /// return \c OFFLOAD_FAIL otherwise.
+  [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
+                                          int64_t Size);
+
+  int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
+  int disassociatePtr(void *HstPtrBegin);
+
+  /// Print information about the transfer from \p HstPtr to \p TgtPtr (or vice
+  /// versa if \p H2D is false). If there is an existing mapping, or if \p Entry
+  /// is set, the associated metadata will be printed as well.
+  void printCopyInfo(void *TgtPtr, void *HstPtr, int64_t Size, bool H2D,
+                     HostDataToTargetTy *Entry,
+                     MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr);
+
+private:
+  DeviceTy &Device;
+};
+
 #endif // OMPTARGET_OPENMP_MAPPING_H

openmp/libomptarget/include/device.h

@@ -56,19 +56,6 @@ struct DeviceTy {
 
   bool HasMappedGlobalData = false;
 
-  /// Host data to device map type with a wrapper key indirection that allows
-  /// concurrent modification of the entries without invalidating the underlying
-  /// entries.
-  using HostDataToTargetListTy =
-      std::set<HostDataToTargetMapKeyTy, std::less<>>;
-
-  /// The HDTTMap is a protected object that can only be accessed by one thread
-  /// at a time.
-  ProtectedObj<HostDataToTargetListTy> HostDataToTargetMap;
-
-  /// The type used to access the HDTT map.
-  using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy;
-
   PendingCtorsDtorsPerLibrary PendingCtorsDtors;
 
   std::mutex PendingGlobalsMtx;
@@ -87,71 +74,8 @@ struct DeviceTy {
   /// Try to initialize the device and return any failure.
   llvm::Error init();
 
-  // Return true if data can be copied to DstDevice directly
-  bool isDataExchangable(const DeviceTy &DstDevice);
-
-  /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures
-  /// exclusive access to the HDTT map.
-  LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
-                             int64_t Size,
-                             HostDataToTargetTy *OwnedTPR = nullptr);
-
-  /// Get the target pointer based on host pointer begin and base. If the
-  /// mapping already exists, the target pointer will be returned directly. In
-  /// addition, if required, the memory region pointed by \p HstPtrBegin of size
-  /// \p Size will also be transferred to the device. If the mapping doesn't
-  /// exist, and if unified shared memory is not enabled, a new mapping will be
-  /// created and the data will also be transferred accordingly. nullptr will be
-  /// returned because of any of following reasons:
-  /// - Data allocation failed;
-  /// - The user tried to do an illegal mapping;
-  /// - Data transfer issue fails.
-  TargetPointerResultTy getTargetPointer(
-      HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
-      int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
-      bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
-      bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
-      AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
-      bool ReleaseHDTTMap = true);
-
-  /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
-  /// ensures exclusive access to the HDTT map.
-  void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
-                       int64_t Size);
-
-  /// Return the target pointer begin (where the data will be moved).
-  /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
-  /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry
-  /// reference counters will be decremented.
-  /// - \p MustContain enforces that the query must not extend beyond an already
-  /// mapped entry to be valid.
-  /// - \p ForceDelete deletes the entry regardless of its reference counting
-  /// (unless it is infinite).
-  /// - \p FromDataEnd tracks the number of threads referencing the entry at
-  /// targetDataEnd for delayed deletion purpose.
-  [[nodiscard]] TargetPointerResultTy
-  getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
-                 bool UseHoldRefCount, bool MustContain = false,
-                 bool ForceDelete = false, bool FromDataEnd = false);
-
-  /// Remove the \p Entry from the data map. Expect the entry's total reference
-  /// count to be zero and the caller thread to be the last one using it. \p
-  /// HDTTMap ensure the caller holds exclusive access and can modify the map.
-  /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c
-  /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling
-  /// this function if the map entry is not expected to exist because \p
-  /// HstPtrBegin uses shared memory.
-  [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
-                                  HostDataToTargetTy *Entry, int64_t Size);
-
-  /// Deallocate the \p Entry from the device memory and delete it. Return \c
-  /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and
-  /// return \c OFFLOAD_FAIL otherwise.
-  [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
-                                          int64_t Size);
-
-  int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
-  int disassociatePtr(void *HstPtrBegin);
+  /// Provide access to the mapping handler.
+  MappingInfoTy &getMappingInfo() { return MappingInfo; }
 
   __tgt_target_table *loadBinary(__tgt_device_image *Img);
 
@@ -166,6 +90,7 @@ struct DeviceTy {
   /// be used (host, shared, device).
   void *allocData(int64_t Size, void *HstPtr = nullptr,
                   int32_t Kind = TARGET_ALLOC_DEFAULT);
+
   /// Deallocates memory which \p TgtPtrBegin points at and returns
   /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. p Kind dictates what
   /// allocator should be used (host, shared, device).
@@ -177,12 +102,16 @@ struct DeviceTy {
   int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
                      AsyncInfoTy &AsyncInfo,
                      HostDataToTargetTy *Entry = nullptr,
-                     DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+                     MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
   // Copy data from device back to host
   int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
                        AsyncInfoTy &AsyncInfo,
                        HostDataToTargetTy *Entry = nullptr,
-                       DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+                       MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
+  // Return true if data can be copied to DstDevice directly
+  bool isDataExchangable(const DeviceTy &DstDevice);
 
   // Copy data from current device to destination device directly
   int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
@@ -250,7 +179,12 @@ struct DeviceTy {
   void deinit();
 
   /// All offload entries available on this device.
-  llvm::DenseMap<llvm::StringRef, OffloadEntryTy *> DeviceOffloadEntries;
+  using DeviceOffloadEntriesMapTy =
+      llvm::DenseMap<llvm::StringRef, OffloadEntryTy *>;
+  ProtectedObj<DeviceOffloadEntriesMapTy> DeviceOffloadEntries;
+
+  /// Handler to collect and organize host-2-device mapping information.
+  MappingInfoTy MappingInfo;
 };
 
 #endif

openmp/libomptarget/src/OpenMP/API.cpp

@@ -178,9 +178,9 @@ EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
   // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
   // lengh array) is mapped instead of the referred storage.
   TargetPointerResultTy TPR =
-      DeviceOrErr->getTgtPtrBegin(const_cast<void *>(Ptr), 1,
-                                  /*UpdateRefCount=*/false,
-                                  /*UseHoldRefCount=*/false);
+      DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+                                                   /*UpdateRefCount=*/false,
+                                                   /*UseHoldRefCount=*/false);
   int Rc = TPR.isPresent();
   DP("Call to omp_target_is_present returns %d\n", Rc);
   return Rc;
@@ -572,8 +572,8 @@ EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
   void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset);
-  int Rc = DeviceOrErr->associatePtr(const_cast<void *>(HostPtr),
-                                     const_cast<void *>(DeviceAddr), Size);
+  int Rc = DeviceOrErr->getMappingInfo().associatePtr(
+      const_cast<void *>(HostPtr), const_cast<void *>(DeviceAddr), Size);
   DP("omp_target_associate_ptr returns %d\n", Rc);
   return Rc;
 }
@@ -599,7 +599,8 @@ EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {
   if (!DeviceOrErr)
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
-  int Rc = DeviceOrErr->disassociatePtr(const_cast<void *>(HostPtr));
+  int Rc = DeviceOrErr->getMappingInfo().disassociatePtr(
+      const_cast<void *>(HostPtr));
   DP("omp_target_disassociate_ptr returns %d\n", Rc);
   return Rc;
 }
@@ -638,9 +639,9 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
   TargetPointerResultTy TPR =
-      DeviceOrErr->getTgtPtrBegin(const_cast<void *>(Ptr), 1,
-                                  /*UpdateRefCount=*/false,
-                                  /*UseHoldRefCount=*/false);
+      DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+                                                   /*UpdateRefCount=*/false,
+                                                   /*UseHoldRefCount=*/false);
   if (!TPR.isPresent()) {
     DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n",
        DPxPTR(Ptr), DeviceNum);