[OpenMP][MI300A] APU: prefault host memory upon memory copy.

On APUs, when copying memory the CPU-allocated memory (e.g., via malloc, mmaps, etc.) is faulted at memory copy time, potentially page-by-page.
This patch prefaults host memory when its size is >2MB, which has been shown as being the threshold after which not prefaulting incurs large overehads (see https://ontrack-internal.amd.com/browse/SWDEV-443343).

Change-Id: Ib95a336b0352a579faf1af7e40cc32d3600a4cdd

Author: carlobertolli

openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp

@@ -2478,6 +2478,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
             // setting default to true here appears to solve random sdma problem
             "LIBOMPTARGET_AMDGPU_USE_MULTIPLE_SDMA_ENGINES", true),
         OMPX_SyncCopyBack("LIBOMPTARGET_SYNC_COPY_BACK", true),
+        OMPX_APUPrefaultMemcopy("LIBOMPTARGET_APU_PREFAULT_MEMCOPY", "true"),
+        OMPX_APUPrefaultMemcopySize("LIBOMPTARGET_APU_PREFAULT_MEMCOPY_SIZE",
+                                    2 * 1024 * 1024), // 2MB
         AMDGPUStreamManager(*this, Agent), AMDGPUEventManager(*this),
         AMDGPUSignalManager(*this), Agent(Agent), HostDevice(HostDevice) {}
 
@@ -3044,6 +3047,15 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    // Prefault GPU page table in XNACK-Enabled case, on APUs,
+    // under the assumption that explicitly allocated memory
+    // will be fully accessed and that on-the-fly individual page faults
+    // perform worse than whole memory faulting.
+    if (OMPX_APUPrefaultMemcopy && Size >= OMPX_APUPrefaultMemcopySize &&
+        IsAPU && IsXnackEnabled)
+      if (auto Err = prepopulatePageTableImpl(const_cast<void *>(HstPtr), Size))
+        return Err;
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -3251,8 +3263,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
 
   Error prepopulatePageTableImpl(void *ptr, int64_t size) override final {
-    // Instruct ROCr that the [ptr, ptr+size-1] pages are
-    // coarse grain
+    // Instruct runtimes that the [ptr, ptr+size-1] pages will be accessed by
+    // devices but should not be migrated (only perform page faults, if needed).
     hsa_amd_svm_attribute_pair_t tt;
     tt.attribute = HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE;
     tt.value = Agent.handle;
@@ -3751,9 +3763,38 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Use ROCm 5.7 interface for multiple SDMA engines
   BoolEnvar OMPX_UseMultipleSdmaEngines;
 
+  /// Value of OMPX_APU_MAPS env var used to force
+  /// automatic zero-copy behavior on non-APU GPUs.
+  BoolEnvar OMPX_ApuMaps;
+
+  /// Value of OMPX_DISABLE_USM_MAPS. Use on MI200
+  /// systems to disable both device memory
+  /// allocations and host-device memory copies upon
+  /// map, and coarse graining of mapped variables.
+  BoolEnvar OMPX_DisableUsmMaps;
+
+  /// Value of OMPX_DISABLE_MAPS. Turns off map table checks
+  /// in libomptarget in unified_shared_memory mode. Legacy:
+  /// never turned to false (unified_shared_memory mode is
+  /// currently always without map checks.
+  BoolEnvar OMPX_NoMapChecks;
+
+  /// Makes warnings turn into fatal errors
+  BoolEnvar OMPX_StrictSanityChecks;
+
   /// Variable to hold synchronous copy back
   BoolEnvar OMPX_SyncCopyBack;
 
+  /// On APUs, this env var indicates whether memory copy
+  /// should be preceded by pre-faulting of host memory,
+  /// to prevent page faults during the copy.
+  BoolEnvar OMPX_APUPrefaultMemcopy;
+
+  /// On APUs, when prefaulting host memory before a copy,
+  /// this env var controls the size after which prefaulting
+  /// is applied.
+  UInt32Envar OMPX_APUPrefaultMemcopySize;
+
   /// Stream manager for AMDGPU streams.
   AMDGPUStreamManagerTy AMDGPUStreamManager;
 
@@ -3798,6 +3839,28 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   // The maximum scratch memory size per thread.
   // See COMPUTE_TMPRING_SIZE.WAVESIZE (divided by threads per wave).
   uint32_t MaxThreadScratchSize;
+
+  /// Is the plugin associated with an APU?
+  bool IsAPU = false;
+
+  // Is the device an MI300X?
+  bool IsEquippedWithMI300X = false;
+
+  // Is the device an MI200?
+  bool IsEquippedWithGFX90A = false;
+
+  /// True if the system is configured with XNACK-Enabled.
+  /// False otherwise.
+  bool IsXnackEnabled = false;
+
+  // Set by OMPX_DISABLE_USM_MAPS environment variable.
+  // If set, fine graned memory is used for maps instead of coarse grained.
+  bool EnableFineGrainedMemory = false;
+
+  /// Set by OMPX_DISABLE_MAPS environment variable.
+  // If false, map checks are performed also in unified_shared_memory mode.
+  // TODO: this feature is non functional.
+  bool NoUSMMapChecks = true;
 };
 
 Error AMDGPUDeviceImageTy::loadExecutable(const AMDGPUDeviceTy &Device) {