[OpenMP] Move logic that needs requirements flags out of plugin.

This patch moves to libomptarget all OpenMP logic that depends on the requirement flags.
This is due to upcoming trunk patch llvm#80345.
The consequence of this change is that we have far less calls to the plugin
during non-initialization phases.

In addition, it
- introduces diagnostic prints for zero-copy modes, and
- make unified_shared_memory program fail on XNACK-Disabled environments
when env variable OMPX_STRICT_SANITY_CHECKS=true.

Note that unified_shared_memory can run on an XNACK-Disabled
system by setting OMPX_EAGER_ZERO_COPY_MAPS, because it supports
accesses to CPU-allocated memory using GPU page table prefaulting,
instead of XNACK.

Change-Id: I252e6557b3db1a13dd0fb1fe55f2f508361d9957

Author: carlobertolli

openmp/libomptarget/include/Shared/Debug.h

@@ -65,6 +65,8 @@ enum OpenMPInfoType : uint32_t {
   OMP_INFOTYPE_AMD_API_TRACE = 0x200,
   // Print whenever data does not have a viable device counterpart.
   OMP_INFOTYPE_EMPTY_MAPPING = 0x0040,
+  // Print diagnostic information for users.
+  OMP_INFOTYPE_USER_DIAGNOSTIC = 0x0080,
   // Enable every flag.
   OMP_INFOTYPE_ALL = 0xffffffff,
 };

openmp/libomptarget/include/Shared/PluginAPI.h

@@ -234,17 +234,16 @@ int32_t __tgt_rtl_initialize_record_replay(int32_t DeviceId, int64_t MemorySize,
 // Return if the system is equipped with an APU
 bool __tgt_rtl_has_apu_device(int32_t DeviceId);
 
-// Returns true, if the system is equipped with a dGPU which supports USM.
+// Returns true if the system is equipped with a dGPU which supports USM.
 bool __tgt_rtl_has_USM_capable_dGPU(int32_t DeviceId);
 
+// Returns true if the system supports unified memory.
+bool __tgt_rtl_supports_unified_memory(int32_t DeviceId);
+
 // Returns true if coarse graining of mapped memory is disabled
 // (it only applies to MI200 GPUs).
 bool __tgt_rtl_is_fine_grained_memory_enabled(int32_t DeviceId);
 
-// Returns true if GPU page table prefaulting is enabled. False
-// otherwise.
-bool __tgt_rtl_requested_prepopulate_gpu_page_table(int32_t DeviceId);
-
 // Check if image is incompatible due to XNACK mismatch.
 void __tgt_rtl_check_invalid_image(__tgt_device_image *Image);
 
@@ -273,6 +272,12 @@ int32_t __tgt_rtl_activate_record_replay(int32_t, uint64_t, void *, bool,
 // Returns true if the device \p DeviceId suggests to use auto zero-copy.
 int32_t __tgt_rtl_use_auto_zero_copy(int32_t DeviceId);
 
+// Performs sanity checks on zero-copy options and prints diagnostic info.
+int32_t __tgt_rtl_zero_copy_sanity_checks_and_diag(int32_t DeviceId,
+                                                   bool isUnifiedSharedMemory,
+                                                   bool isAutoZeroCopy,
+                                                   bool isEagerMaps);
+
 #ifdef __cplusplus
 }
 #endif

openmp/libomptarget/include/Shared/PluginAPI.inc

@@ -51,7 +51,7 @@ PLUGIN_API_HANDLE(initialize_record_replay, false);
 PLUGIN_API_HANDLE(check_invalid_image, true);
 PLUGIN_API_HANDLE(has_apu_device, true);
 PLUGIN_API_HANDLE(has_USM_capable_dGPU, true);
-PLUGIN_API_HANDLE(requested_prepopulate_gpu_page_table, true);
+PLUGIN_API_HANDLE(supports_unified_memory, true);
 PLUGIN_API_HANDLE(is_fine_grained_memory_enabled, true);
 PLUGIN_API_HANDLE(is_system_supporting_managed_memory, true);
 PLUGIN_API_HANDLE(number_of_team_procs, true);
@@ -63,3 +63,4 @@ PLUGIN_API_HANDLE(enable_access_to_all_agents, false);
 PLUGIN_API_HANDLE(release_async_info, false);
 PLUGIN_API_HANDLE(activate_record_replay, false);
 PLUGIN_API_HANDLE(use_auto_zero_copy, false);
+PLUGIN_API_HANDLE(zero_copy_sanity_checks_and_diag, false);

openmp/libomptarget/include/Shared/Requirements.h

@@ -38,7 +38,14 @@ enum OpenMPOffloadingRequiresDirFlags : int64_t {
   /// when running on an APU, the GPU plugin may decide to
   /// run in zero-copy even though the user did not program
   /// their application with unified_shared_memory requirement.
-  OMPX_REQ_AUTO_ZERO_COPY = 0x020
+  OMPX_REQ_AUTO_ZERO_COPY = 0x020,
+  /// Eager Maps is an extension of auto zero-copy and
+  /// unified shared memory. Selected using an environment
+  /// varible OMPX_EAGER_ZERO_COPY_MAPS, it makes memory mapping
+  /// issue a GPU TLB prefaulting action. This allows applications
+  /// using unified memory to run with unified memory support disabled
+  /// (if possible on the target device).
+  OMPX_REQ_EAGER_ZERO_COPY_MAPS = 0x040
 };
 
 class RequirementCollection {

openmp/libomptarget/include/device.h

@@ -49,7 +49,13 @@ struct DeviceTy {
 
   /// Flag to force synchronous data transfers
   /// Controlled via environment flag OMPX_FORCE_SYNC_REGIONS
-  bool ForceSynchronousTargetRegions;
+  bool ForceSynchronousTargetRegions = false;
+
+  /// Flag that indicates whether the user requested eager zero-copy maps
+  /// to execute their application. Even if true, this is only valid on certain
+  /// architectures and configurations, which is checked upon device
+  /// initialization.
+  bool EagerZeroCopyMaps = false;
 
   DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID);
   // DeviceTy is not copyable
@@ -161,6 +167,16 @@ struct DeviceTy {
   /// Ask the device whether the runtime should use auto zero-copy.
   bool useAutoZeroCopy();
 
+  /// Ask the device whether it is an APU.
+  bool checkIfAPU();
+
+  /// Ask the device whether it supports unified memory.
+  bool supportsUnifiedMemory();
+
+  /// Ask the device to perform sanity checks for zero-copy configurations.
+  void zeroCopySanityChecksAndDiag(bool isUnifiedSharedMemory,
+                                   bool isAutoZeroCopy, bool isEagerMaps);
+
 private:
   /// Deinitialize the device (and plugin).
   void deinit();

openmp/libomptarget/plugins-nextgen/amdgpu/src/rtl.cpp

@@ -2504,7 +2504,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
         OMPX_ApuMaps("OMPX_APU_MAPS", false),
         OMPX_DisableUsmMaps("OMPX_DISABLE_USM_MAPS", false),
         OMPX_NoMapChecks("OMPX_DISABLE_MAPS", true),
-        OMPX_EagerApuMaps("OMPX_EAGER_ZERO_COPY_MAPS", false),
+        OMPX_StrictSanityChecks("OMPX_STRICT_SANITY_CHECKS", false),
         AMDGPUStreamManager(*this, Agent), AMDGPUEventManager(*this),
         AMDGPUSignalManager(*this), Agent(Agent), HostDevice(HostDevice) {}
 
@@ -2942,75 +2942,6 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
-  // TODO: clean up the following three functions after removing auto_zero_copy
-  // support and document appropriately.
-  void checkAndAdjustUsmModeForTargetImage(const __tgt_device_image *TgtImage) {
-    assert((TgtImage != nullptr) && "TgtImage is nullptr");
-    assert(!(Plugin::get().getRequiresFlags() & OMP_REQ_UNDEFINED) &&
-           "Requires flags are not set.");
-
-    if (!(IsAPU || hasDGpuWithUsmSupportImpl()))
-      return;
-
-    bool IsXnackRequired =
-        Plugin::get().getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY;
-    utils::XnackBuildMode BinaryXnackMode =
-        utils::extractXnackModeFromBinary(TgtImage);
-
-    if (IsXnackRequired) {
-      handleImageRequiresUsmMode(BinaryXnackMode);
-    } else {
-      handleDefaultMode(BinaryXnackMode);
-    }
-  }
-
-  void handleImageRequiresUsmMode(utils::XnackBuildMode XnackImageMode) {
-    bool IsXnackActiveOnSystem = IsXnackEnabled;
-
-    if ((XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4) ||
-        (XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4 &&
-         IsXnackActiveOnSystem) ||
-        (XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_OFF_V4 &&
-         !IsXnackActiveOnSystem)) {
-      if (OMPX_EagerApuMaps.get() && IsAPU)
-        PrepopulateGPUPageTable = true; // Pre-faulting
-    }
-
-    if (!IsXnackActiveOnSystem &&
-        (XnackImageMode != ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4)) {
-      FAILURE_MESSAGE(
-          "Running a program that requires XNACK on a system where XNACK is "
-          "disabled. This may cause problems when using a OS-allocated pointer "
-          "inside a target region. "
-          "Re-run with HSA_XNACK=1 to remove this warning.\n");
-    }
-  }
-
-  void handleDefaultMode(utils::XnackBuildMode XnackImageMode) {
-    // assuming that copying is required
-    // handled in userAutoZeroCopyImpl
-    // DisableAllocationsForMapsOnApus = false;
-    bool IsXnackActiveOnSystem = IsXnackEnabled;
-
-    if (IsXnackActiveOnSystem && (IsAPU || OMPX_ApuMaps.get()) &&
-        ((XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4) ||
-         (XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4))) {
-      if (IsAPU && OMPX_EagerApuMaps.get()) {
-        PrepopulateGPUPageTable = true; // Pre-faulting
-      }
-      return;
-    }
-
-    if (!IsXnackActiveOnSystem && IsAPU && OMPX_EagerApuMaps.get() &&
-        ((XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4) ||
-         (XnackImageMode == ELF::EF_AMDGPU_FEATURE_XNACK_OFF_V4))) {
-      PrepopulateGPUPageTable = true; // Pre-faulting
-      return;
-    }
-
-    return;
-  }
-
   /// Load the binary image into the device and allocate an image object.
   Expected<DeviceImageTy *> loadBinaryImpl(const __tgt_device_image *TgtImage,
                                            int32_t ImageId) override {
@@ -3022,9 +2953,6 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     // Load the HSA executable.
     if (Error Err = AMDImage->loadExecutable(*this))
       return std::move(Err);
-
-    checkAndAdjustUsmModeForTargetImage(TgtImage);
-
     return AMDImage;
   }
 
@@ -3624,14 +3552,51 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// while it is often not the best on discrete GPUs.
   /// XNACK can be enabled with a kernel boot parameter or with
   /// the HSA_XNACK environment variable.
-  /// ROCm-only behavior: default (non USM, with xnack- or xnack-any)
-  /// and OMPX_EAGER_APU_MAPS is automatic zero-copy with pre-fault.
   bool useAutoZeroCopyImpl() override {
-    return (
-        ((IsAPU || OMPX_ApuMaps) && IsXnackEnabled) ||
-        (IsAPU &&
-         !(Plugin::get().getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY) &&
-         !IsXnackEnabled && OMPX_EagerApuMaps.get()));
+    return ((IsAPU || OMPX_ApuMaps) && IsXnackEnabled);
+  }
+
+  /// Performs sanity checks on the selected zero-copy configuration and prints
+  /// diagnostic information.
+  Error zeroCopySanityChecksAndDiagImpl(bool isUnifiedSharedMemory,
+                                        bool isAutoZeroCopy,
+                                        bool isEagerMaps) override {
+    // Implementation sanity checks: either unified_shared_memory or auto
+    // zero-copy, not both
+    if (isUnifiedSharedMemory && isAutoZeroCopy)
+      return Plugin::error("Internal runtime error: cannot be both "
+                           "unified_shared_memory and auto zero-copy.");
+
+    if (IsXnackEnabled)
+      INFO(OMP_INFOTYPE_USER_DIAGNOSTIC, getDeviceId(), "XNACK is enabled.\n");
+    else
+      INFO(OMP_INFOTYPE_USER_DIAGNOSTIC, getDeviceId(), "XNACK is disabled.\n");
+    if (isUnifiedSharedMemory)
+      INFO(OMP_INFOTYPE_USER_DIAGNOSTIC, getDeviceId(),
+           "Application configured to run in zero-copy using "
+           "unified_shared_memory.\n");
+    else if (isAutoZeroCopy)
+      INFO(
+          OMP_INFOTYPE_USER_DIAGNOSTIC, getDeviceId(),
+          "Application configured to run in zero-copy using auto zero-copy.\n");
+    if (isEagerMaps)
+      INFO(OMP_INFOTYPE_USER_DIAGNOSTIC, getDeviceId(),
+           "Requested pre-faulting of GPU page tables.\n");
+
+    // Sanity checks: selecting unified_shared_memory with XNACK-Disabled
+    // triggers a warning that can be turned into a fatal error using an
+    // environment variable.
+    if (isUnifiedSharedMemory && !IsXnackEnabled) {
+      MESSAGE0(
+          "Running a program that requires XNACK on a system where XNACK is "
+          "disabled. This may cause problems when using an OS-allocated "
+          "pointer "
+          "inside a target region. "
+          "Re-run with HSA_XNACK=1 to remove this warning.");
+      if (OMPX_StrictSanityChecks)
+        llvm_unreachable("User-requested hard stop on sanity check errors.");
+    }
+    return Plugin::success();
   }
 
   /// Getters and setters for stack and heap sizes.
@@ -3845,19 +3810,6 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (OMPX_DisableUsmMaps.get() == true) {
       EnableFineGrainedMemory = true;
     }
-
-    if (IsAPU) {
-      // OMPX_EAGER_ZERO_COPY_MAPS=1 && HSA_XNACK=0 (XNACK-disabled)
-      // && default (non-USM) program
-      if ((OMPX_EagerApuMaps.get() == true) && !IsXnackEnabled &&
-          !(Plugin::get().getRequiresFlags() & OMP_REQ_UNIFIED_SHARED_MEMORY)) {
-        PrepopulateGPUPageTable = true;
-      }
-    }
-  }
-
-  bool requestedPrepopulateGPUPageTableImpl() override final {
-    return PrepopulateGPUPageTable;
   }
 
   bool IsFineGrainedMemoryEnabledImpl() override final {
@@ -3875,6 +3827,10 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return hasGfx90aDevice() || hasMI300xDevice();
   }
 
+  /// Returns whether AMD GPU supports unified memory in
+  /// the current configuration.
+  bool supportsUnifiedMemoryImpl() override final { return IsXnackEnabled; }
+
   /// Envar for controlling the number of HSA queues per device. High number of
   /// queues may degrade performance.
   UInt32Envar OMPX_NumQueues;
@@ -3963,11 +3919,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// currently always without map checks.
   BoolEnvar OMPX_NoMapChecks;
 
-  /// Value of OMPX_EAGER_ZERO_COPY_MAPS. When true, it
-  /// makes the plugin prefault the GPU page table upon
-  /// map. This allows running with XNACK-Disabled and
-  /// use zero-copy.
-  BoolEnvar OMPX_EagerApuMaps;
+  // Makes warnings turn into fatal errors
+  BoolEnvar OMPX_StrictSanityChecks;
 
   /// Stream manager for AMDGPU streams.
   AMDGPUStreamManagerTy AMDGPUStreamManager;
@@ -4030,11 +3983,6 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// False otherwise.
   bool IsXnackEnabled = false;
 
-  /// Set by OMPX_EAGER_ZERO_COPY_MAPS environment variable.
-  /// If set, map clauses provoke prefaulting of the GPU
-  /// page table (applies to limited cases).
-  bool PrepopulateGPUPageTable = false;
-
   // Set by OMPX_DISABLE_USM_MAPS environment variable.
   // If set, fine graned memory is used for maps instead of coarse grained.
   bool EnableFineGrainedMemory = false;

openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h

@@ -826,19 +826,16 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
 
   // Returns true if the system is equipped with an APU.
   // moved in from plugin
-  //  virtual bool hasAPUDevice() { return false; }
   bool hasAPUDevice();
   virtual bool hasAPUDeviceImpl() { return false; }
 
-  // Returns true if the system is equipped with a dGPU that supports USM
-  // virtual bool hasDGpuWithUsmSupport() { return false; }
+  // Returns true if the system is equipped with a dGPU that supports USM/
   bool hasDGpuWithUsmSupport();
   virtual bool hasDGpuWithUsmSupportImpl() { return false; }
 
-  // Returns true if user requested Eager Maps implementation.
-  // virtual bool requestedPrepopulateGPUPageTable() { return false; }
-  bool requestedPrepopulateGPUPageTable();
-  virtual bool requestedPrepopulateGPUPageTableImpl() { return false; }
+  // Returns true if the system supports unified memory.
+  bool supportsUnifiedMemory();
+  virtual bool supportsUnifiedMemoryImpl() { return false; }
 
   // Returns true if coarse graining of mapped variables is
   // disabled on MI200 GPUs.
@@ -952,6 +949,15 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
 
   bool isFastReductionEnabled() const { return IsFastReductionEnabled; }
 
+  /// Performs sanity checks on zero-copy options and prints diagnostic info.
+  Error zeroCopySanityChecksAndDiag(bool isUnifiedSharedMemory,
+                                    bool isAutoZeroCopy, bool isEagerMaps);
+  virtual Error zeroCopySanityChecksAndDiagImpl(bool isUnifiedSharedMemory,
+                                                bool isAutoZeroCopy,
+                                                bool isEagerMaps) {
+    return Error::success();
+  }
+
 private:
   /// Get and set the stack size and heap size for the device. If not used, the
   /// plugin can implement the setters as no-op and setting the output