Add pass to add global to module using new pass manager

Author: Hugh Delaney

libclc/amdgcn-amdhsa/libspirv/atomic/atomic_sub.cl

@@ -18,7 +18,7 @@
           enum MemorySemanticsMask semantics, TYPE val) {                                                                     \
     int atomic_scope = 0, memory_order = 0;                                                                                   \
     GET_ATOMIC_SCOPE_AND_ORDER(scope, atomic_scope, semantics, memory_order)                                                  \
-    return BUILTIN(p, val, memory_order);                                                                                 \
+    return BUILTIN(p, -val, memory_order);                                                                                    \
   }
 
 #define AMDGPU_ATOMIC_SUB(FUNC_NAME, TYPE, TYPE_MANGLED, BUILTIN)              \
@@ -28,11 +28,11 @@
                          BUILTIN)                                              \
   AMDGPU_ATOMIC_SUB_IMPL(FUNC_NAME, TYPE, TYPE_MANGLED, , , 0, BUILTIN)
 
-AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, int, i, __atomic_fetch_sub)
-AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, unsigned int, j, __atomic_fetch_sub)
-AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, long, l, __atomic_fetch_sub)
-AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, unsigned long, m, __atomic_fetch_sub)
-AMDGPU_ATOMIC_SUB(_Z21__spirv_AtomicFSubEXT, float, f, __atomic_fetch_sub)
+AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, int, i, __atomic_fetch_add)
+AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, unsigned int, j, __atomic_fetch_add)
+AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, long, l, __atomic_fetch_add)
+AMDGPU_ATOMIC_SUB(_Z18__spirv_AtomicISub, unsigned long, m, __atomic_fetch_add)
+AMDGPU_ATOMIC_SUB(_Z21__spirv_AtomicFSubEXT, float, f, __atomic_fetch_add)
 
 #undef AMDGPU_ATOMIC
 #undef AMDGPU_ATOMIC_IMPL

llvm/lib/Target/AMDGPU/AMDGPUAddGlobalForAtomicXor.cpp

@@ -0,0 +1,58 @@
+//===- AMDGPUAddGlobalForAtomicXor.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Some AMDGPU atomic instructions require a prefetch in order for them to work
+// properly when using hipMallocManaged. This pass scans a module for the
+// problematic atomic instructions and creates a global PrefetchNeeded if the
+// builtin is present. This allows the prefetch to happen at runtime only if the
+// problematic builtin is chosen.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUAddGlobalForAtomicXor.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+
+using namespace llvm;
+
+#define NEW_GLOBAL_NAME "HipAtomicXorModuleNeedsPrefetch"
+
+namespace {
+
+bool moduleHasAtomicXor(Module &M) {
+  for (auto &F : M) {
+    for (auto &I : instructions(F)) {
+      if (auto *AtomicInst = dyn_cast<AtomicRMWInst>(&I)) {
+        if (AtomicInst->getOperation() == AtomicRMWInst::Xor) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+bool runImpl(Module &M) {
+  if (moduleHasAtomicXor(M)) {
+    LLVMContext &Ctx = M.getContext();
+    new GlobalVariable(M, Type::getInt1Ty(Ctx), true,
+                       GlobalValue::InternalLinkage,
+                       Constant::getAllOnesValue(Type::getInt1Ty(Ctx)),
+                       NEW_GLOBAL_NAME); // FIXME: this seems wrong and bad, is
+                                         // there a better way to make a new
+                                         // GlobalVariable?
+    return true;
+  }
+  return false;
+}
+} // end anonymous namespace
+
+PreservedAnalyses
+AMDGPUAddGlobalForAtomicXorPass::run(Module &M, ModuleAnalysisManager &AM) {
+  return runImpl(M) ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}

llvm/lib/Target/AMDGPU/AMDGPUAddGlobalForAtomicXor.h

@@ -0,0 +1,32 @@
+//===- AMDGPUAddGlobalForAtomicXor.h --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Some AMDGPU atomic instructions require a prefetch in order for them to work
+// properly when using hipMallocManaged. This pass scans a module for the
+// problematic atomic instructions and creates a global PrefetchNeeded if the
+// builtin is present. This allows the prefetch to happen at runtime only if the
+// problematic builtin is chosen.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_ADDGLOBALFORATOMICXOR_H
+#define LLVM_LIB_TARGET_AMDGPU_ADDGLOBALFORATOMICXOR_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class AMDGPUAddGlobalForAtomicXorPass
+    : public PassInfoMixin<AMDGPUAddGlobalForAtomicXorPass> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_ADDGLOBALFORATOMICXOR_H

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

@@ -14,6 +14,7 @@
 
 #include "AMDGPUTargetMachine.h"
 #include "AMDGPU.h"
+#include "AMDGPUAddGlobalForAtomicXor.h"
 #include "AMDGPUAliasAnalysis.h"
 #include "AMDGPUCtorDtorLowering.h"
 #include "AMDGPUExportClustering.h"
@@ -618,6 +619,10 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
           PM.addPass(AMDGPUAlwaysInlinePass());
           return true;
         }
+        if (PassName == "amdgpu-add-global-for-atomic-xor") {
+          PM.addPass(AMDGPUAddGlobalForAtomicXorPass());
+          return true;
+        }
         if (PassName == "amdgpu-lower-module-lds") {
           PM.addPass(AMDGPULowerModuleLDSPass(*this));
           return true;

llvm/lib/Target/AMDGPU/CMakeLists.txt

@@ -41,6 +41,7 @@ tablegen(LLVM InstCombineTables.inc -gen-searchable-tables)
 add_public_tablegen_target(InstCombineTableGen)
 
 add_llvm_target(AMDGPUCodeGen
+  AMDGPUAddGlobalForAtomicXor.cpp
   AMDGPUAliasAnalysis.cpp
   AMDGPUAlwaysInlinePass.cpp
   AMDGPUAnnotateKernelFeatures.cpp