[SPIR-V] Strip convergence intrinsics before ISel (#75948)

The structurizer will require the frontend to emit convergence
intrinsics. Once uses to restructurize the control-flow, those
intrinsics shall be removed, as they cannot be converted to
SPIR-V.

This commit adds a new pass to the SPIR-V backend which strips those
intrinsics.

Those 2 new steps are not limited to Vulkan as OpenCL could
also benefit from not crashing if a convertent operation is in
the IR (even though the frontend doesn't generate such intrinsics).

Signed-off-by: Nathan Gauër <[email protected]>

Author: Keenuts

llvm/lib/Target/SPIRV/CMakeLists.txt

@@ -22,6 +22,7 @@ add_llvm_target(SPIRVCodeGen
   SPIRVGlobalRegistry.cpp
   SPIRVInstrInfo.cpp
   SPIRVInstructionSelector.cpp
+  SPIRVStripConvergentIntrinsics.cpp
   SPIRVISelLowering.cpp
   SPIRVLegalizerInfo.cpp
   SPIRVMCInstLower.cpp

llvm/lib/Target/SPIRV/SPIRV.h

@@ -20,6 +20,7 @@ class InstructionSelector;
 class RegisterBankInfo;
 
 ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM);
+FunctionPass *createSPIRVStripConvergenceIntrinsicsPass();
 FunctionPass *createSPIRVRegularizerPass();
 FunctionPass *createSPIRVPreLegalizerPass();
 FunctionPass *createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM);

llvm/lib/Target/SPIRV/SPIRVStripConvergentIntrinsics.cpp

@@ -0,0 +1,86 @@
+//===-- SPIRVStripConvergentIntrinsics.cpp ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass trims convergence intrinsics as those were only useful when
+// modifying the CFG during IR passes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRV.h"
+#include "SPIRVSubtarget.h"
+#include "SPIRVTargetMachine.h"
+#include "SPIRVUtils.h"
+#include "llvm/CodeGen/IntrinsicLowering.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
+
+using namespace llvm;
+
+namespace llvm {
+void initializeSPIRVStripConvergentIntrinsicsPass(PassRegistry &);
+}
+
+class SPIRVStripConvergentIntrinsics : public FunctionPass {
+public:
+  static char ID;
+
+  SPIRVStripConvergentIntrinsics() : FunctionPass(ID) {
+    initializeSPIRVStripConvergentIntrinsicsPass(
+        *PassRegistry::getPassRegistry());
+  };
+
+  virtual bool runOnFunction(Function &F) override {
+    DenseSet<Instruction *> ToRemove;
+
+    for (BasicBlock &BB : F) {
+      for (Instruction &I : BB) {
+        if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
+          if (II->getIntrinsicID() !=
+                  Intrinsic::experimental_convergence_entry &&
+              II->getIntrinsicID() !=
+                  Intrinsic::experimental_convergence_loop &&
+              II->getIntrinsicID() !=
+                  Intrinsic::experimental_convergence_anchor) {
+            continue;
+          }
+
+          II->replaceAllUsesWith(UndefValue::get(II->getType()));
+          ToRemove.insert(II);
+        } else if (auto *CI = dyn_cast<CallInst>(&I)) {
+          auto OB = CI->getOperandBundle(LLVMContext::OB_convergencectrl);
+          if (!OB.has_value())
+            continue;
+
+          auto *NewCall = CallBase::removeOperandBundle(
+              CI, LLVMContext::OB_convergencectrl, CI);
+          NewCall->copyMetadata(*CI);
+          CI->replaceAllUsesWith(NewCall);
+          ToRemove.insert(CI);
+        }
+      }
+    }
+
+    // All usages must be removed before their definition is removed.
+    for (Instruction *I : ToRemove)
+      I->eraseFromParent();
+
+    return ToRemove.size() != 0;
+  }
+};
+
+char SPIRVStripConvergentIntrinsics::ID = 0;
+INITIALIZE_PASS(SPIRVStripConvergentIntrinsics, "strip-convergent-intrinsics",
+                "SPIRV strip convergent intrinsics", false, false)
+
+FunctionPass *llvm::createSPIRVStripConvergenceIntrinsicsPass() {
+  return new SPIRVStripConvergentIntrinsics();
+}

llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp

@@ -168,6 +168,7 @@ void SPIRVPassConfig::addIRPasses() {
   TargetPassConfig::addIRPasses();
   addPass(createSPIRVRegularizerPass());
   addPass(createSPIRVPrepareFunctionsPass(TM));
+  addPass(createSPIRVStripConvergenceIntrinsicsPass());
 }
 
 void SPIRVPassConfig::addISelPrepare() {

llvm/test/CodeGen/SPIRV/scfg-add-pre-headers.ll

@@ -3,11 +3,14 @@
 ; CHECK-DAG:    %[[#bool:]] = OpTypeBool
 ; CHECK-DAG:    %[[#uint:]] = OpTypeInt 32 0
 ; CHECK-DAG:  %[[#uint_0:]] = OpConstant %[[#uint]] 0
+; CHECK-DAG:                  OpName %[[#main:]] "main"
 
 define void @main() #1 {
   %1 = icmp ne i32 0, 0
+  %t1 = call token @llvm.experimental.convergence.entry()
   br i1 %1, label %l1, label %l2
 
+; CHECK:         %[[#main]] = OpFunction
 ; CHECK:        %[[#cond:]] = OpINotEqual %[[#bool]] %[[#uint_0]] %[[#uint_0]]
 ; CHECK:                      OpBranchConditional %[[#cond]] %[[#l1_pre:]] %[[#l2_pre:]]
 
@@ -18,6 +21,7 @@ define void @main() #1 {
 ; CHECK-NEXT:                 OpBranch %[[#l1_header:]]
 
 l1:
+  %tl1 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
   br i1 %1, label %l1_body, label %l1_end
 ; CHECK-DAG:    %[[#l1_header]] = OpLabel
 ; CHECK-NEXT:                     OpBranchConditional %[[#cond]] %[[#l1_body:]] %[[#l1_end:]]
@@ -33,11 +37,14 @@ l1_continue:
 ; CHECK-NEXT:                      OpBranch %[[#l1_header]]
 
 l1_end:
+  %call = call spir_func i32 @_Z3absi(i32 0) [ "convergencectrl"(token %tl1) ]
   br label %end
 ; CHECK-DAG:   %[[#l1_end]] = OpLabel
+; CHECK-DAG:         %[[#]] = OpFunctionCall
 ; CHECK-NEXT:                 OpBranch %[[#end:]]
 
 l2:
+  %tl2 = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token %t1) ]
   br i1 %1, label %l2_body, label %l2_end
 ; CHECK-DAG:    %[[#l2_header]] = OpLabel
 ; CHECK-NEXT:                     OpBranchConditional %[[#cond]] %[[#l2_body:]] %[[#l2_end:]]
@@ -64,3 +71,11 @@ end:
 }
 
 attributes #1 = { "hlsl.numthreads"="4,8,16" "hlsl.shader"="compute" convergent }
+
+declare token @llvm.experimental.convergence.entry()
+declare token @llvm.experimental.convergence.control()
+declare token @llvm.experimental.convergence.loop()
+
+; This intrinsic is not convergent. This is only because the backend doesn't
+; support convergent operations yet.
+declare spir_func i32 @_Z3absi(i32) convergent