[AMDGPU] Rewrite GFX12 SGPR hazard handling to dedicated pass (#118750)

- Algorithm operates over whole IR to attempt to minimize waits.
- Add support for VALU->VALU SGPR hazards via VA_SDST/VA_VCC.

Author: perlfu

llvm/docs/AMDGPUUsage.rst

@@ -1709,6 +1709,20 @@ The AMDGPU backend supports the following LLVM IR attributes.
                                              as hidden. Hidden arguments are managed by the compiler and are not part of
                                              the explicit arguments supplied by the user.
 
+     "amdgpu-sgpr-hazard-wait"               Disabled SGPR hazard wait insertion if set to 0.
+                                             Exists for testing performance impact of SGPR hazard waits only.
+
+     "amdgpu-sgpr-hazard-boundary-cull"      Enable insertion of SGPR hazard cull sequences at function call boundaries.
+                                             Cull sequence reduces future hazard waits, but has a performance cost.
+
+     "amdgpu-sgpr-hazard-mem-wait-cull"      Enable insertion of SGPR hazard cull sequences before memory waits.
+                                             Cull sequence reduces future hazard waits, but has a performance cost.
+                                             Attempt to amortize cost by overlapping with memory accesses.
+
+     "amdgpu-sgpr-hazard-mem-wait-cull-threshold"
+                                             Sets the number of active SGPR hazards that must be present before
+                                             inserting a cull sequence at a memory wait.
+
      ======================================= ==========================================================
 
 Calling Conventions

llvm/lib/Target/AMDGPU/AMDGPU.h

@@ -463,6 +463,9 @@ void initializeAMDGPUSetWavePriorityPass(PassRegistry &);
 void initializeGCNRewritePartialRegUsesPass(llvm::PassRegistry &);
 extern char &GCNRewritePartialRegUsesID;
 
+void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &);
+extern char &AMDGPUWaitSGPRHazardsLegacyID;
+
 namespace AMDGPU {
 enum TargetIndex {
   TI_CONSTDATA_START,

llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

@@ -29,6 +29,7 @@
 #include "AMDGPUTargetObjectFile.h"
 #include "AMDGPUTargetTransformInfo.h"
 #include "AMDGPUUnifyDivergentExitNodes.h"
+#include "AMDGPUWaitSGPRHazards.h"
 #include "GCNDPPCombine.h"
 #include "GCNIterativeScheduler.h"
 #include "GCNSchedStrategy.h"
@@ -549,6 +550,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeGCNRewritePartialRegUsesPass(*PR);
   initializeGCNRegPressurePrinterPass(*PR);
   initializeAMDGPUPreloadKernArgPrologLegacyPass(*PR);
+  initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
 }
 
 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -1678,6 +1680,8 @@ void GCNPassConfig::addPreEmitPass() {
   // cases.
   addPass(&PostRAHazardRecognizerID);
 
+  addPass(&AMDGPUWaitSGPRHazardsLegacyID);
+
   if (isPassEnabled(EnableInsertDelayAlu, CodeGenOptLevel::Less))
     addPass(&AMDGPUInsertDelayAluID);