[MachineScheduler][RISCV] Release the pending queue base on condition

llvm/include/llvm/CodeGen/MachineScheduler.h

@@ -1068,6 +1068,15 @@ class SchedBoundary {
   /// Dump the state of the information that tracks resource usage.
   void dumpReservedCycles() const;
   void dumpScheduledState() const;
+
+  /// Bump the cycle until the given SU is released from the pending queue.
+  void bumpCycleUntilReleaseSUFromPending(SUnit *SU, unsigned ReadyListLimit) {
+    while (Available.size() < ReadyListLimit &&
+           llvm::find(Pending, SU) != Pending.end()) {
+      bumpCycle(CurrCycle + 1);
+      releasePending();
+    }
+  }
 };
 
 /// Base class for GenericScheduler. This class maintains information about
@@ -1262,6 +1271,8 @@ class GenericScheduler : public GenericSchedulerBase {
     BotCand.SU = nullptr;
   }
 
+  void bumpCycleUntilReleaseSUFromPending(bool IsTop);
+
   void registerRoots() override;
 
 protected:

llvm/include/llvm/CodeGen/TargetRegisterInfo.h

@@ -1190,6 +1190,22 @@ class TargetRegisterInfo : public MCRegisterInfo {
     return false;
   }
 
+  /// Based on the target and current register pressure information from the
+  /// Scheduler, determine whether to release the node in the pending queue
+  virtual bool
+  shouldReleasePendingQueue(MachineFunction &MF,
+                            ArrayRef<unsigned> MaxSetPressure) const {
+    return false;
+  }
+
+  /// For each SUnit, determine whether to release it from the pending queue
+  /// based on the register pressure changes associated with that SUnit.
+  virtual bool shouldReleaseSUFromPendingQueue(MachineFunction &MF,
+                                               ArrayRef<unsigned> PSetID,
+                                               ArrayRef<int> UnitInc) const {
+    return false;
+  }
+
   //===--------------------------------------------------------------------===//
   /// Debug information queries.
 

llvm/lib/CodeGen/MachineScheduler.cpp

@@ -161,6 +161,10 @@ static cl::opt<bool> EnableCyclicPath("misched-cyclicpath", cl::Hidden,
 static cl::opt<bool> EnableMemOpCluster("misched-cluster", cl::Hidden,
                                         cl::desc("Enable memop clustering."),
                                         cl::init(true));
+static cl::opt<bool>
+    EnableReleasePendingQ("misched-release-pending-queue", cl::Hidden,
+                          cl::desc("Release the pending queue"),
+                          cl::init(true));
 static cl::opt<bool>
     ForceFastCluster("force-fast-cluster", cl::Hidden,
                      cl::desc("Switch to fast cluster algorithm with the lost "
@@ -3656,6 +3660,37 @@ void GenericScheduler::pickNodeFromQueue(SchedBoundary &Zone,
   }
 }
 
+void GenericScheduler::bumpCycleUntilReleaseSUFromPending(bool IsTop) {
+  if (!DAG->isTrackingPressure())
+    return;
+  auto ReleasePending = [&](ReadyQueue &Q, const RegPressureTracker &RegP,
+                            ArrayRef<unsigned> MaxSetP, SchedBoundary &SchedB) {
+    for (SUnit *SU : Q) {
+      RegPressureTracker &TempTracker = const_cast<RegPressureTracker &>(RegP);
+      CandPolicy TempPolicy;
+      SchedCandidate TryCand(TempPolicy);
+      initCandidate(TryCand, SU, IsTop, RegP, TempTracker);
+      PressureDiff PDiff = DAG->getPressureDiff(SU);
+      SmallVector<unsigned> PSetIDs;
+      SmallVector<int> UnitIncs;
+      for (const auto &PChange : PDiff) {
+        if (!PChange.isValid())
+          continue;
+        PSetIDs.push_back(PChange.getPSet());
+        UnitIncs.push_back(PChange.getUnitInc());
+      }
+      if (TRI->shouldReleaseSUFromPendingQueue(DAG->MF, PSetIDs, UnitIncs))
+        SchedB.bumpCycleUntilReleaseSUFromPending(SU, ReadyListLimit);
+    }
+  };
+  if (IsTop)
+    ReleasePending(Top.Pending, DAG->getTopRPTracker(),
+                   DAG->getTopRPTracker().getPressure().MaxSetPressure, Top);
+  else
+    ReleasePending(Bot.Pending, DAG->getBotRPTracker(),
+                   DAG->getBotRPTracker().getPressure().MaxSetPressure, Bot);
+}
+
 /// Pick the best candidate node from either the top or bottom queue.
 SUnit *GenericScheduler::pickNodeBidirectional(bool &IsTopNode) {
   // Schedule as far as possible in the direction of no choice. This is most
@@ -3741,6 +3776,14 @@ SUnit *GenericScheduler::pickNode(bool &IsTopNode) {
            Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
     return nullptr;
   }
+
+  if(EnableReleasePendingQ) {
+    if (!RegionPolicy.OnlyBottomUp && TRI->shouldReleasePendingQueue(DAG->MF, DAG->getTopRPTracker().getPressure().MaxSetPressure))
+      bumpCycleUntilReleaseSUFromPending(/*IsTop=*/true);
+    if (!RegionPolicy.OnlyTopDown && TRI->shouldReleasePendingQueue(DAG->MF, DAG->getBotRPTracker().getPressure().MaxSetPressure))
+      bumpCycleUntilReleaseSUFromPending(/*IsTop=*/false);
+  }
+
   SUnit *SU;
   do {
     if (RegionPolicy.OnlyTopDown) {

llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp

@@ -37,6 +37,18 @@ static cl::opt<bool>
                          cl::desc("Disable two address hints for register "
                                   "allocation"));
 
+static cl::opt<unsigned> RVVReleasePendingPressureThreshold(
+    "riscv-release-pending-queue-rvv-pressure-threshold", cl::Hidden,
+    cl::desc("Release pending queue when vector register pressure exceeds "
+             "threshold"),
+    cl::init(7));
+
+static cl::opt<int> UnitIncRVVRegPressureThreshold(
+    "riscv-release-pending-queue-unit-inc-rvv-pressure-threshold", cl::Hidden,
+    cl::desc("Release pending queue when vector register unit pressure "
+             "increase exceeds threshold"),
+    cl::init(-3));
+
 static_assert(RISCV::X1 == RISCV::X0 + 1, "Register list not consecutive");
 static_assert(RISCV::X31 == RISCV::X0 + 31, "Register list not consecutive");
 static_assert(RISCV::F1_H == RISCV::F0_H + 1, "Register list not consecutive");
@@ -954,3 +966,47 @@ bool RISCVRegisterInfo::getRegAllocationHints(
 
   return BaseImplRetVal;
 }
+
+static bool isRVVPressureSetIdx(unsigned Idx) {
+  const static std::vector<unsigned> RVVPressureSetIndices = {
+      RISCV::RegisterPressureSets::VRM8NoV0, RISCV::RegisterPressureSets::VM};
+
+  return llvm::find(RVVPressureSetIndices, Idx) != RVVPressureSetIndices.end();
+}
+
+bool RISCVRegisterInfo::shouldReleasePendingQueue(
+    MachineFunction &MF, ArrayRef<unsigned> MaxSetPressure) const {
+
+  if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
+    return false;
+
+  for (unsigned Idx = 0; Idx < MaxSetPressure.size(); Idx++) {
+    // Consider only the RVV Register, as RVV spilling/reloading has higher
+    // potential costs than hazards.
+    if (!isRVVPressureSetIdx(Idx))
+      continue;
+    if (MaxSetPressure[Idx] + RVVReleasePendingPressureThreshold >
+        getRegPressureSetLimit(MF, Idx)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+bool RISCVRegisterInfo::shouldReleaseSUFromPendingQueue(
+    MachineFunction &MF, ArrayRef<unsigned> PSetID,
+    ArrayRef<int> UnitInc) const {
+
+  if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions())
+    return false;
+
+  for (unsigned Idx = 0; Idx < PSetID.size(); Idx++) {
+    // Consider only the RVV Register, as RVV spilling/reloading has higher
+    // potential costs than hazards.
+    if (!isRVVPressureSetIdx(PSetID[Idx]))
+      continue;
+    if (UnitInc[Idx] < UnitIncRVVRegPressureThreshold)
+      return true;
+  }
+  return false;
+}

llvm/lib/Target/RISCV/RISCVRegisterInfo.h

@@ -144,6 +144,13 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
   static bool isRVVRegClass(const TargetRegisterClass *RC) {
     return RISCVRI::isVRegClass(RC->TSFlags);
   }
+  bool
+  shouldReleasePendingQueue(MachineFunction &MF,
+                            ArrayRef<unsigned> MaxSetPressure) const override;
+
+  bool shouldReleaseSUFromPendingQueue(MachineFunction &MF,
+                                       ArrayRef<unsigned> PSetID,
+                                       ArrayRef<int> UnitInc) const override;
 };
 } // namespace llvm
 

llvm/test/CodeGen/RISCV/rvv/bitreverse-sdnode.ll

@@ -1137,38 +1137,38 @@ define <vscale x 8 x i64> @bitreverse_nxv8i64(<vscale x 8 x i64> %va) {
 ; RV32-NEXT:    li a1, 56
 ; RV32-NEXT:    li a2, 40
 ; RV32-NEXT:    lui a3, 16
-; RV32-NEXT:    lui a4, 4080
-; RV32-NEXT:    addi a5, sp, 8
-; RV32-NEXT:    sw a0, 8(sp)
-; RV32-NEXT:    sw zero, 12(sp)
-; RV32-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli a4, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vsrl.vx v16, v8, a1
 ; RV32-NEXT:    vsrl.vx v24, v8, a2
-; RV32-NEXT:    addi a0, a3, -256
+; RV32-NEXT:    addi a3, a3, -256
 ; RV32-NEXT:    vsll.vx v0, v8, a1
-; RV32-NEXT:    vand.vx v24, v24, a0
+; RV32-NEXT:    vand.vx v24, v24, a3
 ; RV32-NEXT:    vor.vv v16, v24, v16
 ; RV32-NEXT:    addi a1, sp, 16
 ; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vand.vx v16, v8, a0
+; RV32-NEXT:    vand.vx v16, v8, a3
 ; RV32-NEXT:    vsll.vx v16, v16, a2
 ; RV32-NEXT:    vor.vv v16, v0, v16
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vs8r.v v16, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    vlse64.v v0, (a5), zero
-; RV32-NEXT:    vsrl.vi v16, v8, 24
-; RV32-NEXT:    vand.vx v16, v16, a4
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    slli a1, a1, 3
+; RV32-NEXT:    add a1, sp, a1
+; RV32-NEXT:    addi a1, a1, 16
+; RV32-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
+; RV32-NEXT:    vsrl.vi v0, v8, 24
+; RV32-NEXT:    lui a1, 4080
+; RV32-NEXT:    addi a2, sp, 8
+; RV32-NEXT:    sw a0, 8(sp)
+; RV32-NEXT:    sw zero, 12(sp)
+; RV32-NEXT:    vand.vx v0, v0, a1
 ; RV32-NEXT:    vsrl.vi v24, v8, 8
-; RV32-NEXT:    vand.vv v24, v24, v0
-; RV32-NEXT:    vor.vv v16, v24, v16
+; RV32-NEXT:    vlse64.v v16, (a2), zero
+; RV32-NEXT:    vand.vv v24, v24, v16
+; RV32-NEXT:    vor.vv v24, v24, v0
 ; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vor.vv v24, v16, v24
-; RV32-NEXT:    vand.vv v16, v8, v0
-; RV32-NEXT:    vand.vx v8, v8, a4
+; RV32-NEXT:    vl8r.v v0, (a0) # Unknown-size Folded Reload
+; RV32-NEXT:    vor.vv v24, v24, v0
+; RV32-NEXT:    vand.vv v16, v8, v16
+; RV32-NEXT:    vand.vx v8, v8, a1
 ; RV32-NEXT:    vsll.vi v8, v8, 24
 ; RV32-NEXT:    vsll.vi v16, v16, 8
 ; RV32-NEXT:    vor.vv v8, v8, v16