[RISCV] Move RISCVVMV0Elimination past pre-ra scheduling

llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

@@ -445,6 +445,7 @@ FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
 }
 
 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
+  addPass(createRISCVVMV0EliminationPass());
   addPass(createRVVRegAllocPass(false));
   addPass(createRISCVInsertVSETVLIPass());
   if (TM->getOptLevel() != CodeGenOptLevel::None &&
@@ -454,6 +455,7 @@ bool RISCVPassConfig::addRegAssignAndRewriteFast() {
 }
 
 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
+  addPass(createRISCVVMV0EliminationPass());
   addPass(createRVVRegAllocPass(true));
   addPass(createVirtRegRewriter(false));
   addPass(createRISCVInsertVSETVLIPass());
@@ -618,8 +620,6 @@ void RISCVPassConfig::addPreRegAlloc() {
 
   if (TM->getOptLevel() != CodeGenOptLevel::None && EnableMachinePipeliner)
     addPass(&MachinePipelinerID);
-
-  addPass(createRISCVVMV0EliminationPass());
 }
 
 void RISCVPassConfig::addFastRegAlloc() {

llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp

@@ -34,6 +34,9 @@
 #ifndef NDEBUG
 #include "llvm/ADT/PostOrderIterator.h"
 #endif
+#include "llvm/CodeGen/LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveStacks.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 
 using namespace llvm;
@@ -51,15 +54,14 @@ class RISCVVMV0Elimination : public MachineFunctionPass {
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
 
-  MachineFunctionProperties getRequiredProperties() const override {
-    // TODO: We could move this closer to regalloc, out of SSA, which would
-    // allow scheduling past mask operands. We would need to preserve live
-    // intervals.
-    return MachineFunctionProperties().set(
-        MachineFunctionProperties::Property::IsSSA);
+    AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
+    AU.addPreserved<LiveIntervalsWrapperPass>();
+    AU.addPreserved<SlotIndexesWrapperPass>();
+    AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+    AU.addPreserved<LiveStacksWrapperLegacy>();
+
+    MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
 
@@ -88,12 +90,14 @@ bool RISCVVMV0Elimination::runOnMachineFunction(MachineFunction &MF) {
     return false;
 
   MachineRegisterInfo &MRI = MF.getRegInfo();
+  const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
   const TargetInstrInfo *TII = ST->getInstrInfo();
+  auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
+  LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
 
 #ifndef NDEBUG
   // Assert that we won't clobber any existing reads of v0 where we need to
   // insert copies.
-  const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
   ReversePostOrderTraversal<MachineBasicBlock *> RPOT(&*MF.begin());
   for (MachineBasicBlock *MBB : RPOT) {
     bool V0Clobbered = false;
@@ -115,7 +119,6 @@ bool RISCVVMV0Elimination::runOnMachineFunction(MachineFunction &MF) {
 #endif
 
   bool MadeChange = false;
-  SmallVector<MachineInstr *> DeadCopies;
 
   // For any instruction with a vmv0 operand, replace it with a copy to v0.
   for (MachineBasicBlock &MBB : MF) {
@@ -127,33 +130,46 @@ bool RISCVVMV0Elimination::runOnMachineFunction(MachineFunction &MF) {
         if (isVMV0(MCOI)) {
           MachineOperand &MO = MI.getOperand(OpNo);
           Register Src = MO.getReg();
-          assert(MO.isUse() && MO.getSubReg() == RISCV::NoSubRegister &&
-                 Src.isVirtual() && "vmv0 use in unexpected form");
-
-          // Peek through a single copy to match what isel does.
-          if (MachineInstr *SrcMI = MRI.getVRegDef(Src);
-              SrcMI->isCopy() && SrcMI->getOperand(1).getReg().isVirtual() &&
-              SrcMI->getOperand(1).getSubReg() == RISCV::NoSubRegister) {
-            // Delete any dead copys to vmv0 to avoid allocating them.
-            if (MRI.hasOneNonDBGUse(Src))
-              DeadCopies.push_back(SrcMI);
-            Src = SrcMI->getOperand(1).getReg();
+          assert(MO.isUse() && Src.isVirtual() &&
+                 "vmv0 use in unexpected form");
+
+          // If undef don't emit a copy, since the IMPLICIT_DEF Src will no
+          // longer exist at this stage.
+          if (MO.isUndef()) {
+            MO.setReg(RISCV::V0);
+            MadeChange = true;
+            break;
           }
 
-          BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(RISCV::COPY), RISCV::V0)
-              .addReg(Src);
-
+          MachineInstr *Copy = BuildMI(MBB, MI, MI.getDebugLoc(),
+                                       TII->get(RISCV::COPY), RISCV::V0)
+                                   .addReg(Src, 0, MO.getSubReg());
           MO.setReg(RISCV::V0);
+          if (LIS) {
+            LIS->InsertMachineInstrInMaps(*Copy);
+            SlotIndex CopySI = LIS->getInstructionIndex(*Copy).getRegSlot();
+            SlotIndex MISI = LIS->getInstructionIndex(MI).getRegSlot();
+
+            assert(std::distance(TRI->regunits(RISCV::V0).begin(),
+                                 TRI->regunits(RISCV::V0).end()) == 1);
+            unsigned Unit = *TRI->regunits(RISCV::V0).begin();
+
+            // Create a new definition of V0 from Copy To MI.
+            if (LiveRange *LR = LIS->getCachedRegUnit(Unit)) {
+              VNInfo *VNI = LR->getNextValue(CopySI, LIS->getVNInfoAllocator());
+              LR->addSegment(LiveInterval::Segment(CopySI, MISI, VNI));
+            }
+
+            // Shrink Src's interval now that MI doesn't use it.
+            LIS->shrinkToUses(&LIS->getInterval(Src));
+          }
           MadeChange = true;
           break;
         }
       }
     }
   }
 
-  for (MachineInstr *MI : DeadCopies)
-    MI->eraseFromParent();
-
   if (!MadeChange)
     return false;
 

llvm/test/CodeGen/RISCV/O0-pipeline.ll

@@ -43,10 +43,10 @@
 ; CHECK-NEXT:       RISC-V Insert Read/Write CSR Pass
 ; CHECK-NEXT:       RISC-V Insert Write VXRM Pass
 ; CHECK-NEXT:       RISC-V Landing Pad Setup
-; CHECK-NEXT:       RISC-V VMV0 Elimination
 ; CHECK-NEXT:       Init Undef Pass
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
 ; CHECK-NEXT:       Two-Address instruction pass
+; CHECK-NEXT:       RISC-V VMV0 Elimination
 ; CHECK-NEXT:       Fast Register Allocator
 ; CHECK-NEXT:       RISC-V Insert VSETVLI pass
 ; CHECK-NEXT:       Fast Register Allocator

llvm/test/CodeGen/RISCV/O3-pipeline.ll

@@ -128,7 +128,6 @@
 ; CHECK-NEXT:       RISC-V Insert Read/Write CSR Pass
 ; CHECK-NEXT:       RISC-V Insert Write VXRM Pass
 ; CHECK-NEXT:       RISC-V Landing Pad Setup
-; CHECK-NEXT:       RISC-V VMV0 Elimination
 ; CHECK-NEXT:       Detect Dead Lanes
 ; CHECK-NEXT:       Init Undef Pass
 ; CHECK-NEXT:       Process Implicit Definitions
@@ -141,6 +140,7 @@
 ; CHECK-NEXT:       Register Coalescer
 ; CHECK-NEXT:       Rename Disconnected Subregister Components
 ; CHECK-NEXT:       Machine Instruction Scheduler
+; CHECK-NEXT:       RISC-V VMV0 Elimination
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       Debug Variable Analysis
 ; CHECK-NEXT:       Live Stack Slot Analysis