RegAlloc: Do not fatal error if there are no registers in the alloc order

llvm/include/llvm/CodeGen/MachineFunction.h

@@ -186,6 +186,7 @@ class MachineFunctionProperties {
     Selected,
     TiedOpsRewritten,
     FailsVerification,
+    FailedRegAlloc,
     TracksDebugUserValues,
     LastProperty = TracksDebugUserValues,
   };

llvm/lib/CodeGen/MachineFunction.cpp

@@ -102,6 +102,7 @@ static const char *getPropertyName(MachineFunctionProperties::Property Prop) {
   case P::TracksLiveness: return "TracksLiveness";
   case P::TiedOpsRewritten: return "TiedOpsRewritten";
   case P::FailsVerification: return "FailsVerification";
+  case P::FailedRegAlloc: return "FailedRegAlloc";
   case P::TracksDebugUserValues: return "TracksDebugUserValues";
   }
   // clang-format on

llvm/lib/CodeGen/RegAllocBase.cpp

@@ -124,25 +124,10 @@ void RegAllocBase::allocatePhysRegs() {
       }
 
       const TargetRegisterClass *RC = MRI->getRegClass(VirtReg->reg());
-      ArrayRef<MCPhysReg> AllocOrder = RegClassInfo.getOrder(RC);
-      if (AllocOrder.empty()) {
-        report_fatal_error("no registers from class available to allocate");
-      } else {
-        if (MI && MI->isInlineAsm()) {
-          MI->emitInlineAsmError(
-              "inline assembly requires more registers than available");
-        } else {
-          const Function &Fn = VRM->getMachineFunction().getFunction();
-          LLVMContext &Context = Fn.getContext();
-          DiagnosticInfoRegAllocFailure DI(
-              "ran out of registers during register allocation", Fn,
-              MI ? MI->getDebugLoc() : DiagnosticLocation());
-          Context.diagnose(DI);
-        }
-      }
+      AvailablePhysReg = getErrorAssignment(*RC, MI);
 
       // Keep going after reporting the error.
-      VRM->assignVirt2Phys(VirtReg->reg(), AllocOrder.front());
+      VRM->assignVirt2Phys(VirtReg->reg(), AvailablePhysReg);
     } else if (AvailablePhysReg)
       Matrix->assign(*VirtReg, AvailablePhysReg);
 
@@ -192,3 +177,50 @@ void RegAllocBase::enqueue(const LiveInterval *LI) {
                       << " in skipped register class\n");
   }
 }
+
+MCPhysReg RegAllocBase::getErrorAssignment(const TargetRegisterClass &RC,
+                                           const MachineInstr *CtxMI) {
+  MachineFunction &MF = VRM->getMachineFunction();
+
+  // Avoid printing the error for every single instance of the register. It
+  // would be better if this were per register class.
+  bool EmitError = !MF.getProperties().hasProperty(
+      MachineFunctionProperties::Property::FailedRegAlloc);
+  if (EmitError)
+    MF.getProperties().set(MachineFunctionProperties::Property::FailedRegAlloc);
+
+  const Function &Fn = MF.getFunction();
+  LLVMContext &Context = Fn.getContext();
+
+  ArrayRef<MCPhysReg> AllocOrder = RegClassInfo.getOrder(&RC);
+  if (AllocOrder.empty()) {
+    // If the allocation order is empty, it likely means all registers in the
+    // class are reserved. We still to need to pick something, so look at the
+    // underlying class.
+    ArrayRef<MCPhysReg> RawRegs = RC.getRegisters();
+
+    if (EmitError) {
+      DiagnosticInfoRegAllocFailure DI(
+          "no registers from class available to allocate", Fn,
+          CtxMI ? CtxMI->getDebugLoc() : DiagnosticLocation());
+      Context.diagnose(DI);
+    }
+
+    assert(!RawRegs.empty() && "register classes cannot have no registers");
+    return RawRegs.front();
+  }
+
+  if (EmitError) {
+    if (CtxMI && CtxMI->isInlineAsm()) {
+      CtxMI->emitInlineAsmError(
+          "inline assembly requires more registers than available");
+    } else {
+      DiagnosticInfoRegAllocFailure DI(
+          "ran out of registers during register allocation", Fn,
+          CtxMI ? CtxMI->getDebugLoc() : DiagnosticLocation());
+      Context.diagnose(DI);
+    }
+  }
+
+  return AllocOrder.front();
+}

llvm/lib/CodeGen/RegAllocBase.h

@@ -123,6 +123,12 @@ class RegAllocBase {
   virtual MCRegister selectOrSplit(const LiveInterval &VirtReg,
                                    SmallVectorImpl<Register> &splitLVRs) = 0;
 
+  /// Query a physical register to use as a filler in contexts where the
+  /// allocation has failed. This will raise an error, but not abort the
+  /// compilation.
+  MCPhysReg getErrorAssignment(const TargetRegisterClass &RC,
+                               const MachineInstr *CtxMI = nullptr);
+
   // Use this group name for NamedRegionTimer.
   static const char TimerGroupName[];
   static const char TimerGroupDescription[];

llvm/lib/CodeGen/RegAllocFast.cpp

@@ -368,6 +368,9 @@ class RegAllocFastImpl {
                      bool LookAtPhysRegUses = false);
   bool useVirtReg(MachineInstr &MI, MachineOperand &MO, Register VirtReg);
 
+  MCPhysReg getErrorAssignment(const LiveReg &LR, MachineInstr &MI,
+                               const TargetRegisterClass &RC);
+
   MachineBasicBlock::iterator
   getMBBBeginInsertionPoint(MachineBasicBlock &MBB,
                             SmallSet<Register, 2> &PrologLiveIns) const;
@@ -963,22 +966,8 @@ void RegAllocFastImpl::allocVirtReg(MachineInstr &MI, LiveReg &LR,
   if (!BestReg) {
     // Nothing we can do: Report an error and keep going with an invalid
     // allocation.
-    if (MI.isInlineAsm()) {
-      MI.emitInlineAsmError(
-          "inline assembly requires more registers than available");
-    } else {
-      const Function &Fn = MBB->getParent()->getFunction();
-      DiagnosticInfoRegAllocFailure DI(
-          "ran out of registers during register allocation", Fn,
-          MI.getDebugLoc());
-      Fn.getContext().diagnose(DI);
-    }
-
+    LR.PhysReg = getErrorAssignment(LR, MI, RC);
     LR.Error = true;
-    if (!AllocationOrder.empty())
-      LR.PhysReg = AllocationOrder.front();
-    else
-      LR.PhysReg = 0;
     return;
   }
 
@@ -1000,6 +989,7 @@ void RegAllocFastImpl::allocVirtRegUndef(MachineOperand &MO) {
   } else {
     const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
     ArrayRef<MCPhysReg> AllocationOrder = RegClassInfo.getOrder(&RC);
+    // FIXME: This can happen, and should fall back to a reserved entry in RC.
     assert(!AllocationOrder.empty() && "Allocation order must not be empty");
     PhysReg = AllocationOrder[0];
   }
@@ -1074,15 +1064,6 @@ bool RegAllocFastImpl::defineVirtReg(MachineInstr &MI, unsigned OpNum,
   }
   if (LRI->PhysReg == 0) {
     allocVirtReg(MI, *LRI, 0, LookAtPhysRegUses);
-    // If no physical register is available for LRI, we assign one at random
-    // and bail out of this function immediately.
-    if (LRI->Error) {
-      const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
-      ArrayRef<MCPhysReg> AllocationOrder = RegClassInfo.getOrder(&RC);
-      if (AllocationOrder.empty())
-        return setPhysReg(MI, MO, MCRegister::NoRegister);
-      return setPhysReg(MI, MO, *AllocationOrder.begin());
-    }
   } else {
     assert((!isRegUsedInInstr(LRI->PhysReg, LookAtPhysRegUses) || LRI->Error) &&
            "TODO: preassign mismatch");
@@ -1167,13 +1148,6 @@ bool RegAllocFastImpl::useVirtReg(MachineInstr &MI, MachineOperand &MO,
       }
     }
     allocVirtReg(MI, *LRI, Hint, false);
-    if (LRI->Error) {
-      const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
-      ArrayRef<MCPhysReg> AllocationOrder = RegClassInfo.getOrder(&RC);
-      if (AllocationOrder.empty())
-        return setPhysReg(MI, MO, MCRegister::NoRegister);
-      return setPhysReg(MI, MO, *AllocationOrder.begin());
-    }
   }
 
   LRI->LastUse = &MI;
@@ -1185,6 +1159,56 @@ bool RegAllocFastImpl::useVirtReg(MachineInstr &MI, MachineOperand &MO,
   return setPhysReg(MI, MO, LRI->PhysReg);
 }
 
+/// Query a physical register to use as a filler in contexts where the
+/// allocation has failed. This will raise an error, but not abort the
+/// compilation.
+MCPhysReg RegAllocFastImpl::getErrorAssignment(const LiveReg &LR,
+                                               MachineInstr &MI,
+                                               const TargetRegisterClass &RC) {
+  MachineFunction &MF = *MI.getMF();
+
+  // Avoid repeating the error every time a register is used.
+  bool EmitError = !MF.getProperties().hasProperty(
+      MachineFunctionProperties::Property::FailedRegAlloc);
+  if (EmitError)
+    MF.getProperties().set(MachineFunctionProperties::Property::FailedRegAlloc);
+
+  // If the allocation order was empty, all registers in the class were
+  // probably reserved. Fall back to taking the first register in the class,
+  // even if it's reserved.
+  ArrayRef<MCPhysReg> AllocationOrder = RegClassInfo.getOrder(&RC);
+  if (AllocationOrder.empty()) {
+    const Function &Fn = MF.getFunction();
+    if (EmitError) {
+      DiagnosticInfoRegAllocFailure DI(
+          "no registers from class available to allocate", Fn,
+          MI.getDebugLoc());
+      Fn.getContext().diagnose(DI);
+    }
+
+    ArrayRef<MCPhysReg> RawRegs = RC.getRegisters();
+    assert(!RawRegs.empty() && "register classes cannot have no registers");
+    return RawRegs.front();
+  }
+
+  if (!LR.Error && EmitError) {
+    // Nothing we can do: Report an error and keep going with an invalid
+    // allocation.
+    if (MI.isInlineAsm()) {
+      MI.emitInlineAsmError(
+          "inline assembly requires more registers than available");
+    } else {
+      const Function &Fn = MBB->getParent()->getFunction();
+      DiagnosticInfoRegAllocFailure DI(
+          "ran out of registers during register allocation", Fn,
+          MI.getDebugLoc());
+      Fn.getContext().diagnose(DI);
+    }
+  }
+
+  return AllocationOrder.front();
+}
+
 /// Changes operand OpNum in MI the refer the PhysReg, considering subregs.
 /// \return true if MI's MachineOperands were re-arranged/invalidated.
 bool RegAllocFastImpl::setPhysReg(MachineInstr &MI, MachineOperand &MO,

llvm/lib/CodeGen/VirtRegMap.cpp

@@ -88,7 +88,9 @@ void VirtRegMap::assignVirt2Phys(Register virtReg, MCPhysReg physReg) {
   assert(!Virt2PhysMap[virtReg] &&
          "attempt to assign physical register to already mapped "
          "virtual register");
-  assert(!getRegInfo().isReserved(physReg) &&
+  assert((!getRegInfo().isReserved(physReg) ||
+          MF->getProperties().hasProperty(
+              MachineFunctionProperties::Property::FailedRegAlloc)) &&
          "Attempt to map virtReg to a reserved physReg");
   Virt2PhysMap[virtReg] = physReg;
 }
@@ -615,7 +617,10 @@ void VirtRegRewriter::rewrite() {
         assert(Register(PhysReg).isPhysical());
 
         RewriteRegs.insert(PhysReg);
-        assert(!MRI->isReserved(PhysReg) && "Reserved register assignment");
+        assert((!MRI->isReserved(PhysReg) ||
+                MF->getProperties().hasProperty(
+                    MachineFunctionProperties::Property::FailedRegAlloc)) &&
+               "Reserved register assignment");
 
         // Preserve semantics of sub-register operands.
         unsigned SubReg = MO.getSubReg();

llvm/test/CodeGen/AMDGPU/alloc-all-regs-reserved-in-class.mir

@@ -1,12 +1,10 @@
-# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=greedy -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s
+# RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=greedy -verify-machineinstrs -filetype=null %s 2>&1 | FileCheck --implicit-check-not=error %s
 
 # Check that there isn't an assert if we try to allocate a virtual register from
 # a class where all registers are reserved. All AGPRs are reserved on subtargets
 # that do not have them.
 
-# CHECK-NOT: ran out of registers during register allocation
-# CHECK: LLVM ERROR: no registers from class available to allocate
-# CHECK-NOT: ran out of registers during register allocation
+# CHECK: error: <unknown>:0:0: no registers from class available to allocate in function 'use_agpr'
 
 ---
 name: use_agpr

llvm/test/CodeGen/AMDGPU/ran-out-of-registers-error-all-regs-reserved.ll

@@ -1,17 +1,29 @@
-; RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -vgpr-regalloc=greedy -filetype=null %s 2>&1 | FileCheck %s
-; RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -vgpr-regalloc=basic -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -vgpr-regalloc=greedy -verify-machineinstrs=0 -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -vgpr-regalloc=basic -verify-machineinstrs=0 -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -vgpr-regalloc=fast -verify-machineinstrs=0 -filetype=null %s 2>&1 | FileCheck -implicit-check-not=error %s
 
-; TODO: Check regalloc fast when it doesn't assert after failing.
-
-; CHECK: LLVM ERROR: no registers from class available to allocate
+; FIXME: Should pass verifier after failure.
 
 declare <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32, i32, <32 x i32>, i32 immarg, i32 immarg, i32 immarg)
 
+; CHECK: error: <unknown>:0:0: no registers from class available to allocate in function 'no_registers_from_class_available_to_allocate'
 define <32 x i32> @no_registers_from_class_available_to_allocate(<32 x i32> %arg) #0 {
   %ret = call <32 x i32> @llvm.amdgcn.mfma.i32.32x32x4i8(i32 1, i32 2, <32 x i32> %arg, i32 1, i32 2, i32 3)
   ret <32 x i32> %ret
 }
 
+; CHECK: error: <unknown>:0:0: no registers from class available to allocate in function 'no_registers_from_class_available_to_allocate_asm_use'
+define void @no_registers_from_class_available_to_allocate_asm_use(<32 x i32> %arg) #0 {
+  call void asm sideeffect "; use $0", "v"(<32 x i32> %arg)
+  ret void
+}
+
+; CHECK: error: <unknown>:0:0: no registers from class available to allocate in function 'no_registers_from_class_available_to_allocate_asm_def'
+define <32 x i32> @no_registers_from_class_available_to_allocate_asm_def() #0 {
+  %ret = call <32 x i32> asm sideeffect "; def $0", "=v"()
+  ret <32 x i32> %ret
+}
+
 ; FIXME: Special case in fast RA, asserts. Also asserts in greedy
 ; define void @no_registers_from_class_available_to_allocate_undef_asm() #0 {
 ;   call void asm sideeffect "; use $0", "v"(<32 x i32> poison)

llvm/test/CodeGen/AMDGPU/ran-out-of-registers-errors.ll

@@ -12,16 +12,12 @@
 
 
 ; CHECK: error: <unknown>:0:0: ran out of registers during register allocation in function 'ran_out_of_registers_general'
-; BASIC: error: <unknown>:0:0: ran out of registers during register allocation in function 'ran_out_of_registers_general'
-; FAST: error: <unknown>:0:0: ran out of registers during register allocation in function 'ran_out_of_registers_general'
 
 ; DBGINFO-GREEDY: error: {{.*}}:3:1: ran out of registers during register allocation in function 'ran_out_of_registers_general'
 
 ; DBGINFO-BASIC: error: {{.*}}:1:1: ran out of registers during register allocation in function 'ran_out_of_registers_general'
-; DBGINFO-BASIC: error: {{.*}}:3:1: ran out of registers during register allocation in function 'ran_out_of_registers_general'
 
 ; DBGINFO-FAST: error: {{.*}}:3:1: ran out of registers during register allocation in function 'ran_out_of_registers_general'
-; DBGINFO-FAST: error: {{.*}}:1:0: ran out of registers during register allocation in function 'ran_out_of_registers_general'
 define i32 @ran_out_of_registers_general(ptr addrspace(1) %ptr) #0 {
   %ld0 = load volatile i32, ptr addrspace(1) %ptr
   %ld1 = load volatile i32, ptr addrspace(1) %ptr
@@ -49,14 +45,11 @@ define void @ran_out_of_registers_asm_use() #0 {
 ; BASIC: error: inline assembly requires more registers than available at line 23
 
 ; FAST: error: <unknown>:0:0: ran out of registers during register allocation in function '@0'
-; FAST: error: <unknown>:0:0: ran out of registers during register allocation in function '@0'
-
 
 ; DBGINFO-GREEDY: error: inline assembly requires more registers than available at line 23
 ; DBGINFO-BASIC: error: inline assembly requires more registers than available at line 23
 
 ; DBGINFO-FAST: error: {{.*}}:12:1: ran out of registers during register allocation in function '@0'
-; DBGINFO-FAST: error: {{.*}}:9:0: ran out of registers during register allocation in function '@0'
 define i32 @0(ptr addrspace(1) %ptr) #0 {
   %asm = call { i32, i32 } asm sideeffect "; def $0 $1 use $2", "=v,=v,v"(ptr addrspace(1) %ptr), !srcloc !0
   %elt0 = extractvalue { i32, i32 } %asm, 0

llvm/test/CodeGen/AMDGPU/regalloc-illegal-eviction-assert.ll

@@ -1,10 +1,9 @@
-; RUN: not llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs -o - %s 2>%t.err | FileCheck %s
+; RUN: not llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs -o - %s 2>%t.err | FileCheck -implicit-check-not=error %s
 ; RUN: FileCheck -check-prefix=ERR %s < %t.err
 
 ; This testcase would fail on an "illegal eviction". If the assert was
 ; relaxed to allow equivalent cascade numbers, it would infinite loop.
 
-; ERR: error: inline assembly requires more registers than available
 ; ERR: error: inline assembly requires more registers than available
 
 %asm.output = type { <16 x i32>, <8 x i32>, <5 x i32>, <4 x i32>, <16 x i32> }