[RISCV] QCI Interrupt Support (#129957)

This change adds support for `qci-nest` and `qci-nonest` interrupt
attribute values. Both of these are machine-mode interrupts, which use
instructions in Xqciint to push and pop A- and T-registers (and a few
others) from the stack.

In particular:
- `qci-nonest` uses `qc.c.mienter` to save registers at the start of the
function, and uses `qc.c.mileaveret` to restore those registers and
return from the interrupt.
- `qci-nest` uses `qc.c.mienter.nest` to save registers at the start of
the function, and uses `qc.c.mileaveret` to restore those registers and
return from the interrupt.
- `qc.c.mienter` and `qc.c.mienter.nest` both push registers ra, s0
(fp), t0-t6, and a0-a10 onto the stack (as well as some CSRs for the
interrupt context). The difference between these is that
`qc.c.mienter.nest` re-enables M-mode interrupts.
- `qc.c.mileaveret` will restore the registers that were saved by
`qc.c.mienter(.nest)`, and return from the interrupt.

These work for both standard M-mode interrupts and the non-maskable
interrupt CSRs added by Xqciint.

The `qc.c.mienter`, `qc.c.mienter.nest` and `qc.c.mileaveret`
instructions are compatible with push and pop instructions, in as much
as they (mostly) only spill the A- and T-registers, so we can use the
`Zcmp` or `Xqccmp` instructions to spill the S-registers. This
combination (`qci-(no)nest` and `Xqccmp`/`Zcmp`) is not implemented in
this change.

The `qc.c.mienter(.nest)` instructions have a specific register storage
order so they preserve the frame pointer convention linked list past the
current interrupt handler and into the interrupted code and frames if
frame pointers are enabled.

Co-authored-by: Pankaj Gode <[email protected]>

Author: lenary

clang/include/clang/Basic/Attr.td

@@ -2232,8 +2232,8 @@ def RISCVInterrupt : InheritableAttr, TargetSpecificAttr<TargetRISCV> {
   let Spellings = [GCC<"interrupt">];
   let Subjects = SubjectList<[Function]>;
   let Args = [EnumArgument<"Interrupt", "InterruptType", /*is_string=*/true,
-                           ["supervisor", "machine"],
-                           ["supervisor", "machine"],
+                           ["supervisor", "machine", "qci-nest", "qci-nonest"],
+                           ["supervisor", "machine", "qcinest", "qcinonest"],
                            1>];
   let ParseKind = "Interrupt";
   let Documentation = [RISCVInterruptDocs];

clang/include/clang/Basic/AttrDocs.td

@@ -2828,8 +2828,17 @@ targets. This attribute may be attached to a function definition and instructs
 the backend to generate appropriate function entry/exit code so that it can be
 used directly as an interrupt service routine.
 
-Permissible values for this parameter are ``supervisor`` and ``machine``. If
-there is no parameter, then it defaults to ``machine``.
+Permissible values for this parameter are ``supervisor``, ``machine``,
+``qci-nest`` and ``qci-nonest``. If there is no parameter, then it defaults to
+``machine``.
+
+The ``qci-nest`` and ``qci-nonest`` values require Qualcomm's Xqciint extension
+and are used for Machine-mode Interrupts and Machine-mode Non-maskable
+interrupts. These use the following instructions from Xqciint to save and
+restore interrupt state to the stack -- the ``qci-nest`` value will use
+``qc.c.mienter.nest`` and the ``qci-nonest`` value will use ``qc.c.mienter`` to
+begin the interrupt handler. Both of these will use ``qc.c.mileaveret`` to
+restore the state and return to the previous context.
 
 Repeated interrupt attribute on the same declaration will cause a warning
 to be emitted. In case of repeated declarations, the last one prevails.
@@ -2839,6 +2848,7 @@ https://gcc.gnu.org/onlinedocs/gcc/RISC-V-Function-Attributes.html
 https://riscv.org/specifications/privileged-isa/
 The RISC-V Instruction Set Manual Volume II: Privileged Architecture
 Version 1.10.
+https://github.com/quic/riscv-unified-db/releases/tag/Xqci-0.6
   }];
 }
 

clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -12637,8 +12637,9 @@ def err_riscv_builtin_requires_extension : Error<
 def err_riscv_builtin_invalid_lmul : Error<
   "LMUL argument must be in the range [0,3] or [5,7]">;
 def err_riscv_type_requires_extension : Error<
-  "RISC-V type %0 requires the '%1' extension"
->;
+  "RISC-V type %0 requires the '%1' extension">;
+def err_riscv_attribute_interrupt_requires_extension : Error<
+  "RISC-V interrupt attribute '%0' requires extension '%1'">;
 
 def err_std_source_location_impl_not_found : Error<
   "'std::source_location::__impl' was not found; it must be defined before '__builtin_source_location' is called">;

clang/lib/CodeGen/Targets/RISCV.cpp

@@ -833,6 +833,12 @@ class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
     switch (Attr->getInterrupt()) {
     case RISCVInterruptAttr::supervisor: Kind = "supervisor"; break;
     case RISCVInterruptAttr::machine: Kind = "machine"; break;
+    case RISCVInterruptAttr::qcinest:
+      Kind = "qci-nest";
+      break;
+    case RISCVInterruptAttr::qcinonest:
+      Kind = "qci-nonest";
+      break;
     }
 
     Fn->addFnAttr("interrupt", Kind);

clang/lib/Sema/SemaRISCV.cpp

@@ -12,6 +12,7 @@
 
 #include "clang/Sema/SemaRISCV.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/TargetBuiltins.h"
@@ -1475,6 +1476,26 @@ void SemaRISCV::handleInterruptAttr(Decl *D, const ParsedAttr &AL) {
     return;
   }
 
+  switch (Kind) {
+  default:
+    break;
+  case RISCVInterruptAttr::InterruptType::qcinest:
+  case RISCVInterruptAttr::InterruptType::qcinonest: {
+    const TargetInfo &TI = getASTContext().getTargetInfo();
+    llvm::StringMap<bool> FunctionFeatureMap;
+    getASTContext().getFunctionFeatureMap(FunctionFeatureMap,
+                                          dyn_cast<FunctionDecl>(D));
+
+    if (!TI.hasFeature("experimental-xqciint") &&
+        !FunctionFeatureMap.lookup("experimental-xqciint")) {
+      Diag(AL.getLoc(), diag::err_riscv_attribute_interrupt_requires_extension)
+          << Str << "Xqciint";
+      return;
+    }
+    break;
+  }
+  };
+
   D->addAttr(::new (getASTContext())
                  RISCVInterruptAttr(getASTContext(), AL, Kind));
 }

clang/test/Sema/riscv-interrupt-attr-qci.c

@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -triple riscv32-unknown-elf -target-feature +experimental-xqciint -emit-llvm -DCHECK_IR < %s| FileCheck %s
+// RUN: %clang_cc1 %s -triple riscv32-unknown-elf -target-feature +experimental-xqciint -verify=enabled,both -fsyntax-only
+// RUN: %clang_cc1 %s -triple riscv32-unknown-elf -verify=disabled,both -fsyntax-only
+// RUN: %clang_cc1 %s -triple riscv32-unknown-elf -target-feature -experimental-xqciint -verify=disabled,both -fsyntax-only
+// RUN: %clang_cc1 %s -triple riscv64-unknown-elf -verify=disabled,both -fsyntax-only -DRV64
+
+#if defined(CHECK_IR)
+// Test for QCI extension's interrupt attribute support
+// CHECK-LABEL: @foo_nest_interrupt() #0
+// CHECK: ret void
+__attribute__((interrupt("qci-nest")))
+void foo_nest_interrupt(void) {}
+
+// CHECK-LABEL: @foo_nonnest_interrupt() #1
+// CHECK: ret void
+__attribute__((interrupt("qci-nonest")))
+void foo_nonnest_interrupt(void) {}
+
+// CHECK: attributes #0
+// CHECK: "interrupt"="qci-nest"
+// CHECK: attributes #1
+// CHECK: "interrupt"="qci-nonest"
+#else
+// Test for QCI extension's interrupt attribute support
+__attribute__((interrupt("qci-est"))) void foo_nest1(void) {} // both-warning {{'interrupt' attribute argument not supported: qci-est}}
+__attribute__((interrupt("qci-noest"))) void foo_nonest1(void) {} // both-warning {{'interrupt' attribute argument not supported: qci-noest}}
+__attribute__((interrupt(1))) void foo_nest2(void) {} // both-error {{expected string literal as argument of 'interrupt' attribute}}
+__attribute__((interrupt("qci-nest", "qci-nonest"))) void foo1(void) {} // both-error {{'interrupt' attribute takes no more than 1 argument}}
+__attribute__((interrupt("qci-nonest", "qci-nest"))) void foo2(void) {} // both-error {{'interrupt' attribute takes no more than 1 argument}}
+__attribute__((interrupt("", "qci-nonest"))) void foo3(void) {} // both-error {{'interrupt' attribute takes no more than 1 argument}}
+__attribute__((interrupt("", "qci-nest"))) void foo4(void) {} // both-error {{'interrupt' attribute takes no more than 1 argument}}
+__attribute__((interrupt("qci-nonest", 1))) void foo5(void) {} // both-error {{'interrupt' attribute takes no more than 1 argument}}
+__attribute__((interrupt("qci-nest", 1))) void foo6(void) {} // both-error {{'interrupt' attribute takes no more than 1 argument}}
+
+__attribute__((interrupt("qci-nest"))) void foo_nest(void) {} // disabled-error {{RISC-V interrupt attribute 'qci-nest' requires extension 'Xqciint'}}
+__attribute__((interrupt("qci-nonest"))) void foo_nonest(void) {} // disabled-error {{RISC-V interrupt attribute 'qci-nonest' requires extension 'Xqciint'}}
+
+// This tests the errors for the qci interrupts when using
+// `__attribute__((target(...)))` - but they fail on RV64, because you cannot
+// enable xqciint on rv64.
+#if __riscv_xlen == 32
+__attribute__((target("arch=+xqciint"))) __attribute__((interrupt("qci-nest"))) void foo_nest_xqciint(void) {}
+__attribute__((target("arch=+xqciint"))) __attribute__((interrupt("qci-nonest"))) void foo_nonest_xqciint(void) {}
+
+// The attribute order is important, the interrupt attribute must come after the
+// target attribute
+__attribute__((interrupt("qci-nest"))) __attribute__((target("arch=+xqciint"))) void foo_nest_xqciint2(void) {}  // disabled-error {{RISC-V interrupt attribute 'qci-nest' requires extension 'Xqciint'}}
+__attribute__((interrupt("qci-nonest"))) __attribute__((target("arch=+xqciint"))) void foo_nonest_xqciint2(void) {}  // disabled-error {{RISC-V interrupt attribute 'qci-nonest' requires extension 'Xqciint'}}
+#endif
+
+#endif

llvm/lib/Target/RISCV/RISCVFrameLowering.cpp

@@ -121,6 +121,34 @@ static const MCPhysReg FixedCSRFIMap[] = {
     /*s8*/ RISCV::X24, /*s9*/ RISCV::X25, /*s10*/ RISCV::X26,
     /*s11*/ RISCV::X27};
 
+// The number of stack bytes allocated by `QC.C.MIENTER(.NEST)` and popped by
+// `QC.C.MILEAVERET`.
+static constexpr uint64_t QCIInterruptPushAmount = 96;
+
+static const std::pair<MCPhysReg, int8_t> FixedCSRFIQCIInterruptMap[] = {
+    /* -1 is a gap for mepc/qc.mnepc */
+    {/*fp*/ FPReg, -2},
+    /* -3 is a gap for mcause */
+    {/*ra*/ RAReg, -4},
+    /* -5 is reserved */
+    {/*t0*/ RISCV::X5, -6},
+    {/*t1*/ RISCV::X6, -7},
+    {/*t2*/ RISCV::X7, -8},
+    {/*a0*/ RISCV::X10, -9},
+    {/*a1*/ RISCV::X11, -10},
+    {/*a2*/ RISCV::X12, -11},
+    {/*a3*/ RISCV::X13, -12},
+    {/*a4*/ RISCV::X14, -13},
+    {/*a5*/ RISCV::X15, -14},
+    {/*a6*/ RISCV::X16, -15},
+    {/*a7*/ RISCV::X17, -16},
+    {/*t3*/ RISCV::X28, -17},
+    {/*t4*/ RISCV::X29, -18},
+    {/*t5*/ RISCV::X30, -19},
+    {/*t6*/ RISCV::X31, -20},
+    /* -21, -22, -23, -24 are reserved */
+};
+
 // For now we use x3, a.k.a gp, as pointer to shadow call stack.
 // User should not use x3 in their asm.
 static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -382,6 +410,10 @@ void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const {
   // Get the number of bytes to allocate from the FrameInfo.
   uint64_t FrameSize = MFI.getStackSize();
 
+  // QCI Interrupts use at least 96 bytes of stack space
+  if (RVFI->useQCIInterrupt(MF))
+    FrameSize = std::max(FrameSize, QCIInterruptPushAmount);
+
   // Get the alignment.
   Align StackAlign = getStackAlign();
 
@@ -463,6 +495,26 @@ getPushOrLibCallsSavedInfo(const MachineFunction &MF,
   return PushOrLibCallsCSI;
 }
 
+static SmallVector<CalleeSavedInfo, 8>
+getQCISavedInfo(const MachineFunction &MF,
+                const std::vector<CalleeSavedInfo> &CSI) {
+  auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
+
+  SmallVector<CalleeSavedInfo, 8> QCIInterruptCSI;
+  if (!RVFI->useQCIInterrupt(MF))
+    return QCIInterruptCSI;
+
+  for (const auto &CS : CSI) {
+    const auto *FII = llvm::find_if(FixedCSRFIQCIInterruptMap, [&](auto P) {
+      return P.first == CS.getReg();
+    });
+    if (FII != std::end(FixedCSRFIQCIInterruptMap))
+      QCIInterruptCSI.push_back(CS);
+  }
+
+  return QCIInterruptCSI;
+}
+
 void RISCVFrameLowering::allocateAndProbeStackForRVV(
     MachineFunction &MF, MachineBasicBlock &MBB,
     MachineBasicBlock::iterator MBBI, const DebugLoc &DL, int64_t Amount,
@@ -896,8 +948,16 @@ void RISCVFrameLowering::emitPrologue(MachineFunction &MF,
     RealStackSize = FirstSPAdjustAmount;
   }
 
-  if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
-      isPush(FirstFrameSetup->getOpcode())) {
+  if (RVFI->useQCIInterrupt(MF)) {
+    unsigned CFIIndex = MF.addFrameInst(
+        MCCFIInstruction::cfiDefCfaOffset(nullptr, QCIInterruptPushAmount));
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
+        .addCFIIndex(CFIIndex)
+        .setMIFlag(MachineInstr::FrameSetup);
+
+    emitCFIForCSI<CFISaveRegisterEmitter>(MBB, MBBI, getQCISavedInfo(MF, CSI));
+  } else if (RVFI->isPushable(MF) && FirstFrameSetup != MBB.end() &&
+             isPush(FirstFrameSetup->getOpcode())) {
     // Use available stack adjustment in push instruction to allocate additional
     // stack space. Align the stack size down to a multiple of 16. This is
     // needed for RVE.
@@ -1247,7 +1307,7 @@ void RISCVFrameLowering::emitEpilogue(MachineFunction &MF,
 
   // Deallocate stack if StackSize isn't a zero yet
   if (StackSize != 0)
-    deallocateStack(MF, MBB, MBBI, DL, StackSize, 0);
+    deallocateStack(MF, MBB, MBBI, DL, StackSize, RealStackSize - StackSize);
 
   // Emit epilogue for shadow call stack.
   emitSCSEpilogue(MF, MBB, MBBI, DL);
@@ -1737,9 +1797,9 @@ RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const {
   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
   uint64_t StackSize = getStackSizeWithRVVPadding(MF);
 
-  // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved
-  // registers will be pushed by the save-restore libcalls, so we don't have to
-  // split the SP adjustment in this case.
+  // Disable SplitSPAdjust if save-restore libcall, push/pop or QCI interrupts
+  // are used. The callee-saved registers will be pushed by the save-restore
+  // libcalls, so we don't have to split the SP adjustment in this case.
   if (RVFI->getReservedSpillsSize())
     return 0;
 
@@ -1807,8 +1867,9 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
     return true;
 
   auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
-
-  if (RVFI->isPushable(MF)) {
+  if (RVFI->useQCIInterrupt(MF)) {
+    RVFI->setQCIInterruptStackSize(QCIInterruptPushAmount);
+  } else if (RVFI->isPushable(MF)) {
     // Determine how many GPRs we need to push and save it to RVFI.
     unsigned PushedRegNum = getNumPushPopRegs(CSI);
     if (PushedRegNum) {
@@ -1825,8 +1886,20 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
     const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
     unsigned Size = RegInfo->getSpillSize(*RC);
 
-    // This might need a fixed stack slot.
-    if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
+    if (RVFI->useQCIInterrupt(MF)) {
+      const auto *FFI = llvm::find_if(FixedCSRFIQCIInterruptMap, [&](auto P) {
+        return P.first == CS.getReg();
+      });
+      if (FFI != std::end(FixedCSRFIQCIInterruptMap)) {
+        int64_t Offset = FFI->second * (int64_t)Size;
+
+        int FrameIdx = MFI.CreateFixedSpillStackObject(Size, Offset);
+        assert(FrameIdx < 0);
+        CS.setFrameIdx(FrameIdx);
+        continue;
+      }
+      // TODO: QCI Interrupt + Push/Pop
+    } else if (RVFI->useSaveRestoreLibCalls(MF) || RVFI->isPushable(MF)) {
       const auto *FII = llvm::find_if(
           FixedCSRFIMap, [&](MCPhysReg P) { return P == CS.getReg(); });
       unsigned RegNum = std::distance(std::begin(FixedCSRFIMap), FII);
@@ -1862,7 +1935,12 @@ bool RISCVFrameLowering::assignCalleeSavedSpillSlots(
       MFI.setStackID(FrameIdx, TargetStackID::ScalableVector);
   }
 
-  if (RVFI->isPushable(MF)) {
+  if (RVFI->useQCIInterrupt(MF)) {
+    // Allocate a fixed object that covers the entire QCI stack allocation,
+    // because there are gaps which are reserved for future use.
+    MFI.CreateFixedSpillStackObject(
+        QCIInterruptPushAmount, -static_cast<int64_t>(QCIInterruptPushAmount));
+  } else if (RVFI->isPushable(MF)) {
     // Allocate a fixed object that covers all the registers that are pushed.
     if (unsigned PushedRegs = RVFI->getRVPushRegs()) {
       int64_t PushedRegsBytes =
@@ -1892,9 +1970,22 @@ bool RISCVFrameLowering::spillCalleeSavedRegisters(
   if (MI != MBB.end() && !MI->isDebugInstr())
     DL = MI->getDebugLoc();
 
-  // Emit CM.PUSH with base SPimm & evaluate Push stack
   RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
-  if (RVFI->isPushable(*MF)) {
+  if (RVFI->useQCIInterrupt(*MF)) {
+    // Emit QC.C.MIENTER(.NEST)
+    BuildMI(
+        MBB, MI, DL,
+        TII.get(RVFI->getInterruptStackKind(*MF) ==
+                        RISCVMachineFunctionInfo::InterruptStackKind::QCINest
+                    ? RISCV::QC_C_MIENTER_NEST
+                    : RISCV::QC_C_MIENTER))
+        .setMIFlag(MachineInstr::FrameSetup);
+
+    for (auto [Reg, _Offset] : FixedCSRFIQCIInterruptMap)
+      MBB.addLiveIn(Reg);
+    // TODO: Handle QCI Interrupt + Push/Pop
+  } else if (RVFI->isPushable(*MF)) {
+    // Emit CM.PUSH with base SPimm & evaluate Push stack
     unsigned PushedRegNum = RVFI->getRVPushRegs();
     if (PushedRegNum > 0) {
       // Use encoded number to represent registers to spill.
@@ -2051,7 +2142,13 @@ bool RISCVFrameLowering::restoreCalleeSavedRegisters(
   loadRegFromStackSlot(UnmanagedCSI);
 
   RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
-  if (RVFI->isPushable(*MF)) {
+  if (RVFI->useQCIInterrupt(*MF)) {
+    // Don't emit anything here because restoration is handled by
+    // QC.C.MILEAVERET which we already inserted to return.
+    assert(MI->getOpcode() == RISCV::QC_C_MILEAVERET &&
+           "Unexpected QCI Interrupt Return Instruction");
+    // TODO: Handle QCI + Push/Pop
+  } else if (RVFI->isPushable(*MF)) {
     unsigned PushedRegNum = RVFI->getRVPushRegs();
     if (PushedRegNum > 0) {
       unsigned Opcode = getPopOpcode(RVFI->getPushPopKind(*MF));
@@ -2116,6 +2213,11 @@ bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const {
   MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
   const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
 
+  // We do not want QC.C.MILEAVERET to be subject to shrink-wrapping - it must
+  // come in the final block of its function as it both pops and returns.
+  if (RVFI->useQCIInterrupt(*MF))
+    return MBB.succ_empty();
+
   if (!RVFI->useSaveRestoreLibCalls(*MF))
     return true;