[AArch64][PAC] Lower authenticated calls with ptrauth bundles. #85736
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
backend:AArch64
llvm:globalisel
llvm:SelectionDAG
|
@llvm/pr-subscribers-llvm-selectiondag @llvm/pr-subscribers-llvm-globalisel Author: Ahmed Bougacha (ahmedbougacha) ChangesThis adds codegen support for the "ptrauth" operand bundles, which can This allows the generation of combined authenticating calls This is done by threading a PtrAuthInfo descriptor through Note that this also applies to the other forms of indirect calls, This also adopts an x8+ allocation order for GPR64noip, matching GPR64. Patch is 72.74 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/85736.diff 21 Files Affected:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 4c187a3068d823..e02c7435a1b6d2 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -46,6 +46,10 @@ class CallLowering {
virtual void anchor();
public:
+ struct PointerAuthInfo {
+ Register Discriminator;
+ uint64_t Key;
+ };
struct BaseArgInfo {
Type *Ty;
SmallVector<ISD::ArgFlagsTy, 4> Flags;
@@ -125,6 +129,8 @@ class CallLowering {
MDNode *KnownCallees = nullptr;
+ std::optional<PointerAuthInfo> PAI;
+
/// True if the call must be tail call optimized.
bool IsMustTailCall = false;
@@ -587,6 +593,7 @@ class CallLowering {
bool lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &Call,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs, Register SwiftErrorVReg,
+ std::optional<PointerAuthInfo> PAI,
Register ConvergenceCtrlToken,
std::function<unsigned()> GetCalleeReg) const;
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 2f164a460db843..e47a4e12d33cbc 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4290,6 +4290,9 @@ class TargetLowering : public TargetLoweringBase {
/// Return true if the target supports kcfi operand bundles.
virtual bool supportKCFIBundles() const { return false; }
+ /// Return true if the target supports ptrauth operand bundles.
+ virtual bool supportPtrAuthBundles() const { return false; }
+
/// Perform necessary initialization to handle a subset of CSRs explicitly
/// via copies. This function is called at the beginning of instruction
/// selection.
@@ -4401,6 +4404,14 @@ class TargetLowering : public TargetLoweringBase {
llvm_unreachable("Not Implemented");
}
+ /// This structure contains the information necessary for lowering
+ /// pointer-authenticating indirect calls. It is equivalent to the "ptrauth"
+ /// operand bundle found on the call instruction, if any.
+ struct PtrAuthInfo {
+ uint64_t Key;
+ SDValue Discriminator;
+ };
+
/// This structure contains all information that is necessary for lowering
/// calls. It is passed to TLI::LowerCallTo when the SelectionDAG builder
/// needs to lower a call, and targets will see this struct in their LowerCall
@@ -4440,6 +4451,8 @@ class TargetLowering : public TargetLoweringBase {
const ConstantInt *CFIType = nullptr;
SDValue ConvergenceControlToken;
+ std::optional<PtrAuthInfo> PAI;
+
CallLoweringInfo(SelectionDAG &DAG)
: RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false),
DoesNotReturn(false), IsReturnValueUsed(true), IsConvergent(false),
@@ -4562,6 +4575,11 @@ class TargetLowering : public TargetLoweringBase {
return *this;
}
+ CallLoweringInfo &setPtrAuth(PtrAuthInfo Value) {
+ PAI = Value;
+ return *this;
+ }
+
CallLoweringInfo &setIsPostTypeLegalization(bool Value=true) {
IsPostTypeLegalization = Value;
return *this;
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 363fad53b76c35..740a00d8afdd49 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -92,6 +92,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs,
Register SwiftErrorVReg,
+ std::optional<PointerAuthInfo> PAI,
Register ConvergenceCtrlToken,
std::function<unsigned()> GetCalleeReg) const {
CallLoweringInfo Info;
@@ -188,6 +189,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
Info.CallConv = CallConv;
Info.SwiftErrorVReg = SwiftErrorVReg;
+ Info.PAI = PAI;
Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
Info.IsMustTailCall = CB.isMustTailCall();
Info.IsTailCall = CanBeTailCalled;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 47ee2ee507137e..0be2597bf7c54b 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2615,6 +2615,20 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
}
}
+ std::optional<CallLowering::PointerAuthInfo> PAI;
+ if (CB.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ // Functions should never be ptrauth-called directly.
+ assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
+
+ auto PAB = CB.getOperandBundle("ptrauth");
+ Value *Key = PAB->Inputs[0];
+ Value *Discriminator = PAB->Inputs[1];
+
+ Register DiscReg = getOrCreateVReg(*Discriminator);
+ PAI = CallLowering::PointerAuthInfo{DiscReg,
+ cast<ConstantInt>(Key)->getZExtValue()};
+ }
+
Register ConvergenceCtrlToken = 0;
if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
const auto &Token = *Bundle->Inputs[0].get();
@@ -2625,7 +2639,7 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
// optimize into tail calls. Instead, we defer that to selection where a final
// scan is done to check if any instructions are calls.
bool Success = CLI->lowerCall(
- MIRBuilder, CB, Res, Args, SwiftErrorVReg, ConvergenceCtrlToken,
+ MIRBuilder, CB, Res, Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
[&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
// Check if we just inserted a tail call.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f1923a64368f4f..b60be1bb77212e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3300,12 +3300,12 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
const BasicBlock *EHPadBB = I.getSuccessor(1);
MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
- // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
+ // Deopt and ptrauth bundles are lowered in helper functions, and we don't
// have to do anything here to lower funclet bundles.
assert(!I.hasOperandBundlesOtherThan(
{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
- LLVMContext::OB_cfguardtarget,
+ LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
LLVMContext::OB_clang_arc_attachedcall}) &&
"Cannot lower invokes with arbitrary operand bundles yet!");
@@ -3356,6 +3356,8 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
// intrinsic, and right now there are no plans to support other intrinsics
// with deopt state.
LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
+ } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);
} else {
LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
}
@@ -8508,9 +8510,9 @@ SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
}
void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
- bool isTailCall,
- bool isMustTailCall,
- const BasicBlock *EHPadBB) {
+ bool isTailCall, bool isMustTailCall,
+ const BasicBlock *EHPadBB,
+ const TargetLowering::PtrAuthInfo *PAI) {
auto &DL = DAG.getDataLayout();
FunctionType *FTy = CB.getFunctionType();
Type *RetTy = CB.getType();
@@ -8619,6 +8621,15 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
.setCFIType(CFIType)
.setConvergenceControlToken(ConvControlToken);
+
+ // Set the pointer authentication info if we have it.
+ if (PAI) {
+ if (!TLI.supportPtrAuthBundles())
+ report_fatal_error(
+ "This target doesn't support calls with ptrauth operand bundles.");
+ CLI.setPtrAuth(*PAI);
+ }
+
std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
if (Result.first.getNode()) {
@@ -9164,6 +9175,11 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
}
}
+ if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
+ return;
+ }
+
// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
// CFGuardTarget bundles are lowered in LowerCallTo.
@@ -9185,6 +9201,31 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
}
+void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
+ const CallBase &CB, const BasicBlock *EHPadBB) {
+ auto PAB = CB.getOperandBundle("ptrauth");
+ auto *CalleeV = CB.getCalledOperand();
+
+ // Gather the call ptrauth data from the operand bundle:
+ // [ i32 <key>, i64 <discriminator> ]
+ auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
+ Value *Discriminator = PAB->Inputs[1];
+
+ assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
+ assert(Discriminator->getType()->isIntegerTy(64) &&
+ "Invalid ptrauth discriminator");
+
+ // Functions should never be ptrauth-called directly.
+ assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
+
+ // Otherwise, do an authenticated indirect call.
+ TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
+ getValue(Discriminator)};
+
+ LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
+ EHPadBB, &PAI);
+}
+
namespace {
/// AsmOperandInfo - This contains information for each constraint that we are
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index dcf46e0563ff9d..6a3d0276cfd456 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -406,7 +406,8 @@ class SelectionDAGBuilder {
void CopyToExportRegsIfNeeded(const Value *V);
void ExportFromCurrentBlock(const Value *V);
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall,
- bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr);
+ bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr,
+ const TargetLowering::PtrAuthInfo *PAI = nullptr);
// Lower range metadata from 0 to N to assert zext to an integer of nearest
// floor power of two.
@@ -490,6 +491,9 @@ class SelectionDAGBuilder {
bool VarArgDisallowed,
bool ForceVoidReturnTy);
+ void LowerCallSiteWithPtrAuthBundle(const CallBase &CB,
+ const BasicBlock *EHPadBB);
+
/// Returns the type of FrameIndex and TargetFrameIndex nodes.
MVT getFrameIndexTy() {
return DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout());
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 6d34e16fc43401..e4c9503b42e8ef 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -133,6 +133,8 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitSled(const MachineInstr &MI, SledKind Kind);
+ // Emit the sequence for BLRA (authenticate + branch).
+ void emitPtrauthBranch(const MachineInstr *MI);
// Emit the sequence for AUT or AUTPAC.
void emitPtrauthAuthResign(const MachineInstr *MI);
// Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
@@ -1482,6 +1484,10 @@ void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
unsigned AddrDisc,
unsigned &InstsEmitted) {
+ // So far we've used NoRegister in pseudos. Now we need real encodings.
+ if (AddrDisc == AArch64::NoRegister)
+ AddrDisc = AArch64::XZR;
+
// If there is no constant discriminator, there's no blend involved:
// just use the address discriminator register as-is (XZR or not).
if (!Disc)
@@ -1729,6 +1735,39 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
OutStreamer->emitLabel(EndSym);
}
+void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
+ unsigned InstsEmitted = 0;
+
+ unsigned BrTarget = MI->getOperand(0).getReg();
+ auto Key = (AArch64PACKey::ID)MI->getOperand(1).getImm();
+ uint64_t Disc = MI->getOperand(2).getImm();
+ unsigned AddrDisc = MI->getOperand(3).getReg();
+
+ // Compute discriminator into x17
+ assert(isUInt<16>(Disc));
+ unsigned DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, InstsEmitted);
+ bool IsZeroDisc = DiscReg == AArch64::XZR;
+
+ assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+
+ unsigned Opc;
+ if (Key == AArch64PACKey::IA)
+ Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
+ else
+ Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
+
+ MCInst BRInst;
+ BRInst.setOpcode(Opc);
+ BRInst.addOperand(MCOperand::createReg(BrTarget));
+ if (!IsZeroDisc)
+ BRInst.addOperand(MCOperand::createReg(DiscReg));
+ EmitToStreamer(*OutStreamer, BRInst);
+ ++InstsEmitted;
+
+ assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -1869,9 +1908,60 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
emitPtrauthAuthResign(MI);
return;
+ case AArch64::BLRA:
+ emitPtrauthBranch(MI);
+ return;
+
// Tail calls use pseudo instructions so they have the proper code-gen
// attributes (isCall, isReturn, etc.). We lower them to the real
// instruction here.
+ case AArch64::AUTH_TCRETURN:
+ case AArch64::AUTH_TCRETURN_BTI: {
+ const uint64_t Key = MI->getOperand(2).getImm();
+ assert(Key < 2 && "Unknown key kind for authenticating tail-call return");
+ const uint64_t Disc = MI->getOperand(3).getImm();
+ Register AddrDisc = MI->getOperand(4).getReg();
+
+ Register ScratchReg = MI->getOperand(0).getReg() == AArch64::X16
+ ? AArch64::X17
+ : AArch64::X16;
+
+ unsigned DiscReg = AddrDisc;
+ if (Disc) {
+ assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
+
+ if (AddrDisc != AArch64::NoRegister) {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+ .addReg(ScratchReg)
+ .addReg(AArch64::XZR)
+ .addReg(AddrDisc)
+ .addImm(0));
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
+ .addReg(ScratchReg)
+ .addReg(ScratchReg)
+ .addImm(Disc)
+ .addImm(/*shift=*/48));
+ } else {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
+ .addReg(ScratchReg)
+ .addImm(Disc)
+ .addImm(/*shift=*/0));
+ }
+ DiscReg = ScratchReg;
+ }
+
+ const bool isZero = DiscReg == AArch64::NoRegister;
+ const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
+ {AArch64::BRAB, AArch64::BRABZ}};
+
+ MCInst TmpInst;
+ TmpInst.setOpcode(Opcodes[Key][isZero]);
+ TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ if (!isZero)
+ TmpInst.addOperand(MCOperand::createReg(DiscReg));
+ EmitToStreamer(*OutStreamer, TmpInst);
+ return;
+ }
case AArch64::TCRETURNri:
case AArch64::TCRETURNrix16x17:
case AArch64::TCRETURNrix17:
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 03f0778bae59d5..657324d2307c58 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -817,10 +817,44 @@ bool AArch64ExpandPseudo::expandCALL_RVMARKER(
MachineInstr &MI = *MBBI;
MachineOperand &RVTarget = MI.getOperand(0);
assert(RVTarget.isGlobal() && "invalid operand for attached call");
- MachineInstr *OriginalCall =
- createCall(MBB, MBBI, TII, MI.getOperand(1),
- // Regmask starts after the RV and call targets.
- /*RegMaskStartIdx=*/2);
+
+ MachineInstr *OriginalCall = nullptr;
+
+ if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
+ // Pointer auth call.
+ MachineOperand &Key = MI.getOperand(2);
+ assert((Key.getImm() == 0 || Key.getImm() == 1) &&
+ "invalid key for ptrauth call");
+ MachineOperand &IntDisc = MI.getOperand(3);
+ MachineOperand &AddrDisc = MI.getOperand(4);
+
+ OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BLRA))
+ .getInstr();
+ OriginalCall->addOperand(MI.getOperand(1));
+ OriginalCall->addOperand(Key);
+ OriginalCall->addOperand(IntDisc);
+ OriginalCall->addOperand(AddrDisc);
+
+ unsigned RegMaskStartIdx = 5;
+ // Skip register arguments. Those are added during ISel, but are not
+ // needed for the concrete branch.
+ while (!MI.getOperand(RegMaskStartIdx).isRegMask()) {
+ auto MOP = MI.getOperand(RegMaskStartIdx);
+ assert(MOP.isReg() && "can only add register operands");
+ OriginalCall->addOperand(MachineOperand::CreateReg(
+ MOP.getReg(), /*Def=*/false, /*Implicit=*/true, /*isKill=*/false,
+ /*isDead=*/false, /*isUndef=*/MOP.isUndef()));
+ RegMaskStartIdx++;
+ }
+ for (const MachineOperand &MO :
+ llvm::drop_begin(MI.operands(), RegMaskStartIdx))
+ OriginalCall->addOperand(MO);
+ } else {
+ assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
+ OriginalCall = createCall(MBB, MBBI, TII, MI.getOperand(1),
+ // Regmask starts after the RV and call targets.
+ /*RegMaskStartIdx=*/2);
+ }
BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))
.addReg(AArch64::FP, RegState::Define)
@@ -1529,6 +1563,7 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
case AArch64::LDR_PPXI:
return expandSVESpillFill(MBB, MBBI, AArch64::LDR_PXI, 2);
case AArch64::BLR_RVMARKER:
+ case AArch64::BLRA_RVMARKER:
return expandCALL_RVMARKER(MBB, MBBI);
case AArch64::BLR_BTI:
return expandCALL_BTI(MBB, MBBI);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 92f59dbf1e9e52..cb3dd5cf0b22f9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -329,6 +329,40 @@ static bool isZeroingInactiveLanes(SDValue Op) {
}
}
+static std::tuple<SDValue, SDValue>
+extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
+ SDLoc DL(Disc);
+ SDValue AddrDisc;
+ SDValue ConstDisc;
+
+ // If this is a blend, remember the cons...
[truncated]
|
|
@llvm/pr-subscribers-backend-aarch64 Author: Ahmed Bougacha (ahmedbougacha) ChangesThis adds codegen support for the "ptrauth" operand bundles, which can This allows the generation of combined authenticating calls This is done by threading a PtrAuthInfo descriptor through Note that this also applies to the other forms of indirect calls, This also adopts an x8+ allocation order for GPR64noip, matching GPR64. Patch is 72.74 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/85736.diff 21 Files Affected:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 4c187a3068d823..e02c7435a1b6d2 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -46,6 +46,10 @@ class CallLowering {
virtual void anchor();
public:
+ struct PointerAuthInfo {
+ Register Discriminator;
+ uint64_t Key;
+ };
struct BaseArgInfo {
Type *Ty;
SmallVector<ISD::ArgFlagsTy, 4> Flags;
@@ -125,6 +129,8 @@ class CallLowering {
MDNode *KnownCallees = nullptr;
+ std::optional<PointerAuthInfo> PAI;
+
/// True if the call must be tail call optimized.
bool IsMustTailCall = false;
@@ -587,6 +593,7 @@ class CallLowering {
bool lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &Call,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs, Register SwiftErrorVReg,
+ std::optional<PointerAuthInfo> PAI,
Register ConvergenceCtrlToken,
std::function<unsigned()> GetCalleeReg) const;
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 2f164a460db843..e47a4e12d33cbc 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4290,6 +4290,9 @@ class TargetLowering : public TargetLoweringBase {
/// Return true if the target supports kcfi operand bundles.
virtual bool supportKCFIBundles() const { return false; }
+ /// Return true if the target supports ptrauth operand bundles.
+ virtual bool supportPtrAuthBundles() const { return false; }
+
/// Perform necessary initialization to handle a subset of CSRs explicitly
/// via copies. This function is called at the beginning of instruction
/// selection.
@@ -4401,6 +4404,14 @@ class TargetLowering : public TargetLoweringBase {
llvm_unreachable("Not Implemented");
}
+ /// This structure contains the information necessary for lowering
+ /// pointer-authenticating indirect calls. It is equivalent to the "ptrauth"
+ /// operand bundle found on the call instruction, if any.
+ struct PtrAuthInfo {
+ uint64_t Key;
+ SDValue Discriminator;
+ };
+
/// This structure contains all information that is necessary for lowering
/// calls. It is passed to TLI::LowerCallTo when the SelectionDAG builder
/// needs to lower a call, and targets will see this struct in their LowerCall
@@ -4440,6 +4451,8 @@ class TargetLowering : public TargetLoweringBase {
const ConstantInt *CFIType = nullptr;
SDValue ConvergenceControlToken;
+ std::optional<PtrAuthInfo> PAI;
+
CallLoweringInfo(SelectionDAG &DAG)
: RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false),
DoesNotReturn(false), IsReturnValueUsed(true), IsConvergent(false),
@@ -4562,6 +4575,11 @@ class TargetLowering : public TargetLoweringBase {
return *this;
}
+ CallLoweringInfo &setPtrAuth(PtrAuthInfo Value) {
+ PAI = Value;
+ return *this;
+ }
+
CallLoweringInfo &setIsPostTypeLegalization(bool Value=true) {
IsPostTypeLegalization = Value;
return *this;
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 363fad53b76c35..740a00d8afdd49 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -92,6 +92,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
ArrayRef<Register> ResRegs,
ArrayRef<ArrayRef<Register>> ArgRegs,
Register SwiftErrorVReg,
+ std::optional<PointerAuthInfo> PAI,
Register ConvergenceCtrlToken,
std::function<unsigned()> GetCalleeReg) const {
CallLoweringInfo Info;
@@ -188,6 +189,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
Info.KnownCallees = CB.getMetadata(LLVMContext::MD_callees);
Info.CallConv = CallConv;
Info.SwiftErrorVReg = SwiftErrorVReg;
+ Info.PAI = PAI;
Info.ConvergenceCtrlToken = ConvergenceCtrlToken;
Info.IsMustTailCall = CB.isMustTailCall();
Info.IsTailCall = CanBeTailCalled;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 47ee2ee507137e..0be2597bf7c54b 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2615,6 +2615,20 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
}
}
+ std::optional<CallLowering::PointerAuthInfo> PAI;
+ if (CB.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ // Functions should never be ptrauth-called directly.
+ assert(!CB.getCalledFunction() && "invalid direct ptrauth call");
+
+ auto PAB = CB.getOperandBundle("ptrauth");
+ Value *Key = PAB->Inputs[0];
+ Value *Discriminator = PAB->Inputs[1];
+
+ Register DiscReg = getOrCreateVReg(*Discriminator);
+ PAI = CallLowering::PointerAuthInfo{DiscReg,
+ cast<ConstantInt>(Key)->getZExtValue()};
+ }
+
Register ConvergenceCtrlToken = 0;
if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
const auto &Token = *Bundle->Inputs[0].get();
@@ -2625,7 +2639,7 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
// optimize into tail calls. Instead, we defer that to selection where a final
// scan is done to check if any instructions are calls.
bool Success = CLI->lowerCall(
- MIRBuilder, CB, Res, Args, SwiftErrorVReg, ConvergenceCtrlToken,
+ MIRBuilder, CB, Res, Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,
[&]() { return getOrCreateVReg(*CB.getCalledOperand()); });
// Check if we just inserted a tail call.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f1923a64368f4f..b60be1bb77212e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3300,12 +3300,12 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
const BasicBlock *EHPadBB = I.getSuccessor(1);
MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB];
- // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
+ // Deopt and ptrauth bundles are lowered in helper functions, and we don't
// have to do anything here to lower funclet bundles.
assert(!I.hasOperandBundlesOtherThan(
{LLVMContext::OB_deopt, LLVMContext::OB_gc_transition,
LLVMContext::OB_gc_live, LLVMContext::OB_funclet,
- LLVMContext::OB_cfguardtarget,
+ LLVMContext::OB_cfguardtarget, LLVMContext::OB_ptrauth,
LLVMContext::OB_clang_arc_attachedcall}) &&
"Cannot lower invokes with arbitrary operand bundles yet!");
@@ -3356,6 +3356,8 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
// intrinsic, and right now there are no plans to support other intrinsics
// with deopt state.
LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
+ } else if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), EHPadBB);
} else {
LowerCallTo(I, getValue(Callee), false, false, EHPadBB);
}
@@ -8508,9 +8510,9 @@ SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
}
void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
- bool isTailCall,
- bool isMustTailCall,
- const BasicBlock *EHPadBB) {
+ bool isTailCall, bool isMustTailCall,
+ const BasicBlock *EHPadBB,
+ const TargetLowering::PtrAuthInfo *PAI) {
auto &DL = DAG.getDataLayout();
FunctionType *FTy = CB.getFunctionType();
Type *RetTy = CB.getType();
@@ -8619,6 +8621,15 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
.setCFIType(CFIType)
.setConvergenceControlToken(ConvControlToken);
+
+ // Set the pointer authentication info if we have it.
+ if (PAI) {
+ if (!TLI.supportPtrAuthBundles())
+ report_fatal_error(
+ "This target doesn't support calls with ptrauth operand bundles.");
+ CLI.setPtrAuth(*PAI);
+ }
+
std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
if (Result.first.getNode()) {
@@ -9164,6 +9175,11 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
}
}
+ if (I.countOperandBundlesOfType(LLVMContext::OB_ptrauth)) {
+ LowerCallSiteWithPtrAuthBundle(cast<CallBase>(I), /*EHPadBB=*/nullptr);
+ return;
+ }
+
// Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
// have to do anything here to lower funclet bundles.
// CFGuardTarget bundles are lowered in LowerCallTo.
@@ -9185,6 +9201,31 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall());
}
+void SelectionDAGBuilder::LowerCallSiteWithPtrAuthBundle(
+ const CallBase &CB, const BasicBlock *EHPadBB) {
+ auto PAB = CB.getOperandBundle("ptrauth");
+ auto *CalleeV = CB.getCalledOperand();
+
+ // Gather the call ptrauth data from the operand bundle:
+ // [ i32 <key>, i64 <discriminator> ]
+ auto *Key = cast<ConstantInt>(PAB->Inputs[0]);
+ Value *Discriminator = PAB->Inputs[1];
+
+ assert(Key->getType()->isIntegerTy(32) && "Invalid ptrauth key");
+ assert(Discriminator->getType()->isIntegerTy(64) &&
+ "Invalid ptrauth discriminator");
+
+ // Functions should never be ptrauth-called directly.
+ assert(!isa<Function>(CalleeV) && "invalid direct ptrauth call");
+
+ // Otherwise, do an authenticated indirect call.
+ TargetLowering::PtrAuthInfo PAI = {Key->getZExtValue(),
+ getValue(Discriminator)};
+
+ LowerCallTo(CB, getValue(CalleeV), CB.isTailCall(), CB.isMustTailCall(),
+ EHPadBB, &PAI);
+}
+
namespace {
/// AsmOperandInfo - This contains information for each constraint that we are
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index dcf46e0563ff9d..6a3d0276cfd456 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -406,7 +406,8 @@ class SelectionDAGBuilder {
void CopyToExportRegsIfNeeded(const Value *V);
void ExportFromCurrentBlock(const Value *V);
void LowerCallTo(const CallBase &CB, SDValue Callee, bool IsTailCall,
- bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr);
+ bool IsMustTailCall, const BasicBlock *EHPadBB = nullptr,
+ const TargetLowering::PtrAuthInfo *PAI = nullptr);
// Lower range metadata from 0 to N to assert zext to an integer of nearest
// floor power of two.
@@ -490,6 +491,9 @@ class SelectionDAGBuilder {
bool VarArgDisallowed,
bool ForceVoidReturnTy);
+ void LowerCallSiteWithPtrAuthBundle(const CallBase &CB,
+ const BasicBlock *EHPadBB);
+
/// Returns the type of FrameIndex and TargetFrameIndex nodes.
MVT getFrameIndexTy() {
return DAG.getTargetLoweringInfo().getFrameIndexTy(DAG.getDataLayout());
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 6d34e16fc43401..e4c9503b42e8ef 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -133,6 +133,8 @@ class AArch64AsmPrinter : public AsmPrinter {
void emitSled(const MachineInstr &MI, SledKind Kind);
+ // Emit the sequence for BLRA (authenticate + branch).
+ void emitPtrauthBranch(const MachineInstr *MI);
// Emit the sequence for AUT or AUTPAC.
void emitPtrauthAuthResign(const MachineInstr *MI);
// Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
@@ -1482,6 +1484,10 @@ void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
unsigned AddrDisc,
unsigned &InstsEmitted) {
+ // So far we've used NoRegister in pseudos. Now we need real encodings.
+ if (AddrDisc == AArch64::NoRegister)
+ AddrDisc = AArch64::XZR;
+
// If there is no constant discriminator, there's no blend involved:
// just use the address discriminator register as-is (XZR or not).
if (!Disc)
@@ -1729,6 +1735,39 @@ void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
OutStreamer->emitLabel(EndSym);
}
+void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
+ unsigned InstsEmitted = 0;
+
+ unsigned BrTarget = MI->getOperand(0).getReg();
+ auto Key = (AArch64PACKey::ID)MI->getOperand(1).getImm();
+ uint64_t Disc = MI->getOperand(2).getImm();
+ unsigned AddrDisc = MI->getOperand(3).getReg();
+
+ // Compute discriminator into x17
+ assert(isUInt<16>(Disc));
+ unsigned DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, InstsEmitted);
+ bool IsZeroDisc = DiscReg == AArch64::XZR;
+
+ assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
+ "Invalid auth call key");
+
+ unsigned Opc;
+ if (Key == AArch64PACKey::IA)
+ Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
+ else
+ Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
+
+ MCInst BRInst;
+ BRInst.setOpcode(Opc);
+ BRInst.addOperand(MCOperand::createReg(BrTarget));
+ if (!IsZeroDisc)
+ BRInst.addOperand(MCOperand::createReg(DiscReg));
+ EmitToStreamer(*OutStreamer, BRInst);
+ ++InstsEmitted;
+
+ assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
+}
+
// Simple pseudo-instructions have their lowering (with expansion to real
// instructions) auto-generated.
#include "AArch64GenMCPseudoLowering.inc"
@@ -1869,9 +1908,60 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
emitPtrauthAuthResign(MI);
return;
+ case AArch64::BLRA:
+ emitPtrauthBranch(MI);
+ return;
+
// Tail calls use pseudo instructions so they have the proper code-gen
// attributes (isCall, isReturn, etc.). We lower them to the real
// instruction here.
+ case AArch64::AUTH_TCRETURN:
+ case AArch64::AUTH_TCRETURN_BTI: {
+ const uint64_t Key = MI->getOperand(2).getImm();
+ assert(Key < 2 && "Unknown key kind for authenticating tail-call return");
+ const uint64_t Disc = MI->getOperand(3).getImm();
+ Register AddrDisc = MI->getOperand(4).getReg();
+
+ Register ScratchReg = MI->getOperand(0).getReg() == AArch64::X16
+ ? AArch64::X17
+ : AArch64::X16;
+
+ unsigned DiscReg = AddrDisc;
+ if (Disc) {
+ assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
+
+ if (AddrDisc != AArch64::NoRegister) {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+ .addReg(ScratchReg)
+ .addReg(AArch64::XZR)
+ .addReg(AddrDisc)
+ .addImm(0));
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
+ .addReg(ScratchReg)
+ .addReg(ScratchReg)
+ .addImm(Disc)
+ .addImm(/*shift=*/48));
+ } else {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
+ .addReg(ScratchReg)
+ .addImm(Disc)
+ .addImm(/*shift=*/0));
+ }
+ DiscReg = ScratchReg;
+ }
+
+ const bool isZero = DiscReg == AArch64::NoRegister;
+ const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
+ {AArch64::BRAB, AArch64::BRABZ}};
+
+ MCInst TmpInst;
+ TmpInst.setOpcode(Opcodes[Key][isZero]);
+ TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
+ if (!isZero)
+ TmpInst.addOperand(MCOperand::createReg(DiscReg));
+ EmitToStreamer(*OutStreamer, TmpInst);
+ return;
+ }
case AArch64::TCRETURNri:
case AArch64::TCRETURNrix16x17:
case AArch64::TCRETURNrix17:
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 03f0778bae59d5..657324d2307c58 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -817,10 +817,44 @@ bool AArch64ExpandPseudo::expandCALL_RVMARKER(
MachineInstr &MI = *MBBI;
MachineOperand &RVTarget = MI.getOperand(0);
assert(RVTarget.isGlobal() && "invalid operand for attached call");
- MachineInstr *OriginalCall =
- createCall(MBB, MBBI, TII, MI.getOperand(1),
- // Regmask starts after the RV and call targets.
- /*RegMaskStartIdx=*/2);
+
+ MachineInstr *OriginalCall = nullptr;
+
+ if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
+ // Pointer auth call.
+ MachineOperand &Key = MI.getOperand(2);
+ assert((Key.getImm() == 0 || Key.getImm() == 1) &&
+ "invalid key for ptrauth call");
+ MachineOperand &IntDisc = MI.getOperand(3);
+ MachineOperand &AddrDisc = MI.getOperand(4);
+
+ OriginalCall = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::BLRA))
+ .getInstr();
+ OriginalCall->addOperand(MI.getOperand(1));
+ OriginalCall->addOperand(Key);
+ OriginalCall->addOperand(IntDisc);
+ OriginalCall->addOperand(AddrDisc);
+
+ unsigned RegMaskStartIdx = 5;
+ // Skip register arguments. Those are added during ISel, but are not
+ // needed for the concrete branch.
+ while (!MI.getOperand(RegMaskStartIdx).isRegMask()) {
+ auto MOP = MI.getOperand(RegMaskStartIdx);
+ assert(MOP.isReg() && "can only add register operands");
+ OriginalCall->addOperand(MachineOperand::CreateReg(
+ MOP.getReg(), /*Def=*/false, /*Implicit=*/true, /*isKill=*/false,
+ /*isDead=*/false, /*isUndef=*/MOP.isUndef()));
+ RegMaskStartIdx++;
+ }
+ for (const MachineOperand &MO :
+ llvm::drop_begin(MI.operands(), RegMaskStartIdx))
+ OriginalCall->addOperand(MO);
+ } else {
+ assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
+ OriginalCall = createCall(MBB, MBBI, TII, MI.getOperand(1),
+ // Regmask starts after the RV and call targets.
+ /*RegMaskStartIdx=*/2);
+ }
BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORRXrs))
.addReg(AArch64::FP, RegState::Define)
@@ -1529,6 +1563,7 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
case AArch64::LDR_PPXI:
return expandSVESpillFill(MBB, MBBI, AArch64::LDR_PXI, 2);
case AArch64::BLR_RVMARKER:
+ case AArch64::BLRA_RVMARKER:
return expandCALL_RVMARKER(MBB, MBBI);
case AArch64::BLR_BTI:
return expandCALL_BTI(MBB, MBBI);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 92f59dbf1e9e52..cb3dd5cf0b22f9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -329,6 +329,40 @@ static bool isZeroingInactiveLanes(SDValue Op) {
}
}
+static std::tuple<SDValue, SDValue>
+extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
+ SDLoc DL(Disc);
+ SDValue AddrDisc;
+ SDValue ConstDisc;
+
+ // If this is a blend, remember the cons...
[truncated]
|
ahmedbougacha
requested review from
asl,
apazos,
atrosinenko,
kbeyls and
smithp35
|
ping |
ahmedbougacha
requested review from
aaupov,
maksfb,
rafaelauler,
ayermolo,
dcci,
tbaederr and
Endilll
as code owners
ahmedbougacha
changed the base branch from
users/ahmedbougacha/ptrauth-auth-resign-isel
to
main
ahmedbougacha
removed request for
aaupov,
dcci,
maksfb,
rafaelauler,
Endilll,
ayermolo and
tbaederr
ahmedbougacha
changed the base branch from
main
to
users/ahmedbougacha/ptrauth-auth-resign-isel
ahmedbougacha
force-pushed
the
eng/abougacha/ptrauth-call-bundles
branch
from
950508b to
536ab53
Compare
kovdan01
requested changes
May 22, 2024
| @@ -0,0 +1,183 @@ | |||
| ; RUN: llc -mtriple arm64e-apple-darwin -o - -global-isel -global-isel-abort=1 -verify-machineinstrs %s | FileCheck %s --check-prefixes=CHECK | |||
There was a problem hiding this comment.
Please also add tests for ELF: -mtriple aarch64 -mattr=+pauth. Check directives are going be different for some cases, for example, personality encoding would be different, assembly syntax would have minor differences like comment style, function prologue/epilogue might change if we do not need to save frame pointer, labels might have different names, etc.
The call lowering itself looks agnostic to specific triple, but IMHO there should be tests which check that it works both for Apple triples and for a generic ELF triple. Also applies to other test files
There was a problem hiding this comment.
It's not standard practice to test other platforms, but sure, I added ELF checks. Now you folks owe me darwin tests for future PAC changes you make ;) I'll do a pass over the other tests separately
|
Thanks for taking a look; updated |
kovdan01
previously requested changes
May 24, 2024
There was a problem hiding this comment.
@ahmedbougacha In terms of functionality, LGTM (but I'll prefer other guys also looking through the changes). I've left several minor style-related comments.
One more overall style comment which might probably require an additional PR: sometimes identifiers use PtrAuth, sometimes Ptrauth. Is this intentional? If not, I suggest to use the same case style - it would keep things consistent and allow for case-sensitive search.
|
|
||
| if (MI.getOpcode() == AArch64::BLRA_RVMARKER) { | ||
| // Pointer auth call. | ||
| MachineOperand &Key = MI.getOperand(2); |
There was a problem hiding this comment.
Suggested change
| MachineOperand &Key = MI.getOperand(2); | |
| const MachineOperand &Key = MI.getOperand(2); |
There was a problem hiding this comment.
Please see overall comment - this does not seem to be resolved in your latest changes
| MachineOperand &Key = MI.getOperand(2); | ||
| assert((Key.getImm() == 0 || Key.getImm() == 1) && | ||
| "invalid key for ptrauth call"); | ||
| MachineOperand &IntDisc = MI.getOperand(3); |
There was a problem hiding this comment.
Suggested change
| MachineOperand &IntDisc = MI.getOperand(3); | |
| const MachineOperand &IntDisc = MI.getOperand(3); |
There was a problem hiding this comment.
Please see overall comment - this does not seem to be resolved in your latest changes
| assert((Key.getImm() == 0 || Key.getImm() == 1) && | ||
| "invalid key for ptrauth call"); | ||
| MachineOperand &IntDisc = MI.getOperand(3); | ||
| MachineOperand &AddrDisc = MI.getOperand(4); |
There was a problem hiding this comment.
Suggested change
| MachineOperand &AddrDisc = MI.getOperand(4); | |
| const MachineOperand &AddrDisc = MI.getOperand(4); |
There was a problem hiding this comment.
Please see overall comment - this does not seem to be resolved in your latest changes
| @@ -1694,6 +1718,35 @@ let Predicates = [HasPAuth] in { | |||
| def BLRABZ : AuthOneOperand<0b001, 1, "blrabz">; | |||
| } | |||
|
|
|||
| // BLRA pseudo, generalized version of BLRAA/BLRAB/Z. | |||
| // This directly manipulates x16/x17, which are the only registers the OS | |||
| // guarantees are safe to use for sensitive operations. | |||
There was a problem hiding this comment.
Yes, having a more detailed comment here would be nice. As for now, from the comment it was unclear that this is Darwin-specific. It's also probably worth mentioning that it's harmless for non-Darwin, as you said.
Yeah, it's hard to avoid needing case-insensitive search because of the irreconcilable capitalization rules between e.g., enums generated from all-lowercase tablegen, opcode namespaces with a strong all-uppercase tradition, variable/function naming conventions, etc. In general I try to standardize on "ptrauth" regardless of case, and would rather get rid of "pauth" or "pac" first, because they're just objectively worse names. We can indeed pick a consistent ptrauth when the dust settles. |
kovdan01
reviewed
May 30, 2024
There was a problem hiding this comment.
LGTM with a few const-related comments left (actually I've unresolved several previously opened threads). I highly encourage everyone else interested to also look through since while the changes look reasonable and the code does what it's intended to do, there might be some corner cases I've missed or other room for enhancement.
@ahmedbougacha Do I miss some reason why const qualifier should not be applied to pointers/references I've mentioned? Even though it might be obvious for experienced llvm contributors that these values are not modified, it's still IMHO better to explicitly mark them as const so a person not so experienced in this area would not have to guess while reading the code whether the non-const-qualified value is intended to be modified when passed somewhere by pointer/reference or not.
kovdan01
dismissed
their stale review
Changes requested were submitted, no need for this status anymore
73078ec added GPR64noip for hwasan pseudos. Give it an allocation order that prefers allocating from x8 and up, to match GPR64: this allows for easier regalloc, as x0-x7 are likely to be used for parameter passing.
This adds codegen support for the "ptrauth" operand bundles, which can be used to augment indirect calls with the equivalent of an `@llvm.ptrauth.auth` intrinsic call on the call target (possibly preceded by an `@llvm.ptrauth.blend` on the auth discriminator if applicable.) This allows the generation of combined authenticating calls on AArch64 (in the BLRA* PAuth instructions), while avoiding the raw just-authenticated function pointer from being exposed to attackers. This is done by threading a PtrAuthInfo descriptor through the call lowering infrastructure. Note that this also applies to the other forms of indirect calls, notably invokes, rvmarker, and tail calls. Tail-calls in particular bring some additional complexity, with the intersecting register constraints of BTI and PAC discriminator computation.
- test invoke for ELF as well - consolidate GISel/SDAG invoke test - check key/disc validity in consistent ways - misc. nits
- AArch64ExpandPseudos: generalize & use createCall - AArch64InstrInfo: describe x16/x17 usage in pseudo - AArch64InstrInfo: group SDNodes with others - test call for ELF as well - GlobalISel: rename PointerAuthInfo, reorder fields - various nits, auto, const
- assert ptrauth calls aren't used with PAuth_LR - more const nits
ahmedbougacha
force-pushed
the
eng/abougacha/ptrauth-call-bundles
branch
from
e876789 to
5f2ab74
Compare
ahmedbougacha
changed the base branch from
users/ahmedbougacha/ptrauth-auth-resign-isel
to
main
|
I rebased this on top of main and extracted the couple of common helper functions from the auth/resign isel patch, to not depend on it.
Thanks, all of these should be addressed now. I'll leave this open for another day or two if anyone wants to chime in, either way we can iterate further in tree |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This adds codegen support for the "ptrauth" operand bundles, which can
be used to augment indirect calls with the equivalent of an
@llvm.ptrauth.authintrinsic call on the call target (possiblypreceded by an
@llvm.ptrauth.blendon the auth discriminator ifapplicable.)
This allows the generation of combined authenticating calls
on AArch64 (in the BLRA* PAuth instructions), while avoiding
the raw just-authenticated function pointer from being
exposed to attackers.
This is done by threading a PtrAuthInfo descriptor through
the call lowering infrastructure.
Note that this also applies to the other forms of indirect calls,
notably invokes, rvmarker, and tail calls. Tail-calls in particular
bring some additional complexity, with the intersecting register
constraints of BTI and PAC discriminator computation.
This also adopts an x8+ allocation order for GPR64noip, matching GPR64.