Skip to content

Commit 09515f2

Browse files
davidbolvanskydavidbolvansky
committed
[SDAG] Preserve unpredictable metadata, teach X86CmovConversion to respect this metadata
Sometimes an developer would like to have more control over cmov vs branch. We have unpredictable metadata in LLVM IR, but currently it is ignored by X86 backend. Propagate this metadata and avoid cmov->branch conversion in X86CmovConversion for cmov with this metadata. Example: ``` int MaxIndex(int n, int *a) { int t = 0; for (int i = 1; i < n; i++) { // cmov is converted to branch by X86CmovConversion if (a[i] > a[t]) t = i; } return t; } int MaxIndex2(int n, int *a) { int t = 0; for (int i = 1; i < n; i++) { // cmov is preserved if (__builtin_unpredictable(a[i] > a[t])) t = i; } return t; } ``` Reviewed By: nikic Differential Revision: https://reviews.llvm.org/D118118
1 parent 0a21fb3 commit 09515f2

19 files changed

+108
-102
lines changed

‎llvm/include/llvm/CodeGen/MachineInstr.h

Lines changed: 40 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -80,38 +80,39 @@ class MachineInstr
8080
};
8181

8282
enum MIFlag {
83-
NoFlags = 0,
84-
FrameSetup = 1 << 0, // Instruction is used as a part of
85-
// function frame setup code.
86-
FrameDestroy = 1 << 1, // Instruction is used as a part of
87-
// function frame destruction code.
88-
BundledPred = 1 << 2, // Instruction has bundled predecessors.
89-
BundledSucc = 1 << 3, // Instruction has bundled successors.
90-
FmNoNans = 1 << 4, // Instruction does not support Fast
91-
// math nan values.
92-
FmNoInfs = 1 << 5, // Instruction does not support Fast
93-
// math infinity values.
94-
FmNsz = 1 << 6, // Instruction is not required to retain
95-
// signed zero values.
96-
FmArcp = 1 << 7, // Instruction supports Fast math
97-
// reciprocal approximations.
98-
FmContract = 1 << 8, // Instruction supports Fast math
99-
// contraction operations like fma.
100-
FmAfn = 1 << 9, // Instruction may map to Fast math
101-
// intrinsic approximation.
102-
FmReassoc = 1 << 10, // Instruction supports Fast math
103-
// reassociation of operand order.
104-
NoUWrap = 1 << 11, // Instruction supports binary operator
105-
// no unsigned wrap.
106-
NoSWrap = 1 << 12, // Instruction supports binary operator
107-
// no signed wrap.
108-
IsExact = 1 << 13, // Instruction supports division is
109-
// known to be exact.
110-
NoFPExcept = 1 << 14, // Instruction does not raise
111-
// floatint-point exceptions.
112-
NoMerge = 1 << 15, // Passes that drop source location info
113-
// (e.g. branch folding) should skip
114-
// this instruction.
83+
NoFlags = 0,
84+
FrameSetup = 1 << 0, // Instruction is used as a part of
85+
// function frame setup code.
86+
FrameDestroy = 1 << 1, // Instruction is used as a part of
87+
// function frame destruction code.
88+
BundledPred = 1 << 2, // Instruction has bundled predecessors.
89+
BundledSucc = 1 << 3, // Instruction has bundled successors.
90+
FmNoNans = 1 << 4, // Instruction does not support Fast
91+
// math nan values.
92+
FmNoInfs = 1 << 5, // Instruction does not support Fast
93+
// math infinity values.
94+
FmNsz = 1 << 6, // Instruction is not required to retain
95+
// signed zero values.
96+
FmArcp = 1 << 7, // Instruction supports Fast math
97+
// reciprocal approximations.
98+
FmContract = 1 << 8, // Instruction supports Fast math
99+
// contraction operations like fma.
100+
FmAfn = 1 << 9, // Instruction may map to Fast math
101+
// intrinsic approximation.
102+
FmReassoc = 1 << 10, // Instruction supports Fast math
103+
// reassociation of operand order.
104+
NoUWrap = 1 << 11, // Instruction supports binary operator
105+
// no unsigned wrap.
106+
NoSWrap = 1 << 12, // Instruction supports binary operator
107+
// no signed wrap.
108+
IsExact = 1 << 13, // Instruction supports division is
109+
// known to be exact.
110+
NoFPExcept = 1 << 14, // Instruction does not raise
111+
// floatint-point exceptions.
112+
NoMerge = 1 << 15, // Passes that drop source location info
113+
// (e.g. branch folding) should skip
114+
// this instruction.
115+
Unpredictable = 1 << 16, // Instruction with unpredictable condition.
115116
};
116117

117118
private:
@@ -120,12 +121,10 @@ class MachineInstr
120121

121122
// Operands are allocated by an ArrayRecycler.
122123
MachineOperand *Operands = nullptr; // Pointer to the first operand.
123-
uint16_t NumOperands = 0; // Number of operands on instruction.
124-
125-
uint16_t Flags = 0; // Various bits of additional
124+
uint32_t Flags = 0; // Various bits of additional
126125
// information about machine
127126
// instruction.
128-
127+
uint16_t NumOperands = 0; // Number of operands on instruction.
129128
uint8_t AsmPrinterFlags = 0; // Various bits of information used by
130129
// the AsmPrinter to emit helpful
131130
// comments. This is *not* semantic
@@ -357,7 +356,7 @@ class MachineInstr
357356
}
358357

359358
/// Return the MI flags bitvector.
360-
uint16_t getFlags() const {
359+
uint32_t getFlags() const {
361360
return Flags;
362361
}
363362

@@ -368,7 +367,7 @@ class MachineInstr
368367

369368
/// Set a MI flag.
370369
void setFlag(MIFlag Flag) {
371-
Flags |= (uint16_t)Flag;
370+
Flags |= (uint32_t)Flag;
372371
}
373372

374373
void setFlags(unsigned flags) {
@@ -379,7 +378,7 @@ class MachineInstr
379378

380379
/// clearFlag - Clear a MI flag.
381380
void clearFlag(MIFlag Flag) {
382-
Flags &= ~((uint16_t)Flag);
381+
Flags &= ~((uint32_t)Flag);
383382
}
384383

385384
/// Return true if MI is in a bundle (but not the first MI in a bundle).
@@ -1889,9 +1888,9 @@ class MachineInstr
18891888
/// Return the MIFlags which represent both MachineInstrs. This
18901889
/// should be used when merging two MachineInstrs into one. This routine does
18911890
/// not modify the MIFlags of this MachineInstr.
1892-
uint16_t mergeFlagsWith(const MachineInstr& Other) const;
1891+
uint32_t mergeFlagsWith(const MachineInstr& Other) const;
18931892

1894-
static uint16_t copyFlagsFromInstruction(const Instruction &I);
1893+
static uint32_t copyFlagsFromInstruction(const Instruction &I);
18951894

18961895
/// Copy all flags to MachineInst MIFlags
18971896
void copyIRFlags(const Instruction &I);

‎llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -395,14 +395,16 @@ struct SDNodeFlags {
395395
// negative "NoFPExcept" flag here (that defaults to true) makes the flag
396396
// intersection logic more straightforward.
397397
bool NoFPExcept : 1;
398+
// Instructions with attached 'unpredictable' metadata on IR level.
399+
bool Unpredictable : 1;
398400

399401
public:
400402
/// Default constructor turns off all optimization flags.
401403
SDNodeFlags()
402404
: NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
403405
NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
404406
AllowContract(false), ApproximateFuncs(false),
405-
AllowReassociation(false), NoFPExcept(false) {}
407+
AllowReassociation(false), NoFPExcept(false), Unpredictable(false) {}
406408

407409
/// Propagate the fast-math-flags from an IR FPMathOperator.
408410
void copyFMF(const FPMathOperator &FPMO) {
@@ -427,6 +429,7 @@ struct SDNodeFlags {
427429
void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
428430
void setAllowReassociation(bool b) { AllowReassociation = b; }
429431
void setNoFPExcept(bool b) { NoFPExcept = b; }
432+
void setUnpredictable(bool b) { Unpredictable = b; }
430433

431434
// These are accessors for each flag.
432435
bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
@@ -440,6 +443,7 @@ struct SDNodeFlags {
440443
bool hasApproximateFuncs() const { return ApproximateFuncs; }
441444
bool hasAllowReassociation() const { return AllowReassociation; }
442445
bool hasNoFPExcept() const { return NoFPExcept; }
446+
bool hasUnpredictable() const { return Unpredictable; }
443447

444448
/// Clear any flags in this flag set that aren't also set in Flags. All
445449
/// flags will be cleared if Flags are undefined.
@@ -455,6 +459,7 @@ struct SDNodeFlags {
455459
ApproximateFuncs &= Flags.ApproximateFuncs;
456460
AllowReassociation &= Flags.AllowReassociation;
457461
NoFPExcept &= Flags.NoFPExcept;
462+
Unpredictable &= Flags.Unpredictable;
458463
}
459464
};
460465

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
300300
Register Op0 = getOrCreateVReg(*U.getOperand(0));
301301
Register Op1 = getOrCreateVReg(*U.getOperand(1));
302302
Register Res = getOrCreateVReg(U);
303-
uint16_t Flags = 0;
303+
uint32_t Flags = 0;
304304
if (isa<Instruction>(U)) {
305305
const Instruction &I = cast<Instruction>(U);
306306
Flags = MachineInstr::copyFlagsFromInstruction(I);
@@ -314,7 +314,7 @@ bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,
314314
MachineIRBuilder &MIRBuilder) {
315315
Register Op0 = getOrCreateVReg(*U.getOperand(0));
316316
Register Res = getOrCreateVReg(U);
317-
uint16_t Flags = 0;
317+
uint32_t Flags = 0;
318318
if (isa<Instruction>(U)) {
319319
const Instruction &I = cast<Instruction>(U);
320320
Flags = MachineInstr::copyFlagsFromInstruction(I);
@@ -345,7 +345,7 @@ bool IRTranslator::translateCompare(const User &U,
345345
MIRBuilder.buildCopy(
346346
Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
347347
else {
348-
uint16_t Flags = 0;
348+
uint32_t Flags = 0;
349349
if (CI)
350350
Flags = MachineInstr::copyFlagsFromInstruction(*CI);
351351
MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
@@ -1438,7 +1438,7 @@ bool IRTranslator::translateSelect(const User &U,
14381438
ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
14391439
ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
14401440

1441-
uint16_t Flags = 0;
1441+
uint32_t Flags = 0;
14421442
if (const SelectInst *SI = dyn_cast<SelectInst>(&U))
14431443
Flags = MachineInstr::copyFlagsFromInstruction(*SI);
14441444

@@ -1864,7 +1864,7 @@ bool IRTranslator::translateConstrainedFPIntrinsic(
18641864
if (!Opcode)
18651865
return false;
18661866

1867-
unsigned Flags = MachineInstr::copyFlagsFromInstruction(FPI);
1867+
uint32_t Flags = MachineInstr::copyFlagsFromInstruction(FPI);
18681868
if (EB == fp::ExceptionBehavior::ebIgnore)
18691869
Flags |= MachineInstr::NoFPExcept;
18701870

@@ -2370,7 +2370,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
23702370
return CLI->lowerCall(MIRBuilder, Info);
23712371
}
23722372
case Intrinsic::fptrunc_round: {
2373-
unsigned Flags = MachineInstr::copyFlagsFromInstruction(CI);
2373+
uint32_t Flags = MachineInstr::copyFlagsFromInstruction(CI);
23742374

23752375
// Convert the metadata argument to a constant integer
23762376
Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();

‎llvm/lib/CodeGen/MIRParser/MILexer.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
214214
.Case("nsw", MIToken::kw_nsw)
215215
.Case("exact", MIToken::kw_exact)
216216
.Case("nofpexcept", MIToken::kw_nofpexcept)
217+
.Case("unpredictable", MIToken::kw_unpredictable)
217218
.Case("debug-location", MIToken::kw_debug_location)
218219
.Case("debug-instr-number", MIToken::kw_debug_instr_number)
219220
.Case("dbg-instr-ref", MIToken::kw_dbg_instr_ref)

‎llvm/lib/CodeGen/MIRParser/MILexer.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ struct MIToken {
7373
kw_nsw,
7474
kw_exact,
7575
kw_nofpexcept,
76+
kw_unpredictable,
7677
kw_debug_location,
7778
kw_debug_instr_number,
7879
kw_dbg_instr_ref,

‎llvm/lib/CodeGen/MIRParser/MIParser.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1451,7 +1451,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
14511451
Token.is(MIToken::kw_nuw) ||
14521452
Token.is(MIToken::kw_nsw) ||
14531453
Token.is(MIToken::kw_exact) ||
1454-
Token.is(MIToken::kw_nofpexcept)) {
1454+
Token.is(MIToken::kw_nofpexcept) ||
1455+
Token.is(MIToken::kw_unpredictable)) {
14551456
// Mine frame and fast math flags
14561457
if (Token.is(MIToken::kw_frame_setup))
14571458
Flags |= MachineInstr::FrameSetup;
@@ -1479,6 +1480,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
14791480
Flags |= MachineInstr::IsExact;
14801481
if (Token.is(MIToken::kw_nofpexcept))
14811482
Flags |= MachineInstr::NoFPExcept;
1483+
if (Token.is(MIToken::kw_unpredictable))
1484+
Flags |= MachineInstr::Unpredictable;
14821485

14831486
lex();
14841487
}

‎llvm/lib/CodeGen/MIRPrinter.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -801,6 +801,8 @@ void MIPrinter::print(const MachineInstr &MI) {
801801
OS << "nofpexcept ";
802802
if (MI.getFlag(MachineInstr::NoMerge))
803803
OS << "nomerge ";
804+
if (MI.getFlag(MachineInstr::Unpredictable))
805+
OS << "unpredictable ";
804806

805807
OS << TII->getName(MI.getOpcode());
806808
if (I < E)

‎llvm/lib/CodeGen/MachineInstr.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -534,14 +534,14 @@ void MachineInstr::cloneInstrSymbols(MachineFunction &MF,
534534
setPCSections(MF, MI.getPCSections());
535535
}
536536

537-
uint16_t MachineInstr::mergeFlagsWith(const MachineInstr &Other) const {
537+
uint32_t MachineInstr::mergeFlagsWith(const MachineInstr &Other) const {
538538
// For now, the just return the union of the flags. If the flags get more
539539
// complicated over time, we might need more logic here.
540540
return getFlags() | Other.getFlags();
541541
}
542542

543-
uint16_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
544-
uint16_t MIFlags = 0;
543+
uint32_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
544+
uint32_t MIFlags = 0;
545545
// Copy the wrapping flags.
546546
if (const OverflowingBinaryOperator *OB =
547547
dyn_cast<OverflowingBinaryOperator>(&I)) {
@@ -575,6 +575,9 @@ uint16_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
575575
MIFlags |= MachineInstr::MIFlag::FmReassoc;
576576
}
577577

578+
if (I.getMetadata(LLVMContext::MD_unpredictable))
579+
MIFlags |= MachineInstr::MIFlag::Unpredictable;
580+
578581
return MIFlags;
579582
}
580583

‎llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1078,6 +1078,9 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
10781078

10791079
if (Flags.hasNoFPExcept())
10801080
MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
1081+
1082+
if (Flags.hasUnpredictable())
1083+
MI->setFlag(MachineInstr::MIFlag::Unpredictable);
10811084
}
10821085

10831086
// Emit all of the actual operands of this instruction, adding them to the

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3381,6 +3381,9 @@ void SelectionDAGBuilder::visitSelect(const User &I) {
33813381
if (auto *FPOp = dyn_cast<FPMathOperator>(&I))
33823382
Flags.copyFMF(*FPOp);
33833383

3384+
Flags.setUnpredictable(
3385+
cast<SelectInst>(I).getMetadata(LLVMContext::MD_unpredictable));
3386+
33843387
// Min/max matching is only viable if all output VTs are the same.
33853388
if (all_equal(ValueVTs)) {
33863389
EVT VT = ValueVTs[0];

‎llvm/lib/Target/AArch64/AArch64InstrInfo.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6889,7 +6889,7 @@ void AArch64InstrInfo::genAlternativeCodeSequence(
68896889

68906890
// Set the flags on the inserted instructions to be the merged flags of the
68916891
// instructions that we have combined.
6892-
uint16_t Flags = Root.getFlags();
6892+
uint32_t Flags = Root.getFlags();
68936893
if (MUL)
68946894
Flags = Root.mergeFlagsWith(*MUL);
68956895
for (auto *MI : InsInstrs)

‎llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
226226
MachineInstr &NewMI2) const {
227227
// Propagate FP flags from the original instructions.
228228
// But clear poison-generating flags because those may not be valid now.
229-
uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
229+
uint32_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
230230
NewMI1.setFlags(IntersectedFlags);
231231
NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
232232
NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
@@ -239,7 +239,7 @@ void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
239239
}
240240

241241
void PPCInstrInfo::setSpecialOperandAttr(MachineInstr &MI,
242-
uint16_t Flags) const {
242+
uint32_t Flags) const {
243243
MI.setFlags(Flags);
244244
MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
245245
MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
@@ -841,7 +841,7 @@ void PPCInstrInfo::reassociateFMA(
841841
}
842842
}
843843

844-
uint16_t IntersectedFlags = 0;
844+
uint32_t IntersectedFlags = 0;
845845
if (IsILPReassociate)
846846
IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
847847
else

‎llvm/lib/Target/PowerPC/PPCInstrInfo.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -518,7 +518,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
518518

519519
// PowerPC specific version of setSpecialOperandAttr that copies Flags to MI
520520
// and clears nuw, nsw, and exact flags.
521-
void setSpecialOperandAttr(MachineInstr &MI, uint16_t Flags) const;
521+
void setSpecialOperandAttr(MachineInstr &MI, uint32_t Flags) const;
522522

523523
bool isCoalescableExtInstr(const MachineInstr &MI,
524524
Register &SrcReg, Register &DstReg,

‎llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1344,7 +1344,7 @@ void RISCVInstrInfo::setSpecialOperandAttr(MachineInstr &OldMI1,
13441344
MachineInstr &OldMI2,
13451345
MachineInstr &NewMI1,
13461346
MachineInstr &NewMI2) const {
1347-
uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
1347+
uint32_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
13481348
NewMI1.setFlags(IntersectedFlags);
13491349
NewMI2.setFlags(IntersectedFlags);
13501350
}
@@ -1630,7 +1630,7 @@ static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
16301630

16311631
Register DstReg = Dst.getReg();
16321632
unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);
1633-
auto IntersectedFlags = Root.getFlags() & Prev.getFlags();
1633+
uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
16341634
DebugLoc MergedLoc =
16351635
DILocation::getMergedLocation(Root.getDebugLoc(), Prev.getDebugLoc());
16361636

‎llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ Register insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpirvTy,
217217
GR->assignSPIRVTypeToVReg(SpirvTy, NewReg, MIB.getMF());
218218
// Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to keep
219219
// the flags after instruction selection.
220-
const uint16_t Flags = Def->getFlags();
220+
const uint32_t Flags = Def->getFlags();
221221
MIB.buildInstr(SPIRV::ASSIGN_TYPE)
222222
.addDef(Reg)
223223
.addUse(NewReg)

0 commit comments

Comments
 (0)