Skip to content

Commit da16934

Browse files
tianqingwtianqingw
committed
[SelectionDAG][X86] Preserve unpredictable metadata for conditional
branches in SelectionDAG, as well as JCCs generated by X86 backend. This builds on 09515f2, which preserves unpredictable metadata in CodeGen for `select`. This patch does it for conditional branches.
1 parent 65b4a77 commit da16934

File tree

8 files changed

+72
-39
lines changed

8 files changed

+72
-39
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAG.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1165,8 +1165,13 @@ class SelectionDAG {
11651165
SDValue N2, SDValue N3, const SDNodeFlags Flags);
11661166
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
11671167
SDValue N2, SDValue N3, SDValue N4);
1168+
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1169+
SDValue N2, SDValue N3, SDValue N4, const SDNodeFlags Flags);
11681170
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
11691171
SDValue N2, SDValue N3, SDValue N4, SDValue N5);
1172+
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
1173+
SDValue N2, SDValue N3, SDValue N4, SDValue N5,
1174+
const SDNodeFlags Flags);
11701175

11711176
// Specialize again based on number of operands for nodes with a VTList
11721177
// rather than a single VT.

‎llvm/include/llvm/CodeGen/SwitchLoweringUtils.h

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -137,29 +137,34 @@ struct CaseBlock {
137137
SDLoc DL;
138138
DebugLoc DbgLoc;
139139

140-
// Branch weights.
140+
// Branch weights and predictability.
141141
BranchProbability TrueProb, FalseProb;
142+
bool IsUnpredictable;
142143

143144
// Constructor for SelectionDAG.
144145
CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
145146
const Value *cmpmiddle, MachineBasicBlock *truebb,
146147
MachineBasicBlock *falsebb, MachineBasicBlock *me, SDLoc dl,
147148
BranchProbability trueprob = BranchProbability::getUnknown(),
148-
BranchProbability falseprob = BranchProbability::getUnknown())
149+
BranchProbability falseprob = BranchProbability::getUnknown(),
150+
bool isunpredictable = false)
149151
: CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs),
150152
TrueBB(truebb), FalseBB(falsebb), ThisBB(me), DL(dl),
151-
TrueProb(trueprob), FalseProb(falseprob) {}
153+
TrueProb(trueprob), FalseProb(falseprob),
154+
IsUnpredictable(isunpredictable) {}
152155

153156
// Constructor for GISel.
154157
CaseBlock(CmpInst::Predicate pred, bool nocmp, const Value *cmplhs,
155158
const Value *cmprhs, const Value *cmpmiddle,
156159
MachineBasicBlock *truebb, MachineBasicBlock *falsebb,
157160
MachineBasicBlock *me, DebugLoc dl,
158161
BranchProbability trueprob = BranchProbability::getUnknown(),
159-
BranchProbability falseprob = BranchProbability::getUnknown())
162+
BranchProbability falseprob = BranchProbability::getUnknown(),
163+
bool isunpredictable = false)
160164
: PredInfo({pred, nocmp}), CmpLHS(cmplhs), CmpMHS(cmpmiddle),
161165
CmpRHS(cmprhs), TrueBB(truebb), FalseBB(falsebb), ThisBB(me),
162-
DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob) {}
166+
DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob),
167+
IsUnpredictable(isunpredictable) {}
163168
};
164169

165170
struct JumpTable {

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18164,7 +18164,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
1816418164
// nondeterministic jumps).
1816518165
if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) {
1816618166
return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain,
18167-
N1->getOperand(0), N2);
18167+
N1->getOperand(0), N2, N->getFlags());
1816818168
}
1816918169

1817018170
// Variant of the previous fold where there is a SETCC in between:
@@ -18213,7 +18213,8 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
1821318213
if (Updated)
1821418214
return DAG.getNode(
1821518215
ISD::BRCOND, SDLoc(N), MVT::Other, Chain,
18216-
DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2);
18216+
DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2,
18217+
N->getFlags());
1821718218
}
1821818219

1821918220
// If N is a constant we could fold this into a fallthrough or unconditional
@@ -18238,7 +18239,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
1823818239
HandleSDNode ChainHandle(Chain);
1823918240
if (SDValue NewN1 = rebuildSetCC(N1))
1824018241
return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other,
18241-
ChainHandle.getValue(), NewN1, N2);
18242+
ChainHandle.getValue(), NewN1, N2, N->getFlags());
1824218243
}
1824318244

1824418245
return SDValue();

‎llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1065,14 +1065,17 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
10651065
// Create the new machine instruction.
10661066
MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
10671067

1068+
// Transfer IR flags from the SDNode to the MachineInstr
1069+
MachineInstr *MI = MIB.getInstr();
1070+
const SDNodeFlags Flags = Node->getFlags();
1071+
if (Flags.hasUnpredictable())
1072+
MI->setFlag(MachineInstr::MIFlag::Unpredictable);
1073+
10681074
// Add result register values for things that are defined by this
10691075
// instruction.
10701076
if (NumResults) {
10711077
CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
10721078

1073-
// Transfer any IR flags from the SDNode to the MachineInstr
1074-
MachineInstr *MI = MIB.getInstr();
1075-
const SDNodeFlags Flags = Node->getFlags();
10761079
if (Flags.hasNoSignedZeros())
10771080
MI->setFlag(MachineInstr::MIFlag::FmNsz);
10781081

@@ -1105,9 +1108,6 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
11051108

11061109
if (Flags.hasNoFPExcept())
11071110
MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
1108-
1109-
if (Flags.hasUnpredictable())
1110-
MI->setFlag(MachineInstr::MIFlag::Unpredictable);
11111111
}
11121112

11131113
// Emit all of the actual operands of this instruction, adding them to the

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7613,16 +7613,34 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
76137613
}
76147614

76157615
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7616-
SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7616+
SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7617+
const SDNodeFlags Flags) {
76177618
SDValue Ops[] = { N1, N2, N3, N4 };
7618-
return getNode(Opcode, DL, VT, Ops);
7619+
return getNode(Opcode, DL, VT, Ops, Flags);
7620+
}
7621+
7622+
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7623+
SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7624+
SDNodeFlags Flags;
7625+
if (Inserter)
7626+
Flags = Inserter->getFlags();
7627+
return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
76197628
}
76207629

76217630
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
76227631
SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7623-
SDValue N5) {
7632+
SDValue N5, const SDNodeFlags Flags) {
76247633
SDValue Ops[] = { N1, N2, N3, N4, N5 };
7625-
return getNode(Opcode, DL, VT, Ops);
7634+
return getNode(Opcode, DL, VT, Ops, Flags);
7635+
}
7636+
7637+
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7638+
SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7639+
SDValue N5) {
7640+
SDNodeFlags Flags;
7641+
if (Inserter)
7642+
Flags = Inserter->getFlags();
7643+
return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
76267644
}
76277645

76287646
/// getStackArgumentTokenFactor - Compute a TokenFactor to force all

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2818,9 +2818,10 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
28182818
// je foo
28192819
// cmp D, E
28202820
// jle foo
2821+
bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
28212822
const Instruction *BOp = dyn_cast<Instruction>(CondVal);
28222823
if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
2823-
BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
2824+
BOp->hasOneUse() && !IsUnpredictable) {
28242825
Value *Vec;
28252826
const Value *BOp0, *BOp1;
28262827
Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
@@ -2869,7 +2870,9 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
28692870

28702871
// Create a CaseBlock record representing this branch.
28712872
CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2872-
nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2873+
nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
2874+
BranchProbability::getUnknown(), BranchProbability::getUnknown(),
2875+
IsUnpredictable);
28732876

28742877
// Use visitSwitchCase to actually insert the fast branch sequence for this
28752878
// cond branch.
@@ -2957,9 +2960,10 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
29572960
Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
29582961
}
29592962

2960-
SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2961-
MVT::Other, getControlRoot(), Cond,
2962-
DAG.getBasicBlock(CB.TrueBB));
2963+
SDNodeFlags Flags;
2964+
Flags.setUnpredictable(CB.IsUnpredictable);
2965+
SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
2966+
Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
29632967

29642968
setValue(CurInst, BrCond);
29652969

‎llvm/lib/Target/X86/X86ISelLowering.cpp

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24827,14 +24827,14 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
2482724827

2482824828
SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
2482924829
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24830-
Overflow);
24830+
Overflow, Op->getFlags());
2483124831
}
2483224832

2483324833
if (LHS.getSimpleValueType().isInteger()) {
2483424834
SDValue CCVal;
2483524835
SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
2483624836
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24837-
EFLAGS);
24837+
EFLAGS, Op->getFlags());
2483824838
}
2483924839

2484024840
if (CC == ISD::SETOEQ) {
@@ -24860,10 +24860,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
2486024860
DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
2486124861
SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
2486224862
Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
24863-
CCVal, Cmp);
24863+
CCVal, Cmp, Op->getFlags());
2486424864
CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
2486524865
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24866-
Cmp);
24866+
Cmp, Op->getFlags());
2486724867
}
2486824868
}
2486924869
} else if (CC == ISD::SETUNE) {
@@ -24872,18 +24872,18 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
2487224872
// separate test.
2487324873
SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
2487424874
SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
24875-
Chain =
24876-
DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp);
24875+
Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24876+
Cmp, Op->getFlags());
2487724877
CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
2487824878
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24879-
Cmp);
24879+
Cmp, Op->getFlags());
2488024880
} else {
2488124881
X86::CondCode X86Cond =
2488224882
TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
2488324883
SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
2488424884
SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
2488524885
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24886-
Cmp);
24886+
Cmp, Op->getFlags());
2488724887
}
2488824888
}
2488924889

@@ -24894,7 +24894,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
2489424894

2489524895
SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
2489624896
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24897-
Overflow);
24897+
Overflow, Op->getFlags());
2489824898
}
2489924899

2490024900
// Look past the truncate if the high bits are known zero.
@@ -24913,8 +24913,8 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
2491324913

2491424914
SDValue CCVal;
2491524915
SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
24916-
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
24917-
EFLAGS);
24916+
return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, EFLAGS,
24917+
Op->getFlags());
2491824918
}
2491924919

2492024920
// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.

‎llvm/test/CodeGen/X86/unpredictable-brcond.ll

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
2-
; Currently, unpredictable metadata on conditional branches is lost during CodeGen.
2+
; Make sure MIR generated for conditional branch with unpredictable metadata has unpredictable flag.
33
; RUN: llc -mtriple=x86_64-unknown-linux-gnu -stop-after=finalize-isel < %s | FileCheck %s
44

55
define void @cond_branch_1(i1 %cond) {
@@ -11,7 +11,7 @@ define void @cond_branch_1(i1 %cond) {
1111
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
1212
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
1313
; CHECK-NEXT: TEST8ri killed [[COPY1]], 1, implicit-def $eflags
14-
; CHECK-NEXT: JCC_1 %bb.2, 4, implicit $eflags
14+
; CHECK-NEXT: unpredictable JCC_1 %bb.2, 4, implicit $eflags
1515
; CHECK-NEXT: JMP_1 %bb.1
1616
; CHECK-NEXT: {{ $}}
1717
; CHECK-NEXT: bb.1.then:
@@ -51,7 +51,7 @@ define void @cond_branch_2(double %a, double %b, i32 %c, i32 %d) nounwind {
5151
; CHECK-NEXT: [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
5252
; CHECK-NEXT: [[OR8rr:%[0-9]+]]:gr8 = OR8rr [[SETCCr]], killed [[SETCCr1]], implicit-def dead $eflags
5353
; CHECK-NEXT: TEST8rr [[OR8rr]], [[OR8rr]], implicit-def $eflags
54-
; CHECK-NEXT: JCC_1 %bb.2, 5, implicit $eflags
54+
; CHECK-NEXT: unpredictable JCC_1 %bb.2, 5, implicit $eflags
5555
; CHECK-NEXT: JMP_1 %bb.1
5656
; CHECK-NEXT: {{ $}}
5757
; CHECK-NEXT: bb.1.true:
@@ -89,8 +89,8 @@ define void @isint_branch(double %d) nounwind {
8989
; CHECK-NEXT: [[CVTDQ2PDrr:%[0-9]+]]:vr128 = CVTDQ2PDrr killed [[CVTTPD2DQrr]]
9090
; CHECK-NEXT: [[COPY2:%[0-9]+]]:fr64 = COPY [[CVTDQ2PDrr]]
9191
; CHECK-NEXT: nofpexcept UCOMISDrr [[COPY]], killed [[COPY2]], implicit-def $eflags, implicit $mxcsr
92-
; CHECK-NEXT: JCC_1 %bb.2, 5, implicit $eflags
93-
; CHECK-NEXT: JCC_1 %bb.2, 10, implicit $eflags
92+
; CHECK-NEXT: unpredictable JCC_1 %bb.2, 5, implicit $eflags
93+
; CHECK-NEXT: unpredictable JCC_1 %bb.2, 10, implicit $eflags
9494
; CHECK-NEXT: JMP_1 %bb.1
9595
; CHECK-NEXT: {{ $}}
9696
; CHECK-NEXT: bb.1.true:

0 commit comments

Comments
 (0)