Skip to content

Commit fef3b8b

Browse files
ilovepiilovepi
committed
[𝘀𝗽𝗿] changes to main this commit is based on
Created using spr 1.3.4 [skip ci]
1 parent 6992433 commit fef3b8b

29 files changed

+191
-92
lines changed

‎clang/test/CodeGenCXX/attr-likelihood-if-vs-builtin-expect.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,5 +221,5 @@ void tu2(int &i) {
221221
}
222222
}
223223

224-
// CHECK: [[BW_LIKELY]] = !{!"branch_weights", i32 2000, i32 1}
225-
// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", i32 1, i32 2000}
224+
// CHECK: [[BW_LIKELY]] = !{!"branch_weights", !"expected", i32 2000, i32 1}
225+
// CHECK: [[BW_UNLIKELY]] = !{!"branch_weights", !"expected", i32 1, i32 2000}

‎llvm/docs/BranchWeightMetadata.rst

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,14 @@ Supported Instructions
2828

2929
Metadata is only assigned to the conditional branches. There are two extra
3030
operands for the true and the false branch.
31+
We optionally track if the metadata was added by ``__builtin_expect`` or
32+
``__builtin_expect_with_probability`` with an optional field ``!"expected"``.
3133

3234
.. code-block:: none
3335
3436
!0 = !{
3537
!"branch_weights",
38+
[ !"expected", ]
3639
i32 <TRUE_BRANCH_WEIGHT>,
3740
i32 <FALSE_BRANCH_WEIGHT>
3841
}
@@ -47,6 +50,7 @@ is always case #0).
4750
4851
!0 = !{
4952
!"branch_weights",
53+
[ !"expected", ]
5054
i32 <DEFAULT_BRANCH_WEIGHT>
5155
[ , i32 <CASE_BRANCH_WEIGHT> ... ]
5256
}
@@ -60,6 +64,7 @@ Branch weights are assigned to every destination.
6064
6165
!0 = !{
6266
!"branch_weights",
67+
[ !"expected", ]
6368
i32 <LABEL_BRANCH_WEIGHT>
6469
[ , i32 <LABEL_BRANCH_WEIGHT> ... ]
6570
}
@@ -75,6 +80,7 @@ block and entry counts which may not be accurate with sampling.
7580
7681
!0 = !{
7782
!"branch_weights",
83+
[ !"expected", ]
7884
i32 <CALL_BRANCH_WEIGHT>
7985
}
8086
@@ -95,6 +101,7 @@ is used.
95101
96102
!0 = !{
97103
!"branch_weights",
104+
[ !"expected", ]
98105
i32 <INVOKE_NORMAL_WEIGHT>
99106
[ , i32 <INVOKE_UNWIND_WEIGHT> ]
100107
}

‎llvm/include/llvm/IR/MDBuilder.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,11 @@ class MDBuilder {
5959
//===------------------------------------------------------------------===//
6060

6161
/// Return metadata containing two branch weights.
62-
MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight);
62+
/// @param TrueWeight the weight of the true branch
63+
/// @param FalseWeight the weight of the false branch
64+
/// @param Do these weights come from __builtin_expect*
65+
MDNode *createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight,
66+
bool IsExpected = false);
6367

6468
/// Return metadata containing two branch weights, with significant bias
6569
/// towards `true` destination.
@@ -70,7 +74,10 @@ class MDBuilder {
7074
MDNode *createUnlikelyBranchWeights();
7175

7276
/// Return metadata containing a number of branch weights.
73-
MDNode *createBranchWeights(ArrayRef<uint32_t> Weights);
77+
/// @param Weights the weights of all the branches
78+
/// @param Do these weights come from __builtin_expect*
79+
MDNode *createBranchWeights(ArrayRef<uint32_t> Weights,
80+
bool IsExpected = false);
7481

7582
/// Return metadata specifying that a branch or switch is unpredictable.
7683
MDNode *createUnpredictable();

‎llvm/include/llvm/IR/ProfDataUtils.h

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,19 @@ MDNode *getBranchWeightMDNode(const Instruction &I);
5555
/// Nullptr otherwise.
5656
MDNode *getValidBranchWeightMDNode(const Instruction &I);
5757

58+
/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
59+
/// intrinsic
60+
bool hasBranchWeightProvenance(const Instruction &I);
61+
62+
/// Check if Branch Weight Metadata has an "expected" field from an llvm.expect*
63+
/// intrinsic
64+
bool hasBranchWeightProvenance(const MDNode *ProfileData);
65+
66+
/// Return the offset to the first branch weight data
67+
unsigned getBranchWeightOffset(const MDNode *ProfileData);
68+
69+
unsigned getNumBranchWeights(const MDNode &ProfileData);
70+
5871
/// Extract branch weights from MD_prof metadata
5972
///
6073
/// \param ProfileData A pointer to an MDNode.
@@ -111,7 +124,11 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalWeights);
111124

112125
/// Create a new `branch_weights` metadata node and add or overwrite
113126
/// a `prof` metadata reference to instruction `I`.
114-
void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights);
127+
/// \param I the Instruction to set branch weights on.
128+
/// \param Weights an array of weights to set on instruction I.
129+
/// \param IsExpected were these weights added from an llvm.expect* intrinsic.
130+
void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
131+
bool IsExpected);
115132

116133
/// Scaling the profile data attached to 'I' using the ratio of S/T.
117134
void scaleProfData(Instruction &I, uint64_t S, uint64_t T);

‎llvm/lib/CodeGen/CodeGenPrepare.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8866,7 +8866,8 @@ bool CodeGenPrepare::splitBranchCondition(Function &F, ModifyDT &ModifiedDT) {
88668866
scaleWeights(NewTrueWeight, NewFalseWeight);
88678867
Br1->setMetadata(LLVMContext::MD_prof,
88688868
MDBuilder(Br1->getContext())
8869-
.createBranchWeights(TrueWeight, FalseWeight));
8869+
.createBranchWeights(TrueWeight, FalseWeight,
8870+
hasBranchWeightProvenance(*Br1)));
88708871

88718872
NewTrueWeight = TrueWeight;
88728873
NewFalseWeight = 2 * FalseWeight;

‎llvm/lib/IR/Instruction.cpp

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1266,12 +1266,21 @@ Instruction *Instruction::cloneImpl() const {
12661266

12671267
void Instruction::swapProfMetadata() {
12681268
MDNode *ProfileData = getBranchWeightMDNode(*this);
1269-
if (!ProfileData || ProfileData->getNumOperands() != 3)
1269+
if (!isBranchWeightMD(ProfileData))
12701270
return;
12711271

1272-
// The first operand is the name. Fetch them backwards and build a new one.
1273-
Metadata *Ops[] = {ProfileData->getOperand(0), ProfileData->getOperand(2),
1274-
ProfileData->getOperand(1)};
1272+
SmallVector<Metadata *, 4> Ops;
1273+
unsigned FirstIdx = getBranchWeightOffset(ProfileData);
1274+
unsigned SecondIdx = FirstIdx + 1;
1275+
// If there are more weights past the second, we can't swap them
1276+
if (ProfileData->getNumOperands() > SecondIdx + 1)
1277+
return;
1278+
for (unsigned Idx = 0; Idx < FirstIdx; ++Idx) {
1279+
Ops.push_back(ProfileData->getOperand(Idx));
1280+
}
1281+
// Switch the order of the weights
1282+
Ops.push_back(ProfileData->getOperand(SecondIdx));
1283+
Ops.push_back(ProfileData->getOperand(FirstIdx));
12751284
setMetadata(LLVMContext::MD_prof,
12761285
MDNode::get(ProfileData->getContext(), Ops));
12771286
}

‎llvm/lib/IR/Instructions.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5165,7 +5165,7 @@ void SwitchInstProfUpdateWrapper::init() {
51655165
if (!ProfileData)
51665166
return;
51675167

5168-
if (ProfileData->getNumOperands() != SI.getNumSuccessors() + 1) {
5168+
if (getNumBranchWeights(*ProfileData) != SI.getNumSuccessors()) {
51695169
llvm_unreachable("number of prof branch_weights metadata operands does "
51705170
"not correspond to number of succesors");
51715171
}

‎llvm/lib/IR/MDBuilder.cpp

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ MDNode *MDBuilder::createFPMath(float Accuracy) {
3535
}
3636

3737
MDNode *MDBuilder::createBranchWeights(uint32_t TrueWeight,
38-
uint32_t FalseWeight) {
39-
return createBranchWeights({TrueWeight, FalseWeight});
38+
uint32_t FalseWeight, bool IsExpected) {
39+
return createBranchWeights({TrueWeight, FalseWeight}, IsExpected);
4040
}
4141

4242
MDNode *MDBuilder::createLikelyBranchWeights() {
@@ -49,15 +49,19 @@ MDNode *MDBuilder::createUnlikelyBranchWeights() {
4949
return createBranchWeights(1, (1U << 20) - 1);
5050
}
5151

52-
MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights) {
52+
MDNode *MDBuilder::createBranchWeights(ArrayRef<uint32_t> Weights,
53+
bool IsExpected) {
5354
assert(Weights.size() >= 1 && "Need at least one branch weights!");
5455

55-
SmallVector<Metadata *, 4> Vals(Weights.size() + 1);
56+
unsigned int Offset = IsExpected ? 2 : 1;
57+
SmallVector<Metadata *, 4> Vals(Weights.size() + Offset);
5658
Vals[0] = createString("branch_weights");
59+
if (IsExpected)
60+
Vals[1] = createString("expected");
5761

5862
Type *Int32Ty = Type::getInt32Ty(Context);
5963
for (unsigned i = 0, e = Weights.size(); i != e; ++i)
60-
Vals[i + 1] = createConstant(ConstantInt::get(Int32Ty, Weights[i]));
64+
Vals[i + Offset] = createConstant(ConstantInt::get(Int32Ty, Weights[i]));
6165

6266
return MDNode::get(Context, Vals);
6367
}

‎llvm/lib/IR/ProfDataUtils.cpp

Lines changed: 51 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,6 @@ namespace {
4040
// We maintain some constants here to ensure that we access the branch weights
4141
// correctly, and can change the behavior in the future if the layout changes
4242

43-
// The index at which the weights vector starts
44-
constexpr unsigned WeightsIdx = 1;
45-
4643
// the minimum number of operands for MD_prof nodes with branch weights
4744
constexpr unsigned MinBWOps = 3;
4845

@@ -85,6 +82,27 @@ static void extractFromBranchWeightMD(const MDNode *ProfileData,
8582
}
8683
}
8784

85+
template <typename T,
86+
typename = typename std::enable_if<std::is_arithmetic_v<T>>>
87+
static void extractFromBranchWeightMD(const MDNode *ProfileData,
88+
SmallVectorImpl<T> &Weights) {
89+
assert(isBranchWeightMD(ProfileData) && "wrong metadata");
90+
91+
unsigned NOps = ProfileData->getNumOperands();
92+
unsigned WeightsIdx = getBranchWeightOffset(ProfileData);
93+
assert(WeightsIdx < NOps && "Weights Index must be less than NOps.");
94+
Weights.resize(NOps - WeightsIdx);
95+
96+
for (unsigned Idx = WeightsIdx, E = NOps; Idx != E; ++Idx) {
97+
ConstantInt *Weight =
98+
mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));
99+
assert(Weight && "Malformed branch_weight in MD_prof node");
100+
assert(Weight->getValue().getActiveBits() <= (sizeof(T) * 8) &&
101+
"Too many bits for MD_prof branch_weight");
102+
Weights[Idx - WeightsIdx] = Weight->getZExtValue();
103+
}
104+
}
105+
88106
} // namespace
89107

90108
namespace llvm {
@@ -106,6 +124,29 @@ bool hasValidBranchWeightMD(const Instruction &I) {
106124
return getValidBranchWeightMDNode(I);
107125
}
108126

127+
bool hasBranchWeightProvenance(const Instruction &I) {
128+
auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
129+
return hasBranchWeightProvenance(ProfileData);
130+
}
131+
132+
bool hasBranchWeightProvenance(const MDNode *ProfileData) {
133+
if (!isBranchWeightMD(ProfileData))
134+
return false;
135+
auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(1));
136+
// NOTE: if we ever have more types of branch weight provenance,
137+
// we need to check the string value is "expected". For now, we
138+
// supply a more generic API, and avoid the spurious comparisons.
139+
return ProfDataName;
140+
}
141+
142+
unsigned getBranchWeightOffset(const MDNode *ProfileData) {
143+
return hasBranchWeightProvenance(ProfileData) ? 2 : 1;
144+
}
145+
146+
unsigned getNumBranchWeights(const MDNode &ProfileData) {
147+
return ProfileData.getNumOperands() - getBranchWeightOffset(&ProfileData);
148+
}
149+
109150
MDNode *getBranchWeightMDNode(const Instruction &I) {
110151
auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
111152
if (!isBranchWeightMD(ProfileData))
@@ -115,7 +156,7 @@ MDNode *getBranchWeightMDNode(const Instruction &I) {
115156

116157
MDNode *getValidBranchWeightMDNode(const Instruction &I) {
117158
auto *ProfileData = getBranchWeightMDNode(I);
118-
if (ProfileData && ProfileData->getNumOperands() == 1 + I.getNumSuccessors())
159+
if (ProfileData && getNumBranchWeights(*ProfileData) == I.getNumSuccessors())
119160
return ProfileData;
120161
return nullptr;
121162
}
@@ -173,8 +214,9 @@ bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalVal) {
173214
if (!ProfDataName)
174215
return false;
175216

176-
if (ProfDataName->getString() == "branch_weights") {
177-
for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx++) {
217+
if (ProfDataName->getString().equals("branch_weights")) {
218+
unsigned Offset = getBranchWeightOffset(ProfileData);
219+
for (unsigned Idx = Offset; Idx < ProfileData->getNumOperands(); ++Idx) {
178220
auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(Idx));
179221
assert(V && "Malformed branch_weight in MD_prof node");
180222
TotalVal += V->getValue().getZExtValue();
@@ -195,9 +237,10 @@ bool extractProfTotalWeight(const Instruction &I, uint64_t &TotalVal) {
195237
return extractProfTotalWeight(I.getMetadata(LLVMContext::MD_prof), TotalVal);
196238
}
197239

198-
void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights) {
240+
void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
241+
bool IsExpected) {
199242
MDBuilder MDB(I.getContext());
200-
MDNode *BranchWeights = MDB.createBranchWeights(Weights);
243+
MDNode *BranchWeights = MDB.createBranchWeights(Weights, IsExpected);
201244
I.setMetadata(LLVMContext::MD_prof, BranchWeights);
202245
}
203246

‎llvm/lib/IR/Verifier.cpp

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
#include "llvm/IR/Module.h"
105105
#include "llvm/IR/ModuleSlotTracker.h"
106106
#include "llvm/IR/PassManager.h"
107+
#include "llvm/IR/ProfDataUtils.h"
107108
#include "llvm/IR/Statepoint.h"
108109
#include "llvm/IR/Type.h"
109110
#include "llvm/IR/Use.h"
@@ -4785,9 +4786,10 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
47854786
StringRef ProfName = MDS->getString();
47864787

47874788
// Check consistency of !prof branch_weights metadata.
4788-
if (ProfName == "branch_weights") {
4789+
if (ProfName.equals("branch_weights")) {
4790+
unsigned NumBranchWeights = getNumBranchWeights(*MD);
47894791
if (isa<InvokeInst>(&I)) {
4790-
Check(MD->getNumOperands() == 2 || MD->getNumOperands() == 3,
4792+
Check(NumBranchWeights == 1 || NumBranchWeights == 2,
47914793
"Wrong number of InvokeInst branch_weights operands", MD);
47924794
} else {
47934795
unsigned ExpectedNumOperands = 0;
@@ -4807,10 +4809,11 @@ void Verifier::visitProfMetadata(Instruction &I, MDNode *MD) {
48074809
CheckFailed("!prof branch_weights are not allowed for this instruction",
48084810
MD);
48094811

4810-
Check(MD->getNumOperands() == 1 + ExpectedNumOperands,
4812+
Check(NumBranchWeights == ExpectedNumOperands,
48114813
"Wrong number of operands", MD);
48124814
}
4813-
for (unsigned i = 1; i < MD->getNumOperands(); ++i) {
4815+
for (unsigned i = getBranchWeightOffset(MD); i < MD->getNumOperands();
4816+
++i) {
48144817
auto &MDO = MD->getOperand(i);
48154818
Check(MDO, "second operand should not be null", MD);
48164819
Check(mdconst::dyn_extract<ConstantInt>(MDO),

‎llvm/lib/Transforms/IPO/SampleProfile.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1661,7 +1661,8 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
16611661
else if (OverwriteExistingWeights)
16621662
I.setMetadata(LLVMContext::MD_prof, nullptr);
16631663
} else if (!isa<IntrinsicInst>(&I)) {
1664-
setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])});
1664+
setBranchWeights(I, {static_cast<uint32_t>(BlockWeights[BB])},
1665+
/*IsExpected=*/false);
16651666
}
16661667
}
16671668
} else if (OverwriteExistingWeights || ProfileSampleBlockAccurate) {
@@ -1672,7 +1673,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
16721673
if (cast<CallBase>(I).isIndirectCall()) {
16731674
I.setMetadata(LLVMContext::MD_prof, nullptr);
16741675
} else {
1675-
setBranchWeights(I, {uint32_t(0)});
1676+
setBranchWeights(I, {uint32_t(0)}, /*IsExpected=*/false);
16761677
}
16771678
}
16781679
}
@@ -1753,7 +1754,7 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) {
17531754
if (MaxWeight > 0 &&
17541755
(!TI->extractProfTotalWeight(TempWeight) || OverwriteExistingWeights)) {
17551756
LLVM_DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n");
1756-
setBranchWeights(*TI, Weights);
1757+
setBranchWeights(*TI, Weights, /*IsExpected=*/false);
17571758
ORE->emit([&]() {
17581759
return OptimizationRemark(DEBUG_TYPE, "PopularDest", MaxDestInst)
17591760
<< "most popular destination for conditional branches at "

‎llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1878,7 +1878,7 @@ void CHR::fixupBranchesAndSelects(CHRScope *Scope,
18781878
static_cast<uint32_t>(CHRBranchBias.scale(1000)),
18791879
static_cast<uint32_t>(CHRBranchBias.getCompl().scale(1000)),
18801880
};
1881-
setBranchWeights(*MergedBR, Weights);
1881+
setBranchWeights(*MergedBR, Weights, /*IsExpected=*/false);
18821882
CHR_DEBUG(dbgs() << "CHR branch bias " << Weights[0] << ":" << Weights[1]
18831883
<< "\n");
18841884
}

‎llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,8 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee,
259259
promoteCallWithIfThenElse(CB, DirectCallee, BranchWeights);
260260

261261
if (AttachProfToDirectCall) {
262-
setBranchWeights(NewInst, {static_cast<uint32_t>(Count)});
262+
setBranchWeights(NewInst, {static_cast<uint32_t>(Count)},
263+
/*IsExpected=*/false);
263264
}
264265

265266
using namespace ore;

‎llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1422,7 +1422,8 @@ void PGOUseFunc::populateCoverage(IndexedInstrProfReader *PGOReader) {
14221422
for (auto *Succ : successors(&BB))
14231423
Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);
14241424
if (Weights.size() >= 2)
1425-
llvm::setBranchWeights(*BB.getTerminator(), Weights);
1425+
llvm::setBranchWeights(*BB.getTerminator(), Weights,
1426+
/*IsExpected=*/false);
14261427
}
14271428

14281429
unsigned NumCorruptCoverage = 0;
@@ -2206,7 +2207,7 @@ void llvm::setProfMetadata(Module *M, Instruction *TI,
22062207

22072208
misexpect::checkExpectAnnotations(*TI, Weights, /*IsFrontend=*/false);
22082209

2209-
setBranchWeights(*TI, Weights);
2210+
setBranchWeights(*TI, Weights, /*IsExpected=*/false);
22102211
if (EmitBranchProbability) {
22112212
std::string BrCondStr = getBranchCondString(TI);
22122213
if (BrCondStr.empty())

0 commit comments

Comments
 (0)