@@ -1242,7 +1242,8 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
1242
1242
/// \returns
1243
1243
/// - The first instruction found with matching opcode
1244
1244
/// - nullptr if no matching instruction is found
1245
- Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL, unsigned Opcode) {
1245
+ static Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL,
1246
+ unsigned Opcode) {
1246
1247
for (Value *V : VL) {
1247
1248
if (isa<PoisonValue>(V))
1248
1249
continue;
@@ -2071,7 +2072,7 @@ class BoUpSLP {
2071
2072
OptimizationRemarkEmitter *getORE() { return ORE; }
2072
2073
2073
2074
/// This structure holds any data we need about the edges being traversed
2074
- /// during buildTree_rec (). We keep track of:
2075
+ /// during buildTreeRec (). We keep track of:
2075
2076
/// (i) the user TreeEntry index, and
2076
2077
/// (ii) the index of the edge.
2077
2078
struct EdgeInfo {
@@ -3491,8 +3492,8 @@ class BoUpSLP {
3491
3492
OrdersType &ReorderIndices) const;
3492
3493
3493
3494
/// This is the recursive part of buildTree.
3494
- void buildTree_rec (ArrayRef<Value *> Roots, unsigned Depth,
3495
- const EdgeInfo &EI, unsigned InterleaveFactor = 0);
3495
+ void buildTreeRec (ArrayRef<Value *> Roots, unsigned Depth, const EdgeInfo &EI ,
3496
+ unsigned InterleaveFactor = 0);
3496
3497
3497
3498
/// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
3498
3499
/// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -3810,7 +3811,7 @@ class BoUpSLP {
3810
3811
private:
3811
3812
/// The operands of each instruction in each lane Operands[op_index][lane].
3812
3813
/// Note: This helps avoid the replication of the code that performs the
3813
- /// reordering of operands during buildTree_rec () and vectorizeTree().
3814
+ /// reordering of operands during buildTreeRec () and vectorizeTree().
3814
3815
SmallVector<ValueList, 2> Operands;
3815
3816
3816
3817
/// MainOp and AltOp are recorded inside. S should be obtained from
@@ -4590,10 +4591,10 @@ class BoUpSLP {
4590
4591
};
4591
4592
4592
4593
#ifndef NDEBUG
4593
- friend inline raw_ostream &operator<<(raw_ostream &os ,
4594
+ friend inline raw_ostream &operator<<(raw_ostream &OS ,
4594
4595
const BoUpSLP::ScheduleData &SD) {
4595
- SD.dump(os );
4596
- return os ;
4596
+ SD.dump(OS );
4597
+ return OS ;
4597
4598
}
4598
4599
#endif
4599
4600
@@ -4694,10 +4695,10 @@ class BoUpSLP {
4694
4695
};
4695
4696
4696
4697
#ifndef NDEBUG
4697
- friend inline raw_ostream &operator<<(raw_ostream &os ,
4698
+ friend inline raw_ostream &operator<<(raw_ostream &OS ,
4698
4699
const BoUpSLP::ScheduleBundle &Bundle) {
4699
- Bundle.dump(os );
4700
- return os ;
4700
+ Bundle.dump(OS );
4701
+ return OS ;
4701
4702
}
4702
4703
#endif
4703
4704
@@ -8070,14 +8071,14 @@ void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
8070
8071
UserIgnoreList = &UserIgnoreLst;
8071
8072
if (!allSameType(Roots))
8072
8073
return;
8073
- buildTree_rec (Roots, 0, EdgeInfo());
8074
+ buildTreeRec (Roots, 0, EdgeInfo());
8074
8075
}
8075
8076
8076
8077
void BoUpSLP::buildTree(ArrayRef<Value *> Roots) {
8077
8078
deleteTree();
8078
8079
if (!allSameType(Roots))
8079
8080
return;
8080
- buildTree_rec (Roots, 0, EdgeInfo());
8081
+ buildTreeRec (Roots, 0, EdgeInfo());
8081
8082
}
8082
8083
8083
8084
/// Tries to find subvector of loads and builds new vector of only loads if can
@@ -8628,7 +8629,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
8628
8629
}))
8629
8630
continue;
8630
8631
unsigned Sz = VectorizableTree.size();
8631
- buildTree_rec (SubSlice, 0, EdgeInfo(), InterleaveFactor);
8632
+ buildTreeRec (SubSlice, 0, EdgeInfo(), InterleaveFactor);
8632
8633
if (Sz == VectorizableTree.size()) {
8633
8634
IsVectorized = false;
8634
8635
// Try non-interleaved vectorization with smaller vector
@@ -8683,7 +8684,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
8683
8684
inversePermutation(E.ReorderIndices, ReorderMask);
8684
8685
reorderScalars(GatheredScalars, ReorderMask);
8685
8686
}
8686
- buildTree_rec (GatheredScalars, 0, EdgeInfo());
8687
+ buildTreeRec (GatheredScalars, 0, EdgeInfo());
8687
8688
}
8688
8689
// If no new entries created, consider it as no gathered loads entries must be
8689
8690
// handled.
@@ -9977,9 +9978,9 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
9977
9978
return true;
9978
9979
}
9979
9980
9980
- void BoUpSLP::buildTree_rec (ArrayRef<Value *> VL, unsigned Depth,
9981
- const EdgeInfo &UserTreeIdx,
9982
- unsigned InterleaveFactor) {
9981
+ void BoUpSLP::buildTreeRec (ArrayRef<Value *> VL, unsigned Depth,
9982
+ const EdgeInfo &UserTreeIdx,
9983
+ unsigned InterleaveFactor) {
9983
9984
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
9984
9985
9985
9986
SmallVector<int> ReuseShuffleIndices;
@@ -10022,7 +10023,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10022
10023
} else {
10023
10024
TE->CombinedEntriesWithIndices.emplace_back(VectorizableTree.size(),
10024
10025
Idx == 0 ? 0 : Op1.size());
10025
- buildTree_rec (Op, Depth, {TE, Idx});
10026
+ buildTreeRec (Op, Depth, {TE, Idx});
10026
10027
}
10027
10028
};
10028
10029
AddNode(Op1, 0);
@@ -10115,12 +10116,12 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10115
10116
continue;
10116
10117
InstructionsState S = getSameOpcode(Op, *TLI);
10117
10118
if ((!S || S.getOpcode() != Instruction::PHI) || S.isAltShuffle())
10118
- buildTree_rec (Op, Depth + 1, {TE, I});
10119
+ buildTreeRec (Op, Depth + 1, {TE, I});
10119
10120
else
10120
10121
PHIOps.push_back(I);
10121
10122
}
10122
10123
for (unsigned I : PHIOps)
10123
- buildTree_rec (Operands[I], Depth + 1, {TE, I});
10124
+ buildTreeRec (Operands[I], Depth + 1, {TE, I});
10124
10125
};
10125
10126
switch (ShuffleOrOp) {
10126
10127
case Instruction::PHI: {
@@ -10164,7 +10165,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10164
10165
"(ExtractValueInst/ExtractElementInst).\n";
10165
10166
TE->dump());
10166
10167
// This is a special case, as it does not gather, but at the same time
10167
- // we are not extending buildTree_rec () towards the operands.
10168
+ // we are not extending buildTreeRec () towards the operands.
10168
10169
TE->setOperand(*this);
10169
10170
return;
10170
10171
}
@@ -10197,7 +10198,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10197
10198
TE->dump());
10198
10199
10199
10200
TE->setOperand(*this);
10200
- buildTree_rec (TE->getOperand(1), Depth + 1, {TE, 1});
10201
+ buildTreeRec (TE->getOperand(1), Depth + 1, {TE, 1});
10201
10202
return;
10202
10203
}
10203
10204
case Instruction::Load: {
@@ -10253,7 +10254,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10253
10254
}
10254
10255
TE->setOperand(*this);
10255
10256
if (State == TreeEntry::ScatterVectorize)
10256
- buildTree_rec (PointerOps, Depth + 1, {TE, 0});
10257
+ buildTreeRec (PointerOps, Depth + 1, {TE, 0});
10257
10258
return;
10258
10259
}
10259
10260
case Instruction::ZExt:
@@ -10294,7 +10295,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10294
10295
10295
10296
TE->setOperand(*this);
10296
10297
for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
10297
- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10298
+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
10298
10299
if (ShuffleOrOp == Instruction::Trunc) {
10299
10300
ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
10300
10301
} else if (ShuffleOrOp == Instruction::SIToFP ||
@@ -10349,8 +10350,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10349
10350
}
10350
10351
TE->setOperand(0, Left);
10351
10352
TE->setOperand(1, Right);
10352
- buildTree_rec (Left, Depth + 1, {TE, 0});
10353
- buildTree_rec (Right, Depth + 1, {TE, 1});
10353
+ buildTreeRec (Left, Depth + 1, {TE, 0});
10354
+ buildTreeRec (Right, Depth + 1, {TE, 1});
10354
10355
if (ShuffleOrOp == Instruction::ICmp) {
10355
10356
unsigned NumSignBits0 =
10356
10357
ComputeNumSignBits(VL0->getOperand(0), *DL, 0, AC, nullptr, DT);
@@ -10395,7 +10396,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10395
10396
10396
10397
TE->setOperand(*this, isa<BinaryOperator>(VL0) && isCommutative(VL0));
10397
10398
for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
10398
- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10399
+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
10399
10400
return;
10400
10401
}
10401
10402
case Instruction::GetElementPtr: {
@@ -10451,7 +10452,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10451
10452
TE->setOperand(IndexIdx, Operands.back());
10452
10453
10453
10454
for (unsigned I = 0, Ops = Operands.size(); I < Ops; ++I)
10454
- buildTree_rec (Operands[I], Depth + 1, {TE, I});
10455
+ buildTreeRec (Operands[I], Depth + 1, {TE, I});
10455
10456
return;
10456
10457
}
10457
10458
case Instruction::Store: {
@@ -10468,7 +10469,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10468
10469
dbgs() << "SLP: added a new TreeEntry (jumbled StoreInst).\n";
10469
10470
TE->dump());
10470
10471
TE->setOperand(*this);
10471
- buildTree_rec (TE->getOperand(0), Depth + 1, {TE, 0});
10472
+ buildTreeRec (TE->getOperand(0), Depth + 1, {TE, 0});
10472
10473
return;
10473
10474
}
10474
10475
case Instruction::Call: {
@@ -10487,7 +10488,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10487
10488
// vectorize it.
10488
10489
if (isVectorIntrinsicWithScalarOpAtArg(ID, I, TTI))
10489
10490
continue;
10490
- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10491
+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
10491
10492
}
10492
10493
return;
10493
10494
}
@@ -10540,14 +10541,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
10540
10541
}
10541
10542
TE->setOperand(0, Left);
10542
10543
TE->setOperand(1, Right);
10543
- buildTree_rec (Left, Depth + 1, {TE, 0});
10544
- buildTree_rec (Right, Depth + 1, {TE, 1});
10544
+ buildTreeRec (Left, Depth + 1, {TE, 0});
10545
+ buildTreeRec (Right, Depth + 1, {TE, 1});
10545
10546
return;
10546
10547
}
10547
10548
10548
10549
TE->setOperand(*this, isa<BinaryOperator>(VL0) || CI);
10549
10550
for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
10550
- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10551
+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
10551
10552
return;
10552
10553
}
10553
10554
default:
@@ -11626,7 +11627,7 @@ void BoUpSLP::transformNodes() {
11626
11627
unsigned PrevSize = VectorizableTree.size();
11627
11628
[[maybe_unused]] unsigned PrevEntriesSize =
11628
11629
LoadEntriesToVectorize.size();
11629
- buildTree_rec (Slice, 0, EdgeInfo(&E, UINT_MAX));
11630
+ buildTreeRec (Slice, 0, EdgeInfo(&E, UINT_MAX));
11630
11631
if (PrevSize + 1 == VectorizableTree.size() && !SameTE &&
11631
11632
VectorizableTree[PrevSize]->isGather() &&
11632
11633
VectorizableTree[PrevSize]->hasState() &&
@@ -15970,7 +15971,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
15970
15971
15971
15972
// LastInst can still be null at this point if there's either not an entry
15972
15973
// for BB in BlocksSchedules or there's no ScheduleData available for
15973
- // VL.back(). This can be the case if buildTree_rec aborts for various
15974
+ // VL.back(). This can be the case if buildTreeRec aborts for various
15974
15975
// reasons (e.g., the maximum recursion depth is reached, the maximum region
15975
15976
// size is reached, etc.). ScheduleData is initialized in the scheduling
15976
15977
// "dry-run".
@@ -15981,10 +15982,10 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
15981
15982
// last instruction in program order, LastInst will be set to Front, and we
15982
15983
// will visit all the remaining instructions in the block.
15983
15984
//
15984
- // One of the reasons we exit early from buildTree_rec is to place an upper
15985
+ // One of the reasons we exit early from buildTreeRec is to place an upper
15985
15986
// bound on compile-time. Thus, taking an additional compile-time hit here is
15986
15987
// not ideal. However, this should be exceedingly rare since it requires that
15987
- // we both exit early from buildTree_rec and that the bundle be out-of-order
15988
+ // we both exit early from buildTreeRec and that the bundle be out-of-order
15988
15989
// (causing us to iterate all the way to the end of the block).
15989
15990
if (!Res)
15990
15991
Res = FindLastInst();
@@ -21436,8 +21437,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
21436
21437
// not be useful.
21437
21438
R.getORE()->emit([&]() {
21438
21439
std::string TypeStr;
21439
- llvm::raw_string_ostream rso (TypeStr);
21440
- Ty->print(rso );
21440
+ llvm::raw_string_ostream OS (TypeStr);
21441
+ Ty->print(OS );
21441
21442
return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0)
21442
21443
<< "Cannot SLP vectorize list: type "
21443
21444
<< TypeStr + " is unsupported by vectorizer";
@@ -23331,20 +23332,20 @@ static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
23331
23332
} while (true);
23332
23333
}
23333
23334
23334
- static void findBuildAggregate_rec (Instruction *LastInsertInst,
23335
- TargetTransformInfo *TTI,
23336
- SmallVectorImpl<Value *> &BuildVectorOpds,
23337
- SmallVectorImpl<Value *> &InsertElts,
23338
- unsigned OperandOffset, const BoUpSLP &R) {
23335
+ static void findBuildAggregateRec (Instruction *LastInsertInst,
23336
+ TargetTransformInfo *TTI,
23337
+ SmallVectorImpl<Value *> &BuildVectorOpds,
23338
+ SmallVectorImpl<Value *> &InsertElts,
23339
+ unsigned OperandOffset, const BoUpSLP &R) {
23339
23340
do {
23340
23341
Value *InsertedOperand = LastInsertInst->getOperand(1);
23341
23342
std::optional<unsigned> OperandIndex =
23342
23343
getElementIndex(LastInsertInst, OperandOffset);
23343
23344
if (!OperandIndex || R.isDeleted(LastInsertInst))
23344
23345
return;
23345
23346
if (isa<InsertElementInst, InsertValueInst>(InsertedOperand)) {
23346
- findBuildAggregate_rec (cast<Instruction>(InsertedOperand), TTI,
23347
- BuildVectorOpds, InsertElts, *OperandIndex, R);
23347
+ findBuildAggregateRec (cast<Instruction>(InsertedOperand), TTI,
23348
+ BuildVectorOpds, InsertElts, *OperandIndex, R);
23348
23349
23349
23350
} else {
23350
23351
BuildVectorOpds[*OperandIndex] = InsertedOperand;
@@ -23389,8 +23390,7 @@ static bool findBuildAggregate(Instruction *LastInsertInst,
23389
23390
BuildVectorOpds.resize(*AggregateSize);
23390
23391
InsertElts.resize(*AggregateSize);
23391
23392
23392
- findBuildAggregate_rec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0,
23393
- R);
23393
+ findBuildAggregateRec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0, R);
23394
23394
llvm::erase(BuildVectorOpds, nullptr);
23395
23395
llvm::erase(InsertElts, nullptr);
23396
23396
if (BuildVectorOpds.size() >= 2)
0 commit comments