Skip to content

Commit d8a2f21

Browse files
committed
[SLP] Use named structs in vectorizeStores() (NFC)
This is a mostly straightforward replacement of the previous std::pair<int, std::set<std::pair<...>>> data structure with slightly more readable alternatives.
1 parent 5c02f1a commit d8a2f21

File tree

1 file changed

+61
-45
lines changed

1 file changed

+61
-45
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 61 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -19923,31 +19923,51 @@ bool SLPVectorizerPass::vectorizeStores(
1992319923
BoUpSLP::ValueSet VectorizedStores;
1992419924
bool Changed = false;
1992519925

19926+
/// A store instruction and the distance of its address to a base pointer.
19927+
struct CandidateInstr {
19928+
CandidateInstr(unsigned InstrIdx, int DistToBasePtr)
19929+
: InstrIdx(InstrIdx), DistToBasePtr(DistToBasePtr) {}
19930+
unsigned InstrIdx;
19931+
int DistToBasePtr;
19932+
};
1992619933
struct StoreDistCompare {
19927-
bool operator()(const std::pair<unsigned, int> &Op1,
19928-
const std::pair<unsigned, int> &Op2) const {
19929-
return Op1.second < Op2.second;
19934+
bool operator()(const CandidateInstr &Op1,
19935+
const CandidateInstr &Op2) const {
19936+
return Op1.DistToBasePtr < Op2.DistToBasePtr;
1993019937
}
1993119938
};
19932-
// A set of pairs (index of store in Stores array ref, Distance of the store
19933-
// address relative to base store address in units).
19934-
using StoreIndexToDistSet =
19935-
std::set<std::pair<unsigned, int>, StoreDistCompare>;
19936-
auto TryToVectorize = [&](const StoreIndexToDistSet &Set) {
19939+
19940+
/// A group of store instructions that we'll try to bundle together.
19941+
/// They are ordered using their signed distance to the address of this
19942+
/// set's BaseInstr.
19943+
struct CandidateBundle {
19944+
CandidateBundle(unsigned BaseInstrIdx) { reset(BaseInstrIdx); }
19945+
void reset(unsigned NewBaseInstr) {
19946+
BaseInstrIdx = NewBaseInstr;
19947+
Instrs.clear();
19948+
Instrs.emplace(NewBaseInstr, 0);
19949+
}
19950+
// TODO: This should probably just be an std::map
19951+
using CandidateSet = std::set<CandidateInstr, StoreDistCompare>;
19952+
unsigned BaseInstrIdx;
19953+
CandidateSet Instrs;
19954+
};
19955+
19956+
auto TryToVectorize = [&](const CandidateBundle::CandidateSet &StoreSeq) {
1993719957
int PrevDist = -1;
1993819958
BoUpSLP::ValueList Operands;
1993919959
// Collect the chain into a list.
19940-
for (auto [Idx, Data] : enumerate(Set)) {
19941-
if (Operands.empty() || Data.second - PrevDist == 1) {
19942-
Operands.push_back(Stores[Data.first]);
19943-
PrevDist = Data.second;
19944-
if (Idx != Set.size() - 1)
19960+
for (auto [Idx, Data] : enumerate(StoreSeq)) {
19961+
if (Operands.empty() || Data.DistToBasePtr - PrevDist == 1) {
19962+
Operands.push_back(Stores[Data.InstrIdx]);
19963+
PrevDist = Data.DistToBasePtr;
19964+
if (Idx != StoreSeq.size() - 1)
1994519965
continue;
1994619966
}
1994719967
auto E = make_scope_exit([&, &DataVar = Data]() {
1994819968
Operands.clear();
19949-
Operands.push_back(Stores[DataVar.first]);
19950-
PrevDist = DataVar.second;
19969+
Operands.push_back(Stores[DataVar.InstrIdx]);
19970+
PrevDist = DataVar.DistToBasePtr;
1995119971
});
1995219972

1995319973
if (Operands.size() <= 1 ||
@@ -20214,7 +20234,8 @@ bool SLPVectorizerPass::vectorizeStores(
2021420234
// Need to store the index of the very first store separately, since the set
2021520235
// may be reordered after the insertion and the first store may be moved. This
2021620236
// container allows to reduce number of calls of getPointersDiff() function.
20217-
SmallVector<std::pair<unsigned, StoreIndexToDistSet>> SortedStores;
20237+
SmallVector<CandidateBundle> SortedStores;
20238+
2021820239
// Inserts the specified store SI with the given index Idx to the set of the
2021920240
// stores. If the store with the same distance is found already - stop
2022020241
// insertion, try to vectorize already found stores. If some stores from this
@@ -20248,56 +20269,51 @@ bool SLPVectorizerPass::vectorizeStores(
2024820269
// dependencies and no need to waste compile time to try to vectorize them.
2024920270
// - Try to vectorize the sequence {1, {1, 0}, {3, 2}}.
2025020271
auto FillStoresSet = [&](unsigned Idx, StoreInst *SI) {
20251-
for (std::pair<unsigned, StoreIndexToDistSet> &Set : SortedStores) {
20272+
for (CandidateBundle &StoreSeq : SortedStores) {
2025220273
std::optional<int> Diff = getPointersDiff(
20253-
Stores[Set.first]->getValueOperand()->getType(),
20254-
Stores[Set.first]->getPointerOperand(),
20274+
Stores[StoreSeq.BaseInstrIdx]->getValueOperand()->getType(),
20275+
Stores[StoreSeq.BaseInstrIdx]->getPointerOperand(),
2025520276
SI->getValueOperand()->getType(), SI->getPointerOperand(), *DL, *SE,
2025620277
/*StrictCheck=*/true);
2025720278
if (!Diff)
2025820279
continue;
20259-
auto It = Set.second.find(std::make_pair(Idx, *Diff));
20260-
if (It == Set.second.end()) {
20261-
Set.second.emplace(Idx, *Diff);
20280+
auto It = StoreSeq.Instrs.find({Idx, *Diff});
20281+
if (It == StoreSeq.Instrs.end()) {
20282+
StoreSeq.Instrs.emplace(Idx, *Diff);
2026220283
return;
2026320284
}
2026420285
// Try to vectorize the first found set to avoid duplicate analysis.
20265-
TryToVectorize(Set.second);
20266-
unsigned ItIdx = It->first;
20267-
int ItDist = It->second;
20268-
StoreIndexToDistSet PrevSet;
20269-
copy_if(Set.second, std::inserter(PrevSet, PrevSet.end()),
20270-
[&](const std::pair<unsigned, int> &Pair) {
20271-
return Pair.first > ItIdx;
20272-
});
20273-
Set.second.clear();
20274-
Set.first = Idx;
20275-
Set.second.emplace(Idx, 0);
20286+
TryToVectorize(StoreSeq.Instrs);
20287+
unsigned ItIdx = It->InstrIdx;
20288+
int ItDist = It->DistToBasePtr;
20289+
CandidateBundle::CandidateSet PrevSet;
20290+
copy_if(StoreSeq.Instrs, std::inserter(PrevSet, PrevSet.end()),
20291+
[&](const CandidateInstr &I) { return I.InstrIdx > ItIdx; });
20292+
StoreSeq.reset(Idx);
2027620293
// Insert stores that followed previous match to try to vectorize them
2027720294
// with this store.
2027820295
unsigned StartIdx = ItIdx + 1;
2027920296
SmallBitVector UsedStores(Idx - StartIdx);
2028020297
// Distances to previously found dup store (or this store, since they
2028120298
// store to the same addresses).
2028220299
SmallVector<int> Dists(Idx - StartIdx, 0);
20283-
for (const std::pair<unsigned, int> &Pair : reverse(PrevSet)) {
20300+
for (const CandidateInstr &Store : reverse(PrevSet)) {
2028420301
// Do not try to vectorize sequences, we already tried.
20285-
if (VectorizedStores.contains(Stores[Pair.first]))
20302+
if (VectorizedStores.contains(Stores[Store.InstrIdx]))
2028620303
break;
20287-
unsigned BI = Pair.first - StartIdx;
20304+
unsigned BI = Store.InstrIdx - StartIdx;
2028820305
UsedStores.set(BI);
20289-
Dists[BI] = Pair.second - ItDist;
20306+
Dists[BI] = Store.DistToBasePtr - ItDist;
2029020307
}
2029120308
for (unsigned I = StartIdx; I < Idx; ++I) {
2029220309
unsigned BI = I - StartIdx;
2029320310
if (UsedStores.test(BI))
20294-
Set.second.emplace(I, Dists[BI]);
20311+
StoreSeq.Instrs.emplace(I, Dists[BI]);
2029520312
}
2029620313
return;
2029720314
}
20298-
auto &Res = SortedStores.emplace_back();
20299-
Res.first = Idx;
20300-
Res.second.emplace(Idx, 0);
20315+
// We did not find a comparable store, start a new sequence.
20316+
SortedStores.emplace_back(Idx);
2030120317
};
2030220318
Type *PrevValTy = nullptr;
2030320319
for (auto [I, SI] : enumerate(Stores)) {
@@ -20307,17 +20323,17 @@ bool SLPVectorizerPass::vectorizeStores(
2030720323
PrevValTy = SI->getValueOperand()->getType();
2030820324
// Check that we do not try to vectorize stores of different types.
2030920325
if (PrevValTy != SI->getValueOperand()->getType()) {
20310-
for (auto &Set : SortedStores)
20311-
TryToVectorize(Set.second);
20326+
for (CandidateBundle &StoreSeq : SortedStores)
20327+
TryToVectorize(StoreSeq.Instrs);
2031220328
SortedStores.clear();
2031320329
PrevValTy = SI->getValueOperand()->getType();
2031420330
}
2031520331
FillStoresSet(I, SI);
2031620332
}
2031720333

2031820334
// Final vectorization attempt.
20319-
for (auto &Set : SortedStores)
20320-
TryToVectorize(Set.second);
20335+
for (CandidateBundle &StoreSeq : SortedStores)
20336+
TryToVectorize(StoreSeq.Instrs);
2032120337

2032220338
return Changed;
2032320339
}

0 commit comments

Comments
 (0)