@@ -884,6 +884,42 @@ class InstructionsState {
884
884
static InstructionsState invalid() { return {nullptr, nullptr}; }
885
885
};
886
886
887
+ /// A reference to a ld/st instruction and the distance of its address to a base
888
+ /// pointer.
889
+ struct MemInstAndPtrDist {
890
+ MemInstAndPtrDist(unsigned InstrIdx, int DistToBasePtr)
891
+ : InstrIdx(InstrIdx), DistToBasePtr(DistToBasePtr) {}
892
+ unsigned InstrIdx;
893
+ int DistToBasePtr;
894
+ };
895
+ struct PtrDistCompare {
896
+ bool operator()(const MemInstAndPtrDist &Op1,
897
+ const MemInstAndPtrDist &Op2) const {
898
+ return Op1.DistToBasePtr < Op2.DistToBasePtr;
899
+ }
900
+ };
901
+
902
+ /// A group of memory instructions that we'll try to bundle together using
903
+ /// vector ops. They are ordered using their signed distance to the address of
904
+ /// this group's BaseInstr.
905
+ struct RelatedMemInsts {
906
+ RelatedMemInsts(unsigned BaseInstrIdx) { reset(BaseInstrIdx); }
907
+ void reset(unsigned NewBaseInstr) {
908
+ BaseInstrIdx = NewBaseInstr;
909
+ Instrs.clear();
910
+ Instrs.emplace(NewBaseInstr, 0);
911
+ }
912
+
913
+ // TODO: This should probably just be an std::map
914
+ using InstSet = std::set<MemInstAndPtrDist, PtrDistCompare>;
915
+
916
+ /// The index of the Base instruction, i.e. the one with a 0 pointer distance.
917
+ unsigned BaseInstrIdx;
918
+
919
+ /// Maps an instruction index to a pointer distance from \p BaseInstrIdx.
920
+ InstSet Instrs;
921
+ };
922
+
887
923
} // end anonymous namespace
888
924
889
925
/// \returns true if \p Opcode is allowed as part of the main/alternate
@@ -19923,37 +19959,7 @@ bool SLPVectorizerPass::vectorizeStores(
19923
19959
BoUpSLP::ValueSet VectorizedStores;
19924
19960
bool Changed = false;
19925
19961
19926
- /// A store instruction and the distance of its address to a base pointer.
19927
- struct CandidateInstr {
19928
- CandidateInstr(unsigned InstrIdx, int DistToBasePtr)
19929
- : InstrIdx(InstrIdx), DistToBasePtr(DistToBasePtr) {}
19930
- unsigned InstrIdx;
19931
- int DistToBasePtr;
19932
- };
19933
- struct StoreDistCompare {
19934
- bool operator()(const CandidateInstr &Op1,
19935
- const CandidateInstr &Op2) const {
19936
- return Op1.DistToBasePtr < Op2.DistToBasePtr;
19937
- }
19938
- };
19939
-
19940
- /// A group of store instructions that we'll try to bundle together.
19941
- /// They are ordered using their signed distance to the address of this
19942
- /// set's BaseInstr.
19943
- struct CandidateBundle {
19944
- CandidateBundle(unsigned BaseInstrIdx) { reset(BaseInstrIdx); }
19945
- void reset(unsigned NewBaseInstr) {
19946
- BaseInstrIdx = NewBaseInstr;
19947
- Instrs.clear();
19948
- Instrs.emplace(NewBaseInstr, 0);
19949
- }
19950
- // TODO: This should probably just be an std::map
19951
- using CandidateSet = std::set<CandidateInstr, StoreDistCompare>;
19952
- unsigned BaseInstrIdx;
19953
- CandidateSet Instrs;
19954
- };
19955
-
19956
- auto TryToVectorize = [&](const CandidateBundle::CandidateSet &StoreSeq) {
19962
+ auto TryToVectorize = [&](const RelatedMemInsts::InstSet &StoreSeq) {
19957
19963
int PrevDist = -1;
19958
19964
BoUpSLP::ValueList Operands;
19959
19965
// Collect the chain into a list.
@@ -20234,7 +20240,7 @@ bool SLPVectorizerPass::vectorizeStores(
20234
20240
// Need to store the index of the very first store separately, since the set
20235
20241
// may be reordered after the insertion and the first store may be moved. This
20236
20242
// container allows to reduce number of calls of getPointersDiff() function.
20237
- SmallVector<CandidateBundle > SortedStores;
20243
+ SmallVector<RelatedMemInsts > SortedStores;
20238
20244
20239
20245
// Inserts the specified store SI with the given index Idx to the set of the
20240
20246
// stores. If the store with the same distance is found already - stop
@@ -20269,7 +20275,7 @@ bool SLPVectorizerPass::vectorizeStores(
20269
20275
// dependencies and no need to waste compile time to try to vectorize them.
20270
20276
// - Try to vectorize the sequence {1, {1, 0}, {3, 2}}.
20271
20277
auto FillStoresSet = [&](unsigned Idx, StoreInst *SI) {
20272
- for (CandidateBundle &StoreSeq : SortedStores) {
20278
+ for (RelatedMemInsts &StoreSeq : SortedStores) {
20273
20279
std::optional<int> Diff = getPointersDiff(
20274
20280
Stores[StoreSeq.BaseInstrIdx]->getValueOperand()->getType(),
20275
20281
Stores[StoreSeq.BaseInstrIdx]->getPointerOperand(),
@@ -20286,9 +20292,9 @@ bool SLPVectorizerPass::vectorizeStores(
20286
20292
TryToVectorize(StoreSeq.Instrs);
20287
20293
unsigned ItIdx = It->InstrIdx;
20288
20294
int ItDist = It->DistToBasePtr;
20289
- CandidateBundle::CandidateSet PrevSet;
20295
+ RelatedMemInsts::InstSet PrevSet;
20290
20296
copy_if(StoreSeq.Instrs, std::inserter(PrevSet, PrevSet.end()),
20291
- [&](const CandidateInstr &I) { return I.InstrIdx > ItIdx; });
20297
+ [&](const MemInstAndPtrDist &I) { return I.InstrIdx > ItIdx; });
20292
20298
StoreSeq.reset(Idx);
20293
20299
// Insert stores that followed previous match to try to vectorize them
20294
20300
// with this store.
@@ -20297,7 +20303,7 @@ bool SLPVectorizerPass::vectorizeStores(
20297
20303
// Distances to previously found dup store (or this store, since they
20298
20304
// store to the same addresses).
20299
20305
SmallVector<int> Dists(Idx - StartIdx, 0);
20300
- for (const CandidateInstr &Store : reverse(PrevSet)) {
20306
+ for (const MemInstAndPtrDist &Store : reverse(PrevSet)) {
20301
20307
// Do not try to vectorize sequences, we already tried.
20302
20308
if (VectorizedStores.contains(Stores[Store.InstrIdx]))
20303
20309
break;
@@ -20323,7 +20329,7 @@ bool SLPVectorizerPass::vectorizeStores(
20323
20329
PrevValTy = SI->getValueOperand()->getType();
20324
20330
// Check that we do not try to vectorize stores of different types.
20325
20331
if (PrevValTy != SI->getValueOperand()->getType()) {
20326
- for (CandidateBundle &StoreSeq : SortedStores)
20332
+ for (RelatedMemInsts &StoreSeq : SortedStores)
20327
20333
TryToVectorize(StoreSeq.Instrs);
20328
20334
SortedStores.clear();
20329
20335
PrevValTy = SI->getValueOperand()->getType();
@@ -20332,7 +20338,7 @@ bool SLPVectorizerPass::vectorizeStores(
20332
20338
}
20333
20339
20334
20340
// Final vectorization attempt.
20335
- for (CandidateBundle &StoreSeq : SortedStores)
20341
+ for (RelatedMemInsts &StoreSeq : SortedStores)
20336
20342
TryToVectorize(StoreSeq.Instrs);
20337
20343
20338
20344
return Changed;
0 commit comments