Skip to content

Commit 62db1c8

Browse files
committed
[SLP]Better decision making on whether to try stores packs for vectorization
Since the stores are sorted by distance, comparing the indices in the original array and early exit, if the index is less than the index of the last store, not always the best strategy. Better to remove such stores explicitly to try better to check for the vectorization opportunity. Fixes llvm#115008
1 parent bdf8e30 commit 62db1c8

File tree

2 files changed

+11
-11
lines changed

2 files changed

+11
-11
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18515,25 +18515,30 @@ bool SLPVectorizerPass::vectorizeStores(
1851518515
}
1851618516
// Try to vectorize the first found set to avoid duplicate analysis.
1851718517
TryToVectorize(Set.second);
18518+
unsigned ItIdx = It->first;
18519+
int ItDist = It->second;
1851818520
StoreIndexToDistSet PrevSet;
18519-
PrevSet.swap(Set.second);
18521+
copy_if(Set.second, std::inserter(PrevSet, PrevSet.end()),
18522+
[&](const std::pair<unsigned, int> &Pair) {
18523+
return Pair.first > ItIdx;
18524+
});
18525+
Set.second.clear();
1852018526
Set.first = Idx;
1852118527
Set.second.emplace(Idx, 0);
1852218528
// Insert stores that followed previous match to try to vectorize them
1852318529
// with this store.
18524-
unsigned StartIdx = It->first + 1;
18530+
unsigned StartIdx = ItIdx + 1;
1852518531
SmallBitVector UsedStores(Idx - StartIdx);
1852618532
// Distances to previously found dup store (or this store, since they
1852718533
// store to the same addresses).
1852818534
SmallVector<int> Dists(Idx - StartIdx, 0);
1852918535
for (const std::pair<unsigned, int> &Pair : reverse(PrevSet)) {
1853018536
// Do not try to vectorize sequences, we already tried.
18531-
if (Pair.first <= It->first ||
18532-
VectorizedStores.contains(Stores[Pair.first]))
18537+
if (VectorizedStores.contains(Stores[Pair.first]))
1853318538
break;
1853418539
unsigned BI = Pair.first - StartIdx;
1853518540
UsedStores.set(BI);
18536-
Dists[BI] = Pair.second - It->second;
18541+
Dists[BI] = Pair.second - ItDist;
1853718542
}
1853818543
for (unsigned I = StartIdx; I < Idx; ++I) {
1853918544
unsigned BI = I - StartIdx;

llvm/test/Transforms/SLPVectorizer/RISCV/repeated-address-store.ll

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,7 @@ define void @test(ptr %dest) {
66
; CHECK-SAME: ptr [[DEST:%.*]]) #[[ATTR0:[0-9]+]] {
77
; CHECK-NEXT: [[ENTRY:.*:]]
88
; CHECK-NEXT: [[INC3:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 3
9-
; CHECK-NEXT: store i32 1, ptr [[INC3]], align 2
10-
; CHECK-NEXT: store i32 1, ptr [[DEST]], align 4
11-
; CHECK-NEXT: [[INC1:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 1
12-
; CHECK-NEXT: store i32 1, ptr [[INC1]], align 2
13-
; CHECK-NEXT: [[INC2:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 2
14-
; CHECK-NEXT: store i32 1, ptr [[INC2]], align 2
9+
; CHECK-NEXT: store <4 x i32> splat (i32 1), ptr [[DEST]], align 4
1510
; CHECK-NEXT: store i32 2, ptr [[DEST]], align 2
1611
; CHECK-NEXT: store i32 1, ptr [[INC3]], align 2
1712
; CHECK-NEXT: ret void

0 commit comments

Comments
 (0)