Skip to content

Commit 92ae2ca

Browse files
committed
[SLP][NFC]Improve BottomTopTop reordering of orders for multi-iterations
attempts, NFC. If several iterations of reodering of orders is required, need to use different algorithm.
1 parent 313d33e commit 92ae2ca

File tree

1 file changed

+31
-7
lines changed

1 file changed

+31
-7
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3732,22 +3732,45 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
37323732
/// the original order of the scalars. Procedure transforms the provided order
37333733
/// in accordance with the given \p Mask. If the resulting \p Order is just an
37343734
/// identity order, \p Order is cleared.
3735-
static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
3735+
static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
3736+
bool BottomOrder = false) {
37363737
assert(!Mask.empty() && "Expected non-empty mask.");
3738+
unsigned Sz = Mask.size();
3739+
if (BottomOrder) {
3740+
SmallVector<unsigned> PrevOrder;
3741+
if (Order.empty()) {
3742+
PrevOrder.resize(Sz);
3743+
std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
3744+
} else {
3745+
PrevOrder.swap(Order);
3746+
}
3747+
Order.assign(Sz, Sz);
3748+
for (unsigned I = 0; I < Sz; ++I)
3749+
if (Mask[I] != PoisonMaskElem)
3750+
Order[I] = PrevOrder[Mask[I]];
3751+
if (all_of(enumerate(Order), [&](const auto &Data) {
3752+
return Data.value() == Sz || Data.index() == Data.value();
3753+
})) {
3754+
Order.clear();
3755+
return;
3756+
}
3757+
fixupOrderingIndices(Order);
3758+
return;
3759+
}
37373760
SmallVector<int> MaskOrder;
37383761
if (Order.empty()) {
3739-
MaskOrder.resize(Mask.size());
3762+
MaskOrder.resize(Sz);
37403763
std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
37413764
} else {
37423765
inversePermutation(Order, MaskOrder);
37433766
}
37443767
reorderReuses(MaskOrder, Mask);
3745-
if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size())) {
3768+
if (ShuffleVectorInst::isIdentityMask(MaskOrder, Sz)) {
37463769
Order.clear();
37473770
return;
37483771
}
3749-
Order.assign(Mask.size(), Mask.size());
3750-
for (unsigned I = 0, E = Mask.size(); I < E; ++I)
3772+
Order.assign(Sz, Sz);
3773+
for (unsigned I = 0; I < Sz; ++I)
37513774
if (MaskOrder[I] != PoisonMaskElem)
37523775
Order[MaskOrder[I]] = I;
37533776
fixupOrderingIndices(Order);
@@ -4880,7 +4903,8 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
48804903
Data.first->isAltShuffle() ||
48814904
Data.first->State == TreeEntry::PossibleStridedVectorize) {
48824905
reorderScalars(Data.first->Scalars, Mask);
4883-
reorderOrder(Data.first->ReorderIndices, MaskOrder);
4906+
reorderOrder(Data.first->ReorderIndices, MaskOrder,
4907+
/*BottomOrder=*/true);
48844908
if (Data.first->ReuseShuffleIndices.empty() &&
48854909
!Data.first->ReorderIndices.empty() &&
48864910
!Data.first->isAltShuffle()) {
@@ -4889,7 +4913,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
48894913
OrderedEntries.insert(Data.first);
48904914
}
48914915
} else {
4892-
reorderOrder(Data.first->ReorderIndices, Mask);
4916+
reorderOrder(Data.first->ReorderIndices, Mask, /*BottomOrder=*/true);
48934917
}
48944918
}
48954919
}

0 commit comments

Comments
 (0)