@@ -3732,22 +3732,45 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
3732
3732
/// the original order of the scalars. Procedure transforms the provided order
3733
3733
/// in accordance with the given \p Mask. If the resulting \p Order is just an
3734
3734
/// identity order, \p Order is cleared.
3735
- static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask) {
3735
+ static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
3736
+ bool BottomOrder = false) {
3736
3737
assert(!Mask.empty() && "Expected non-empty mask.");
3738
+ unsigned Sz = Mask.size();
3739
+ if (BottomOrder) {
3740
+ SmallVector<unsigned> PrevOrder;
3741
+ if (Order.empty()) {
3742
+ PrevOrder.resize(Sz);
3743
+ std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
3744
+ } else {
3745
+ PrevOrder.swap(Order);
3746
+ }
3747
+ Order.assign(Sz, Sz);
3748
+ for (unsigned I = 0; I < Sz; ++I)
3749
+ if (Mask[I] != PoisonMaskElem)
3750
+ Order[I] = PrevOrder[Mask[I]];
3751
+ if (all_of(enumerate(Order), [&](const auto &Data) {
3752
+ return Data.value() == Sz || Data.index() == Data.value();
3753
+ })) {
3754
+ Order.clear();
3755
+ return;
3756
+ }
3757
+ fixupOrderingIndices(Order);
3758
+ return;
3759
+ }
3737
3760
SmallVector<int> MaskOrder;
3738
3761
if (Order.empty()) {
3739
- MaskOrder.resize(Mask.size() );
3762
+ MaskOrder.resize(Sz );
3740
3763
std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
3741
3764
} else {
3742
3765
inversePermutation(Order, MaskOrder);
3743
3766
}
3744
3767
reorderReuses(MaskOrder, Mask);
3745
- if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size() )) {
3768
+ if (ShuffleVectorInst::isIdentityMask(MaskOrder, Sz )) {
3746
3769
Order.clear();
3747
3770
return;
3748
3771
}
3749
- Order.assign(Mask.size(), Mask.size() );
3750
- for (unsigned I = 0, E = Mask.size() ; I < E ; ++I)
3772
+ Order.assign(Sz, Sz );
3773
+ for (unsigned I = 0; I < Sz ; ++I)
3751
3774
if (MaskOrder[I] != PoisonMaskElem)
3752
3775
Order[MaskOrder[I]] = I;
3753
3776
fixupOrderingIndices(Order);
@@ -4880,7 +4903,8 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
4880
4903
Data.first->isAltShuffle() ||
4881
4904
Data.first->State == TreeEntry::PossibleStridedVectorize) {
4882
4905
reorderScalars(Data.first->Scalars, Mask);
4883
- reorderOrder(Data.first->ReorderIndices, MaskOrder);
4906
+ reorderOrder(Data.first->ReorderIndices, MaskOrder,
4907
+ /*BottomOrder=*/true);
4884
4908
if (Data.first->ReuseShuffleIndices.empty() &&
4885
4909
!Data.first->ReorderIndices.empty() &&
4886
4910
!Data.first->isAltShuffle()) {
@@ -4889,7 +4913,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
4889
4913
OrderedEntries.insert(Data.first);
4890
4914
}
4891
4915
} else {
4892
- reorderOrder(Data.first->ReorderIndices, Mask);
4916
+ reorderOrder(Data.first->ReorderIndices, Mask, /*BottomOrder=*/true );
4893
4917
}
4894
4918
}
4895
4919
}
0 commit comments