Skip to content

Commit 45e41f9

Browse files
[SLP] Compute a shuffle mask for SK_InsertSubvector (#85408)
This is the third of a series of small patches to compute shuffle masks for the couple of cases where we call getShuffleCost without one. My goal is to add an invariant that all calls to getShuffleCost for fixed length vectors have a mask. After this change, there is one SK_InsertSubvector case left. I excluded it from this patch just because I thought it worthy of individual attention and review. --------- Co-authored-by: Alexey Bataev <[email protected]>
1 parent 92b5601 commit 45e41f9

File tree

1 file changed

+11
-4
lines changed

1 file changed

+11
-4
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4295,9 +4295,12 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
42954295
llvm_unreachable(
42964296
"Expected only consecutive, strided or masked gather loads.");
42974297
}
4298+
SmallVector<int> ShuffleMask(VL.size());
4299+
for (int Idx : seq<int>(0, VL.size()))
4300+
ShuffleMask[Idx] = Idx / VF == I ? VL.size() + Idx % VF : Idx;
42984301
VecLdCost +=
42994302
TTI.getShuffleCost(TTI ::SK_InsertSubvector, VecTy,
4300-
std::nullopt, CostKind, I * VF, SubVecTy);
4303+
ShuffleMask, CostKind, I * VF, SubVecTy);
43014304
}
43024305
// If masked gather cost is higher - better to vectorize, so
43034306
// consider it as a gather node. It will be better estimated
@@ -7401,7 +7404,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
74017404
Index + NumSrcElts <= static_cast<int>(Mask.size()))
74027405
return TTI.getShuffleCost(
74037406
TTI::SK_InsertSubvector,
7404-
FixedVectorType::get(Tp->getElementType(), Mask.size()), std::nullopt,
7407+
FixedVectorType::get(Tp->getElementType(), Mask.size()), Mask,
74057408
TTI::TCK_RecipThroughput, Index, Tp);
74067409
}
74077410
return TTI.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args);
@@ -7674,9 +7677,13 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
76747677
}
76757678
if (NeedInsertSubvectorAnalysis) {
76767679
// Add the cost for the subvectors insert.
7677-
for (int I = VF, E = VL.size(); I < E; I += VF)
7680+
SmallVector<int> ShuffleMask(VL.size());
7681+
for (int I = VF, E = VL.size(); I < E; I += VF) {
7682+
for (int Idx : seq<int>(0, E))
7683+
ShuffleMask[Idx] = Idx / VF == I ? E + Idx % VF : Idx;
76787684
GatherCost += TTI.getShuffleCost(TTI::SK_InsertSubvector, VecTy,
7679-
std::nullopt, CostKind, I, LoadTy);
7685+
ShuffleMask, CostKind, I, LoadTy);
7686+
}
76807687
}
76817688
GatherCost -= ScalarsCost;
76827689
}

0 commit comments

Comments
 (0)