Skip to content

Commit 0674ed7

Browse files
[SLP] Compute a shuffle mask for getGatherCost (#85330)
This is the second of a series of small patches to compute shuffle masks for the couple of cases where we call getShuffleCost without one. My goal is to add an invariant that all calls to getShuffleCost for fixed length vectors have a mask. --------- Co-authored-by: Alexey Bataev <[email protected]>
1 parent 45e41f9 commit 0674ed7

File tree

1 file changed

+14
-7
lines changed

1 file changed

+14
-7
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10388,7 +10388,7 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL,
1038810388
// Check if the same elements are inserted several times and count them as
1038910389
// shuffle candidates.
1039010390
APInt ShuffledElements = APInt::getZero(VL.size());
10391-
DenseSet<Value *> UniqueElements;
10391+
DenseMap<Value *, unsigned> UniqueElements;
1039210392
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1039310393
InstructionCost Cost;
1039410394
auto EstimateInsertCost = [&](unsigned I, Value *V) {
@@ -10397,27 +10397,34 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL,
1039710397
TTI->getVectorInstrCost(Instruction::InsertElement, VecTy, CostKind,
1039810398
I, Constant::getNullValue(VecTy), V);
1039910399
};
10400+
SmallVector<int> ShuffleMask(VL.size(), PoisonMaskElem);
1040010401
for (unsigned I = 0, E = VL.size(); I < E; ++I) {
1040110402
Value *V = VL[I];
1040210403
// No need to shuffle duplicates for constants.
1040310404
if ((ForPoisonSrc && isConstant(V)) || isa<UndefValue>(V)) {
1040410405
ShuffledElements.setBit(I);
10406+
ShuffleMask[I] = isa<PoisonValue>(V) ? PoisonMaskElem : I;
1040510407
continue;
1040610408
}
10407-
if (!UniqueElements.insert(V).second) {
10408-
DuplicateNonConst = true;
10409-
ShuffledElements.setBit(I);
10409+
10410+
auto Res = UniqueElements.try_emplace(V, I);
10411+
if (Res.second) {
10412+
EstimateInsertCost(I, V);
10413+
ShuffleMask[I] = I;
1041010414
continue;
1041110415
}
10412-
EstimateInsertCost(I, V);
10416+
10417+
DuplicateNonConst = true;
10418+
ShuffledElements.setBit(I);
10419+
ShuffleMask[I] = Res.first->second;
1041310420
}
1041410421
if (ForPoisonSrc)
1041510422
Cost =
1041610423
TTI->getScalarizationOverhead(VecTy, ~ShuffledElements, /*Insert*/ true,
1041710424
/*Extract*/ false, CostKind);
1041810425
if (DuplicateNonConst)
10419-
Cost +=
10420-
TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, VecTy);
10426+
Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
10427+
VecTy, ShuffleMask);
1042110428
return Cost;
1042210429
}
1042310430

0 commit comments

Comments
 (0)