[SLP][REVEC] Add getScalarizationOverhead helper function to reduce error when REVEC is enabled. #128530
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
error when REVEC is enabled.
|
@llvm/pr-subscribers-vectorizers Author: Han-Kuan Chen (HanKuanChen) ChangesFull diff: https://github.com/llvm/llvm-project/pull/128530.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3d660b63309d4..60b4e0263f8bd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5014,6 +5014,39 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
return TTI.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args);
}
+/// This is similar to TargetTransformInfo::getScalarizationOverhead, but if
+/// ScalarTy is a FixedVectorType, a vector will be inserted or extracted
+/// instead of a scalar.
+static InstructionCost getScalarizationOverhead(const TargetTransformInfo &TTI,
+ Type *ScalarTy, VectorType *Ty,
+ const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<Value *> VL = {}) {
+ assert(!isa<ScalableVectorType>(Ty) &&
+ "ScalableVectorType is not supported.");
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ assert(SLPReVec && "Only supported by REVEC.");
+ // If ScalarTy is FixedVectorType, we should use CreateInsertVector instead
+ // of CreateInsertElement.
+ unsigned ScalarTyNumElements = VecTy->getNumElements();
+ InstructionCost Cost = 0;
+ for (unsigned I : seq(DemandedElts.getBitWidth())) {
+ if (!DemandedElts[I])
+ continue;
+ if (Insert)
+ Cost += getShuffleCost(TTI, TTI::SK_InsertSubvector, Ty, {}, CostKind,
+ I * ScalarTyNumElements, VecTy);
+ if (Extract)
+ Cost += getShuffleCost(TTI, TTI::SK_ExtractSubvector, Ty, {}, CostKind,
+ I * ScalarTyNumElements, VecTy);
+ }
+ return Cost;
+ }
+ return TTI.getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
+ CostKind, VL);
+}
+
/// Correctly creates insert_subvector, checking that the index is multiple of
/// the subvectors length. Otherwise, generates shuffle using \p Generator or
/// using default shuffle.
@@ -5240,8 +5273,9 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
/*VariableMask=*/false, CommonAlignment, CostKind) +
(ProfitableGatherPointers ? 0 : VectorGEPCost);
InstructionCost GatherCost =
- TTI.getScalarizationOverhead(VecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind) +
+ getScalarizationOverhead(TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind) +
ScalarLoadsCost;
// The list of loads is small or perform partial check already - directly
// compare masked gather cost and gather cost.
@@ -5294,10 +5328,10 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
// Can be vectorized later as a serie of loads/insertelements.
InstructionCost VecLdCost = 0;
if (!DemandedElts.isZero()) {
- VecLdCost =
- TTI.getScalarizationOverhead(VecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind) +
- ScalarGEPCost;
+ VecLdCost = getScalarizationOverhead(TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind) +
+ ScalarGEPCost;
for (unsigned Idx : seq<unsigned>(VL.size()))
if (DemandedElts[Idx])
VecLdCost +=
@@ -5323,13 +5357,14 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
return getUnderlyingObject(V) !=
getUnderlyingObject(PointerOps.front());
}))
- VectorGEPCost += TTI.getScalarizationOverhead(
- SubVecTy, APInt::getAllOnes(VF),
+ VectorGEPCost += getScalarizationOverhead(
+ TTI, ScalarTy, SubVecTy, APInt::getAllOnes(VF),
/*Insert=*/true, /*Extract=*/false, CostKind);
else
VectorGEPCost +=
- TTI.getScalarizationOverhead(
- SubVecTy, APInt::getOneBitSet(ScalarTyNumElements * VF, 0),
+ getScalarizationOverhead(
+ TTI, ScalarTy, SubVecTy,
+ APInt::getOneBitSet(ScalarTyNumElements * VF, 0),
/*Insert=*/true, /*Extract=*/false, CostKind) +
::getShuffleCost(TTI, TTI::SK_Broadcast, SubVecTy, {},
CostKind);
@@ -9912,20 +9947,9 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
Cost += ::getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, {}, CostKind,
Idx, getWidenedType(ScalarTy, Sz));
}
- if (auto *FTy = dyn_cast<FixedVectorType>(ScalarTy)) {
- assert(SLPReVec && "Only supported by REVEC.");
- // If ScalarTy is FixedVectorType, we should use CreateInsertVector instead
- // of CreateInsertElement.
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- for (unsigned I : seq<unsigned>(TE.Scalars.size()))
- if (DemandedElts[I])
- Cost +=
- TTI->getShuffleCost(TTI::SK_InsertSubvector, VecTy, std::nullopt,
- CostKind, I * ScalarTyNumElements, FTy);
- } else {
- Cost += TTI->getScalarizationOverhead(VecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind);
- }
+ Cost += getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind);
int Sz = TE.Scalars.size();
SmallVector<int> ReorderMask(TE.ReorderIndices.begin(),
TE.ReorderIndices.end());
@@ -9954,8 +9978,9 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
ReorderMask[I] = I + Sz;
}
}
- InstructionCost BVCost = TTI->getScalarizationOverhead(
- VecTy, DemandedElts, /*Insert=*/true, /*Extract=*/false, CostKind);
+ InstructionCost BVCost =
+ getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true, /*Extract=*/false, CostKind);
if (!DemandedElts.isAllOnes())
BVCost += ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, ReorderMask);
if (Cost >= BVCost) {
@@ -11603,9 +11628,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(Offset < NumElts && "Failed to find vector index offset");
InstructionCost Cost = 0;
- Cost -= TTI->getScalarizationOverhead(SrcVecTy, DemandedElts,
- /*Insert*/ true, /*Extract*/ false,
- CostKind);
+ Cost -=
+ getScalarizationOverhead(*TTI, ScalarTy, SrcVecTy, DemandedElts,
+ /*Insert*/ true, /*Extract*/ false, CostKind);
// First cost - resize to actual vector size if not identity shuffle or
// need to shift the vector.
@@ -13780,8 +13805,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
}
if (!IsIdentity)
FirstShuffleCost = GetShuffleCost(FirstMask, Entries.front(), VecTy);
- FirstShuffleCost += TTI->getScalarizationOverhead(
- MaskVecTy, DemandedElts, /*Insert=*/true,
+ FirstShuffleCost += getScalarizationOverhead(
+ *TTI, VL.front()->getType(), MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
}
InstructionCost SecondShuffleCost = 0;
@@ -13805,17 +13830,17 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
}
if (!IsIdentity)
SecondShuffleCost = GetShuffleCost(SecondMask, Entries[1], VecTy);
- SecondShuffleCost += TTI->getScalarizationOverhead(
- MaskVecTy, DemandedElts, /*Insert=*/true,
+ SecondShuffleCost += getScalarizationOverhead(
+ *TTI, VL.front()->getType(), MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
}
APInt DemandedElts = APInt::getAllOnes(SubMask.size());
for (auto [I, Idx] : enumerate(SubMask))
if (Idx == PoisonMaskElem)
DemandedElts.clearBit(I);
- InstructionCost BuildVectorCost =
- TTI->getScalarizationOverhead(MaskVecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind);
+ InstructionCost BuildVectorCost = getScalarizationOverhead(
+ *TTI, VL.front()->getType(), MaskVecTy, DemandedElts, /*Insert=*/true,
+ /*Extract=*/false, CostKind);
const TreeEntry *BestEntry = nullptr;
if (FirstShuffleCost < ShuffleCost) {
std::for_each(std::next(Mask.begin(), Part * VL.size()),
@@ -13968,45 +13993,15 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
ShuffledElements.setBit(I);
ShuffleMask[I] = Res.first->second;
}
- if (!DemandedElements.isZero()) {
- if (isa<FixedVectorType>(ScalarTy)) {
- assert(SLPReVec && "Only supported by REVEC.");
- // We don't need to insert elements one by one. Instead, we can insert the
- // entire vector into the destination.
- Cost = 0;
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- for (unsigned I : seq<unsigned>(VL.size()))
- if (DemandedElements[I])
- Cost += ::getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, {},
- CostKind, I * ScalarTyNumElements,
- cast<FixedVectorType>(ScalarTy));
- } else {
- Cost += TTI->getScalarizationOverhead(VecTy, DemandedElements,
- /*Insert=*/true,
- /*Extract=*/false, CostKind, VL);
- }
- }
- if (ForPoisonSrc) {
- if (isa<FixedVectorType>(ScalarTy)) {
- assert(SLPReVec && "Only supported by REVEC.");
- // We don't need to insert elements one by one. Instead, we can insert the
- // entire vector into the destination.
- assert(DemandedElements.isZero() &&
- "Need to consider the cost from DemandedElements.");
- Cost = 0;
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- for (unsigned I : seq<unsigned>(VL.size()))
- if (!ShuffledElements[I])
- Cost += TTI->getShuffleCost(
- TTI::SK_InsertSubvector, VecTy, std::nullopt, CostKind,
- I * ScalarTyNumElements, cast<FixedVectorType>(ScalarTy));
- } else {
- Cost = TTI->getScalarizationOverhead(VecTy,
- /*DemandedElts*/ ~ShuffledElements,
- /*Insert*/ true,
- /*Extract*/ false, CostKind, VL);
- }
- }
+ if (!DemandedElements.isZero())
+ Cost += getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElements,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind, VL);
+ if (ForPoisonSrc)
+ Cost = getScalarizationOverhead(*TTI, ScalarTy, VecTy,
+ /*DemandedElts*/ ~ShuffledElements,
+ /*Insert*/ true,
+ /*Extract*/ false, CostKind, VL);
if (DuplicateNonConst)
Cost += ::getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteSingleSrc,
VecTy, ShuffleMask);
|
|
@llvm/pr-subscribers-llvm-transforms Author: Han-Kuan Chen (HanKuanChen) ChangesFull diff: https://github.com/llvm/llvm-project/pull/128530.diff 1 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3d660b63309d4..60b4e0263f8bd 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5014,6 +5014,39 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
return TTI.getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args);
}
+/// This is similar to TargetTransformInfo::getScalarizationOverhead, but if
+/// ScalarTy is a FixedVectorType, a vector will be inserted or extracted
+/// instead of a scalar.
+static InstructionCost getScalarizationOverhead(const TargetTransformInfo &TTI,
+ Type *ScalarTy, VectorType *Ty,
+ const APInt &DemandedElts,
+ bool Insert, bool Extract,
+ TTI::TargetCostKind CostKind,
+ ArrayRef<Value *> VL = {}) {
+ assert(!isa<ScalableVectorType>(Ty) &&
+ "ScalableVectorType is not supported.");
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ assert(SLPReVec && "Only supported by REVEC.");
+ // If ScalarTy is FixedVectorType, we should use CreateInsertVector instead
+ // of CreateInsertElement.
+ unsigned ScalarTyNumElements = VecTy->getNumElements();
+ InstructionCost Cost = 0;
+ for (unsigned I : seq(DemandedElts.getBitWidth())) {
+ if (!DemandedElts[I])
+ continue;
+ if (Insert)
+ Cost += getShuffleCost(TTI, TTI::SK_InsertSubvector, Ty, {}, CostKind,
+ I * ScalarTyNumElements, VecTy);
+ if (Extract)
+ Cost += getShuffleCost(TTI, TTI::SK_ExtractSubvector, Ty, {}, CostKind,
+ I * ScalarTyNumElements, VecTy);
+ }
+ return Cost;
+ }
+ return TTI.getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
+ CostKind, VL);
+}
+
/// Correctly creates insert_subvector, checking that the index is multiple of
/// the subvectors length. Otherwise, generates shuffle using \p Generator or
/// using default shuffle.
@@ -5240,8 +5273,9 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
/*VariableMask=*/false, CommonAlignment, CostKind) +
(ProfitableGatherPointers ? 0 : VectorGEPCost);
InstructionCost GatherCost =
- TTI.getScalarizationOverhead(VecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind) +
+ getScalarizationOverhead(TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind) +
ScalarLoadsCost;
// The list of loads is small or perform partial check already - directly
// compare masked gather cost and gather cost.
@@ -5294,10 +5328,10 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
// Can be vectorized later as a serie of loads/insertelements.
InstructionCost VecLdCost = 0;
if (!DemandedElts.isZero()) {
- VecLdCost =
- TTI.getScalarizationOverhead(VecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind) +
- ScalarGEPCost;
+ VecLdCost = getScalarizationOverhead(TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind) +
+ ScalarGEPCost;
for (unsigned Idx : seq<unsigned>(VL.size()))
if (DemandedElts[Idx])
VecLdCost +=
@@ -5323,13 +5357,14 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
return getUnderlyingObject(V) !=
getUnderlyingObject(PointerOps.front());
}))
- VectorGEPCost += TTI.getScalarizationOverhead(
- SubVecTy, APInt::getAllOnes(VF),
+ VectorGEPCost += getScalarizationOverhead(
+ TTI, ScalarTy, SubVecTy, APInt::getAllOnes(VF),
/*Insert=*/true, /*Extract=*/false, CostKind);
else
VectorGEPCost +=
- TTI.getScalarizationOverhead(
- SubVecTy, APInt::getOneBitSet(ScalarTyNumElements * VF, 0),
+ getScalarizationOverhead(
+ TTI, ScalarTy, SubVecTy,
+ APInt::getOneBitSet(ScalarTyNumElements * VF, 0),
/*Insert=*/true, /*Extract=*/false, CostKind) +
::getShuffleCost(TTI, TTI::SK_Broadcast, SubVecTy, {},
CostKind);
@@ -9912,20 +9947,9 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
Cost += ::getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, {}, CostKind,
Idx, getWidenedType(ScalarTy, Sz));
}
- if (auto *FTy = dyn_cast<FixedVectorType>(ScalarTy)) {
- assert(SLPReVec && "Only supported by REVEC.");
- // If ScalarTy is FixedVectorType, we should use CreateInsertVector instead
- // of CreateInsertElement.
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- for (unsigned I : seq<unsigned>(TE.Scalars.size()))
- if (DemandedElts[I])
- Cost +=
- TTI->getShuffleCost(TTI::SK_InsertSubvector, VecTy, std::nullopt,
- CostKind, I * ScalarTyNumElements, FTy);
- } else {
- Cost += TTI->getScalarizationOverhead(VecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind);
- }
+ Cost += getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind);
int Sz = TE.Scalars.size();
SmallVector<int> ReorderMask(TE.ReorderIndices.begin(),
TE.ReorderIndices.end());
@@ -9954,8 +9978,9 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
ReorderMask[I] = I + Sz;
}
}
- InstructionCost BVCost = TTI->getScalarizationOverhead(
- VecTy, DemandedElts, /*Insert=*/true, /*Extract=*/false, CostKind);
+ InstructionCost BVCost =
+ getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElts,
+ /*Insert=*/true, /*Extract=*/false, CostKind);
if (!DemandedElts.isAllOnes())
BVCost += ::getShuffleCost(*TTI, TTI::SK_PermuteTwoSrc, VecTy, ReorderMask);
if (Cost >= BVCost) {
@@ -11603,9 +11628,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
assert(Offset < NumElts && "Failed to find vector index offset");
InstructionCost Cost = 0;
- Cost -= TTI->getScalarizationOverhead(SrcVecTy, DemandedElts,
- /*Insert*/ true, /*Extract*/ false,
- CostKind);
+ Cost -=
+ getScalarizationOverhead(*TTI, ScalarTy, SrcVecTy, DemandedElts,
+ /*Insert*/ true, /*Extract*/ false, CostKind);
// First cost - resize to actual vector size if not identity shuffle or
// need to shift the vector.
@@ -13780,8 +13805,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
}
if (!IsIdentity)
FirstShuffleCost = GetShuffleCost(FirstMask, Entries.front(), VecTy);
- FirstShuffleCost += TTI->getScalarizationOverhead(
- MaskVecTy, DemandedElts, /*Insert=*/true,
+ FirstShuffleCost += getScalarizationOverhead(
+ *TTI, VL.front()->getType(), MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
}
InstructionCost SecondShuffleCost = 0;
@@ -13805,17 +13830,17 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
}
if (!IsIdentity)
SecondShuffleCost = GetShuffleCost(SecondMask, Entries[1], VecTy);
- SecondShuffleCost += TTI->getScalarizationOverhead(
- MaskVecTy, DemandedElts, /*Insert=*/true,
+ SecondShuffleCost += getScalarizationOverhead(
+ *TTI, VL.front()->getType(), MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
}
APInt DemandedElts = APInt::getAllOnes(SubMask.size());
for (auto [I, Idx] : enumerate(SubMask))
if (Idx == PoisonMaskElem)
DemandedElts.clearBit(I);
- InstructionCost BuildVectorCost =
- TTI->getScalarizationOverhead(MaskVecTy, DemandedElts, /*Insert=*/true,
- /*Extract=*/false, CostKind);
+ InstructionCost BuildVectorCost = getScalarizationOverhead(
+ *TTI, VL.front()->getType(), MaskVecTy, DemandedElts, /*Insert=*/true,
+ /*Extract=*/false, CostKind);
const TreeEntry *BestEntry = nullptr;
if (FirstShuffleCost < ShuffleCost) {
std::for_each(std::next(Mask.begin(), Part * VL.size()),
@@ -13968,45 +13993,15 @@ InstructionCost BoUpSLP::getGatherCost(ArrayRef<Value *> VL, bool ForPoisonSrc,
ShuffledElements.setBit(I);
ShuffleMask[I] = Res.first->second;
}
- if (!DemandedElements.isZero()) {
- if (isa<FixedVectorType>(ScalarTy)) {
- assert(SLPReVec && "Only supported by REVEC.");
- // We don't need to insert elements one by one. Instead, we can insert the
- // entire vector into the destination.
- Cost = 0;
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- for (unsigned I : seq<unsigned>(VL.size()))
- if (DemandedElements[I])
- Cost += ::getShuffleCost(*TTI, TTI::SK_InsertSubvector, VecTy, {},
- CostKind, I * ScalarTyNumElements,
- cast<FixedVectorType>(ScalarTy));
- } else {
- Cost += TTI->getScalarizationOverhead(VecTy, DemandedElements,
- /*Insert=*/true,
- /*Extract=*/false, CostKind, VL);
- }
- }
- if (ForPoisonSrc) {
- if (isa<FixedVectorType>(ScalarTy)) {
- assert(SLPReVec && "Only supported by REVEC.");
- // We don't need to insert elements one by one. Instead, we can insert the
- // entire vector into the destination.
- assert(DemandedElements.isZero() &&
- "Need to consider the cost from DemandedElements.");
- Cost = 0;
- unsigned ScalarTyNumElements = getNumElements(ScalarTy);
- for (unsigned I : seq<unsigned>(VL.size()))
- if (!ShuffledElements[I])
- Cost += TTI->getShuffleCost(
- TTI::SK_InsertSubvector, VecTy, std::nullopt, CostKind,
- I * ScalarTyNumElements, cast<FixedVectorType>(ScalarTy));
- } else {
- Cost = TTI->getScalarizationOverhead(VecTy,
- /*DemandedElts*/ ~ShuffledElements,
- /*Insert*/ true,
- /*Extract*/ false, CostKind, VL);
- }
- }
+ if (!DemandedElements.isZero())
+ Cost += getScalarizationOverhead(*TTI, ScalarTy, VecTy, DemandedElements,
+ /*Insert=*/true,
+ /*Extract=*/false, CostKind, VL);
+ if (ForPoisonSrc)
+ Cost = getScalarizationOverhead(*TTI, ScalarTy, VecTy,
+ /*DemandedElts*/ ~ShuffledElements,
+ /*Insert*/ true,
+ /*Extract*/ false, CostKind, VL);
if (DuplicateNonConst)
Cost += ::getShuffleCost(*TTI, TargetTransformInfo::SK_PermuteSingleSrc,
VecTy, ShuffleMask);
|
When ScalarTy is FixedVectorType, VecTy->getNumElements() is different from ScalarTy is NOT FixedVectorType. However, VF is same.
| TTI.getScalarizationOverhead( | ||
| PtrVecTy, APInt::getOneBitSet(VecTy->getNumElements(), 0), | ||
| /*Insert=*/true, /*Extract=*/false, CostKind) + | ||
| TTI.getScalarizationOverhead(PtrVecTy, APInt::getOneBitSet(Sz, 0), |
There was a problem hiding this comment.
This is not a part of this change, the common function is not used
There was a problem hiding this comment.
They share DemandedElts in Line 5278.
InstructionCost GatherCost =
getScalarizationOverhead(TTI, ScalarTy, VecTy, DemandedElts,
/*Insert=*/true,
/*Extract=*/false, CostKind) +
ScalarLoadsCost;
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
No description provided.