Skip to content

Commit 233fb98

Browse files
committed
[ARM] Improve bitwise reduction costs
This adds some basic and/or/xor reduction costs for NEON/MVE, handling them like other reductions where vector operations are used to reduce to legal sizes, followed by an optional VREV+VAND/VORR/VEOR step and scalarization from there.
1 parent 48987fb commit 233fb98

File tree

2 files changed

+118
-90
lines changed

2 files changed

+118
-90
lines changed

llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1707,6 +1707,34 @@ ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
17071707
getArithmeticInstrCost(Opcode, ValTy->getElementType(), CostKind);
17081708
}
17091709

1710+
if ((ISD == ISD::AND || ISD == ISD::OR || ISD == ISD::XOR) &&
1711+
(EltSize == 64 || EltSize == 32 || EltSize == 16 || EltSize == 8)) {
1712+
unsigned NumElts = cast<FixedVectorType>(ValTy)->getNumElements();
1713+
unsigned VecLimit =
1714+
ST->hasMVEIntegerOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1715+
InstructionCost VecCost = 0;
1716+
while (isPowerOf2_32(NumElts) && NumElts * EltSize > VecLimit) {
1717+
Type *VecTy = FixedVectorType::get(ValTy->getElementType(), NumElts / 2);
1718+
VecCost += getArithmeticInstrCost(Opcode, VecTy, CostKind);
1719+
NumElts /= 2;
1720+
}
1721+
// For i16/i8, MVE will perform a VREV + VORR/VAND/VEOR for the 64bit vector
1722+
// step.
1723+
if (ST->hasMVEIntegerOps() && ValVT.getScalarSizeInBits() <= 16 &&
1724+
NumElts * EltSize == 64) {
1725+
Type *VecTy = FixedVectorType::get(ValTy->getElementType(), NumElts);
1726+
VecCost += ST->getMVEVectorCostFactor(CostKind) +
1727+
getArithmeticInstrCost(Opcode, VecTy, CostKind);
1728+
NumElts /= 2;
1729+
}
1730+
1731+
// From here we extract the elements and perform the and/or/xor.
1732+
InstructionCost ExtractCost = NumElts;
1733+
return VecCost + ExtractCost +
1734+
(NumElts - 1) * getArithmeticInstrCost(
1735+
Opcode, ValTy->getElementType(), CostKind);
1736+
}
1737+
17101738
if (!ST->hasMVEIntegerOps() || !ValVT.isSimple() || ISD != ISD::ADD ||
17111739
TTI::requiresOrderedReduction(FMF))
17121740
return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind);

0 commit comments

Comments
 (0)