95
95
#include <cctype>
96
96
#include <cstdint>
97
97
#include <cstdlib>
98
+ #include <deque>
98
99
#include <iterator>
99
100
#include <limits>
100
101
#include <optional>
@@ -17503,6 +17504,8 @@ static SDValue performVecReduceAddCombineWithUADDLP(SDNode *N,
17503
17504
return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i32, UADDLP);
17504
17505
}
17505
17506
17507
+ // Turn [sign|zero]_extend(vecreduce_add()) into SVE's SADDV|UADDV
17508
+ // instructions.
17506
17509
static SDValue
17507
17510
performVecReduceAddExtCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
17508
17511
const AArch64TargetLowering &TLI) {
@@ -17513,53 +17516,43 @@ performVecReduceAddExtCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
17513
17516
17514
17517
SelectionDAG &DAG = DCI.DAG;
17515
17518
auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
17516
- SDNode *ZEXT = N->getOperand(0).getNode();
17517
- EVT VecVT = ZEXT->getOperand(0).getValueType();
17519
+ SDValue VecOp = N->getOperand(0).getOperand(0);
17518
17520
SDLoc DL(N);
17519
17521
17520
- SDValue VecOp = ZEXT->getOperand(0);
17521
- VecVT = VecOp.getValueType();
17522
- bool IsScalableType = VecVT.isScalableVector();
17523
- SmallVector<SDValue, 2> ResultValues;
17522
+ bool IsScalableType = VecOp.getValueType().isScalableVector();
17523
+ std::deque<SDValue> ResultValues;
17524
+ ResultValues.push_back(VecOp);
17524
17525
17525
- if (!TLI.isTypeLegal(VecVT)) {
17526
- SmallVector<SDValue, 2> PrevValues;
17527
- PrevValues.push_back(VecOp);
17526
+ // Split the input vectors if not legal.
17527
+ while (!TLI.isTypeLegal(ResultValues.front().getValueType())) {
17528
+ if (!ResultValues.front()
17529
+ .getValueType()
17530
+ .getVectorElementCount()
17531
+ .isKnownEven())
17532
+ return SDValue();
17533
+ EVT CurVT = ResultValues.front().getValueType();
17528
17534
while (true) {
17529
-
17530
- if (!VecVT.isScalableVector() &&
17531
- !PrevValues[0].getValueType().getVectorElementCount().isKnownEven())
17532
- return SDValue();
17533
-
17534
- for (SDValue Vec : PrevValues) {
17535
- SDValue Lo, Hi;
17536
- std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
17537
- ResultValues.push_back(Lo);
17538
- ResultValues.push_back(Hi);
17539
- }
17540
- if (TLI.isTypeLegal(ResultValues[0].getValueType()))
17535
+ SDValue Vec = ResultValues.front();
17536
+ if (Vec.getValueType() != CurVT)
17541
17537
break;
17542
- PrevValues.clear();
17543
- std::copy(ResultValues.begin(), ResultValues.end(),
17544
- std::back_inserter(PrevValues));
17545
- ResultValues.clear();
17538
+ ResultValues.pop_front();
17539
+ SDValue Lo, Hi;
17540
+ std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
17541
+ ResultValues.push_back(Lo);
17542
+ ResultValues.push_back(Hi);
17546
17543
}
17547
- } else {
17548
- ResultValues.push_back(VecOp);
17549
17544
}
17550
- SDNode *VecRed = N;
17551
- EVT ElemType = VecRed->getValueType(0);
17552
- SmallVector<SDValue, 2> Results;
17553
17545
17546
+ EVT ElemType = N->getValueType(0);
17547
+ SmallVector<SDValue, 2> Results;
17554
17548
if (!IsScalableType &&
17555
17549
!TLI.useSVEForFixedLengthVectorVT(
17556
17550
ResultValues[0].getValueType(),
17557
17551
/*OverrideNEON=*/Subtarget.useSVEForFixedLengthVectors(
17558
17552
ResultValues[0].getValueType())))
17559
17553
return SDValue();
17560
17554
17561
- for (unsigned Num = 0; Num < ResultValues.size(); ++Num) {
17562
- SDValue Reg = ResultValues[Num];
17555
+ for (SDValue Reg : ResultValues) {
17563
17556
EVT RdxVT = Reg->getValueType(0);
17564
17557
SDValue Pg = getPredicateForVector(DAG, DL, RdxVT);
17565
17558
if (!IsScalableType) {
0 commit comments