@@ -12895,7 +12895,7 @@ class HorizontalReduction {
12895
12895
HorizontalReduction() = default;
12896
12896
12897
12897
/// Try to find a reduction tree.
12898
- bool matchAssociativeReduction(Instruction *Root,
12898
+ bool matchAssociativeReduction(BoUpSLP &R, Instruction *Root,
12899
12899
ScalarEvolution &SE, const DataLayout &DL,
12900
12900
const TargetLibraryInfo &TLI) {
12901
12901
RdxKind = HorizontalReduction::getRdxKind(Root);
@@ -12924,11 +12924,10 @@ class HorizontalReduction {
12924
12924
// Checks if the operands of the \p TreeN instruction are also reduction
12925
12925
// operations or should be treated as reduced values or an extra argument,
12926
12926
// which is not part of the reduction.
12927
- auto &&CheckOperands = [this, IsCmpSelMinMax,
12928
- BB](Instruction *TreeN,
12929
- SmallVectorImpl<Value *> &ExtraArgs,
12930
- SmallVectorImpl<Value *> &PossibleReducedVals,
12931
- SmallVectorImpl<Instruction *> &ReductionOps) {
12927
+ auto CheckOperands = [&](Instruction *TreeN,
12928
+ SmallVectorImpl<Value *> &ExtraArgs,
12929
+ SmallVectorImpl<Value *> &PossibleReducedVals,
12930
+ SmallVectorImpl<Instruction *> &ReductionOps) {
12932
12931
for (int I = getFirstOperandIndex(TreeN),
12933
12932
End = getNumberOfOperands(TreeN);
12934
12933
I < End; ++I) {
@@ -12943,10 +12942,14 @@ class HorizontalReduction {
12943
12942
}
12944
12943
// If the edge is not an instruction, or it is different from the main
12945
12944
// reduction opcode or has too many uses - possible reduced value.
12945
+ // Also, do not try to reduce const values, if the operation is not
12946
+ // foldable.
12946
12947
if (!EdgeInst || getRdxKind(EdgeInst) != RdxKind ||
12947
12948
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
12948
12949
!hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
12949
- !isVectorizable(RdxKind, EdgeInst)) {
12950
+ !isVectorizable(RdxKind, EdgeInst) ||
12951
+ (R.isAnalyzedReductionRoot(EdgeInst) &&
12952
+ all_of(EdgeInst->operands(), Constant::classof))) {
12950
12953
PossibleReducedVals.push_back(EdgeVal);
12951
12954
continue;
12952
12955
}
@@ -13230,9 +13233,11 @@ class HorizontalReduction {
13230
13233
allConstant(Candidates)) {
13231
13234
Value *Res = Candidates.front();
13232
13235
++VectorizedVals.try_emplace(Candidates.front(), 0).first->getSecond();
13233
- for (Value *V : ArrayRef(Candidates).drop_front()) {
13234
- Res = createOp(Builder, RdxKind, Res, V, "const.rdx", ReductionOps);
13235
- ++VectorizedVals.try_emplace(V, 0).first->getSecond();
13236
+ for (Value *VC : ArrayRef(Candidates).drop_front()) {
13237
+ Res = createOp(Builder, RdxKind, Res, VC, "const.rdx", ReductionOps);
13238
+ ++VectorizedVals.try_emplace(VC, 0).first->getSecond();
13239
+ if (auto *ResI = dyn_cast<Instruction>(Res))
13240
+ V.analyzedReductionRoot(ResI);
13236
13241
}
13237
13242
VectorizedTree = GetNewVectorizedTree(VectorizedTree, Res);
13238
13243
continue;
@@ -14143,7 +14148,7 @@ bool SLPVectorizerPass::vectorizeHorReduction(
14143
14148
if (!isReductionCandidate(Inst))
14144
14149
return nullptr;
14145
14150
HorizontalReduction HorRdx;
14146
- if (!HorRdx.matchAssociativeReduction(Inst, *SE, *DL, *TLI))
14151
+ if (!HorRdx.matchAssociativeReduction(R, Inst, *SE, *DL, *TLI))
14147
14152
return nullptr;
14148
14153
return HorRdx.tryToReduce(R, TTI, *TLI);
14149
14154
};
0 commit comments