Skip to content

Commit d266f47

Browse files
committed
[LV] Fix the cost of in-loop minmax reduction.
1 parent d4e46f0 commit d266f47

File tree

1 file changed

+11
-3
lines changed

1 file changed

+11
-3
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6138,12 +6138,20 @@ LoopVectorizationCostModel::getReductionPatternCost(
61386138
const RecurrenceDescriptor &RdxDesc =
61396139
Legal->getReductionVars().find(cast<PHINode>(ReductionPhi))->second;
61406140

6141-
InstructionCost BaseCost = TTI.getArithmeticReductionCost(
6142-
RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind);
6141+
InstructionCost BaseCost;
6142+
RecurKind RK = RdxDesc.getRecurrenceKind();
6143+
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
6144+
Intrinsic::ID MinMaxID = getMinMaxReductionIntrinsicOp(RK);
6145+
BaseCost = TTI.getMinMaxReductionCost(MinMaxID, VectorTy,
6146+
RdxDesc.getFastMathFlags(), CostKind);
6147+
} else {
6148+
BaseCost = TTI.getArithmeticReductionCost(
6149+
RdxDesc.getOpcode(), VectorTy, RdxDesc.getFastMathFlags(), CostKind);
6150+
}
61436151

61446152
// For a call to the llvm.fmuladd intrinsic we need to add the cost of a
61456153
// normal fmul instruction to the cost of the fadd reduction.
6146-
if (RdxDesc.getRecurrenceKind() == RecurKind::FMulAdd)
6154+
if (RK == RecurKind::FMulAdd)
61476155
BaseCost +=
61486156
TTI.getArithmeticInstrCost(Instruction::FMul, VectorTy, CostKind);
61496157

0 commit comments

Comments
 (0)