Skip to content

Commit 0e3c791

Browse files
authored
Revert "[LV][VPlan] Remove any-of reduction from precomputeCost. NFC" (#117280)
Reverts #117109 Some test cases need to update.
1 parent 764cfd7 commit 0e3c791

File tree

1 file changed

+22
-2
lines changed

1 file changed

+22
-2
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7303,14 +7303,34 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
73037303

73047304
// The legacy cost model has special logic to compute the cost of in-loop
73057305
// reductions, which may be smaller than the sum of all instructions involved
7306-
// in the reduction.
7306+
// in the reduction. For AnyOf reductions, VPlan codegen may remove the select
7307+
// which the legacy cost model uses to assign cost. Pre-compute their costs
7308+
// for now.
73077309
// TODO: Switch to costing based on VPlan once the logic has been ported.
73087310
for (const auto &[RedPhi, RdxDesc] : Legal->getReductionVars()) {
73097311
if (ForceTargetInstructionCost.getNumOccurrences())
73107312
continue;
73117313

7312-
if (!CM.isInLoopReduction(RedPhi))
7314+
if (!CM.isInLoopReduction(RedPhi) &&
7315+
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
7316+
RdxDesc.getRecurrenceKind()))
7317+
continue;
7318+
7319+
// AnyOf reduction codegen may remove the select. To match the legacy cost
7320+
// model, pre-compute the cost for AnyOf reductions here.
7321+
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
7322+
RdxDesc.getRecurrenceKind())) {
7323+
auto *Select = cast<SelectInst>(*find_if(
7324+
RedPhi->users(), [](User *U) { return isa<SelectInst>(U); }));
7325+
assert(!CostCtx.SkipCostComputation.contains(Select) &&
7326+
"reduction op visited multiple times");
7327+
CostCtx.SkipCostComputation.insert(Select);
7328+
auto ReductionCost = CostCtx.getLegacyCost(Select, VF);
7329+
LLVM_DEBUG(dbgs() << "Cost of " << ReductionCost << " for VF " << VF
7330+
<< ":\n any-of reduction " << *Select << "\n");
7331+
Cost += ReductionCost;
73137332
continue;
7333+
}
73147334

73157335
const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop);
73167336
SetVector<Instruction *> ChainOpsAndOperands(ChainOps.begin(),

0 commit comments

Comments
 (0)