@@ -7303,14 +7303,34 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
7303
7303
7304
7304
// The legacy cost model has special logic to compute the cost of in-loop
7305
7305
// reductions, which may be smaller than the sum of all instructions involved
7306
- // in the reduction.
7306
+ // in the reduction. For AnyOf reductions, VPlan codegen may remove the select
7307
+ // which the legacy cost model uses to assign cost. Pre-compute their costs
7308
+ // for now.
7307
7309
// TODO: Switch to costing based on VPlan once the logic has been ported.
7308
7310
for (const auto &[RedPhi, RdxDesc] : Legal->getReductionVars ()) {
7309
7311
if (ForceTargetInstructionCost.getNumOccurrences ())
7310
7312
continue ;
7311
7313
7312
- if (!CM.isInLoopReduction (RedPhi))
7314
+ if (!CM.isInLoopReduction (RedPhi) &&
7315
+ !RecurrenceDescriptor::isAnyOfRecurrenceKind (
7316
+ RdxDesc.getRecurrenceKind ()))
7317
+ continue ;
7318
+
7319
+ // AnyOf reduction codegen may remove the select. To match the legacy cost
7320
+ // model, pre-compute the cost for AnyOf reductions here.
7321
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind (
7322
+ RdxDesc.getRecurrenceKind ())) {
7323
+ auto *Select = cast<SelectInst>(*find_if (
7324
+ RedPhi->users (), [](User *U) { return isa<SelectInst>(U); }));
7325
+ assert (!CostCtx.SkipCostComputation .contains (Select) &&
7326
+ " reduction op visited multiple times" );
7327
+ CostCtx.SkipCostComputation .insert (Select);
7328
+ auto ReductionCost = CostCtx.getLegacyCost (Select, VF);
7329
+ LLVM_DEBUG (dbgs () << " Cost of " << ReductionCost << " for VF " << VF
7330
+ << " :\n any-of reduction " << *Select << " \n " );
7331
+ Cost += ReductionCost;
7313
7332
continue ;
7333
+ }
7314
7334
7315
7335
const auto &ChainOps = RdxDesc.getReductionOpChain (RedPhi, OrigLoop);
7316
7336
SetVector<Instruction *> ChainOpsAndOperands (ChainOps.begin (),
0 commit comments