@@ -8963,16 +8963,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
8963
8963
" AnyOf reductions are not allowed for in-loop reductions" );
8964
8964
8965
8965
// Collect the chain of "link" recipes for the reduction starting at PhiR.
8966
- SetVector<VPRecipeBase *> Worklist;
8966
+ SetVector<VPSingleDefRecipe *> Worklist;
8967
8967
Worklist.insert (PhiR);
8968
8968
for (unsigned I = 0 ; I != Worklist.size (); ++I) {
8969
- VPRecipeBase *Cur = Worklist[I];
8970
- for (VPUser *U : Cur->getVPSingleValue ()->users ()) {
8971
- auto *UserRecipe = dyn_cast<VPRecipeBase>(U);
8972
- if (!UserRecipe)
8969
+ VPSingleDefRecipe *Cur = Worklist[I];
8970
+ for (VPUser *U : Cur->users ()) {
8971
+ auto *UserRecipe = dyn_cast<VPSingleDefRecipe>(U);
8972
+ if (!UserRecipe) {
8973
+ assert (isa<VPLiveOut>(U) &&
8974
+ " U must either be a VPSingleDef or VPLiveOut" );
8973
8975
continue ;
8974
- assert (UserRecipe->getNumDefinedValues () == 1 &&
8975
- " recipes must define exactly one result value" );
8976
+ }
8976
8977
Worklist.insert (UserRecipe);
8977
8978
}
8978
8979
}
@@ -8982,10 +8983,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
8982
8983
// (PreviousLink) to tell which of the two operands of a Link will remain
8983
8984
// scalar and which will be reduced. For minmax by select(cmp), Link will be
8984
8985
// the select instructions.
8985
- VPRecipeBase *PreviousLink = PhiR; // Aka Worklist[0].
8986
- for (VPRecipeBase *CurrentLink : Worklist.getArrayRef ().drop_front ()) {
8987
- VPValue *PreviousLinkV = PreviousLink->getVPSingleValue ();
8988
-
8986
+ VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
8987
+ for (VPSingleDefRecipe *CurrentLink : Worklist.getArrayRef ().drop_front ()) {
8989
8988
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr ();
8990
8989
8991
8990
// Index of the first operand which holds a non-mask vector operand.
@@ -9000,7 +8999,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
9000
8999
" Expected instruction to be a call to the llvm.fmuladd intrinsic" );
9001
9000
assert (((MinVF.isScalar () && isa<VPReplicateRecipe>(CurrentLink)) ||
9002
9001
isa<VPWidenCallRecipe>(CurrentLink)) &&
9003
- CurrentLink->getOperand (2 ) == PreviousLinkV &&
9002
+ CurrentLink->getOperand (2 ) == PreviousLink &&
9004
9003
" expected a call where the previous link is the added operand" );
9005
9004
9006
9005
// If the instruction is a call to the llvm.fmuladd intrinsic then we
@@ -9031,15 +9030,15 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
9031
9030
// Note that for non-commutable operands (cmp-selects), the semantics of
9032
9031
// the cmp-select are captured in the recurrence kind.
9033
9032
unsigned VecOpId =
9034
- CurrentLink->getOperand (IndexOfFirstOperand) == PreviousLinkV
9033
+ CurrentLink->getOperand (IndexOfFirstOperand) == PreviousLink
9035
9034
? IndexOfFirstOperand + 1
9036
9035
: IndexOfFirstOperand;
9037
9036
VecOp = CurrentLink->getOperand (VecOpId);
9038
- assert (VecOp != PreviousLinkV &&
9037
+ assert (VecOp != PreviousLink &&
9039
9038
CurrentLink->getOperand (CurrentLink->getNumOperands () - 1 -
9040
9039
(VecOpId - IndexOfFirstOperand)) ==
9041
- PreviousLinkV &&
9042
- " PreviousLinkV must be the operand other than VecOp" );
9040
+ PreviousLink &&
9041
+ " PreviousLink must be the operand other than VecOp" );
9043
9042
}
9044
9043
9045
9044
BasicBlock *BB = CurrentLinkI->getParent ();
@@ -9051,13 +9050,13 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
9051
9050
}
9052
9051
9053
9052
VPReductionRecipe *RedRecipe = new VPReductionRecipe (
9054
- RdxDesc, CurrentLinkI, PreviousLinkV , VecOp, CondOp);
9053
+ RdxDesc, CurrentLinkI, PreviousLink , VecOp, CondOp);
9055
9054
// Append the recipe to the end of the VPBasicBlock because we need to
9056
9055
// ensure that it comes after all of it's inputs, including CondOp.
9057
9056
// Note that this transformation may leave over dead recipes (including
9058
9057
// CurrentLink), which will be cleaned by a later VPlan transform.
9059
9058
LinkVPBB->appendRecipe (RedRecipe);
9060
- CurrentLink->getVPSingleValue ()-> replaceAllUsesWith (RedRecipe);
9059
+ CurrentLink->replaceAllUsesWith (RedRecipe);
9061
9060
PreviousLink = RedRecipe;
9062
9061
}
9063
9062
}
0 commit comments