Skip to content

Commit abdb61f

Browse files
authored
[VPlan] Introduce VPSingleDefRecipe. (#77023)
This patch introduces a new common base class for recipes defining a single result VPValue. This has been discussed/mentioned at various previous reviews as potential follow-up and helps to replace various getVPSingleValue calls. PR: #77023
1 parent 2759cfa commit abdb61f

File tree

4 files changed

+163
-102
lines changed

4 files changed

+163
-102
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8963,16 +8963,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
89638963
"AnyOf reductions are not allowed for in-loop reductions");
89648964

89658965
// Collect the chain of "link" recipes for the reduction starting at PhiR.
8966-
SetVector<VPRecipeBase *> Worklist;
8966+
SetVector<VPSingleDefRecipe *> Worklist;
89678967
Worklist.insert(PhiR);
89688968
for (unsigned I = 0; I != Worklist.size(); ++I) {
8969-
VPRecipeBase *Cur = Worklist[I];
8970-
for (VPUser *U : Cur->getVPSingleValue()->users()) {
8971-
auto *UserRecipe = dyn_cast<VPRecipeBase>(U);
8972-
if (!UserRecipe)
8969+
VPSingleDefRecipe *Cur = Worklist[I];
8970+
for (VPUser *U : Cur->users()) {
8971+
auto *UserRecipe = dyn_cast<VPSingleDefRecipe>(U);
8972+
if (!UserRecipe) {
8973+
assert(isa<VPLiveOut>(U) &&
8974+
"U must either be a VPSingleDef or VPLiveOut");
89738975
continue;
8974-
assert(UserRecipe->getNumDefinedValues() == 1 &&
8975-
"recipes must define exactly one result value");
8976+
}
89768977
Worklist.insert(UserRecipe);
89778978
}
89788979
}
@@ -8982,10 +8983,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
89828983
// (PreviousLink) to tell which of the two operands of a Link will remain
89838984
// scalar and which will be reduced. For minmax by select(cmp), Link will be
89848985
// the select instructions.
8985-
VPRecipeBase *PreviousLink = PhiR; // Aka Worklist[0].
8986-
for (VPRecipeBase *CurrentLink : Worklist.getArrayRef().drop_front()) {
8987-
VPValue *PreviousLinkV = PreviousLink->getVPSingleValue();
8988-
8986+
VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
8987+
for (VPSingleDefRecipe *CurrentLink : Worklist.getArrayRef().drop_front()) {
89898988
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
89908989

89918990
// Index of the first operand which holds a non-mask vector operand.
@@ -9000,7 +8999,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
90008999
"Expected instruction to be a call to the llvm.fmuladd intrinsic");
90019000
assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
90029001
isa<VPWidenCallRecipe>(CurrentLink)) &&
9003-
CurrentLink->getOperand(2) == PreviousLinkV &&
9002+
CurrentLink->getOperand(2) == PreviousLink &&
90049003
"expected a call where the previous link is the added operand");
90059004

90069005
// If the instruction is a call to the llvm.fmuladd intrinsic then we
@@ -9031,15 +9030,15 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
90319030
// Note that for non-commutable operands (cmp-selects), the semantics of
90329031
// the cmp-select are captured in the recurrence kind.
90339032
unsigned VecOpId =
9034-
CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLinkV
9033+
CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
90359034
? IndexOfFirstOperand + 1
90369035
: IndexOfFirstOperand;
90379036
VecOp = CurrentLink->getOperand(VecOpId);
9038-
assert(VecOp != PreviousLinkV &&
9037+
assert(VecOp != PreviousLink &&
90399038
CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
90409039
(VecOpId - IndexOfFirstOperand)) ==
9041-
PreviousLinkV &&
9042-
"PreviousLinkV must be the operand other than VecOp");
9040+
PreviousLink &&
9041+
"PreviousLink must be the operand other than VecOp");
90439042
}
90449043

90459044
BasicBlock *BB = CurrentLinkI->getParent();
@@ -9051,13 +9050,13 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
90519050
}
90529051

90539052
VPReductionRecipe *RedRecipe = new VPReductionRecipe(
9054-
RdxDesc, CurrentLinkI, PreviousLinkV, VecOp, CondOp);
9053+
RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp);
90559054
// Append the recipe to the end of the VPBasicBlock because we need to
90569055
// ensure that it comes after all of it's inputs, including CondOp.
90579056
// Note that this transformation may leave over dead recipes (including
90589057
// CurrentLink), which will be cleaned by a later VPlan transform.
90599058
LinkVPBB->appendRecipe(RedRecipe);
9060-
CurrentLink->getVPSingleValue()->replaceAllUsesWith(RedRecipe);
9059+
CurrentLink->replaceAllUsesWith(RedRecipe);
90619060
PreviousLink = RedRecipe;
90629061
}
90639062
}

0 commit comments

Comments
 (0)