Skip to content

Commit c7209cb

Browse files
committed
[LV] Assert that there's a resume phi for epilogue loops (NFC).
This patch adds an assert to createAndCollectMergePhiForReduction to make sure there is a resume phi when vectorizing the epilogue loop. This is needed to set the resume value from the main vector loop. This assertion guards against the issue caused the revert of #78304.
1 parent 86ab10c commit c7209cb

File tree

1 file changed

+8
-4
lines changed

1 file changed

+8
-4
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7450,7 +7450,8 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
74507450
static void createAndCollectMergePhiForReduction(
74517451
VPInstruction *RedResult,
74527452
DenseMap<const RecurrenceDescriptor *, Value *> &ReductionResumeValues,
7453-
VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock) {
7453+
VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock,
7454+
bool VectorizingEpilogue) {
74547455
if (!RedResult ||
74557456
RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
74567457
return;
@@ -7463,6 +7464,9 @@ static void createAndCollectMergePhiForReduction(
74637464
State.get(RedResult, VPIteration(State.UF - 1, VPLane::getFirstLane()));
74647465
auto *ResumePhi =
74657466
dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
7467+
assert((!VectorizingEpilogue || ResumePhi) &&
7468+
"when vectorizing the epilogue loop, we need a resume phi from main "
7469+
"vector loop");
74667470

74677471
// TODO: bc.merge.rdx should not be created here, instead it should be
74687472
// modeled in VPlan.
@@ -7588,9 +7592,9 @@ LoopVectorizationPlanner::executePlan(
75887592
auto *ExitVPBB =
75897593
cast<VPBasicBlock>(BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
75907594
for (VPRecipeBase &R : *ExitVPBB) {
7591-
createAndCollectMergePhiForReduction(dyn_cast<VPInstruction>(&R),
7592-
ReductionResumeValues, State, OrigLoop,
7593-
State.CFG.VPBB2IRBB[ExitVPBB]);
7595+
createAndCollectMergePhiForReduction(
7596+
dyn_cast<VPInstruction>(&R), ReductionResumeValues, State, OrigLoop,
7597+
State.CFG.VPBB2IRBB[ExitVPBB], ExpandedSCEVs);
75947598
}
75957599

75967600
// 2.6. Maintain Loop Hints

0 commit comments

Comments
 (0)