Skip to content

Commit cb2560d

Browse files
authored
[VPlan] Verify plan before optimizations. NFC (#122678)
I've been exploring verifying the VPlan before and after the EVL transformation steps, and noticed that the VPlan comes out in an invalid state between construction and optimisation. In adjustRecipesForReductions, we leave behind some dead recipes which are invalid: 1) When we replace a link with a reduction recipe, the old link ends up becoming a use-before-def: WIDEN ir<%l7> = add ir<%sum.02>, ir<%indvars.iv>.1 WIDEN ir<%l8> = add ir<%l7>.1, ir<%l3> WIDEN ir<%l9> = add ir<%l8>.1, ir<%l5> ... REDUCE ir<%l7>.1 = ir<%sum.02> + reduce.add (ir<%indvars.iv>.1) REDUCE ir<%l8>.1 = ir<%l7>.1 + reduce.add (ir<%l3>) REDUCE ir<%l9>.1 = ir<%l8>.1 + reduce.add (ir<%l5>) 2) When transforming an AnyOf reduction phi to a boolean, we leave behind a select with mismatching operand types, which will trigger the assertions in VTypeAnalysis after #122679 This adds an extra verification step and deletes the dead recipes eagerly to keep the plan valid.
1 parent 9f114af commit cb2560d

File tree

1 file changed

+11
-2
lines changed

1 file changed

+11
-2
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9624,6 +9624,8 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
96249624
VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
96259625
WithoutRuntimeCheck);
96269626
}
9627+
9628+
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
96279629
return Plan;
96289630
}
96299631

@@ -9698,6 +9700,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
96989700
VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
96999701
VPBasicBlock *Header = VectorLoopRegion->getEntryBasicBlock();
97009702
VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
9703+
SmallVector<VPRecipeBase *> ToDelete;
9704+
97019705
for (VPRecipeBase &R : Header->phis()) {
97029706
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
97039707
if (!PhiR || !PhiR->isInLoop() || (MinVF.isScalar() && !PhiR->isOrdered()))
@@ -9817,10 +9821,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
98179821
CM.useOrderedReductions(RdxDesc), CurrentLinkI->getDebugLoc());
98189822
// Append the recipe to the end of the VPBasicBlock because we need to
98199823
// ensure that it comes after all of it's inputs, including CondOp.
9820-
// Note that this transformation may leave over dead recipes (including
9821-
// CurrentLink), which will be cleaned by a later VPlan transform.
9824+
// Delete CurrentLink as it will be invalid if its operand is replaced
9825+
// with a reduction defined at the bottom of the block in the next link.
98229826
LinkVPBB->appendRecipe(RedRecipe);
98239827
CurrentLink->replaceAllUsesWith(RedRecipe);
9828+
ToDelete.push_back(CurrentLink);
98249829
PreviousLink = RedRecipe;
98259830
}
98269831
}
@@ -9935,6 +9940,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
99359940
Cmp = Builder.createNot(Cmp);
99369941
VPValue *Or = Builder.createOr(PhiR, Cmp);
99379942
Select->getVPSingleValue()->replaceAllUsesWith(Or);
9943+
// Delete Select now that it has invalid types.
9944+
ToDelete.push_back(Select);
99389945

99399946
// Convert the reduction phi to operate on bools.
99409947
PhiR->setOperand(0, Plan->getOrAddLiveIn(ConstantInt::getFalse(
@@ -9952,6 +9959,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
99529959
}
99539960

99549961
VPlanTransforms::clearReductionWrapFlags(*Plan);
9962+
for (VPRecipeBase *R : ToDelete)
9963+
R->eraseFromParent();
99559964
}
99569965

99579966
void VPDerivedIVRecipe::execute(VPTransformState &State) {

0 commit comments

Comments
 (0)