@@ -626,14 +626,12 @@ static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
626
626
static void legalizeAndOptimizeInductions (VPlan &Plan) {
627
627
using namespace llvm ::VPlanPatternMatch;
628
628
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion ()->getEntryBasicBlock ();
629
- SmallVector<VPWidenInductionRecipe *, 4 > InductionPhis;
630
- for (VPRecipeBase &R : HeaderVPBB->phis ())
631
- if (auto *IV = dyn_cast<VPWidenInductionRecipe>(&R))
632
- InductionPhis.push_back (IV);
633
-
634
629
bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly ();
635
- VPBuilder Builder;
636
- for (VPWidenInductionRecipe *PhiR : reverse (InductionPhis)) {
630
+ VPBuilder Builder (HeaderVPBB, HeaderVPBB->getFirstNonPhi ());
631
+ for (VPRecipeBase &Phi : HeaderVPBB->phis ()) {
632
+ auto *PhiR = dyn_cast<VPWidenInductionRecipe>(&Phi);
633
+ if (!PhiR)
634
+ continue ;
637
635
// Try to narrow wide and replicating recipes to uniform recipes, based on
638
636
// VPlan analysis.
639
637
// TODO: Apply to all recipes in the future, to replace legacy uniformity
@@ -643,8 +641,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
643
641
auto *Def = dyn_cast<VPSingleDefRecipe>(U);
644
642
auto *RepR = dyn_cast<VPReplicateRecipe>(U);
645
643
// Skip recipes that shouldn't be narrowed.
646
- if (!Def ||
647
- !isa<VPReplicateRecipe, VPWidenRecipe, VPWidenGEPRecipe>(Def) ||
644
+ if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
648
645
Def->getNumUsers () == 0 || !Def->getUnderlyingValue () ||
649
646
(RepR && (RepR->isSingleScalar () || RepR->isPredicated ())))
650
647
continue ;
@@ -657,13 +654,11 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
657
654
Def->operands (), /* IsUniform*/ true );
658
655
Clone->insertAfter (Def);
659
656
Def->replaceAllUsesWith (Clone);
660
- Def->eraseFromParent ();
661
657
}
662
658
663
- Builder.setInsertPoint (HeaderVPBB, HeaderVPBB->getFirstNonPhi ());
664
659
// Replace wide pointer inductions which have only their scalars used by
665
660
// PtrAdd(IndStart, ScalarIVSteps (0, Step)).
666
- if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(PhiR )) {
661
+ if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi )) {
667
662
if (!PtrIV->onlyScalarsGenerated (Plan.hasScalableVF ()))
668
663
continue ;
669
664
@@ -684,7 +679,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
684
679
685
680
// Replace widened induction with scalar steps for users that only use
686
681
// scalars.
687
- auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(PhiR );
682
+ auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(&Phi );
688
683
if (HasOnlyVectorVFs && none_of (WideIV->users (), [WideIV](VPUser *U) {
689
684
return U->usesScalars (WideIV);
690
685
}))
0 commit comments