!fixup update after merge.

Author: fhahn

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -2895,14 +2895,18 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
 
   // Don't apply optimizations below when no vector region remains, as they all
   // require a vector loop at the moment.
-  if (!State.Plan->getVectorLoopRegion())
+  VPBasicBlock *HeaderVPBB = find_singleton<VPBasicBlock>(
+      vp_depth_first_shallow(State.Plan->getEntry()),
+      [](VPBlockBase *VPB, bool) {
+        auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+        return VPBB && VPBB->isHeader() ? VPBB : nullptr;
+      });
+  if (!HeaderVPBB)
     return;
 
   for (Instruction *PI : PredicatedInstructions)
     sinkScalarOperands(&*PI);
 
-  VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>(
-      State.Plan->getVectorPreheader()->getSingleSuccessor());
   BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
 
   // Remove redundant induction instructions.
@@ -7710,7 +7714,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
       ILV.getTripCount(),
       ILV.getOrCreateVectorTripCount(ILV.LoopVectorPreHeader), State);
   replaceVPBBWithIRVPBB(VectorPH, State.CFG.PrevBB);
-  VPlanTransforms::prepareToExecute(BestVPlan);
 
   BestVPlan.execute(&State);
 
@@ -7736,15 +7739,18 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   // 2.6. Maintain Loop Hints
   // Keep all loop hints from the original loop on the vector loop (we'll
   // replace the vectorizer-specific hints below).
-  if (auto *LoopRegion = BestVPlan.getVectorLoopRegion()) {
+  VPBasicBlock *HeaderVPBB = find_singleton<VPBasicBlock>(
+      vp_depth_first_shallow(BestVPlan.getEntry()), [](VPBlockBase *VPB, bool) {
+        auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+        return VPBB && VPBB->isHeader() ? VPBB : nullptr;
+      });
+  if (HeaderVPBB) {
     MDNode *OrigLoopID = OrigLoop->getLoopID();
 
     std::optional<MDNode *> VectorizedLoopID =
         makeFollowupLoopID(OrigLoopID, {LLVMLoopVectorizeFollowupAll,
                                         LLVMLoopVectorizeFollowupVectorized});
 
-  VPBasicBlock *HeaderVPBB =
-      cast<VPBasicBlock>(BestVPlan.getVectorPreheader()->getSingleSuccessor());
     Loop *L = LI->getLoopFor(State.CFG.VPBB2IRBB[HeaderVPBB]);
     if (VectorizedLoopID) {
       L->setLoopID(*VectorizedLoopID);
@@ -7770,7 +7776,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   ILV.printDebugTracesAtEnd();
 
   // 4. Adjust branch weight of the branch in the middle block.
-  if (BestVPlan.getVectorLoopRegion()) {
+  if (HeaderVPBB) {
     auto *MiddleVPBB = BestVPlan.getMiddleBlock();
     auto *MiddleTerm =
         cast<BranchInst>(State.CFG.VPBB2IRBB[MiddleVPBB]->getTerminator());

llvm/lib/Transforms/Vectorize/VPlan.cpp

@@ -487,17 +487,17 @@ void VPBasicBlock::execute(VPTransformState *State) {
 
   if (isHeader()) {
     // Create and register the new vector loop.
-    State->CurrentVectorLoop = State->LI->AllocateLoop();
+    State->CurrentParentLoop = State->LI->AllocateLoop();
     BasicBlock *VectorPH =
         State->CFG.VPBB2IRBB[cast<VPBasicBlock>(getPredecessors()[0])];
     Loop *ParentLoop = State->LI->getLoopFor(VectorPH);
 
     // Insert the new loop into the loop nest and register the new basic blocks
     // before calling any utilities such as SCEV that require valid LoopInfo.
     if (ParentLoop)
-      ParentLoop->addChildLoop(State->CurrentVectorLoop);
+      ParentLoop->addChildLoop(State->CurrentParentLoop);
     else
-      State->LI->addTopLevelLoop(State->CurrentVectorLoop);
+      State->LI->addTopLevelLoop(State->CurrentParentLoop);
   }
 
   auto IsReplicateRegion = [](VPBlockBase *BB) {
@@ -823,10 +823,6 @@ void VPRegionBlock::removeRegion() {
 
   VPBlockUtils::connectBlocks(Preheader, Header);
   VPBlockUtils::connectBlocks(Exiting, Middle);
-
-  // Set LoopRegion's Entry to nullptr, as the CFG from LoopRegion shouldn't
-  // be deleted when the region is deleted.
-  Entry = nullptr;
 }
 
 VPlan::VPlan(Loop *L) {
@@ -1029,13 +1025,17 @@ void VPlan::execute(VPTransformState *State) {
         Phi = cast<PHINode>(GEP->getPointerOperand());
       }
 
-      auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
-      bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
-                         (isa<VPReductionPHIRecipe>(PhiR) &&
-                          cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
-      Value *Phi = State->get(PhiR, NeedsScalar);
-      Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
-      cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
+      Phi->setIncomingBlock(1, VectorLatchBB);
+
+      // Move the last step to the end of the latch block. This ensures
+      // consistent placement of all induction updates.
+      Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
+      Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
+
+      // Use the steps for the last part as backedge value for the induction.
+      if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
+        Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
+      continue;
     }
 
     auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
@@ -1045,6 +1045,7 @@ void VPlan::execute(VPTransformState *State) {
     Value *Phi = State->get(PhiR, NeedsScalar);
     Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
     cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
+    }
   }
 }
 
@@ -1419,7 +1420,9 @@ bool VPValue::isDefinedOutsideLoopRegions() const {
 
   const VPBasicBlock *DefVPBB = DefR->getParent();
   auto *Plan = DefVPBB->getPlan();
-  return DefVPBB == Plan->getPreheader() || DefVPBB == Plan->getEntry();
+  if (Plan->getVectorLoopRegion())
+    return !DefR->getParent()->getEnclosingLoopRegion();
+  return DefVPBB == Plan->getEntry();
 }
 void VPValue::replaceAllUsesWith(VPValue *New) {
   replaceUsesWithIf(New, [](VPUser &, unsigned) { return true; });

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2057,7 +2057,6 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
     R->removeRegion();
     // Add explicit backedge.
     VPBlockUtils::connectBlocks(Latch, Header);
-    delete R;
   }
 
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(