@@ -357,18 +357,9 @@ static void addReplicateRegions(VPlan &Plan) {
357
357
}
358
358
}
359
359
360
- void VPlanTransforms::createAndOptimizeReplicateRegions (VPlan &Plan) {
361
- // Convert masked VPReplicateRecipes to if-then region blocks.
362
- addReplicateRegions (Plan);
363
-
364
- bool ShouldSimplify = true ;
365
- while (ShouldSimplify) {
366
- ShouldSimplify = sinkScalarOperands (Plan);
367
- ShouldSimplify |= mergeReplicateRegionsIntoSuccessors (Plan);
368
- ShouldSimplify |= VPlanTransforms::mergeBlocksIntoPredecessors (Plan);
369
- }
370
- }
371
- bool VPlanTransforms::mergeBlocksIntoPredecessors (VPlan &Plan) {
360
+ // / Remove redundant VPBasicBlocks by merging them into their predecessor if
361
+ // / the predecessor has a single successor.
362
+ static bool mergeBlocksIntoPredecessors (VPlan &Plan) {
372
363
SmallVector<VPBasicBlock *> WorkList;
373
364
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
374
365
vp_depth_first_deep (Plan.getEntry ()))) {
@@ -395,7 +386,25 @@ bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
395
386
return !WorkList.empty ();
396
387
}
397
388
398
- void VPlanTransforms::removeRedundantInductionCasts (VPlan &Plan) {
389
+ void VPlanTransforms::createAndOptimizeReplicateRegions (VPlan &Plan) {
390
+ // Convert masked VPReplicateRecipes to if-then region blocks.
391
+ addReplicateRegions (Plan);
392
+
393
+ bool ShouldSimplify = true ;
394
+ while (ShouldSimplify) {
395
+ ShouldSimplify = sinkScalarOperands (Plan);
396
+ ShouldSimplify |= mergeReplicateRegionsIntoSuccessors (Plan);
397
+ ShouldSimplify |= mergeBlocksIntoPredecessors (Plan);
398
+ }
399
+ }
400
+
401
+ // / Remove redundant casts of inductions.
402
+ // /
403
+ // / Such redundant casts are casts of induction variables that can be ignored,
404
+ // / because we already proved that the casted phi is equal to the uncasted phi
405
+ // / in the vectorized loop. There is no need to vectorize the cast - the same
406
+ // / value can be used for both the phi and casts in the vector loop.
407
+ static void removeRedundantInductionCasts (VPlan &Plan) {
399
408
for (auto &Phi : Plan.getVectorLoopRegion ()->getEntryBasicBlock ()->phis ()) {
400
409
auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
401
410
if (!IV || IV->getTruncInst ())
@@ -426,7 +435,9 @@ void VPlanTransforms::removeRedundantInductionCasts(VPlan &Plan) {
426
435
}
427
436
}
428
437
429
- void VPlanTransforms::removeRedundantCanonicalIVs (VPlan &Plan) {
438
+ // / Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
439
+ // / recipe, if it exists.
440
+ static void removeRedundantCanonicalIVs (VPlan &Plan) {
430
441
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV ();
431
442
VPWidenCanonicalIVRecipe *WidenNewIV = nullptr ;
432
443
for (VPUser *U : CanonicalIV->users ()) {
@@ -462,7 +473,7 @@ void VPlanTransforms::removeRedundantCanonicalIVs(VPlan &Plan) {
462
473
}
463
474
}
464
475
465
- void VPlanTransforms:: removeDeadRecipes (VPlan &Plan) {
476
+ static void removeDeadRecipes (VPlan &Plan) {
466
477
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT (
467
478
Plan.getEntry ());
468
479
@@ -531,7 +542,11 @@ static VPValue *createScalarIVSteps(VPlan &Plan, const InductionDescriptor &ID,
531
542
return Steps;
532
543
}
533
544
534
- void VPlanTransforms::optimizeInductions (VPlan &Plan, ScalarEvolution &SE) {
545
+ // / If any user of a VPWidenIntOrFpInductionRecipe needs scalar values,
546
+ // / provide them by building scalar steps off of the canonical scalar IV and
547
+ // / update the original IV's users. This is an optional optimization to reduce
548
+ // / the needs of vector extracts.
549
+ static void optimizeInductions (VPlan &Plan, ScalarEvolution &SE) {
535
550
SmallVector<VPRecipeBase *> ToRemove;
536
551
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion ()->getEntryBasicBlock ();
537
552
bool HasOnlyVectorVFs = !Plan.hasVF (ElementCount::getFixed (1 ));
@@ -560,7 +575,9 @@ void VPlanTransforms::optimizeInductions(VPlan &Plan, ScalarEvolution &SE) {
560
575
}
561
576
}
562
577
563
- void VPlanTransforms::removeRedundantExpandSCEVRecipes (VPlan &Plan) {
578
+ // / Remove redundant EpxandSCEVRecipes in \p Plan's entry block by replacing
579
+ // / them with already existing recipes expanding the same SCEV expression.
580
+ static void removeRedundantExpandSCEVRecipes (VPlan &Plan) {
564
581
DenseMap<const SCEV *, VPValue *> SCEV2VPV;
565
582
566
583
for (VPRecipeBase &R :
0 commit comments