[VPlan] Construct immutable VPIRBBs for exit blocks at construction(NFC) #128374
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-vectorizers Author: Florian Hahn (fhahn) ChangesConstract immutable VPIRBasicBlocks for all exit blocks up front and keep a list of them. Same as the scalar header, they are leaf nodes of the VPlan and won't change. Some exit blocks may be unreachable, e.g. if the scalar epilogue always executes or depending on optimizations. This simplifies both the way we retrieve the exit blocks as well as hooking up the exit blocks. Full diff: https://github.com/llvm/llvm-project/pull/128374.diff 5 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7b346bfd67c0d..e50c7d785bce7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9140,6 +9140,10 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
VPlan &Plan) {
SetVector<VPIRInstruction *> ExitUsersToFix;
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
+ // Nothing to do for unreachable exit blocks.
+ if (ExitVPBB->getNumPredecessors() == 0)
+ continue;
+
for (VPRecipeBase &R : *ExitVPBB) {
auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
if (!ExitIRI)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index cd111365c134c..a05fda5f4fdff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -851,6 +851,11 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
VPlan::VPlan(Loop *L) {
setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
ScalarHeader = createVPIRBasicBlock(L->getHeader());
+
+ SmallVector<BasicBlock *> IRExitBlocks;
+ L->getExitBlocks(IRExitBlocks);
+ for (BasicBlock *EB : IRExitBlocks)
+ ExitBlocks.push_back(createVPIRBasicBlock(EB));
}
VPlan::~VPlan() {
@@ -932,7 +937,7 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
// we unconditionally branch to the scalar preheader. Do nothing.
// 3) Otherwise, construct a runtime check.
BasicBlock *IRExitBlock = TheLoop->getUniqueLatchExitBlock();
- auto *VPExitBlock = Plan->createVPIRBasicBlock(IRExitBlock);
+ VPIRBasicBlock *VPExitBlock = Plan->getExitBlock(IRExitBlock);
// The connection order corresponds to the operands of the conditional branch.
VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
@@ -984,6 +989,12 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
}
}
+VPIRBasicBlock *VPlan::getExitBlock(BasicBlock *IRBB) const {
+ return *find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) {
+ return VPIRBB->getIRBasicBlock() == IRBB;
+ });
+}
+
bool VPlan::isExitBlock(VPBlockBase *VPBB) {
return isa<VPIRBasicBlock>(VPBB) && VPBB->getNumSuccessors() == 0;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d86914f0fb026..a964b7f2e935f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3426,6 +3426,11 @@ class VPlan {
/// VPIRBasicBlock wrapping the header of the original scalar loop.
VPIRBasicBlock *ScalarHeader;
+ /// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original
+ /// scalar loop. Note that some exit blocks may be unreachable, e.g. if the
+ /// scalar epilogue always executes
+ SmallVector<VPIRBasicBlock *, 2> ExitBlocks;
+
/// Holds the VFs applicable to this VPlan.
SmallSetVector<ElementCount, 2> VFs;
@@ -3559,11 +3564,13 @@ class VPlan {
/// Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; }
- /// Return an iterator range over the VPIRBasicBlock wrapping the exit blocks
- /// of the VPlan, that is leaf nodes except the scalar header. Defined in
- /// VPlanHCFG, as the definition of the type needs access to the definitions
- /// of VPBlockShallowTraversalWrapper.
- auto getExitBlocks();
+ /// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of
+ /// the original scalar loop.
+ ArrayRef<VPIRBasicBlock *> getExitBlocks() const { return ExitBlocks; }
+
+ /// Return the VPIRBasicBlock corresponding to \p IRBB. \p IRBB must be an
+ /// exit block.
+ VPIRBasicBlock *getExitBlock(BasicBlock *IRBB) const;
/// Returns true if \p VPBB is an exit block.
bool isExitBlock(VPBlockBase *VPBB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanCFG.h b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
index 8fbdacd1ea771..a1014c398789f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanCFG.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
@@ -307,15 +307,6 @@ template <> struct GraphTraits<VPlan *> {
}
};
-inline auto VPlan::getExitBlocks() {
- VPBlockBase *ScalarHeader = getScalarHeader();
- return make_filter_range(
- VPBlockUtils::blocksOnly<VPIRBasicBlock>(
- vp_depth_first_shallow(getVectorLoopRegion()->getSingleSuccessor())),
- [ScalarHeader](VPIRBasicBlock *VPIRBB) {
- return VPIRBB != ScalarHeader && VPIRBB->getNumSuccessors() == 0;
- });
-}
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANCFG_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ce81b2e147df8..3dcc4f6c169b4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2049,18 +2049,9 @@ void VPlanTransforms::handleUncountableEarlyExit(
cast<BranchInst>(UncountableExitingBlock->getTerminator());
BasicBlock *TrueSucc = EarlyExitingBranch->getSuccessor(0);
BasicBlock *FalseSucc = EarlyExitingBranch->getSuccessor(1);
-
- // The early exit block may or may not be the same as the "countable" exit
- // block. Creates a new VPIRBB for the early exit block in case it is distinct
- // from the countable exit block.
- // TODO: Introduce both exit blocks during VPlan skeleton construction.
- VPIRBasicBlock *VPEarlyExitBlock;
- if (OrigLoop->getUniqueExitBlock()) {
- VPEarlyExitBlock = cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
- } else {
- VPEarlyExitBlock = Plan.createVPIRBasicBlock(
- !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
- }
+ BasicBlock *EarlyExitIRBB =
+ !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc;
+ VPIRBasicBlock *VPEarlyExitBlock = Plan.getExitBlock(EarlyExitIRBB);
VPValue *EarlyExitNotTakenCond = RecipeBuilder.getBlockInMask(
OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
|
|
@llvm/pr-subscribers-llvm-transforms Author: Florian Hahn (fhahn) ChangesConstract immutable VPIRBasicBlocks for all exit blocks up front and keep a list of them. Same as the scalar header, they are leaf nodes of the VPlan and won't change. Some exit blocks may be unreachable, e.g. if the scalar epilogue always executes or depending on optimizations. This simplifies both the way we retrieve the exit blocks as well as hooking up the exit blocks. Full diff: https://github.com/llvm/llvm-project/pull/128374.diff 5 Files Affected:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7b346bfd67c0d..e50c7d785bce7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9140,6 +9140,10 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder,
VPlan &Plan) {
SetVector<VPIRInstruction *> ExitUsersToFix;
for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
+ // Nothing to do for unreachable exit blocks.
+ if (ExitVPBB->getNumPredecessors() == 0)
+ continue;
+
for (VPRecipeBase &R : *ExitVPBB) {
auto *ExitIRI = dyn_cast<VPIRInstruction>(&R);
if (!ExitIRI)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index cd111365c134c..a05fda5f4fdff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -851,6 +851,11 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
VPlan::VPlan(Loop *L) {
setEntry(createVPIRBasicBlock(L->getLoopPreheader()));
ScalarHeader = createVPIRBasicBlock(L->getHeader());
+
+ SmallVector<BasicBlock *> IRExitBlocks;
+ L->getExitBlocks(IRExitBlocks);
+ for (BasicBlock *EB : IRExitBlocks)
+ ExitBlocks.push_back(createVPIRBasicBlock(EB));
}
VPlan::~VPlan() {
@@ -932,7 +937,7 @@ VPlanPtr VPlan::createInitialVPlan(Type *InductionTy,
// we unconditionally branch to the scalar preheader. Do nothing.
// 3) Otherwise, construct a runtime check.
BasicBlock *IRExitBlock = TheLoop->getUniqueLatchExitBlock();
- auto *VPExitBlock = Plan->createVPIRBasicBlock(IRExitBlock);
+ VPIRBasicBlock *VPExitBlock = Plan->getExitBlock(IRExitBlock);
// The connection order corresponds to the operands of the conditional branch.
VPBlockUtils::insertBlockAfter(VPExitBlock, MiddleVPBB);
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
@@ -984,6 +989,12 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
}
}
+VPIRBasicBlock *VPlan::getExitBlock(BasicBlock *IRBB) const {
+ return *find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) {
+ return VPIRBB->getIRBasicBlock() == IRBB;
+ });
+}
+
bool VPlan::isExitBlock(VPBlockBase *VPBB) {
return isa<VPIRBasicBlock>(VPBB) && VPBB->getNumSuccessors() == 0;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index d86914f0fb026..a964b7f2e935f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3426,6 +3426,11 @@ class VPlan {
/// VPIRBasicBlock wrapping the header of the original scalar loop.
VPIRBasicBlock *ScalarHeader;
+ /// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original
+ /// scalar loop. Note that some exit blocks may be unreachable, e.g. if the
+ /// scalar epilogue always executes
+ SmallVector<VPIRBasicBlock *, 2> ExitBlocks;
+
/// Holds the VFs applicable to this VPlan.
SmallSetVector<ElementCount, 2> VFs;
@@ -3559,11 +3564,13 @@ class VPlan {
/// Return the VPIRBasicBlock wrapping the header of the scalar loop.
VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; }
- /// Return an iterator range over the VPIRBasicBlock wrapping the exit blocks
- /// of the VPlan, that is leaf nodes except the scalar header. Defined in
- /// VPlanHCFG, as the definition of the type needs access to the definitions
- /// of VPBlockShallowTraversalWrapper.
- auto getExitBlocks();
+ /// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of
+ /// the original scalar loop.
+ ArrayRef<VPIRBasicBlock *> getExitBlocks() const { return ExitBlocks; }
+
+ /// Return the VPIRBasicBlock corresponding to \p IRBB. \p IRBB must be an
+ /// exit block.
+ VPIRBasicBlock *getExitBlock(BasicBlock *IRBB) const;
/// Returns true if \p VPBB is an exit block.
bool isExitBlock(VPBlockBase *VPBB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanCFG.h b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
index 8fbdacd1ea771..a1014c398789f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanCFG.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanCFG.h
@@ -307,15 +307,6 @@ template <> struct GraphTraits<VPlan *> {
}
};
-inline auto VPlan::getExitBlocks() {
- VPBlockBase *ScalarHeader = getScalarHeader();
- return make_filter_range(
- VPBlockUtils::blocksOnly<VPIRBasicBlock>(
- vp_depth_first_shallow(getVectorLoopRegion()->getSingleSuccessor())),
- [ScalarHeader](VPIRBasicBlock *VPIRBB) {
- return VPIRBB != ScalarHeader && VPIRBB->getNumSuccessors() == 0;
- });
-}
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANCFG_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ce81b2e147df8..3dcc4f6c169b4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2049,18 +2049,9 @@ void VPlanTransforms::handleUncountableEarlyExit(
cast<BranchInst>(UncountableExitingBlock->getTerminator());
BasicBlock *TrueSucc = EarlyExitingBranch->getSuccessor(0);
BasicBlock *FalseSucc = EarlyExitingBranch->getSuccessor(1);
-
- // The early exit block may or may not be the same as the "countable" exit
- // block. Creates a new VPIRBB for the early exit block in case it is distinct
- // from the countable exit block.
- // TODO: Introduce both exit blocks during VPlan skeleton construction.
- VPIRBasicBlock *VPEarlyExitBlock;
- if (OrigLoop->getUniqueExitBlock()) {
- VPEarlyExitBlock = cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
- } else {
- VPEarlyExitBlock = Plan.createVPIRBasicBlock(
- !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
- }
+ BasicBlock *EarlyExitIRBB =
+ !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc;
+ VPIRBasicBlock *VPEarlyExitBlock = Plan.getExitBlock(EarlyExitIRBB);
VPValue *EarlyExitNotTakenCond = RecipeBuilder.getBlockInMask(
OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
|
david-arm
approved these changes
Feb 24, 2025
ayalz
reviewed
Feb 24, 2025
| @@ -9140,6 +9140,10 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder, | |||
| VPlan &Plan) { | |||
| SetVector<VPIRInstruction *> ExitUsersToFix; | |||
| for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) { | |||
| // Nothing to do for unreachable exit blocks. | |||
There was a problem hiding this comment.
All exit blocks are expected to be reachable, in the long run.
The exit blocks of Plan include only exit.block and optionally latch.exit, as depicted in
https://llvm.org/docs/Vectorizers.html#epilogue-vectorization and https://llvm.org/docs/Vectorizers.html#early-exit-vectorization, where the scalar header is expected to eventually reach the above exit blocks.
There was a problem hiding this comment.
Yep, once we connect the exits also to the scalar loop
Comment on lines
993
to
994
| return *find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) { | ||
| return VPIRBB->getIRBasicBlock() == IRBB; | ||
| }); |
There was a problem hiding this comment.
Better to hold the (one or two) exit blocks in a map keyed by their IRBB, than by a SmallVector?
There was a problem hiding this comment.
Left as vector for now, as getExitBlock should not be needed soonish and be replaced by connecting the blocks during initial VPlan construction.
| auto getExitBlocks(); | ||
| /// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of | ||
| /// the original scalar loop. | ||
| ArrayRef<VPIRBasicBlock *> getExitBlocks() const { return ExitBlocks; } |
There was a problem hiding this comment.
This admittedly may be more involved if ExitBlocks are heald as an IRBBToVPExitBlock map.
There was a problem hiding this comment.
Left as vector for now, as getExitBlock should not be needed soonish and be replaced by connecting the blocks during initial VPlan construction.
| @@ -3426,6 +3426,11 @@ class VPlan { | |||
| /// VPIRBasicBlock wrapping the header of the original scalar loop. | |||
| VPIRBasicBlock *ScalarHeader; | |||
|
|
|||
| /// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original | |||
| /// scalar loop. Note that some exit blocks may be unreachable, e.g. if the | |||
| /// scalar epilogue always executes | |||
There was a problem hiding this comment.
Suggested change
| /// scalar epilogue always executes | |
| /// scalar epilogue always executes. |
| @@ -3426,6 +3426,11 @@ class VPlan { | |||
| /// VPIRBasicBlock wrapping the header of the original scalar loop. | |||
| VPIRBasicBlock *ScalarHeader; | |||
|
|
|||
| /// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original | |||
| /// scalar loop. Note that some exit blocks may be unreachable, e.g. if the | |||
There was a problem hiding this comment.
All exit blocks all expected to be reachable in the long run. If scalar epilog always executes, the exit block will be reached only from the scalar epilog (currently unmodelled), w/o a direct edge from middle block to the exit block.
There was a problem hiding this comment.
Yep, added at the moment to clarify
Comment on lines
+3571
to
+3572
| /// Return the VPIRBasicBlock corresponding to \p IRBB. \p IRBB must be an | ||
| /// exit block. |
There was a problem hiding this comment.
"must" - if IRBB is not an exit block, an assert is triggered, or null is returned?
There was a problem hiding this comment.
Added an assert, rather than just relying on dereferenceing the end iterator.
Constract immutable VPIRBasicBlocks for all exit blocks up front and keep a list of them. Same as the scalar header, they are leaf nodes of the VPlan and won't change. Some exit blocks may be unreachable, e.g. if the scalar epilogue always executes or depending on optimizations. This simplifies both the way we retrieve the exit blocks as well as hooking up the exit blocks.
fhahn
force-pushed
the
vplan-keep-list-of-exit-blocks
branch
from
e950e32 to
d35f56a
Compare
llvm-sync bot
pushed a commit
to arm/arm-toolchain
that referenced
this pull request
Feb 25, 2025
…struction(NFC) (#128374) Constract immutable VPIRBasicBlocks for all exit blocks up front and keep a list of them. Same as the scalar header, they are leaf nodes of the VPlan and won't change. Some exit blocks may be unreachable, e.g. if the scalar epilogue always executes or depending on optimizations. This simplifies both the way we retrieve the exit blocks as well as hooking up the exit blocks. PR: llvm/llvm-project#128374
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Constract immutable VPIRBasicBlocks for all exit blocks up front and keep a list of them. Same as the scalar header, they are leaf nodes of the VPlan and won't change. Some exit blocks may be unreachable, e.g. if the scalar epilogue always executes or depending on optimizations.
This simplifies both the way we retrieve the exit blocks as well as hooking up the exit blocks.