Skip to content

Commit 4d3148d

Browse files
authored
[LV][EVL] Fix the check for legality of folding with EVL. (#125678)
The current legality check for folding with EVL has incomplete verification for VF. This patch fixes the VF check, ensuring that tail folding with EVL is enabled only when a scalable VF is available. This allows loops that prefer tail folding with EVL but cannot use scalable VF vectorization to still be vectorized using a fixed VF, rather than abandoning vectorization entirely.
1 parent 99099cd commit 4d3148d

File tree

3 files changed

+343
-84
lines changed

3 files changed

+343
-84
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1454,9 +1454,10 @@ class LoopVectorizationCostModel {
14541454
// FIXME: Investigate opportunity for fixed vector factor.
14551455
// FIXME: support fixed-order recurrences by fixing splice of non VFxUF
14561456
// penultimate EVL.
1457-
bool EVLIsLegal =
1458-
UserIC <= 1 && TTI.hasActiveVectorLength(0, nullptr, Align()) &&
1459-
!EnableVPlanNativePath && Legal->getFixedOrderRecurrences().empty();
1457+
bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1458+
TTI.hasActiveVectorLength(0, nullptr, Align()) &&
1459+
!EnableVPlanNativePath &&
1460+
Legal->getFixedOrderRecurrences().empty();
14601461
if (!EVLIsLegal) {
14611462
// If for some reason EVL mode is unsupported, fallback to
14621463
// DataWithoutLaneMask to try to vectorize the loop with folded tail
@@ -4109,7 +4110,8 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
41094110
// found modulo the vectorization factor is not zero, try to fold the tail
41104111
// by masking.
41114112
// FIXME: look for a smaller MaxVF that does divide TC rather than masking.
4112-
setTailFoldingStyles(MaxFactors.ScalableVF.isScalable(), UserIC);
4113+
bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
4114+
setTailFoldingStyles(ContainsScalableVF, UserIC);
41134115
if (foldTailByMasking()) {
41144116
if (getTailFoldingStyle() == TailFoldingStyle::DataWithEVL) {
41154117
LLVM_DEBUG(
@@ -4120,8 +4122,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
41204122
// Tail folded loop using VP intrinsics restricts the VF to be scalable
41214123
// for now.
41224124
// TODO: extend it for fixed vectors, if required.
4123-
assert(MaxFactors.ScalableVF.isScalable() &&
4124-
"Expected scalable vector factor.");
4125+
assert(ContainsScalableVF && "Expected scalable vector factor.");
41254126

41264127
MaxFactors.FixedVF = ElementCount::getFixed(1);
41274128
}

0 commit comments

Comments
 (0)