Skip to content

[LV][NFC]Preselect folding style while chosing the max VF, NFC. #81885

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1509,15 +1509,27 @@ class LoopVectorizationCostModel {
}

/// Returns the TailFoldingStyle that is best for the current loop.
TailFoldingStyle
getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
if (!CanFoldTailByMasking)
return TailFoldingStyle::None;
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
return IVUpdateMayOverflow ? ChosenTailFoldingStyle.first
: ChosenTailFoldingStyle.second;
}

/// Selects and saves TailFoldingStyle for 2 options - if IV may overflow or
/// not.
void selectTailFoldinStyle() {
if (!Legal->prepareToFoldTailByMasking())
return;

if (ForceTailFoldingStyle.getNumOccurrences())
return ForceTailFoldingStyle;
if (ForceTailFoldingStyle.getNumOccurrences()) {
ChosenTailFoldingStyle.first = ChosenTailFoldingStyle.second =
ForceTailFoldingStyle;
return;
}

return TTI.getPreferredTailFoldingStyle(IVUpdateMayOverflow);
ChosenTailFoldingStyle.first =
TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true);
ChosenTailFoldingStyle.second =
TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false);
}

/// Returns true if all loop blocks should be masked to fold tail loop.
Expand Down Expand Up @@ -1674,8 +1686,10 @@ class LoopVectorizationCostModel {
/// iterations to execute in the scalar loop.
ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;

/// All blocks of loop are to be masked to fold tail of scalar iterations.
bool CanFoldTailByMasking = false;
/// Control finally chosen tail folding style. The first element is used if IV
/// update may overflow, the second element - if it does not.
std::pair<TailFoldingStyle, TailFoldingStyle> ChosenTailFoldingStyle =
std::make_pair(TailFoldingStyle::None, TailFoldingStyle::None);

/// A map holding scalar costs for different vectorization factors. The
/// presence of a cost for an instruction in the mapping indicates that the
Expand Down Expand Up @@ -4632,10 +4646,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// found modulo the vectorization factor is not zero, try to fold the tail
// by masking.
// FIXME: look for a smaller MaxVF that does divide TC rather than masking.
if (Legal->prepareToFoldTailByMasking()) {
CanFoldTailByMasking = true;
selectTailFoldinStyle();
if (foldTailByMasking())
return MaxFactors;
}

// If there was a tail-folding hint/switch, but we can't fold the tail by
// masking, fallback to a vectorization with a scalar epilogue.
Expand Down