Skip to content

[SLP] Simplify buildTree() (NFC) #138833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 8, 2025
Merged

[SLP] Simplify buildTree() (NFC) #138833

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b4aa315
buildTree_rec(): Move common code to newGatherTreeEntry() function
gbossu Apr 14, 2025
3337529
buildTree_rec(): Single point of definition for legality characteristics
gbossu Apr 14, 2025
6a33a7b
fixup! buildTree_rec(): Single point of definition for legality chara…
gbossu May 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 67 additions & 49 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4063,6 +4063,15 @@ class BoUpSLP {
}
#endif

/// Create a new gather TreeEntry
TreeEntry *newGatherTreeEntry(ArrayRef<Value *> VL,
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {}) {
auto Invalid = ScheduleBundle::invalid();
return newTreeEntry(VL, Invalid, S, UserTreeIdx, ReuseShuffleIndices);
}

/// Create a new VectorizableTree entry.
TreeEntry *newTreeEntry(ArrayRef<Value *> VL, ScheduleBundle &Bundle,
const InstructionsState &S,
Expand Down Expand Up @@ -4251,13 +4260,34 @@ class BoUpSLP {
bool areAltOperandsProfitable(const InstructionsState &S,
ArrayRef<Value *> VL) const;

/// Contains all the outputs of legality analysis for a list of values to
/// vectorize.
class ScalarsVectorizationLegality {
InstructionsState S;
bool IsLegal;
bool TryToFindDuplicates;
bool TrySplitVectorize;

public:
ScalarsVectorizationLegality(InstructionsState S, bool IsLegal,
bool TryToFindDuplicates = true,
bool TrySplitVectorize = false)
: S(S), IsLegal(IsLegal), TryToFindDuplicates(TryToFindDuplicates),
TrySplitVectorize(TrySplitVectorize) {
assert((!IsLegal || (S.valid() && TryToFindDuplicates)) &&
"Inconsistent state");
}
const InstructionsState &getInstructionsState() const { return S; };
bool isLegal() const { return IsLegal; }
bool tryToFindDuplicates() const { return TryToFindDuplicates; }
bool trySplitVectorize() const { return TrySplitVectorize; }
};

/// Checks if the specified list of the instructions/values can be vectorized
/// in general.
bool isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
const EdgeInfo &UserTreeIdx,
InstructionsState &S,
bool &TryToFindDuplicates,
bool &TrySplitVectorize) const;
ScalarsVectorizationLegality
getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
const EdgeInfo &UserTreeIdx) const;

/// Checks if the specified list of the instructions/values can be vectorized
/// and fills required data before actual scheduling of the instructions.
Expand Down Expand Up @@ -9734,25 +9764,21 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
return true;
}

bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
const EdgeInfo &UserTreeIdx,
InstructionsState &S,
bool &TryToFindDuplicates,
bool &TrySplitVectorize) const {
BoUpSLP::ScalarsVectorizationLegality
BoUpSLP::getScalarsVectorizationLegality(ArrayRef<Value *> VL, unsigned Depth,
const EdgeInfo &UserTreeIdx) const {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");

S = getSameOpcode(VL, *TLI);
TryToFindDuplicates = true;
TrySplitVectorize = false;
InstructionsState S = getSameOpcode(VL, *TLI);

// Don't go into catchswitch blocks, which can happen with PHIs.
// Such blocks can only have PHIs and the catchswitch. There is no
// place to insert a shuffle if we need to, so just avoid that issue.
if (S && isa<CatchSwitchInst>(S.getMainOp()->getParent()->getTerminator())) {
LLVM_DEBUG(dbgs() << "SLP: bundle in catchswitch block.\n");
// Do not try to pack to avoid extra instructions here.
TryToFindDuplicates = false;
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/false);
}

// Check if this is a duplicate of another entry.
Expand All @@ -9762,14 +9788,14 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (E->isSame(VL)) {
LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.getMainOp()
<< ".\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
SmallPtrSet<Value *, 8> Values(llvm::from_range, E->Scalars);
if (all_of(VL, [&](Value *V) {
return isa<PoisonValue>(V) || Values.contains(V);
})) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
Expand All @@ -9786,23 +9812,23 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
cast<Instruction>(I)->getOpcode() == S.getOpcode();
})))) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}

// Don't handle scalable vectors
if (S && S.getOpcode() == Instruction::ExtractElement &&
isa<ScalableVectorType>(
cast<ExtractElementInst>(S.getMainOp())->getVectorOperandType())) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}

// Don't handle vectors.
if (!SLPReVec && getValueType(VL.front())->isVectorTy()) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
// Do not try to pack to avoid extra instructions here.
TryToFindDuplicates = false;
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/false);
}

// If all of the operands are identical or constant we have a simple solution.
Expand Down Expand Up @@ -9892,11 +9918,12 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (!S) {
LLVM_DEBUG(dbgs() << "SLP: Try split and if failed, gathering due to "
"C,S,B,O, small shuffle. \n");
TrySplitVectorize = true;
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/true,
/*TrySplitVectorize=*/true);
}
LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O, small shuffle. \n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}

// Don't vectorize ephemeral values.
Expand All @@ -9906,8 +9933,8 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
<< ") is ephemeral.\n");
// Do not try to pack to avoid extra instructions here.
TryToFindDuplicates = false;
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false,
/*TryToFindDuplicates=*/false);
}
}
}
Expand Down Expand Up @@ -9956,7 +9983,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
if (PreferScalarize) {
LLVM_DEBUG(dbgs() << "SLP: The instructions are in tree and alternate "
"node is not profitable.\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}

Expand All @@ -9965,7 +9992,7 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
for (Value *V : VL) {
if (UserIgnoreList->contains(V)) {
LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
}
}
Expand Down Expand Up @@ -9995,9 +10022,9 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
// Do not vectorize EH and non-returning blocks, not profitable in most
// cases.
LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
return false;
return ScalarsVectorizationLegality(S, /*IsLegal=*/false);
}
return true;
return ScalarsVectorizationLegality(S, /*IsLegal=*/true);
}

void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
Expand All @@ -10008,7 +10035,6 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
SmallVector<int> ReuseShuffleIndices;
SmallVector<Value *> VL(VLRef.begin(), VLRef.end());

InstructionsState S = InstructionsState::invalid();
// Tries to build split node.
auto TrySplitNode = [&](const InstructionsState &LocalState) {
SmallVector<Value *> Op1, Op2;
Expand Down Expand Up @@ -10042,22 +10068,20 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
return true;
};

bool TryToPackDuplicates;
bool TrySplitVectorize;
if (!isLegalToVectorizeScalars(VL, Depth, UserTreeIdx, S, TryToPackDuplicates,
TrySplitVectorize)) {
if (TrySplitVectorize) {
ScalarsVectorizationLegality Legality =
getScalarsVectorizationLegality(VL, Depth, UserTreeIdx);
const InstructionsState &S = Legality.getInstructionsState();
if (!Legality.isLegal()) {
if (Legality.trySplitVectorize()) {
auto [MainOp, AltOp] = getMainAltOpsNoStateVL(VL);
// Last chance to try to vectorize alternate node.
if (MainOp && AltOp && TrySplitNode(InstructionsState(MainOp, AltOp)))
return;
}
if (TryToPackDuplicates)
if (Legality.tryToFindDuplicates())
tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx);

auto Invalid = ScheduleBundle::invalid();
newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}

Expand All @@ -10068,9 +10092,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Check that every instruction appears once in this bundle.
if (!tryToFindDuplicates(VL, ReuseShuffleIndices, *TTI, *TLI, S, UserTreeIdx,
/*TryPad=*/true)) {
auto Invalid = ScheduleBundle::invalid();
newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}

Expand All @@ -10083,9 +10105,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
TreeEntry::EntryState State = getScalarsVectorizationState(
S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
if (State == TreeEntry::NeedToGather) {
auto Invalid = ScheduleBundle::invalid();
newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}

Expand All @@ -10109,9 +10129,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
auto Invalid = ScheduleBundle::invalid();
newTreeEntry(VL, Invalid /*not vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
BS.ScheduleRegionSize < BS.ScheduleRegionSizeLimit)
Expand Down