Skip to content

Commit e4ef651

Browse files
committed
[VPlan] Simplify VPReductionPHIRecipe::execute (NFC).
Simplify VPReductionPHIRecipe::execute by handling the simple cases first, by directly using State.get() to the appropriate start value.
1 parent c46927f commit e4ef651

File tree

1 file changed

+26
-46
lines changed

1 file changed

+26
-46
lines changed

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 26 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -3835,24 +3835,17 @@ void VPFirstOrderRecurrencePHIRecipe::print(raw_ostream &O, const Twine &Indent,
38353835
#endif
38363836

38373837
void VPReductionPHIRecipe::execute(VPTransformState &State) {
3838-
auto &Builder = State.Builder;
3839-
38403838
// If this phi is fed by a scaled reduction then it should output a
38413839
// vector with fewer elements than the VF.
38423840
ElementCount VF = State.VF.divideCoefficientBy(VFScaleFactor);
38433841

3844-
// Reductions do not have to start at zero. They can start with
3845-
// any loop invariant values.
3846-
VPValue *StartVPV = getStartValue();
3847-
Value *StartV = StartVPV->getLiveInIRValue();
3848-
38493842
// In order to support recurrences we need to be able to vectorize Phi nodes.
38503843
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
38513844
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
38523845
// this value when we vectorize all of the instructions that use the PHI.
3846+
auto *ScalarTy = State.TypeAnalysis.inferScalarType(this);
38533847
bool ScalarPHI = State.VF.isScalar() || IsInLoop;
3854-
Type *VecTy =
3855-
ScalarPHI ? StartV->getType() : VectorType::get(StartV->getType(), VF);
3848+
Type *VecTy = ScalarPHI ? ScalarTy : VectorType::get(ScalarTy, VF);
38563849

38573850
BasicBlock *HeaderBB = State.CFG.PrevBB;
38583851
assert(State.CurrentParentLoop->getHeader() == HeaderBB &&
@@ -3863,22 +3856,18 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
38633856

38643857
BasicBlock *VectorPH =
38653858
State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));
3859+
// Create start and identity vector values for the reduction in the preheader.
3860+
// TODO: Introduce recipes in VPlan preheader to create initial values.
3861+
IRBuilderBase::InsertPointGuard IPBuilder(State.Builder);
3862+
State.Builder.SetInsertPoint(VectorPH->getTerminator());
38663863

3867-
Value *Iden = nullptr;
3864+
// Reductions do not have to start at zero. They can start with
3865+
// any loop invariant values.
3866+
VPValue *StartVPV = getStartValue();
38683867
RecurKind RK = RdxDesc.getRecurrenceKind();
3869-
unsigned CurrentPart = getUnrollPart(*this);
3870-
38713868
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK) ||
3872-
RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
3873-
// MinMax and AnyOf reductions have the start value as their identity.
3874-
if (ScalarPHI) {
3875-
Iden = StartV;
3876-
} else {
3877-
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
3878-
Builder.SetInsertPoint(VectorPH->getTerminator());
3879-
StartV = Iden = State.get(StartVPV);
3880-
}
3881-
} else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
3869+
RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) ||
3870+
RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) {
38823871
// [I|F]FindLastIV will use a sentinel value to initialize the reduction
38833872
// phi or the resume value from the main vector loop when vectorizing the
38843873
// epilogue loop. In the exit block, ComputeReductionResult will generate
@@ -3888,33 +3877,24 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
38883877
// TODO: The sentinel value is not always necessary. When the start value is
38893878
// a constant, and smaller than the start value of the induction variable,
38903879
// the start value can be directly used to initialize the reduction phi.
3891-
Iden = StartV;
3892-
if (!ScalarPHI) {
3893-
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
3894-
Builder.SetInsertPoint(VectorPH->getTerminator());
3895-
StartV = Iden = Builder.CreateVectorSplat(State.VF, Iden);
3896-
}
3897-
} else {
3898-
Iden = llvm::getRecurrenceIdentity(RK, VecTy->getScalarType(),
3899-
RdxDesc.getFastMathFlags());
3900-
3901-
if (!ScalarPHI) {
3902-
if (CurrentPart == 0) {
3903-
// Create start and identity vector values for the reduction in the
3904-
// preheader.
3905-
// TODO: Introduce recipes in VPlan preheader to create initial values.
3906-
Iden = Builder.CreateVectorSplat(VF, Iden);
3907-
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
3908-
Builder.SetInsertPoint(VectorPH->getTerminator());
3909-
Constant *Zero = Builder.getInt32(0);
3910-
StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
3911-
} else {
3912-
Iden = Builder.CreateVectorSplat(VF, Iden);
3913-
}
3880+
Phi->addIncoming(State.get(StartVPV, ScalarPHI), VectorPH);
3881+
return;
3882+
}
3883+
3884+
Value *Iden = getRecurrenceIdentity(RK, VecTy->getScalarType(),
3885+
RdxDesc.getFastMathFlags());
3886+
unsigned CurrentPart = getUnrollPart(*this);
3887+
Value *StartV = StartVPV->getLiveInIRValue();
3888+
if (!ScalarPHI) {
3889+
if (CurrentPart == 0) {
3890+
Iden = State.Builder.CreateVectorSplat(VF, Iden);
3891+
Constant *Zero = State.Builder.getInt32(0);
3892+
StartV = State.Builder.CreateInsertElement(Iden, StartV, Zero);
3893+
} else {
3894+
Iden = State.Builder.CreateVectorSplat(VF, Iden);
39143895
}
39153896
}
39163897

3917-
Phi = cast<PHINode>(State.get(this, IsInLoop));
39183898
Value *StartVal = (CurrentPart == 0) ? StartV : Iden;
39193899
Phi->addIncoming(StartVal, VectorPH);
39203900
}

0 commit comments

Comments
 (0)