@@ -3835,24 +3835,17 @@ void VPFirstOrderRecurrencePHIRecipe::print(raw_ostream &O, const Twine &Indent,
3835
3835
#endif
3836
3836
3837
3837
void VPReductionPHIRecipe::execute (VPTransformState &State) {
3838
- auto &Builder = State.Builder ;
3839
-
3840
3838
// If this phi is fed by a scaled reduction then it should output a
3841
3839
// vector with fewer elements than the VF.
3842
3840
ElementCount VF = State.VF .divideCoefficientBy (VFScaleFactor);
3843
3841
3844
- // Reductions do not have to start at zero. They can start with
3845
- // any loop invariant values.
3846
- VPValue *StartVPV = getStartValue ();
3847
- Value *StartV = StartVPV->getLiveInIRValue ();
3848
-
3849
3842
// In order to support recurrences we need to be able to vectorize Phi nodes.
3850
3843
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
3851
3844
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
3852
3845
// this value when we vectorize all of the instructions that use the PHI.
3846
+ auto *ScalarTy = State.TypeAnalysis .inferScalarType (this );
3853
3847
bool ScalarPHI = State.VF .isScalar () || IsInLoop;
3854
- Type *VecTy =
3855
- ScalarPHI ? StartV->getType () : VectorType::get (StartV->getType (), VF);
3848
+ Type *VecTy = ScalarPHI ? ScalarTy : VectorType::get (ScalarTy, VF);
3856
3849
3857
3850
BasicBlock *HeaderBB = State.CFG .PrevBB ;
3858
3851
assert (State.CurrentParentLoop ->getHeader () == HeaderBB &&
@@ -3863,22 +3856,18 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
3863
3856
3864
3857
BasicBlock *VectorPH =
3865
3858
State.CFG .VPBB2IRBB .at (getParent ()->getCFGPredecessor (0 ));
3859
+ // Create start and identity vector values for the reduction in the preheader.
3860
+ // TODO: Introduce recipes in VPlan preheader to create initial values.
3861
+ IRBuilderBase::InsertPointGuard IPBuilder (State.Builder );
3862
+ State.Builder .SetInsertPoint (VectorPH->getTerminator ());
3866
3863
3867
- Value *Iden = nullptr ;
3864
+ // Reductions do not have to start at zero. They can start with
3865
+ // any loop invariant values.
3866
+ VPValue *StartVPV = getStartValue ();
3868
3867
RecurKind RK = RdxDesc.getRecurrenceKind ();
3869
- unsigned CurrentPart = getUnrollPart (*this );
3870
-
3871
3868
if (RecurrenceDescriptor::isMinMaxRecurrenceKind (RK) ||
3872
- RecurrenceDescriptor::isAnyOfRecurrenceKind (RK)) {
3873
- // MinMax and AnyOf reductions have the start value as their identity.
3874
- if (ScalarPHI) {
3875
- Iden = StartV;
3876
- } else {
3877
- IRBuilderBase::InsertPointGuard IPBuilder (Builder);
3878
- Builder.SetInsertPoint (VectorPH->getTerminator ());
3879
- StartV = Iden = State.get (StartVPV);
3880
- }
3881
- } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind (RK)) {
3869
+ RecurrenceDescriptor::isAnyOfRecurrenceKind (RK) ||
3870
+ RecurrenceDescriptor::isFindLastIVRecurrenceKind (RK)) {
3882
3871
// [I|F]FindLastIV will use a sentinel value to initialize the reduction
3883
3872
// phi or the resume value from the main vector loop when vectorizing the
3884
3873
// epilogue loop. In the exit block, ComputeReductionResult will generate
@@ -3888,33 +3877,24 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
3888
3877
// TODO: The sentinel value is not always necessary. When the start value is
3889
3878
// a constant, and smaller than the start value of the induction variable,
3890
3879
// the start value can be directly used to initialize the reduction phi.
3891
- Iden = StartV;
3892
- if (!ScalarPHI) {
3893
- IRBuilderBase::InsertPointGuard IPBuilder (Builder);
3894
- Builder.SetInsertPoint (VectorPH->getTerminator ());
3895
- StartV = Iden = Builder.CreateVectorSplat (State.VF , Iden);
3896
- }
3897
- } else {
3898
- Iden = llvm::getRecurrenceIdentity (RK, VecTy->getScalarType (),
3899
- RdxDesc.getFastMathFlags ());
3900
-
3901
- if (!ScalarPHI) {
3902
- if (CurrentPart == 0 ) {
3903
- // Create start and identity vector values for the reduction in the
3904
- // preheader.
3905
- // TODO: Introduce recipes in VPlan preheader to create initial values.
3906
- Iden = Builder.CreateVectorSplat (VF, Iden);
3907
- IRBuilderBase::InsertPointGuard IPBuilder (Builder);
3908
- Builder.SetInsertPoint (VectorPH->getTerminator ());
3909
- Constant *Zero = Builder.getInt32 (0 );
3910
- StartV = Builder.CreateInsertElement (Iden, StartV, Zero);
3911
- } else {
3912
- Iden = Builder.CreateVectorSplat (VF, Iden);
3913
- }
3880
+ Phi->addIncoming (State.get (StartVPV, ScalarPHI), VectorPH);
3881
+ return ;
3882
+ }
3883
+
3884
+ Value *Iden = getRecurrenceIdentity (RK, VecTy->getScalarType (),
3885
+ RdxDesc.getFastMathFlags ());
3886
+ unsigned CurrentPart = getUnrollPart (*this );
3887
+ Value *StartV = StartVPV->getLiveInIRValue ();
3888
+ if (!ScalarPHI) {
3889
+ if (CurrentPart == 0 ) {
3890
+ Iden = State.Builder .CreateVectorSplat (VF, Iden);
3891
+ Constant *Zero = State.Builder .getInt32 (0 );
3892
+ StartV = State.Builder .CreateInsertElement (Iden, StartV, Zero);
3893
+ } else {
3894
+ Iden = State.Builder .CreateVectorSplat (VF, Iden);
3914
3895
}
3915
3896
}
3916
3897
3917
- Phi = cast<PHINode>(State.get (this , IsInLoop));
3918
3898
Value *StartVal = (CurrentPart == 0 ) ? StartV : Iden;
3919
3899
Phi->addIncoming (StartVal, VectorPH);
3920
3900
}
0 commit comments