@@ -9141,7 +9141,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
9141
9141
continue ;
9142
9142
9143
9143
const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor ();
9144
- auto *Result = PhiR->getBackedgeValue ()-> getDefiningRecipe ();
9144
+ auto *NewExitingVPV = PhiR->getBackedgeValue ();
9145
9145
// If tail is folded by masking, introduce selects between the phi
9146
9146
// and the live-out instruction of each reduction, at the beginning of the
9147
9147
// dedicated latch block.
@@ -9151,21 +9151,20 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
9151
9151
VPValue *Red = PhiR->getBackedgeValue ();
9152
9152
assert (Red->getDefiningRecipe ()->getParent () != LatchVPBB &&
9153
9153
" reduction recipe must be defined before latch" );
9154
- FastMathFlags FMFs = RdxDesc.getFastMathFlags ();
9155
9154
Type *PhiTy = PhiR->getOperand (0 )->getLiveInIRValue ()->getType ();
9156
- Result =
9155
+ std::optional<FastMathFlags> FMFs =
9157
9156
PhiTy->isFloatingPointTy ()
9158
- ? new VPInstruction (Instruction::Select, {Cond, Red, PhiR}, FMFs )
9159
- : new VPInstruction (Instruction::Select, {Cond, Red, PhiR}) ;
9160
- Result-> insertBefore (&* Builder.getInsertPoint () );
9161
- Red->replaceUsesWithIf (
9162
- Result-> getVPSingleValue (),
9163
- [](VPUser &U, unsigned ) { return isa<VPLiveOut>(&U); });
9157
+ ? std::make_optional (RdxDesc. getFastMathFlags () )
9158
+ : std::nullopt ;
9159
+ NewExitingVPV = Builder.createSelect (Cond, Red, PhiR, {}, " " , FMFs );
9160
+ Red->replaceUsesWithIf (NewExitingVPV, [](VPUser &U, unsigned ) {
9161
+ return isa<VPLiveOut>(&U);
9162
+ });
9164
9163
if (PreferPredicatedReductionSelect ||
9165
9164
TTI.preferPredicatedReductionSelect (
9166
9165
PhiR->getRecurrenceDescriptor ().getOpcode (), PhiTy,
9167
9166
TargetTransformInfo::ReductionFlags ()))
9168
- PhiR->setOperand (1 , Result-> getVPSingleValue () );
9167
+ PhiR->setOperand (1 , NewExitingVPV );
9169
9168
}
9170
9169
// If the vector reduction can be performed in a smaller type, we truncate
9171
9170
// then extend the loop exit value to enable InstCombine to evaluate the
@@ -9174,17 +9173,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
9174
9173
if (MinVF.isVector () && PhiTy != RdxDesc.getRecurrenceType ()) {
9175
9174
assert (!PhiR->isInLoop () && " Unexpected truncated inloop reduction!" );
9176
9175
Type *RdxTy = RdxDesc.getRecurrenceType ();
9177
- auto *Trunc = new VPWidenCastRecipe (Instruction::Trunc,
9178
- Result-> getVPSingleValue () , RdxTy);
9176
+ auto *Trunc =
9177
+ new VPWidenCastRecipe (Instruction::Trunc, NewExitingVPV , RdxTy);
9179
9178
auto *Extnd =
9180
9179
RdxDesc.isSigned ()
9181
9180
? new VPWidenCastRecipe (Instruction::SExt, Trunc, PhiTy)
9182
9181
: new VPWidenCastRecipe (Instruction::ZExt, Trunc, PhiTy);
9183
9182
9184
- Trunc->insertAfter (Result );
9183
+ Trunc->insertAfter (NewExitingVPV-> getDefiningRecipe () );
9185
9184
Extnd->insertAfter (Trunc);
9186
- Result-> getVPSingleValue () ->replaceAllUsesWith (Extnd);
9187
- Trunc->setOperand (0 , Result-> getVPSingleValue () );
9185
+ NewExitingVPV ->replaceAllUsesWith (Extnd);
9186
+ Trunc->setOperand (0 , NewExitingVPV );
9188
9187
}
9189
9188
}
9190
9189
0 commit comments