Skip to content

Commit 927ed49

Browse files
committed
[VPlan] Truncate/Extend ComputeReductionResult at construction (NFC).
Instead of looking up the narrower reduction type via getRecurrenceType we can generate the needed extend directly at constructiond re-use the truncated value from the loop.
1 parent ecff028 commit 927ed49

File tree

6 files changed

+36
-55
lines changed

6 files changed

+36
-55
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 29 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -7260,7 +7260,10 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
72607260
// Get the VPInstruction computing the reduction result in the middle block.
72617261
// The first operand may not be from the middle block if it is not connected
72627262
// to the scalar preheader. In that case, there's nothing to fix.
7263-
auto *EpiRedResult = dyn_cast<VPInstruction>(EpiResumePhiR->getOperand(0));
7263+
VPValue *Incoming = EpiResumePhiR->getOperand(0);
7264+
match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
7265+
VPlanPatternMatch::m_VPValue(Incoming)));
7266+
auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
72647267
if (!EpiRedResult ||
72657268
(EpiRedResult->getOpcode() != VPInstruction::ComputeAnyOfResult &&
72667269
EpiRedResult->getOpcode() != VPInstruction::ComputeReductionResult &&
@@ -9211,28 +9214,6 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92119214
PhiR->setOperand(1, NewExitingVPV);
92129215
}
92139216

9214-
// If the vector reduction can be performed in a smaller type, we truncate
9215-
// then extend the loop exit value to enable InstCombine to evaluate the
9216-
// entire expression in the smaller type.
9217-
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9218-
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9219-
RdxDesc.getRecurrenceKind())) {
9220-
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9221-
Type *RdxTy = RdxDesc.getRecurrenceType();
9222-
auto *Trunc =
9223-
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9224-
auto *Extnd =
9225-
RdxDesc.isSigned()
9226-
? new VPWidenCastRecipe(Instruction::SExt, Trunc, PhiTy)
9227-
: new VPWidenCastRecipe(Instruction::ZExt, Trunc, PhiTy);
9228-
9229-
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9230-
Extnd->insertAfter(Trunc);
9231-
if (PhiR->getOperand(1) == NewExitingVPV)
9232-
PhiR->setOperand(1, Extnd->getVPSingleValue());
9233-
NewExitingVPV = Extnd;
9234-
}
9235-
92369217
// We want code in the middle block to appear to execute on the location of
92379218
// the scalar loop's latch terminator because: (a) it is all compiler
92389219
// generated, (b) these instructions are always executed after evaluating
@@ -9271,6 +9252,31 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
92719252
Builder.createNaryOp(VPInstruction::ComputeReductionResult,
92729253
{PhiR, NewExitingVPV}, Flags, ExitDL);
92739254
}
9255+
// If the vector reduction can be performed in a smaller type, we truncate
9256+
// then extend the loop exit value to enable InstCombine to evaluate the
9257+
// entire expression in the smaller type.
9258+
if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType() &&
9259+
!RecurrenceDescriptor::isAnyOfRecurrenceKind(
9260+
RdxDesc.getRecurrenceKind())) {
9261+
assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
9262+
Type *RdxTy = RdxDesc.getRecurrenceType();
9263+
auto *Trunc =
9264+
new VPWidenCastRecipe(Instruction::Trunc, NewExitingVPV, RdxTy);
9265+
Instruction::CastOps ExtendOpc =
9266+
RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
9267+
auto *Extnd = new VPWidenCastRecipe(ExtendOpc, Trunc, PhiTy);
9268+
Trunc->insertAfter(NewExitingVPV->getDefiningRecipe());
9269+
Extnd->insertAfter(Trunc);
9270+
if (PhiR->getOperand(1) == NewExitingVPV)
9271+
PhiR->setOperand(1, Extnd->getVPSingleValue());
9272+
9273+
// Update ComputeReductionResult with the truncated exiting value and
9274+
// extend its result.
9275+
FinalReductionResult->setOperand(1, Trunc);
9276+
FinalReductionResult =
9277+
Builder.createScalarCast(ExtendOpc, FinalReductionResult, PhiTy, {});
9278+
}
9279+
92749280
// Update all users outside the vector region. Also replace redundant
92759281
// ExtractLastElement.
92769282
for (auto *U : to_vector(OrigExitingVPV->users())) {

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 0 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -731,7 +731,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
731731
assert(!RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) &&
732732
"should be handled by ComputeFindLastIVResult");
733733

734-
Type *ResultTy = State.TypeAnalysis.inferScalarType(this);
735734
// The recipe's operands are the reduction phi, followed by one operand for
736735
// each part of the reduction.
737736
unsigned UF = getNumOperands() - 1;
@@ -743,15 +742,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
743742
if (hasFastMathFlags())
744743
Builder.setFastMathFlags(getFastMathFlags());
745744

746-
// If the vector reduction can be performed in a smaller type, we truncate
747-
// then extend the loop exit value to enable InstCombine to evaluate the
748-
// entire expression in the smaller type.
749-
// TODO: Handle this in truncateToMinBW.
750-
if (State.VF.isVector() && ResultTy != RdxDesc.getRecurrenceType()) {
751-
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), State.VF);
752-
for (unsigned Part = 0; Part < UF; ++Part)
753-
RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);
754-
}
755745
// Reduce all of the unrolled parts into a single vector.
756746
Value *ReducedPartRdx = RdxParts[0];
757747
if (PhiR->isOrdered()) {
@@ -776,13 +766,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
776766
// All ops in the reduction inherit fast-math-flags from the recurrence
777767
// descriptor.
778768
ReducedPartRdx = createSimpleReduction(Builder, ReducedPartRdx, RK);
779-
780-
// If the reduction can be performed in a smaller type, we need to extend
781-
// the reduction to the wider type before we branch to the original loop.
782-
if (ResultTy != RdxDesc.getRecurrenceType())
783-
ReducedPartRdx = RdxDesc.isSigned()
784-
? Builder.CreateSExt(ReducedPartRdx, ResultTy)
785-
: Builder.CreateZExt(ReducedPartRdx, ResultTy);
786769
}
787770

788771
return ReducedPartRdx;

llvm/test/Transforms/LoopVectorize/X86/cost-model.ll

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,8 +1167,7 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
11671167
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
11681168
; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
11691169
; CHECK: middle.block:
1170-
; CHECK-NEXT: [[TMP10:%.*]] = trunc <16 x i32> [[TMP7]] to <16 x i1>
1171-
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP10]])
1170+
; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
11721171
; CHECK-NEXT: [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
11731172
; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[VEC_EPILOG_PH]]
11741173
; CHECK: scalar.ph:

llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,8 +208,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
208208
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
209209
; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
210210
; CHECK: middle.block:
211-
; CHECK-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i16>
212-
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP9]])
211+
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP6]])
213212
; CHECK-NEXT: [[TMP11:%.*]] = zext i16 [[TMP10]] to i32
214213
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
215214
; CHECK: vec.epilog.iter.check:
@@ -234,8 +233,7 @@ define i16 @reduction_or_trunc(ptr noalias nocapture %ptr) {
234233
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT4]], 256
235234
; CHECK-NEXT: br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
236235
; CHECK: vec.epilog.middle.block:
237-
; CHECK-NEXT: [[TMP22:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i16>
238-
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP22]])
236+
; CHECK-NEXT: [[TMP23:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP19]])
239237
; CHECK-NEXT: [[TMP24:%.*]] = zext i16 [[TMP23]] to i32
240238
; CHECK-NEXT: br i1 true, label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
241239
; CHECK: vec.epilog.scalar.ph:

llvm/test/Transforms/LoopVectorize/reduction-small-size.ll

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
2525
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
2626
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
2727
; CHECK: middle.block:
28-
; CHECK-NEXT: [[TMP6:%.*]] = trunc <4 x i32> [[TMP4]] to <4 x i8>
29-
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP6]])
28+
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP3]])
3029
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
3130
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
3231
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -104,8 +103,7 @@ define i8 @PR34687_no_undef(i1 %c, i32 %x, i32 %n) {
104103
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
105104
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
106105
; CHECK: middle.block:
107-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
108-
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP8]])
106+
; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP5]])
109107
; CHECK-NEXT: [[TMP10:%.*]] = zext i8 [[TMP9]] to i32
110108
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
111109
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -183,8 +181,7 @@ define i32 @PR35734(i32 %x, i32 %y) {
183181
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
184182
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
185183
; CHECK: middle.block:
186-
; CHECK-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i1>
187-
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP8]])
184+
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.add.v4i1(<4 x i1> [[TMP5]])
188185
; CHECK-NEXT: [[TMP10:%.*]] = sext i1 [[TMP9]] to i32
189186
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
190187
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]

llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
4343
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
4444
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4545
; CHECK: middle.block:
46-
; CHECK-NEXT: [[TMP37:%.*]] = trunc <vscale x 8 x i32> [[TMP34]] to <vscale x 8 x i8>
47-
; CHECK-NEXT: [[TMP38:%.*]] = trunc <vscale x 8 x i32> [[TMP36]] to <vscale x 8 x i8>
48-
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP38]], [[TMP37]]
46+
; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 8 x i8> [[TMP35]], [[TMP33]]
4947
; CHECK-NEXT: [[TMP39:%.*]] = call i8 @llvm.vector.reduce.add.nxv8i8(<vscale x 8 x i8> [[BIN_RDX]])
5048
; CHECK-NEXT: [[TMP40:%.*]] = zext i8 [[TMP39]] to i32
5149
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 256, [[N_VEC]]

0 commit comments

Comments
 (0)