Skip to content

Commit 9094aa1

Browse files
fhahnfhahn
committed
[VPlan] Model FOR resume value extraction in VPlan.
This patch introduces a new ExtractRecurrenceResume VPInstruction opcode to extract the value of a FOR to be used as resume value for the ph in the scalar loop. Note that it takes a VPFirstOrderRecurrencePHIRecipe as operand. This is needed at the moment to conveniently let fixFixedOrderRecurrence still handle creating and patching up the phis in the scalar pre-header and header. As ExtractRecurrenceResume won't have any users yet, it is marked as having side-effects until the phi in the scalar pre-header is also created and managed by VPlan. Depends on #93395
1 parent 5a26ba2 commit 9094aa1

20 files changed

+156
-139
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 16 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -608,8 +608,7 @@ class InnerLoopVectorizer {
608608

609609
/// Create the exit value of first order recurrences in the middle block and
610610
/// update their users.
611-
void fixFixedOrderRecurrence(VPFirstOrderRecurrencePHIRecipe *PhiR,
612-
VPTransformState &State);
611+
void fixFixedOrderRecurrence(VPInstruction *VPI, VPTransformState &State);
613612

614613
/// Iteratively sink the scalarized operands of a predicated instruction into
615614
/// the block that was created for it.
@@ -3391,16 +3390,18 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
33913390
fixNonInductionPHIs(Plan, State);
33923391

33933392
// At this point every instruction in the original loop is widened to a
3394-
// vector form. Now we need to fix the recurrences in the loop. These PHI
3395-
// nodes are currently empty because we did not want to introduce cycles.
3396-
// This is the second stage of vectorizing recurrences. Note that fixing
3397-
// reduction phis are already modeled in VPlan.
3398-
// TODO: Also model fixing fixed-order recurrence phis in VPlan.
3393+
// vector form. Note that fixing reduction phis, as well as extracting the
3394+
// exit and resume values for fixed-order recurrences are already modeled in
3395+
// VPlan. All that remains to do here is creating a phi in the scalar
3396+
// pre-header for each fixed-rder recurrence resume value.
3397+
// TODO: Also model creating phis in the scalar pre-header in VPlan.
33993398
VPRegionBlock *VectorRegion = State.Plan->getVectorLoopRegion();
3400-
VPBasicBlock *HeaderVPBB = VectorRegion->getEntryBasicBlock();
3401-
for (VPRecipeBase &R : HeaderVPBB->phis()) {
3402-
if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
3403-
fixFixedOrderRecurrence(FOR, State);
3399+
VPBasicBlock *ExitVPBB =
3400+
cast<VPBasicBlock>(VectorRegion->getSingleSuccessor());
3401+
for (VPRecipeBase &R : *ExitVPBB) {
3402+
if (auto *VPI = dyn_cast<VPInstruction>(&R))
3403+
if (VPI->getOpcode() == VPInstruction::ExtractRecurrenceResume)
3404+
fixFixedOrderRecurrence(VPI, State);
34043405
}
34053406

34063407
// Forget the original basic block.
@@ -3469,73 +3470,13 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State,
34693470
VF.getKnownMinValue() * UF);
34703471
}
34713472

3472-
void InnerLoopVectorizer::fixFixedOrderRecurrence(
3473-
VPFirstOrderRecurrencePHIRecipe *PhiR, VPTransformState &State) {
3474-
// This is the second phase of vectorizing first-order recurrences. An
3475-
// overview of the transformation is described below. Suppose we have the
3476-
// following loop.
3477-
//
3478-
// for (int i = 0; i < n; ++i)
3479-
// b[i] = a[i] - a[i - 1];
3480-
//
3481-
// There is a first-order recurrence on "a". For this loop, the shorthand
3482-
// scalar IR looks like:
3483-
//
3484-
// scalar.ph:
3485-
// s_init = a[-1]
3486-
// br scalar.body
3487-
//
3488-
// scalar.body:
3489-
// i = phi [0, scalar.ph], [i+1, scalar.body]
3490-
// s1 = phi [s_init, scalar.ph], [s2, scalar.body]
3491-
// s2 = a[i]
3492-
// b[i] = s2 - s1
3493-
// br cond, scalar.body, ...
3494-
//
3495-
// In this example, s1 is a recurrence because it's value depends on the
3496-
// previous iteration. In the first phase of vectorization, we created a
3497-
// vector phi v1 for s1. We now complete the vectorization and produce the
3498-
// shorthand vector IR shown below (for VF = 4, UF = 1).
3499-
//
3500-
// vector.ph:
3501-
// v_init = vector(..., ..., ..., a[-1])
3502-
// br vector.body
3503-
//
3504-
// vector.body
3505-
// i = phi [0, vector.ph], [i+4, vector.body]
3506-
// v1 = phi [v_init, vector.ph], [v2, vector.body]
3507-
// v2 = a[i, i+1, i+2, i+3];
3508-
// v3 = vector(v1(3), v2(0, 1, 2))
3509-
// b[i, i+1, i+2, i+3] = v2 - v3
3510-
// br cond, vector.body, middle.block
3511-
//
3512-
// middle.block:
3513-
// x = v2(3)
3514-
// br scalar.ph
3515-
//
3516-
// scalar.ph:
3517-
// s_init = phi [x, middle.block], [a[-1], otherwise]
3518-
// br scalar.body
3519-
//
3520-
// After execution completes the vector loop, we extract the next value of
3521-
// the recurrence (x) to use as the initial value in the scalar loop.
3522-
3473+
void InnerLoopVectorizer::fixFixedOrderRecurrence(VPInstruction *VPI,
3474+
VPTransformState &State) {
35233475
// Extract the last vector element in the middle block. This will be the
35243476
// initial value for the recurrence when jumping to the scalar loop.
3525-
VPValue *PreviousDef = PhiR->getBackedgeValue();
3526-
Value *Incoming = State.get(PreviousDef, UF - 1);
3527-
auto *ExtractForScalar = Incoming;
3528-
auto *IdxTy = Builder.getInt32Ty();
3529-
Value *RuntimeVF = nullptr;
3530-
if (VF.isVector()) {
3531-
auto *One = ConstantInt::get(IdxTy, 1);
3532-
Builder.SetInsertPoint(LoopMiddleBlock->getTerminator());
3533-
RuntimeVF = getRuntimeVF(Builder, IdxTy, VF);
3534-
auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
3535-
ExtractForScalar =
3536-
Builder.CreateExtractElement(Incoming, LastIdx, "vector.recur.extract");
3537-
}
3477+
Value *ExtractForScalar = State.get(VPI, UF - 1, true);
35383478

3479+
auto *PhiR = cast<VPFirstOrderRecurrencePHIRecipe>(VPI->getOperand(0));
35393480
// Fix the initial value of the original recurrence in the scalar loop.
35403481
Builder.SetInsertPoint(LoopScalarPreHeader, LoopScalarPreHeader->begin());
35413482
PHINode *Phi = cast<PHINode>(PhiR->getUnderlyingValue());

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1180,6 +1180,7 @@ class VPInstruction : public VPRecipeWithIRFlags {
11801180
BranchOnCond,
11811181
ComputeReductionResult,
11821182
ExtractRecurrenceResult,
1183+
ExtractRecurrenceResume,
11831184
LogicalAnd, // Non-poison propagating logical And.
11841185
// Add an offset in bytes (second operand) to a base pointer (first
11851186
// operand). Only generates scalar values (either for the first lane only or
@@ -3614,7 +3615,8 @@ inline bool isUniformAfterVectorization(VPValue *VPV) {
36143615
return all_of(GEP->operands(), isUniformAfterVectorization);
36153616
if (auto *VPI = dyn_cast<VPInstruction>(Def))
36163617
return VPI->getOpcode() == VPInstruction::ComputeReductionResult ||
3617-
VPI->getOpcode() == VPInstruction::ExtractRecurrenceResult;
3618+
VPI->getOpcode() == VPInstruction::ExtractRecurrenceResult ||
3619+
VPI->getOpcode() == VPInstruction::ExtractRecurrenceResume;
36183620
return false;
36193621
}
36203622
} // end namespace vputils

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
302302
case VPInstruction::CanonicalIVIncrementForPart:
303303
case VPInstruction::ComputeReductionResult:
304304
case VPInstruction::ExtractRecurrenceResult:
305+
case VPInstruction::ExtractRecurrenceResume:
305306
case VPInstruction::PtrAdd:
306307
case VPInstruction::ExplicitVectorLength:
307308
return true;
@@ -581,6 +582,18 @@ Value *VPInstruction::generatePerPart(VPTransformState &State, unsigned Part) {
581582
Res->setName(Name);
582583
return Res;
583584
}
585+
case VPInstruction::ExtractRecurrenceResume: {
586+
if (Part != 0)
587+
return State.get(this, 0, /*IsScalar*/ true);
588+
589+
auto *PhiR = cast<VPFirstOrderRecurrencePHIRecipe>(getOperand(0));
590+
VPValue *PreviousDef = PhiR->getBackedgeValue();
591+
Value *Res =
592+
State.get(PreviousDef, VPIteration(State.UF - 1,
593+
VPLane::getLastLaneForVF(State.VF)));
594+
Res->setName(Name);
595+
return Res;
596+
}
584597
case VPInstruction::LogicalAnd: {
585598
Value *A = State.get(getOperand(0), Part);
586599
Value *B = State.get(getOperand(1), Part);
@@ -716,6 +729,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
716729
case VPInstruction::BranchOnCount:
717730
O << "branch-on-count";
718731
break;
732+
case VPInstruction::ExtractRecurrenceResume:
733+
O << "extract-recurrence-resume";
734+
break;
719735
case VPInstruction::ExtractRecurrenceResult:
720736
O << "extract-recurrence-result";
721737
break;

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -846,11 +846,62 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
846846
// all users.
847847
RecurSplice->setOperand(0, FOR);
848848

849+
// This is the second phase of vectorizing first-order recurrences. An
850+
// overview of the transformation is described below. Suppose we have the
851+
// following loop.
852+
//
853+
// for (int i = 0; i < n; ++i)
854+
// b[i] = a[i] - a[i - 1];
855+
//
856+
// There is a first-order recurrence on "a". For this loop, the shorthand
857+
// scalar IR looks like:
858+
//
859+
// scalar.ph:
860+
// s_init = a[-1]
861+
// br scalar.body
862+
//
863+
// scalar.body:
864+
// i = phi [0, scalar.ph], [i+1, scalar.body]
865+
// s1 = phi [s_init, scalar.ph], [s2, scalar.body]
866+
// s2 = a[i]
867+
// b[i] = s2 - s1
868+
// br cond, scalar.body, ...
869+
//
870+
// In this example, s1 is a recurrence because it's value depends on the
871+
// previous iteration. In the first phase of vectorization, we created a
872+
// vector phi v1 for s1. We now complete the vectorization and produce the
873+
// shorthand vector IR shown below (for VF = 4, UF = 1).
874+
//
875+
// vector.ph:
876+
// v_init = vector(..., ..., ..., a[-1])
877+
// br vector.body
878+
//
879+
// vector.body
880+
// i = phi [0, vector.ph], [i+4, vector.body]
881+
// v1 = phi [v_init, vector.ph], [v2, vector.body]
882+
// v2 = a[i, i+1, i+2, i+3];
883+
// v3 = vector(v1(3), v2(0, 1, 2))
884+
// b[i, i+1, i+2, i+3] = v2 - v3
885+
// br cond, vector.body, middle.block
886+
//
887+
// middle.block:
888+
// x = v2(3)
889+
// br scalar.ph
890+
//
891+
// scalar.ph:
892+
// s_init = phi [x, middle.block], [a[-1], otherwise]
893+
// br scalar.body
894+
//
895+
// After execution completes the vector loop, we extract the next value of
896+
// the recurrence (x) to use as the initial value in the scalar loop. This
897+
// is modeled by ExtractRecurrenceResume.
849898
auto *Result = cast<VPInstruction>(MiddleBuilder.createNaryOp(
850899
VPInstruction::ExtractRecurrenceResult, {FOR->getBackedgeValue()}, {},
851900
"vector.recur.extract.for.phi"));
852901
RecurSplice->replaceUsesWithIf(
853902
Result, [](VPUser &U, unsigned) { return isa<VPLiveOut>(&U); });
903+
MiddleBuilder.createNaryOp(VPInstruction::ExtractRecurrenceResume, {FOR},
904+
{}, "vector.recur.extract");
854905
}
855906
return true;
856907
}

‎llvm/test/Transforms/LoopVectorize/AArch64/fixed-order-recurrence.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
4747
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4848
; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4949
; CHECK: middle.block:
50-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
5150
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD1]], i32 15
51+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
5252
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
5353
; CHECK: scalar.ph:
5454
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
@@ -154,10 +154,10 @@ define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
154154
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
155155
; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
156156
; CHECK: middle.block:
157-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
158157
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD5]], i32 15
159158
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <16 x i8> [[TMP8]], i32 15
160159
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT9:%.*]] = extractelement <16 x i8> [[TMP10]], i32 15
160+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
161161
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
162162
; CHECK: scalar.ph:
163163
; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT9]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,8 @@ define i64 @pointer_induction_only(ptr %start, ptr %end) {
127127
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
128128
; CHECK: middle.block:
129129
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
130-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
131130
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
131+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
132132
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
133133
; CHECK: scalar.ph:
134134
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -786,8 +786,8 @@ define void @add_phifail(ptr noalias nocapture readonly %p, ptr noalias nocaptur
786786
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
787787
; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
788788
; CHECK: middle.block:
789-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
790789
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP4]], i32 15
790+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
791791
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
792792
; CHECK: scalar.ph:
793793
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
@@ -871,8 +871,8 @@ define i8 @add_phifail2(ptr noalias nocapture readonly %p, ptr noalias nocapture
871871
; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
872872
; CHECK: middle.block:
873873
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <16 x i32> [[TMP6]], i32 14
874-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
875874
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i32> [[TMP6]], i32 15
875+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
876876
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
877877
; CHECK: scalar.ph:
878878
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,6 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
100100
; DEFAULT: middle.block:
101101
; DEFAULT-NEXT: [[BIN_RDX:%.*]] = or <vscale x 4 x i32> [[TMP58]], [[TMP57]]
102102
; DEFAULT-NEXT: [[TMP60:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
103-
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
104103
; DEFAULT-NEXT: [[TMP61:%.*]] = call i32 @llvm.vscale.i32()
105104
; DEFAULT-NEXT: [[TMP62:%.*]] = mul i32 [[TMP61]], 4
106105
; DEFAULT-NEXT: [[TMP63:%.*]] = sub i32 [[TMP62]], 1
@@ -109,6 +108,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
109108
; DEFAULT-NEXT: [[TMP65:%.*]] = mul i32 [[TMP64]], 4
110109
; DEFAULT-NEXT: [[TMP66:%.*]] = sub i32 [[TMP65]], 1
111110
; DEFAULT-NEXT: [[VECTOR_RECUR_EXTRACT13:%.*]] = extractelement <vscale x 4 x i32> [[TMP20]], i32 [[TMP66]]
111+
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
112112
; DEFAULT-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
113113
; DEFAULT: scalar.ph:
114114
; DEFAULT-NEXT: [[SCALAR_RECUR_INIT14:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT13]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1509,11 +1509,11 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) #1 {
15091509
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
15101510
; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
15111511
; CHECK: middle.block:
1512-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
15131512
; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
15141513
; CHECK-NEXT: [[TMP31:%.*]] = shl nuw nsw i32 [[TMP30]], 2
15151514
; CHECK-NEXT: [[TMP32:%.*]] = add nsw i32 [[TMP31]], -1
15161515
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_MASKED_GATHER4]], i32 [[TMP32]]
1516+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
15171517
; CHECK-NEXT: br i1 [[CMP_N]], label [[END:%.*]], label [[SCALAR_PH]]
15181518
; CHECK: scalar.ph:
15191519
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[DOTPRE]], [[VECTOR_MEMCHECK]] ], [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ define void @firstorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
4747
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
4848
; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
4949
; CHECK: middle.block:
50-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
5150
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD1]], i32 15
51+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
5252
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
5353
; CHECK: scalar.ph:
5454
; CHECK-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]
@@ -154,10 +154,10 @@ define void @thirdorderrec(ptr nocapture noundef readonly %x, ptr noalias nocapt
154154
; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
155155
; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
156156
; CHECK: middle.block:
157-
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
158157
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <16 x i8> [[WIDE_LOAD5]], i32 15
159158
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <16 x i8> [[TMP8]], i32 15
160159
; CHECK-NEXT: [[VECTOR_RECUR_EXTRACT9:%.*]] = extractelement <16 x i8> [[TMP10]], i32 15
160+
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
161161
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]]
162162
; CHECK: scalar.ph:
163163
; CHECK-NEXT: [[SCALAR_RECUR_INIT10:%.*]] = phi i8 [ [[DOTPRE]], [[FOR_BODY_PREHEADER]] ], [ [[VECTOR_RECUR_EXTRACT9]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/X86/pr72969.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,8 +83,8 @@ define void @test(ptr %p) {
8383
; VEC-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
8484
; VEC-NEXT: br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
8585
; VEC: middle.block:
86-
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
8786
; VEC-NEXT: [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[TMP28]], i32 3
87+
; VEC-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP4]], [[N_VEC]]
8888
; VEC-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
8989
; VEC: scalar.ph:
9090
; VEC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 1, [[VECTOR_SCEVCHECK]] ], [ 1, [[ENTRY:%.*]] ], [ [[VECTOR_RECUR_EXTRACT]], [[MIDDLE_BLOCK]] ]

‎llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) {
3333
; CHECK-NEXT: Successor(s): middle.block
3434
; CHECK-EMPTY:
3535
; CHECK-NEXT: middle.block:
36+
; CHECK-NEXT: EMIT vp<{{.+}}> = extract-recurrence-resume ir<%for.1>
37+
; CHECK-NEXT: EMIT vp<{{.+}}> = extract-recurrence-resume ir<%for.2>
3638
; CHECK-NEXT: No successors
3739
; CHECK-NEXT: }
3840
;
@@ -89,6 +91,9 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) {
8991
; CHECK-NEXT: Successor(s): middle.block
9092
; CHECK-EMPTY:
9193
; CHECK-NEXT: middle.block:
94+
; CHECK-NEXT: EMIT vp<{{.+}}> = extract-recurrence-resume ir<%for.1>
95+
; CHECK-NEXT: EMIT vp<{{.+}}> = extract-recurrence-resume ir<%for.2>
96+
; CHECK-NEXT: EMIT vp<{{.+}}> = extract-recurrence-resume ir<%for.3>
9297
; CHECK-NEXT: No successors
9398
; CHECK-NEXT: }
9499
;

0 commit comments

Comments
 (0)