Skip to content

Commit f79f490

Browse files
lukel97lukel97
committed
[VPlan] Pass VF as operand to VPWidenPointerInductionRecipe
Similarly to VPWidenIntOrFpInductionRecipe, if we want to support it in EVL tail folding we need to increment the induction by EVL steps instead of VF steps, but currently the VF is hard-wired in VPWidenPointerInductionRecipe. This adds an operand for the VF and plumbs it through, so that we can swap it out in VPlanTransforms::tryAddExplicitVectorLength further down the line.
1 parent e4372c4 commit f79f490

File tree

6 files changed

+30
-36
lines changed

6 files changed

+30
-36
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8403,7 +8403,7 @@ VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
84038403
VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep(),
84048404
*PSE.getSE());
84058405
return new VPWidenPointerInductionRecipe(
8406-
Phi, Operands[0], Step, *II,
8406+
Phi, Operands[0], Step, &Plan.getVF(), *II,
84078407
LoopVectorizationPlanner::getDecisionAndClampRange(
84088408
[&](ElementCount VF) {
84098409
return CM.isScalarAfterVectorization(Phi, VF);

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2198,25 +2198,28 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe {
21982198
};
21992199

22002200
class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
2201-
public VPUnrollPartAccessor<3> {
2201+
public VPUnrollPartAccessor<4> {
22022202
bool IsScalarAfterVectorization;
22032203

22042204
public:
22052205
/// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
22062206
/// Start.
22072207
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
2208-
const InductionDescriptor &IndDesc,
2208+
VPValue *VF, const InductionDescriptor &IndDesc,
22092209
bool IsScalarAfterVectorization, DebugLoc DL)
22102210
: VPWidenInductionRecipe(VPDef::VPWidenPointerInductionSC, Phi, Start,
22112211
Step, IndDesc, DL),
2212-
IsScalarAfterVectorization(IsScalarAfterVectorization) {}
2212+
IsScalarAfterVectorization(IsScalarAfterVectorization) {
2213+
addOperand(VF);
2214+
}
22132215

22142216
~VPWidenPointerInductionRecipe() override = default;
22152217

22162218
VPWidenPointerInductionRecipe *clone() override {
22172219
return new VPWidenPointerInductionRecipe(
22182220
cast<PHINode>(getUnderlyingInstr()), getOperand(0), getOperand(1),
2219-
getInductionDescriptor(), IsScalarAfterVectorization, getDebugLoc());
2221+
getOperand(2), getInductionDescriptor(), IsScalarAfterVectorization,
2222+
getDebugLoc());
22202223
}
22212224

22222225
VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC)
@@ -2231,7 +2234,7 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
22312234
/// the first unrolled part, if it exists. Returns itself if unrolling did not
22322235
/// take place.
22332236
VPValue *getFirstUnrolledPartOperand() {
2234-
return getUnrollPart(*this) == 0 ? this : getOperand(2);
2237+
return getUnrollPart(*this) == 0 ? this : getOperand(3);
22352238
}
22362239

22372240
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3169,7 +3169,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
31693169
BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint();
31703170
Value *ScalarStepValue = State.get(getStepValue(), VPLane(0));
31713171
Type *PhiType = State.TypeAnalysis.inferScalarType(getStepValue());
3172-
Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF);
3172+
Value *RuntimeVF = State.get(getOperand(2), true);
31733173
// Add induction update using an incorrect block temporarily. The phi node
31743174
// will be fixed after VPlan execution. Note that at this point the latch
31753175
// block cannot be used, as it does not exist yet.
@@ -3214,19 +3214,21 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
32143214
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
32153215
void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent,
32163216
VPSlotTracker &SlotTracker) const {
3217-
assert((getNumOperands() == 2 || getNumOperands() == 4) &&
3217+
assert((getNumOperands() == 3 || getNumOperands() == 5) &&
32183218
"unexpected number of operands");
32193219
O << Indent << "EMIT ";
32203220
printAsOperand(O, SlotTracker);
32213221
O << " = WIDEN-POINTER-INDUCTION ";
32223222
getStartValue()->printAsOperand(O, SlotTracker);
32233223
O << ", ";
32243224
getStepValue()->printAsOperand(O, SlotTracker);
3225-
if (getNumOperands() == 4) {
3226-
O << ", ";
3227-
getOperand(2)->printAsOperand(O, SlotTracker);
3225+
O << ", ";
3226+
getOperand(2)->printAsOperand(O, SlotTracker);
3227+
if (getNumOperands() == 5) {
32283228
O << ", ";
32293229
getOperand(3)->printAsOperand(O, SlotTracker);
3230+
O << ", ";
3231+
getOperand(4)->printAsOperand(O, SlotTracker);
32303232
}
32313233
}
32323234
#endif

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ target triple = "aarch64-unknown-linux-gnu"
99
; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 1
1010
;
1111
; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' {
12+
; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
1213
; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
1314
; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count
1415
; CHECK-NEXT: Live-in ir<%N> = original trip-count
@@ -24,7 +25,7 @@ target triple = "aarch64-unknown-linux-gnu"
2425
; CHECK-NEXT: <x1> vector loop: {
2526
; CHECK-NEXT: vector.body:
2627
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION
27-
; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, ir<1>
28+
; CHECK-NEXT: EMIT ir<%ptr.iv.2> = WIDEN-POINTER-INDUCTION ir<%start.2>, ir<1>, vp<[[VF]]>
2829
; CHECK-NEXT: vp<[[PTR_IDX:%.+]]> = DERIVED-IV ir<0> + vp<[[CAN_IV]]> * ir<8>
2930
; CHECK-NEXT: vp<[[PTR_IDX_STEPS:%.+]]> = SCALAR-STEPS vp<[[PTR_IDX]]>, ir<8>
3031
; CHECK-NEXT: EMIT vp<[[PTR_IV_1:%.+]]> = ptradd ir<%start.1>, vp<[[PTR_IDX_STEPS]]>
@@ -67,11 +68,9 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
6768
; CHECK: vector.body:
6869
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
6970
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
70-
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
71-
; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2
72-
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 1
71+
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP6]], 1
7372
; CHECK-NEXT: [[TMP10:%.*]] = mul i64 1, [[TMP9]]
74-
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 0
73+
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP6]], 0
7574
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
7675
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
7776
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
@@ -160,11 +159,9 @@ define void @pointer_induction(ptr noalias %start, i64 %N) {
160159
; CHECK: vector.body:
161160
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
162161
; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
163-
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
164-
; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2
165-
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 1
162+
; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP6]], 1
166163
; CHECK-NEXT: [[TMP10:%.*]] = mul i64 1, [[TMP9]]
167-
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 0
164+
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP6]], 0
168165
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
169166
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
170167
; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -240,8 +240,7 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
240240
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
241241
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
242242
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
243-
; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
244-
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 3
243+
; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP5]], 3
245244
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
246245
; CHECK-NEXT: [[TMP10:%.*]] = shl <vscale x 2 x i64> [[TMP9]], splat (i64 2)
247246
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP10]]
@@ -315,8 +314,7 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr %
315314
; CHECK: vector.body:
316315
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR:%.*]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
317316
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
318-
; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
319-
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
317+
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP0]], 2
320318
; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
321319
; CHECK-NEXT: [[TMP5:%.*]] = shl <vscale x 2 x i64> [[TMP4]], splat (i64 1)
322320
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP5]]

‎llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -170,11 +170,9 @@ define void @single_constant_stride_ptr_iv(ptr %p) {
170170
; CHECK: vector.body:
171171
; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
172172
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
173-
; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
174-
; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4
175-
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 1
173+
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 1
176174
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 8, [[TMP11]]
177-
; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP10]], 0
175+
; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP8]], 0
178176
; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
179177
; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
180178
; CHECK-NEXT: [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -740,11 +738,9 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
740738
; STRIDED-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
741739
; STRIDED-NEXT: [[POINTER_PHI11:%.*]] = phi ptr [ [[P2]], [[VECTOR_PH]] ], [ [[PTR_IND12:%.*]], [[VECTOR_BODY]] ]
742740
; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
743-
; STRIDED-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
744-
; STRIDED-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4
745-
; STRIDED-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 1
741+
; STRIDED-NEXT: [[TMP16:%.*]] = mul i64 [[TMP13]], 1
746742
; STRIDED-NEXT: [[TMP17:%.*]] = mul i64 [[STRIDE]], [[TMP16]]
747-
; STRIDED-NEXT: [[TMP18:%.*]] = mul i64 [[TMP15]], 0
743+
; STRIDED-NEXT: [[TMP18:%.*]] = mul i64 [[TMP13]], 0
748744
; STRIDED-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP18]], i64 0
749745
; STRIDED-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
750746
; STRIDED-NEXT: [[TMP19:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -753,11 +749,9 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
753749
; STRIDED-NEXT: [[DOTSPLAT10:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
754750
; STRIDED-NEXT: [[TMP21:%.*]] = mul <vscale x 4 x i64> [[TMP20]], [[DOTSPLAT10]]
755751
; STRIDED-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP21]]
756-
; STRIDED-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
757-
; STRIDED-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4
758-
; STRIDED-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 1
752+
; STRIDED-NEXT: [[TMP24:%.*]] = mul i64 [[TMP13]], 1
759753
; STRIDED-NEXT: [[TMP25:%.*]] = mul i64 [[STRIDE]], [[TMP24]]
760-
; STRIDED-NEXT: [[TMP26:%.*]] = mul i64 [[TMP23]], 0
754+
; STRIDED-NEXT: [[TMP26:%.*]] = mul i64 [[TMP13]], 0
761755
; STRIDED-NEXT: [[DOTSPLATINSERT13:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP26]], i64 0
762756
; STRIDED-NEXT: [[DOTSPLAT14:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT13]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
763757
; STRIDED-NEXT: [[TMP27:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()

0 commit comments

Comments
 (0)