Skip to content

Commit 1769186

Browse files
david-armdavid-arm
committed
Address review comments
1 parent 8720f8a commit 1769186

8 files changed

+88
-8
lines changed

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -prefer-inloop-reductions -S | FileCheck %s
33

44
;
@@ -69,6 +69,10 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
6969
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
7070
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
7171
; CHECK: vec.epilog.scalar.ph:
72+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
73+
; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
74+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
75+
; CHECK: for.body:
7276
;
7377
entry:
7478
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -S | FileCheck %s
33

44
;
@@ -69,6 +69,10 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
6969
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
7070
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
7171
; CHECK: vec.epilog.scalar.ph:
72+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
73+
; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
74+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
75+
; CHECK: for.body:
7276
;
7377
entry:
7478
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -S | FileCheck %s
33

44
;
@@ -64,6 +64,10 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
6464
; CHECK-NEXT: [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC4]]
6565
; CHECK-NEXT: br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
6666
; CHECK: vec.epilog.scalar.ph:
67+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
68+
; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ]
69+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
70+
; CHECK: for.body:
6771
;
6872
entry:
6973
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^loop:"
22
; REQUIRES: asserts
33
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -epilogue-vectorization-minimum-VF=0 -prefer-predicate-over-epilogue=scalar-epilogue \
44
; RUN: -debug-only=loop-vectorize -force-target-instruction-cost=1 -S 2>%t | FileCheck %s --check-prefix=CHECK
@@ -80,6 +80,9 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
8080
; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 1024, [[N_VEC3]]
8181
; CHECK-NEXT: br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
8282
; CHECK: vec.epilog.scalar.ph:
83+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
84+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
85+
; CHECK: for.body:
8386
;
8487
; CHECK-VF8-LABEL: @main_vf_vscale_x_16(
8588
; CHECK-VF8-NEXT: iter.check:
@@ -130,6 +133,9 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
130133
; CHECK-VF8: vec.epilog.middle.block:
131134
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
132135
; CHECK-VF8: vec.epilog.scalar.ph:
136+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
137+
; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]
138+
; CHECK-VF8: for.body:
133139
;
134140
entry:
135141
br label %for.body
@@ -211,6 +217,9 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
211217
; CHECK: vec.epilog.middle.block:
212218
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
213219
; CHECK: vec.epilog.scalar.ph:
220+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
221+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
222+
; CHECK: for.body:
214223
;
215224
; CHECK-VF8-LABEL: @main_vf_vscale_x_2(
216225
; CHECK-VF8-NEXT: iter.check:
@@ -261,6 +270,9 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
261270
; CHECK-VF8: vec.epilog.middle.block:
262271
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
263272
; CHECK-VF8: vec.epilog.scalar.ph:
273+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
274+
; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]
275+
; CHECK-VF8: for.body:
264276
;
265277
entry:
266278
br label %for.body
@@ -343,6 +355,10 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
343355
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 10000, [[N_VEC3]]
344356
; CHECK-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
345357
; CHECK: vec.epilog.scalar.ph:
358+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
359+
; CHECK-NEXT: [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
360+
; CHECK-NEXT: br label [[LOOP:%.*]]
361+
; CHECK: loop:
346362
;
347363
; CHECK-VF8-LABEL: @test_pr57912_pointer_induction(
348364
; CHECK-VF8-NEXT: iter.check:
@@ -395,6 +411,10 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
395411
; CHECK-VF8: vec.epilog.middle.block:
396412
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
397413
; CHECK-VF8: vec.epilog.scalar.ph:
414+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
415+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
416+
; CHECK-VF8-NEXT: br label [[LOOP:%.*]]
417+
; CHECK-VF8: loop:
398418
;
399419
entry:
400420
br label %loop

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^loop:"
22
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown -mattr=+sve -S %s | FileCheck %s
33

44
define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
@@ -44,6 +44,10 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
4444
; CHECK-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[IND_END]], i64 -8
4545
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
4646
; CHECK: scalar.ph:
47+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY:%.*]] ]
48+
; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]
49+
; CHECK-NEXT: br label [[LOOP:%.*]]
50+
; CHECK: loop:
4751
;
4852
entry:
4953
br label %loop

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^while.body:"
22
; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
33
; RUN: < %s | FileCheck %s --check-prefix=CHECK
44
; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
@@ -45,6 +45,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
4545
; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
4646
; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
4747
; CHECK: scalar.ph:
48+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
49+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
50+
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
51+
; CHECK: while.body:
4852
;
4953
; CHECK-IN-LOOP-LABEL: @add_reduction_i32(
5054
; CHECK-IN-LOOP-NEXT: entry:
@@ -84,6 +88,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
8488
; CHECK-IN-LOOP: middle.block:
8589
; CHECK-IN-LOOP-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
8690
; CHECK-IN-LOOP: scalar.ph:
91+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
92+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
93+
; CHECK-IN-LOOP-NEXT: br label [[WHILE_BODY:%.*]]
94+
; CHECK-IN-LOOP: while.body:
8795
;
8896
entry:
8997
br label %while.body
@@ -140,6 +148,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
140148
; CHECK: middle.block:
141149
; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
142150
; CHECK: scalar.ph:
151+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
152+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
153+
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
154+
; CHECK: while.body:
143155
;
144156
; CHECK-IN-LOOP-LABEL: @add_reduction_f32(
145157
; CHECK-IN-LOOP-NEXT: entry:
@@ -178,6 +190,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
178190
; CHECK-IN-LOOP: middle.block:
179191
; CHECK-IN-LOOP-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
180192
; CHECK-IN-LOOP: scalar.ph:
193+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
194+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
195+
; CHECK-IN-LOOP-NEXT: br label [[WHILE_BODY:%.*]]
196+
; CHECK-IN-LOOP: while.body:
181197
;
182198
entry:
183199
br label %while.body
@@ -240,6 +256,10 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
240256
; CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP20]])
241257
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
242258
; CHECK: scalar.ph:
259+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
260+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ]
261+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
262+
; CHECK: for.body:
243263
;
244264
; CHECK-IN-LOOP-LABEL: @cond_xor_reduction(
245265
; CHECK-IN-LOOP-NEXT: entry:
@@ -283,6 +303,10 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
283303
; CHECK-IN-LOOP: middle.block:
284304
; CHECK-IN-LOOP-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
285305
; CHECK-IN-LOOP: scalar.ph:
306+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
307+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ]
308+
; CHECK-IN-LOOP-NEXT: br label [[FOR_BODY:%.*]]
309+
; CHECK-IN-LOOP: for.body:
286310
;
287311
entry:
288312
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -passes=loop-vectorize,dce,instcombine -S \
33
; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
44

@@ -68,6 +68,10 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
6868
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
6969
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
7070
; CHECK: scalar.ph:
71+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[C]], [[ENTRY:%.*]] ]
72+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
73+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
74+
; CHECK: for.body:
7175
;
7276
entry:
7377
br label %for.body
@@ -148,6 +152,11 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no
148152
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
149153
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
150154
; CHECK: scalar.ph:
155+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
156+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[SRC]], [[ENTRY]] ]
157+
; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[DST]], [[ENTRY]] ]
158+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
159+
; CHECK: for.body:
151160
;
152161
entry:
153162
br label %for.body
@@ -219,6 +228,12 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
219228
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
220229
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
221230
; CHECK: scalar.ph:
231+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
232+
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
233+
; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
234+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
235+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
236+
; CHECK: for.body:
222237
;
223238
entry:
224239
br label %for.body
@@ -267,6 +282,8 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr %
267282
; CHECK: middle.block:
268283
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
269284
; CHECK: scalar.ph:
285+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
286+
; CHECK: for.body:
270287
;
271288
entry:
272289
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:" --version 3
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --version 3
22
; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -epilogue-vectorization-minimum-VF=4 -debug-only=loop-vectorize -force-vector-interleave=1 -S 2>&1 | FileCheck %s
33
; REQUIRES: asserts
44

@@ -66,6 +66,9 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
6666
; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[N_MOD_VF2]], 0
6767
; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
6868
; CHECK: vec.epilog.scalar.ph:
69+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
70+
; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
71+
; CHECK: for.body:
6972
;
7073
entry:
7174
br label %for.body

0 commit comments

Comments
 (0)