Skip to content

Commit 4c02390

Browse files
david-armdavid-arm
committed
Address review comments
1 parent f19c3d1 commit 4c02390

8 files changed

+88
-8
lines changed

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -prefer-inloop-reductions -S | FileCheck %s
33

44
;
@@ -71,6 +71,10 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
7171
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
7272
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
7373
; CHECK: vec.epilog.scalar.ph:
74+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
75+
; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
76+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
77+
; CHECK: for.body:
7478
;
7579
entry:
7680
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -S | FileCheck %s
33

44
;
@@ -71,6 +71,10 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
7171
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
7272
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
7373
; CHECK: vec.epilog.scalar.ph:
74+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
75+
; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
76+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
77+
; CHECK: for.body:
7478
;
7579
entry:
7680
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -S | FileCheck %s
33

44
;
@@ -66,6 +66,10 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
6666
; CHECK-NEXT: [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC4]]
6767
; CHECK-NEXT: br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
6868
; CHECK: vec.epilog.scalar.ph:
69+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
70+
; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ]
71+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
72+
; CHECK: for.body:
6973
;
7074
entry:
7175
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^loop:"
22
; REQUIRES: asserts
33
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -epilogue-vectorization-minimum-VF=0 -prefer-predicate-over-epilogue=scalar-epilogue \
44
; RUN: -debug-only=loop-vectorize -force-target-instruction-cost=1 -S 2>%t | FileCheck %s --check-prefix=CHECK
@@ -82,6 +82,9 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
8282
; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 1024, [[N_VEC3]]
8383
; CHECK-NEXT: br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
8484
; CHECK: vec.epilog.scalar.ph:
85+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
86+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
87+
; CHECK: for.body:
8588
;
8689
; CHECK-VF8-LABEL: @main_vf_vscale_x_16(
8790
; CHECK-VF8-NEXT: iter.check:
@@ -134,6 +137,9 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
134137
; CHECK-VF8: vec.epilog.middle.block:
135138
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
136139
; CHECK-VF8: vec.epilog.scalar.ph:
140+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
141+
; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]
142+
; CHECK-VF8: for.body:
137143
;
138144
entry:
139145
br label %for.body
@@ -217,6 +223,9 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
217223
; CHECK: vec.epilog.middle.block:
218224
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
219225
; CHECK: vec.epilog.scalar.ph:
226+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
227+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
228+
; CHECK: for.body:
220229
;
221230
; CHECK-VF8-LABEL: @main_vf_vscale_x_2(
222231
; CHECK-VF8-NEXT: iter.check:
@@ -269,6 +278,9 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
269278
; CHECK-VF8: vec.epilog.middle.block:
270279
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
271280
; CHECK-VF8: vec.epilog.scalar.ph:
281+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
282+
; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]
283+
; CHECK-VF8: for.body:
272284
;
273285
entry:
274286
br label %for.body
@@ -353,6 +365,10 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
353365
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 10000, [[N_VEC3]]
354366
; CHECK-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
355367
; CHECK: vec.epilog.scalar.ph:
368+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
369+
; CHECK-NEXT: [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
370+
; CHECK-NEXT: br label [[LOOP:%.*]]
371+
; CHECK: loop:
356372
;
357373
; CHECK-VF8-LABEL: @test_pr57912_pointer_induction(
358374
; CHECK-VF8-NEXT: iter.check:
@@ -407,6 +423,10 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
407423
; CHECK-VF8: vec.epilog.middle.block:
408424
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
409425
; CHECK-VF8: vec.epilog.scalar.ph:
426+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
427+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
428+
; CHECK-VF8-NEXT: br label [[LOOP:%.*]]
429+
; CHECK-VF8: loop:
410430
;
411431
entry:
412432
br label %loop

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^loop:"
22
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown -mattr=+sve -S %s | FileCheck %s
33

44
define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
@@ -45,6 +45,10 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
4545
; CHECK-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[IND_END]], i64 -8
4646
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
4747
; CHECK: scalar.ph:
48+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY:%.*]] ]
49+
; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]
50+
; CHECK-NEXT: br label [[LOOP:%.*]]
51+
; CHECK: loop:
4852
;
4953
entry:
5054
br label %loop

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^while.body:"
22
; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
33
; RUN: < %s | FileCheck %s --check-prefix=CHECK
44
; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
@@ -46,6 +46,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
4646
; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
4747
; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
4848
; CHECK: scalar.ph:
49+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
50+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
51+
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
52+
; CHECK: while.body:
4953
;
5054
; CHECK-IN-LOOP-LABEL: @add_reduction_i32(
5155
; CHECK-IN-LOOP-NEXT: entry:
@@ -86,6 +90,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
8690
; CHECK-IN-LOOP: middle.block:
8791
; CHECK-IN-LOOP-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
8892
; CHECK-IN-LOOP: scalar.ph:
93+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
94+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
95+
; CHECK-IN-LOOP-NEXT: br label [[WHILE_BODY:%.*]]
96+
; CHECK-IN-LOOP: while.body:
8997
;
9098
entry:
9199
br label %while.body
@@ -143,6 +151,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
143151
; CHECK: middle.block:
144152
; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
145153
; CHECK: scalar.ph:
154+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
155+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
156+
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
157+
; CHECK: while.body:
146158
;
147159
; CHECK-IN-LOOP-LABEL: @add_reduction_f32(
148160
; CHECK-IN-LOOP-NEXT: entry:
@@ -182,6 +194,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
182194
; CHECK-IN-LOOP: middle.block:
183195
; CHECK-IN-LOOP-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
184196
; CHECK-IN-LOOP: scalar.ph:
197+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
198+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
199+
; CHECK-IN-LOOP-NEXT: br label [[WHILE_BODY:%.*]]
200+
; CHECK-IN-LOOP: while.body:
185201
;
186202
entry:
187203
br label %while.body
@@ -245,6 +261,10 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
245261
; CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP20]])
246262
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
247263
; CHECK: scalar.ph:
264+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
265+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ]
266+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
267+
; CHECK: for.body:
248268
;
249269
; CHECK-IN-LOOP-LABEL: @cond_xor_reduction(
250270
; CHECK-IN-LOOP-NEXT: entry:
@@ -289,6 +309,10 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
289309
; CHECK-IN-LOOP: middle.block:
290310
; CHECK-IN-LOOP-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
291311
; CHECK-IN-LOOP: scalar.ph:
312+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
313+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ]
314+
; CHECK-IN-LOOP-NEXT: br label [[FOR_BODY:%.*]]
315+
; CHECK-IN-LOOP: for.body:
292316
;
293317
entry:
294318
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -passes=loop-vectorize,dce,instcombine -S \
33
; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
44

@@ -69,6 +69,10 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
6969
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
7070
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
7171
; CHECK: scalar.ph:
72+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[C]], [[ENTRY:%.*]] ]
73+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
74+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
75+
; CHECK: for.body:
7276
;
7377
entry:
7478
br label %for.body
@@ -149,6 +153,11 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no
149153
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
150154
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
151155
; CHECK: scalar.ph:
156+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
157+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[SRC]], [[ENTRY]] ]
158+
; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[DST]], [[ENTRY]] ]
159+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
160+
; CHECK: for.body:
152161
;
153162
entry:
154163
br label %for.body
@@ -220,6 +229,12 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
220229
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
221230
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
222231
; CHECK: scalar.ph:
232+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
233+
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
234+
; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
235+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
236+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
237+
; CHECK: for.body:
223238
;
224239
entry:
225240
br label %for.body
@@ -268,6 +283,8 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr %
268283
; CHECK: middle.block:
269284
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
270285
; CHECK: scalar.ph:
286+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
287+
; CHECK: for.body:
271288
;
272289
entry:
273290
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:" --version 3
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --version 3
22
; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -epilogue-vectorization-minimum-VF=4 -debug-only=loop-vectorize -force-vector-interleave=1 -S 2>&1 | FileCheck %s
33
; REQUIRES: asserts
44

@@ -66,6 +66,9 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
6666
; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[N_MOD_VF2]], 0
6767
; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
6868
; CHECK: vec.epilog.scalar.ph:
69+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
70+
; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
71+
; CHECK: for.body:
6972
;
7073
entry:
7174
br label %for.body

0 commit comments

Comments
 (0)