Skip to content

Commit ad22752

Browse files
david-armdavid-arm
committed
Address review comments
1 parent 0644b83 commit ad22752

8 files changed

+88
-8
lines changed

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -prefer-inloop-reductions -S | FileCheck %s
33

44
;
@@ -70,6 +70,10 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
7070
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
7171
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
7272
; CHECK: vec.epilog.scalar.ph:
73+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
74+
; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP27]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[BIN_RDX]], [[VEC_EPILOG_ITER_CHECK]] ], [ 1, [[ITER_CHECK]] ]
75+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
76+
; CHECK: for.body:
7377
;
7478
entry:
7579
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -S | FileCheck %s
33

44
;
@@ -70,6 +70,10 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
7070
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 [[N]], [[N_VEC5]]
7171
; CHECK-NEXT: br i1 [[CMP_N6]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
7272
; CHECK: vec.epilog.scalar.ph:
73+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
74+
; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i64 [ [[TMP28]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], [[VEC_EPILOG_ITER_CHECK]] ], [ 5, [[ITER_CHECK]] ]
75+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
76+
; CHECK: for.body:
7377
;
7478
entry:
7579
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt < %s -passes=loop-vectorize -mtriple aarch64-unknown-linux-gnu -mattr=+sve -epilogue-vectorization-force-VF=2 -S | FileCheck %s
33

44
;
@@ -65,6 +65,10 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
6565
; CHECK-NEXT: [[CMP_N5:%.*]] = icmp eq i64 [[N]], [[N_VEC4]]
6666
; CHECK-NEXT: br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]]
6767
; CHECK: vec.epilog.scalar.ph:
68+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
69+
; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ]
70+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
71+
; CHECK: for.body:
6872
;
6973
entry:
7074
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^loop:"
22
; REQUIRES: asserts
33
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -epilogue-vectorization-minimum-VF=0 -prefer-predicate-over-epilogue=scalar-epilogue \
44
; RUN: -debug-only=loop-vectorize -force-target-instruction-cost=1 -S 2>%t | FileCheck %s --check-prefix=CHECK
@@ -81,6 +81,9 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
8181
; CHECK-NEXT: [[CMP_N4:%.*]] = icmp eq i64 1024, [[N_VEC3]]
8282
; CHECK-NEXT: br i1 [[CMP_N4]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
8383
; CHECK: vec.epilog.scalar.ph:
84+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
85+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
86+
; CHECK: for.body:
8487
;
8588
; CHECK-VF8-LABEL: @main_vf_vscale_x_16(
8689
; CHECK-VF8-NEXT: iter.check:
@@ -132,6 +135,9 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
132135
; CHECK-VF8: vec.epilog.middle.block:
133136
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
134137
; CHECK-VF8: vec.epilog.scalar.ph:
138+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
139+
; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]
140+
; CHECK-VF8: for.body:
135141
;
136142
entry:
137143
br label %for.body
@@ -214,6 +220,9 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
214220
; CHECK: vec.epilog.middle.block:
215221
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
216222
; CHECK: vec.epilog.scalar.ph:
223+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
224+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
225+
; CHECK: for.body:
217226
;
218227
; CHECK-VF8-LABEL: @main_vf_vscale_x_2(
219228
; CHECK-VF8-NEXT: iter.check:
@@ -265,6 +274,9 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
265274
; CHECK-VF8: vec.epilog.middle.block:
266275
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
267276
; CHECK-VF8: vec.epilog.scalar.ph:
277+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
278+
; CHECK-VF8-NEXT: br label [[FOR_BODY:%.*]]
279+
; CHECK-VF8: for.body:
268280
;
269281
entry:
270282
br label %for.body
@@ -348,6 +360,10 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
348360
; CHECK-NEXT: [[CMP_N6:%.*]] = icmp eq i64 10000, [[N_VEC3]]
349361
; CHECK-NEXT: br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
350362
; CHECK: vec.epilog.scalar.ph:
363+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
364+
; CHECK-NEXT: [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END4]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
365+
; CHECK-NEXT: br label [[LOOP:%.*]]
366+
; CHECK: loop:
351367
;
352368
; CHECK-VF8-LABEL: @test_pr57912_pointer_induction(
353369
; CHECK-VF8-NEXT: iter.check:
@@ -401,6 +417,10 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
401417
; CHECK-VF8: vec.epilog.middle.block:
402418
; CHECK-VF8-NEXT: br i1 true, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
403419
; CHECK-VF8: vec.epilog.scalar.ph:
420+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 10000, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
421+
; CHECK-VF8-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END1]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], [[ITER_CHECK]] ]
422+
; CHECK-VF8-NEXT: br label [[LOOP:%.*]]
423+
; CHECK-VF8: loop:
404424
;
405425
entry:
406426
br label %loop

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^loop:"
22
; RUN: opt -passes=loop-vectorize -mtriple=aarch64-unknown -mattr=+sve -S %s | FileCheck %s
33

44
define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
@@ -45,6 +45,10 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
4545
; CHECK-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[IND_END]], i64 -8
4646
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
4747
; CHECK: scalar.ph:
48+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[START_1]], [[ENTRY:%.*]] ]
49+
; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[IND_END3]], [[MIDDLE_BLOCK]] ], [ [[START_2]], [[ENTRY]] ]
50+
; CHECK-NEXT: br label [[LOOP:%.*]]
51+
; CHECK: loop:
4852
;
4953
entry:
5054
br label %loop

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --filter-out-after "^while.body:"
22
; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
33
; RUN: < %s | FileCheck %s --check-prefix=CHECK
44
; RUN: opt -S -hints-allow-reordering=false -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
@@ -45,6 +45,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
4545
; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
4646
; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
4747
; CHECK: scalar.ph:
48+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
49+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
50+
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
51+
; CHECK: while.body:
4852
;
4953
; CHECK-IN-LOOP-LABEL: @add_reduction_i32(
5054
; CHECK-IN-LOOP-NEXT: entry:
@@ -84,6 +88,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
8488
; CHECK-IN-LOOP: middle.block:
8589
; CHECK-IN-LOOP-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
8690
; CHECK-IN-LOOP: scalar.ph:
91+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
92+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
93+
; CHECK-IN-LOOP-NEXT: br label [[WHILE_BODY:%.*]]
94+
; CHECK-IN-LOOP: while.body:
8795
;
8896
entry:
8997
br label %while.body
@@ -140,6 +148,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
140148
; CHECK: middle.block:
141149
; CHECK-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
142150
; CHECK: scalar.ph:
151+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
152+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
153+
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
154+
; CHECK: while.body:
143155
;
144156
; CHECK-IN-LOOP-LABEL: @add_reduction_f32(
145157
; CHECK-IN-LOOP-NEXT: entry:
@@ -178,6 +190,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
178190
; CHECK-IN-LOOP: middle.block:
179191
; CHECK-IN-LOOP-NEXT: br i1 true, label [[WHILE_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
180192
; CHECK-IN-LOOP: scalar.ph:
193+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
194+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
195+
; CHECK-IN-LOOP-NEXT: br label [[WHILE_BODY:%.*]]
196+
; CHECK-IN-LOOP: while.body:
181197
;
182198
entry:
183199
br label %while.body
@@ -240,6 +256,10 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
240256
; CHECK-NEXT: [[TMP25:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP20]])
241257
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
242258
; CHECK: scalar.ph:
259+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
260+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP25]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ]
261+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
262+
; CHECK: for.body:
243263
;
244264
; CHECK-IN-LOOP-LABEL: @cond_xor_reduction(
245265
; CHECK-IN-LOOP-NEXT: entry:
@@ -283,6 +303,10 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
283303
; CHECK-IN-LOOP: middle.block:
284304
; CHECK-IN-LOOP-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
285305
; CHECK-IN-LOOP: scalar.ph:
306+
; CHECK-IN-LOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
307+
; CHECK-IN-LOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ]
308+
; CHECK-IN-LOOP-NEXT: br label [[FOR_BODY:%.*]]
309+
; CHECK-IN-LOOP: for.body:
286310
;
287311
entry:
288312
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:"
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:"
22
; RUN: opt -mtriple aarch64-linux-gnu -mattr=+sve -passes=loop-vectorize,dce,instcombine -S \
33
; RUN: -prefer-predicate-over-epilogue=scalar-epilogue < %s | FileCheck %s
44

@@ -69,6 +69,10 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
6969
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
7070
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
7171
; CHECK: scalar.ph:
72+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[C]], [[ENTRY:%.*]] ]
73+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
74+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
75+
; CHECK: for.body:
7276
;
7377
entry:
7478
br label %for.body
@@ -149,6 +153,11 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no
149153
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
150154
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
151155
; CHECK: scalar.ph:
156+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
157+
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[SRC]], [[ENTRY]] ]
158+
; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[DST]], [[ENTRY]] ]
159+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
160+
; CHECK: for.body:
152161
;
153162
entry:
154163
br label %for.body
@@ -220,6 +229,12 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
220229
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
221230
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
222231
; CHECK: scalar.ph:
232+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
233+
; CHECK-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[A]], [[ENTRY]] ]
234+
; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[IND_END2]], [[MIDDLE_BLOCK]] ], [ [[B]], [[ENTRY]] ]
235+
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
236+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
237+
; CHECK: for.body:
223238
;
224239
entry:
225240
br label %for.body
@@ -268,6 +283,8 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr %
268283
; CHECK: middle.block:
269284
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
270285
; CHECK: scalar.ph:
286+
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
287+
; CHECK: for.body:
271288
;
272289
entry:
273290
br label %for.body

‎llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^vec.epilog.scalar.ph:" --version 3
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^for.body:" --version 3
22
; RUN: opt < %s -mattr=+sve2 -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -sve-gather-overhead=2 -sve-scatter-overhead=2 -epilogue-vectorization-minimum-VF=4 -debug-only=loop-vectorize -force-vector-interleave=1 -S 2>&1 | FileCheck %s
33
; REQUIRES: asserts
44

@@ -66,6 +66,9 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
6666
; CHECK-NEXT: [[CMP_N7:%.*]] = icmp eq i64 [[N_MOD_VF2]], 0
6767
; CHECK-NEXT: br i1 [[CMP_N7]], label [[FOR_EXIT]], label [[SCALAR_PH]]
6868
; CHECK: vec.epilog.scalar.ph:
69+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
70+
; CHECK-NEXT: br label [[FOR_BODY1:%.*]]
71+
; CHECK: for.body:
6972
;
7073
entry:
7174
br label %for.body

0 commit comments

Comments
 (0)