Skip to content

Commit 2ab910c

Browse files
fhahnfhahn
committed
[LV] Check pointer user are in loop when checking for uniform pointers.
Widening decisions are not set for users outside the loop. Avoid crashing by only calling isVectorizedMemAccessUse for users in the loop. Fixes #102934.
1 parent 2913e71 commit 2ab910c

File tree

2 files changed

+167
-1
lines changed

2 files changed

+167
-1
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3705,7 +3705,8 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
37053705
auto *I = cast<Instruction>(V);
37063706
auto UsersAreMemAccesses =
37073707
llvm::all_of(I->users(), [&](User *U) -> bool {
3708-
return isVectorizedMemAccessUse(cast<Instruction>(U), V);
3708+
auto *UI = cast<Instruction>(U);
3709+
return TheLoop->contains(UI) && isVectorizedMemAccessUse(UI, V);
37093710
});
37103711
if (UsersAreMemAccesses)
37113712
addToWorklistIfAllowed(I);

‎llvm/test/Transforms/LoopVectorize/X86/gep-use-outside-loop.ll

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt -p loop-vectorize -mtriple=x86_64-apple-macosx -mcpu=skylake-avx512 -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
3+
4+
; Test case for https://github.com/llvm/llvm-project/issues/102934.
5+
define void @gep_use_in_dead_block(ptr noalias %dst, ptr %src) {
6+
; CHECK-LABEL: define void @gep_use_in_dead_block(
7+
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
8+
; CHECK-NEXT: [[ENTRY:.*]]:
9+
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
10+
; CHECK: [[VECTOR_PH]]:
11+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
12+
; CHECK: [[VECTOR_BODY]]:
13+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
14+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
15+
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
16+
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
17+
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
18+
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP0]]
19+
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i16, ptr [[TMP4]], i32 0
20+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP5]], align 2
21+
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD]], <i16 10, i16 10, i16 10, i16 10>
22+
; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[TMP6]], <i1 true, i1 true, i1 true, i1 true>
23+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP0]]
24+
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP1]]
25+
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP2]]
26+
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i16, ptr [[DST]], i64 [[TMP3]]
27+
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i16, ptr [[TMP8]], i32 0
28+
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> zeroinitializer, ptr [[TMP12]], i32 2, <4 x i1> [[TMP7]])
29+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
30+
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
31+
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
32+
; CHECK: [[MIDDLE_BLOCK]]:
33+
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
34+
; CHECK: [[SCALAR_PH]]:
35+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
36+
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
37+
; CHECK: [[LOOP_HEADER]]:
38+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
39+
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
40+
; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 2
41+
; CHECK-NEXT: [[C:%.*]] = icmp eq i16 [[L]], 10
42+
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
43+
; CHECK: [[THEN]]:
44+
; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV]]
45+
; CHECK-NEXT: store i16 0, ptr [[GEP_DST]], align 2
46+
; CHECK-NEXT: br label %[[LOOP_LATCH]]
47+
; CHECK: [[DEAD:.*]]:
48+
; CHECK-NEXT: store i16 0, ptr [[GEP_DST]], align 2
49+
; CHECK-NEXT: br label %[[DEAD]]
50+
; CHECK: [[LOOP_LATCH]]:
51+
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
52+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 99
53+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
54+
; CHECK: [[EXIT]]:
55+
; CHECK-NEXT: ret void
56+
;
57+
entry:
58+
br label %loop.header
59+
60+
loop.header:
61+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
62+
%gep.src = getelementptr i16, ptr %src, i64 %iv
63+
%l = load i16, ptr %gep.src
64+
%c = icmp eq i16 %l, 10
65+
br i1 %c, label %loop.latch, label %then
66+
67+
then:
68+
%gep.dst = getelementptr i16, ptr %dst, i64 %iv
69+
store i16 0, ptr %gep.dst, align 2
70+
br label %loop.latch
71+
72+
dead:
73+
store i16 0, ptr %gep.dst, align 2
74+
br label %dead
75+
76+
loop.latch:
77+
%iv.next = add i64 %iv, 1
78+
%ec = icmp eq i64 %iv.next, 99
79+
br i1 %ec, label %exit, label %loop.header
80+
81+
exit:
82+
ret void
83+
}
84+
85+
define void @gep_use_outside_loop(ptr noalias %dst, ptr %src) {
86+
; CHECK-LABEL: define void @gep_use_outside_loop(
87+
; CHECK-SAME: ptr noalias [[DST:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
88+
; CHECK-NEXT: [[ENTRY:.*]]:
89+
; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
90+
; CHECK: [[VECTOR_PH]]:
91+
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
92+
; CHECK: [[VECTOR_BODY]]:
93+
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
94+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
95+
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
96+
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i16, ptr [[DST]], <4 x i64> [[VEC_IND]]
97+
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP0]]
98+
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[TMP2]], i32 0
99+
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP3]], align 2
100+
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD]], <i16 10, i16 10, i16 10, i16 10>
101+
; CHECK-NEXT: [[TMP5:%.*]] = xor <4 x i1> [[TMP4]], <i1 true, i1 true, i1 true, i1 true>
102+
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP1]], i32 0
103+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[TMP6]], i32 0
104+
; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> zeroinitializer, ptr [[TMP7]], i32 2, <4 x i1> [[TMP5]])
105+
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
106+
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
107+
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
108+
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
109+
; CHECK: [[MIDDLE_BLOCK]]:
110+
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x ptr> [[TMP1]], i32 3
111+
; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
112+
; CHECK: [[SCALAR_PH]]:
113+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
114+
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
115+
; CHECK: [[LOOP_HEADER]]:
116+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
117+
; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV]]
118+
; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
119+
; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 2
120+
; CHECK-NEXT: [[C:%.*]] = icmp eq i16 [[L]], 10
121+
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
122+
; CHECK: [[THEN]]:
123+
; CHECK-NEXT: store i16 0, ptr [[GEP_DST]], align 2
124+
; CHECK-NEXT: br label %[[LOOP_LATCH]]
125+
; CHECK: [[LOOP_LATCH]]:
126+
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
127+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 99
128+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
129+
; CHECK: [[EXIT]]:
130+
; CHECK-NEXT: [[GEP_DST_LCSSA:%.*]] = phi ptr [ [[GEP_DST]], %[[LOOP_LATCH]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
131+
; CHECK-NEXT: store i16 0, ptr [[GEP_DST_LCSSA]], align 2
132+
; CHECK-NEXT: ret void
133+
;
134+
entry:
135+
br label %loop.header
136+
137+
loop.header:
138+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
139+
%gep.dst = getelementptr i16, ptr %dst, i64 %iv
140+
%gep.src = getelementptr i16, ptr %src, i64 %iv
141+
%l = load i16, ptr %gep.src
142+
%c = icmp eq i16 %l, 10
143+
br i1 %c, label %loop.latch, label %then
144+
145+
then:
146+
store i16 0, ptr %gep.dst, align 2
147+
br label %loop.latch
148+
149+
loop.latch:
150+
%iv.next = add i64 %iv, 1
151+
%ec = icmp eq i64 %iv.next, 99
152+
br i1 %ec, label %exit, label %loop.header
153+
154+
exit:
155+
store i16 0, ptr %gep.dst, align 2
156+
ret void
157+
}
158+
;.
159+
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
160+
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
161+
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
162+
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
163+
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
164+
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
165+
;.

0 commit comments

Comments
 (0)