Skip to content

Commit 1b377db

Browse files
fhahnfhahn
authored
[LAA] Check accesses don't overlap early to determine NoDep (#92307)
Use getStartAndEndForAccess to compute the start and end of both src and sink (factored out to helper in bce3680). If they do not overlap (i.e. SrcEnd <= SinkStart || SinkEnd <= SrcStart), there is no dependence, regardless of stride. PR: #92307
1 parent 7cee61c commit 1b377db

File tree

6 files changed

+320
-140
lines changed

6 files changed

+320
-140
lines changed

‎llvm/lib/Analysis/LoopAccessAnalysis.cpp

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -213,9 +213,7 @@ getStartAndEndForAccess(const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy,
213213

214214
if (SE->isLoopInvariant(PtrExpr, Lp)) {
215215
ScStart = ScEnd = PtrExpr;
216-
} else {
217-
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrExpr);
218-
assert(AR && "Invalid addrec expression");
216+
} else if (auto *AR = dyn_cast<SCEVAddRecExpr>(PtrExpr)) {
219217
const SCEV *Ex = PSE.getBackedgeTakenCount();
220218

221219
ScStart = AR->getStart();
@@ -234,7 +232,9 @@ getStartAndEndForAccess(const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy,
234232
ScStart = SE->getUMinExpr(ScStart, ScEnd);
235233
ScEnd = SE->getUMaxExpr(AR->getStart(), ScEnd);
236234
}
237-
}
235+
} else
236+
return {SE->getCouldNotCompute(), SE->getCouldNotCompute()};
237+
238238
assert(SE->isLoopInvariant(ScStart, Lp) && "ScStart needs to be invariant");
239239
assert(SE->isLoopInvariant(ScEnd, Lp)&& "ScEnd needs to be invariant");
240240

@@ -256,6 +256,9 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr,
256256
bool NeedsFreeze) {
257257
const auto &[ScStart, ScEnd] =
258258
getStartAndEndForAccess(Lp, PtrExpr, AccessTy, PSE);
259+
assert(!isa<SCEVCouldNotCompute>(ScStart) &&
260+
!isa<SCEVCouldNotCompute>(ScEnd) &&
261+
"must be able to compute both start and end expressions");
259262
Pointers.emplace_back(Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, PtrExpr,
260263
NeedsFreeze);
261264
}
@@ -1987,6 +1990,23 @@ getDependenceDistanceStrideAndSize(
19871990
InnermostLoop))
19881991
return MemoryDepChecker::Dependence::IndirectUnsafe;
19891992

1993+
// Check if we can prove that Sink only accesses memory after Src's end or
1994+
// vice versa.
1995+
const auto &[SrcStart, SrcEnd] =
1996+
getStartAndEndForAccess(InnermostLoop, Src, ATy, PSE);
1997+
const auto &[SinkStart, SinkEnd] =
1998+
getStartAndEndForAccess(InnermostLoop, Sink, BTy, PSE);
1999+
2000+
if (!isa<SCEVCouldNotCompute>(SrcStart) &&
2001+
!isa<SCEVCouldNotCompute>(SrcEnd) &&
2002+
!isa<SCEVCouldNotCompute>(SinkStart) &&
2003+
!isa<SCEVCouldNotCompute>(SinkEnd)) {
2004+
if (SE.isKnownPredicate(CmpInst::ICMP_ULE, SrcEnd, SinkStart))
2005+
return MemoryDepChecker::Dependence::NoDep;
2006+
if (SE.isKnownPredicate(CmpInst::ICMP_ULE, SinkEnd, SrcStart))
2007+
return MemoryDepChecker::Dependence::NoDep;
2008+
}
2009+
19902010
// Need accesses with constant strides and the same direction. We don't want
19912011
// to vectorize "A[B[i]] += ..." and similar code or pointer arithmetic that
19922012
// could wrap in the address space.

‎llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -117,16 +117,8 @@ exit:
117117
; CHECK-LABEL: function 'neg_dist_dep_type_size_equivalence':
118118
; CHECK-NEXT: loop:
119119
; CHECK-NEXT: Report: unsafe dependent memory operations in loop.
120-
; CHECK-NEXT: Unknown data dependence.
120+
; CHECK-NEXT: Backward loop carried data dependence that prevents store-to-load forwarding.
121121
; CHECK-NEXT: Dependences:
122-
; CHECK-NEXT: Unknown:
123-
; CHECK-NEXT: %ld.f64 = load double, ptr %gep.iv, align 8 ->
124-
; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n.i64, align 8
125-
; CHECK-EMPTY:
126-
; CHECK-NEXT: Unknown:
127-
; CHECK-NEXT: %ld.i64 = load i64, ptr %gep.iv, align 8 ->
128-
; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n.i64, align 8
129-
; CHECK-EMPTY:
130122
; CHECK-NEXT: BackwardVectorizableButPreventsForwarding:
131123
; CHECK-NEXT: %ld.f64 = load double, ptr %gep.iv, align 8 ->
132124
; CHECK-NEXT: store double %val, ptr %gep.iv.101.i64, align 8
@@ -137,7 +129,7 @@ exit:
137129
; CHECK-EMPTY:
138130
; CHECK-NEXT: Unknown:
139131
; CHECK-NEXT: store double %val, ptr %gep.iv.101.i64, align 8 ->
140-
; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n.i64, align 8
132+
; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n.i64, align 8
141133
; CHECK-EMPTY:
142134
; CHECK-NEXT: Run-time memory checks:
143135
; CHECK-NEXT: Grouped accesses:

‎llvm/test/Analysis/LoopAccessAnalysis/invariant-dependence-before.ll

Lines changed: 8 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,8 @@
44
define void @test_invar_dependence_before_positive_strided_access_1(ptr %a) {
55
; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_1'
66
; CHECK-NEXT: loop:
7-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
8-
; CHECK-NEXT: Unknown data dependence.
7+
; CHECK-NEXT: Memory dependences are safe
98
; CHECK-NEXT: Dependences:
10-
; CHECK-NEXT: Unknown:
11-
; CHECK-NEXT: %l = load i32, ptr %a, align 4 ->
12-
; CHECK-NEXT: store i32 %l, ptr %gep, align 4
13-
; CHECK-EMPTY:
149
; CHECK-NEXT: Run-time memory checks:
1510
; CHECK-NEXT: Grouped accesses:
1611
; CHECK-EMPTY:
@@ -39,13 +34,8 @@ exit:
3934
define void @test_invar_dependence_before_positive_strided_access_2(ptr %a) {
4035
; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_2'
4136
; CHECK-NEXT: loop:
42-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
43-
; CHECK-NEXT: Unknown data dependence.
37+
; CHECK-NEXT: Memory dependences are safe
4438
; CHECK-NEXT: Dependences:
45-
; CHECK-NEXT: Unknown:
46-
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
47-
; CHECK-NEXT: store i32 %l, ptr %a, align 4
48-
; CHECK-EMPTY:
4939
; CHECK-NEXT: Run-time memory checks:
5040
; CHECK-NEXT: Grouped accesses:
5141
; CHECK-EMPTY:
@@ -144,13 +134,8 @@ exit:
144134
define void @test_invar_dependence_before_positive_strided_access_1_different_access_sizes(ptr %a) {
145135
; CHECK-LABEL: 'test_invar_dependence_before_positive_strided_access_1_different_access_sizes'
146136
; CHECK-NEXT: loop:
147-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
148-
; CHECK-NEXT: Unknown data dependence.
137+
; CHECK-NEXT: Memory dependences are safe
149138
; CHECK-NEXT: Dependences:
150-
; CHECK-NEXT: Unknown:
151-
; CHECK-NEXT: %l = load i32, ptr %a, align 4 ->
152-
; CHECK-NEXT: store i8 %t, ptr %gep, align 1
153-
; CHECK-EMPTY:
154139
; CHECK-NEXT: Run-time memory checks:
155140
; CHECK-NEXT: Grouped accesses:
156141
; CHECK-EMPTY:
@@ -216,13 +201,8 @@ exit:
216201
define void @test_invar_dependence_before_negative_strided_access_1(ptr %a) {
217202
; CHECK-LABEL: 'test_invar_dependence_before_negative_strided_access_1'
218203
; CHECK-NEXT: loop:
219-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
220-
; CHECK-NEXT: Unknown data dependence.
204+
; CHECK-NEXT: Memory dependences are safe
221205
; CHECK-NEXT: Dependences:
222-
; CHECK-NEXT: Unknown:
223-
; CHECK-NEXT: %l = load i32, ptr %a, align 4 ->
224-
; CHECK-NEXT: store i32 %l, ptr %gep, align 4
225-
; CHECK-EMPTY:
226206
; CHECK-NEXT: Run-time memory checks:
227207
; CHECK-NEXT: Grouped accesses:
228208
; CHECK-EMPTY:
@@ -251,13 +231,8 @@ exit:
251231
define void @test_invar_dependence_before_negative_strided_access_2(ptr %a) {
252232
; CHECK-LABEL: 'test_invar_dependence_before_negative_strided_access_2'
253233
; CHECK-NEXT: loop:
254-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
255-
; CHECK-NEXT: Unknown data dependence.
234+
; CHECK-NEXT: Memory dependences are safe
256235
; CHECK-NEXT: Dependences:
257-
; CHECK-NEXT: Unknown:
258-
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
259-
; CHECK-NEXT: store i32 %l, ptr %a, align 4
260-
; CHECK-EMPTY:
261236
; CHECK-NEXT: Run-time memory checks:
262237
; CHECK-NEXT: Grouped accesses:
263238
; CHECK-EMPTY:
@@ -357,13 +332,8 @@ exit:
357332
define void @test_both_invar_before_1(ptr %a) {
358333
; CHECK-LABEL: 'test_both_invar_before_1'
359334
; CHECK-NEXT: loop:
360-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
361-
; CHECK-NEXT: Unknown data dependence.
335+
; CHECK-NEXT: Memory dependences are safe
362336
; CHECK-NEXT: Dependences:
363-
; CHECK-NEXT: Unknown:
364-
; CHECK-NEXT: %l = load i32, ptr %a, align 4 ->
365-
; CHECK-NEXT: store i32 %l, ptr %gep.off, align 4
366-
; CHECK-EMPTY:
367337
; CHECK-NEXT: Run-time memory checks:
368338
; CHECK-NEXT: Grouped accesses:
369339
; CHECK-EMPTY:
@@ -391,13 +361,8 @@ exit:
391361
define void @test_both_invar_before_2(ptr %a) {
392362
; CHECK-LABEL: 'test_both_invar_before_2'
393363
; CHECK-NEXT: loop:
394-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
395-
; CHECK-NEXT: Unknown data dependence.
364+
; CHECK-NEXT: Memory dependences are safe
396365
; CHECK-NEXT: Dependences:
397-
; CHECK-NEXT: Unknown:
398-
; CHECK-NEXT: %l = load i32, ptr %gep.off, align 4 ->
399-
; CHECK-NEXT: store i32 %l, ptr %a, align 4
400-
; CHECK-EMPTY:
401366
; CHECK-NEXT: Run-time memory checks:
402367
; CHECK-NEXT: Grouped accesses:
403368
; CHECK-EMPTY:
@@ -758,13 +723,8 @@ exit:
758723
define void @test_invar_vector_dependence_before_positive_strided_access_1(ptr %a) {
759724
; CHECK-LABEL: 'test_invar_vector_dependence_before_positive_strided_access_1'
760725
; CHECK-NEXT: loop:
761-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
762-
; CHECK-NEXT: Unknown data dependence.
726+
; CHECK-NEXT: Memory dependences are safe
763727
; CHECK-NEXT: Dependences:
764-
; CHECK-NEXT: Unknown:
765-
; CHECK-NEXT: %l = load <4 x i8>, ptr %a, align 4 ->
766-
; CHECK-NEXT: store i32 0, ptr %gep, align 4
767-
; CHECK-EMPTY:
768728
; CHECK-NEXT: Run-time memory checks:
769729
; CHECK-NEXT: Grouped accesses:
770730
; CHECK-EMPTY:

‎llvm/test/Analysis/LoopAccessAnalysis/loop-invariant-dep-with-backedge-taken-count.ll

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,8 @@
77
define void @test_distance_greater_than_BTC_100(ptr %a) {
88
; CHECK-LABEL: 'test_distance_greater_than_BTC_100'
99
; CHECK-NEXT: loop:
10-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
11-
; CHECK-NEXT: Unknown data dependence.
10+
; CHECK-NEXT: Memory dependences are safe
1211
; CHECK-NEXT: Dependences:
13-
; CHECK-NEXT: Unknown:
14-
; CHECK-NEXT: %l = load i32, ptr %gep.x, align 4 ->
15-
; CHECK-NEXT: store i32 %l, ptr %gep, align 4
16-
; CHECK-EMPTY:
1712
; CHECK-NEXT: Run-time memory checks:
1813
; CHECK-NEXT: Grouped accesses:
1914
; CHECK-EMPTY:
@@ -112,13 +107,8 @@ exit:
112107
define void @test_distance_greater_than_BTC_10000(ptr %a) {
113108
; CHECK-LABEL: 'test_distance_greater_than_BTC_10000'
114109
; CHECK-NEXT: loop:
115-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
116-
; CHECK-NEXT: Unknown data dependence.
110+
; CHECK-NEXT: Memory dependences are safe
117111
; CHECK-NEXT: Dependences:
118-
; CHECK-NEXT: Unknown:
119-
; CHECK-NEXT: %l = load i32, ptr %gep.x, align 4 ->
120-
; CHECK-NEXT: store i32 %l, ptr %gep, align 4
121-
; CHECK-EMPTY:
122112
; CHECK-NEXT: Run-time memory checks:
123113
; CHECK-NEXT: Grouped accesses:
124114
; CHECK-EMPTY:

‎llvm/test/Analysis/LoopAccessAnalysis/non-constant-strides-backward.ll

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,8 @@ exit:
4545
define void @different_non_constant_strides_known_backward_distance_larger_than_trip_count(ptr %A) {
4646
; CHECK-LABEL: 'different_non_constant_strides_known_backward_distance_larger_than_trip_count'
4747
; CHECK-NEXT: loop:
48-
; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
49-
; CHECK-NEXT: Unknown data dependence.
48+
; CHECK-NEXT: Memory dependences are safe
5049
; CHECK-NEXT: Dependences:
51-
; CHECK-NEXT: Unknown:
52-
; CHECK-NEXT: %l = load i32, ptr %gep, align 4 ->
53-
; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4
54-
; CHECK-EMPTY:
5550
; CHECK-NEXT: Run-time memory checks:
5651
; CHECK-NEXT: Grouped accesses:
5752
; CHECK-EMPTY:

0 commit comments

Comments
 (0)