Skip to content

Commit 92d8bd6

Browse files
fhahnfhahn
committed
!fixup check object size to determine no-wrap.
1 parent 198b3b1 commit 92d8bd6

File tree

5 files changed

+67
-73
lines changed

5 files changed

+67
-73
lines changed

‎llvm/lib/Analysis/Loads.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,8 @@ bool llvm::isDereferenceableAndAlignedInLoop(
333333

334334
// Try to get the access size.
335335
const SCEV *PtrDiff = SE.getMinusSCEV(AccessEnd, AccessStart);
336+
if (isa<SCEVCouldNotCompute>(PtrDiff))
337+
return false;
336338
APInt MaxPtrDiff = SE.getUnsignedRangeMax(PtrDiff);
337339

338340
Value *Base = nullptr;

‎llvm/lib/Analysis/LoopAccessAnalysis.cpp

Lines changed: 51 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,51 @@ RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup(
188188
Members.push_back(Index);
189189
}
190190

191+
/// Return true, if evaluating \p AR at \p MaxBTC cannot wrap, because \p AR at
192+
/// \p MaxBTC is guaranteed inbounds of the accessed object.
193+
static bool evaluateAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR,
194+
const SCEV *MaxBTC,
195+
ScalarEvolution &SE,
196+
const DataLayout &DL) {
197+
auto *PointerBase = SE.getPointerBase(AR->getStart());
198+
auto *StartPtr = dyn_cast<SCEVUnknown>(PointerBase);
199+
if (!StartPtr)
200+
return false;
201+
bool CheckForNonNull, CheckForFreed;
202+
uint64_t DerefBytes = StartPtr->getValue()->getPointerDereferenceableBytes(
203+
DL, CheckForNonNull, CheckForFreed);
204+
205+
if (CheckForNonNull || CheckForFreed)
206+
return false;
207+
208+
const SCEV *Step = AR->getStepRecurrence(SE);
209+
Type *WiderTy = SE.getWiderType(MaxBTC->getType(), Step->getType());
210+
Step = SE.getNoopOrSignExtend(Step, WiderTy);
211+
MaxBTC = SE.getNoopOrSignExtend(MaxBTC, WiderTy);
212+
if (SE.isKnownPositive(Step)) {
213+
// For positive steps, check if (AR->getStart() - StartPtr) + MaxBTC <=
214+
// DerefBytes / Step
215+
return SE.isKnownPredicate(
216+
CmpInst::ICMP_ULE,
217+
SE.getAddExpr(SE.getMinusSCEV(AR->getStart(), StartPtr), MaxBTC),
218+
SE.getUDivExpr(SE.getConstant(WiderTy, DerefBytes), Step));
219+
}
220+
if (SE.isKnownNegative(Step)) {
221+
// For negative steps, check using StartOffset == AR->getStart() - StartPtr:
222+
// * StartOffset >= MaxBTC * Step
223+
// * AND StartOffset <= DerefBytes / Step
224+
auto *StartOffset = SE.getMinusSCEV(AR->getStart(), StartPtr);
225+
return SE.isKnownPredicate(
226+
CmpInst::ICMP_UGE, StartOffset,
227+
SE.getMulExpr(MaxBTC, SE.getNegativeSCEV(Step))) &&
228+
SE.isKnownPredicate(
229+
CmpInst::ICMP_ULE, StartOffset,
230+
SE.getUDivExpr(SE.getConstant(WiderTy, DerefBytes),
231+
SE.getNegativeSCEV(Step)));
232+
}
233+
return false;
234+
}
235+
191236
std::pair<const SCEV *, const SCEV *> llvm::getStartAndEndForAccess(
192237
const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, const SCEV *BTC,
193238
const SCEV *MaxBTC, ScalarEvolution *SE,
@@ -226,12 +271,12 @@ std::pair<const SCEV *, const SCEV *> llvm::getStartAndEndForAccess(
226271
// will get incremented by EltSize before returning, so this effectively
227272
// sets ScEnd to unsigned max. Note that LAA separately checks that
228273
// accesses cannot not wrap, so unsigned max represents an upper bound.
229-
// TODO: Use additional information to determine no-wrap including
230-
// size/dereferencability info from the accessed object.
231-
ScEnd = AR->evaluateAtIteration(MaxBTC, *SE);
232-
if (!SE->isKnownNonNegative(SE->getMinusSCEV(ScEnd, ScStart)))
233-
ScEnd = SE->getNegativeSCEV(
234-
SE->getAddExpr(EltSizeSCEV, SE->getOne(EltSizeSCEV->getType())));
274+
ScEnd = SE->getAddExpr(
275+
SE->getNegativeSCEV(EltSizeSCEV),
276+
SE->getSCEV(ConstantExpr::getIntToPtr(
277+
ConstantInt::get(EltSizeSCEV->getType(), -1), AR->getType())));
278+
if (evaluateAddRecAtMaxBTCWillNotWrap(AR, MaxBTC, *SE, DL))
279+
ScEnd = AR->evaluateAtIteration(MaxBTC, *SE);
235280
}
236281
const SCEV *Step = AR->getStepRecurrence(*SE);
237282

‎llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_not
7575
; CHECK-NEXT: (Low: %B High: (2000 + %B))
7676
; CHECK-NEXT: Member: {%B,+,4}<nuw><%loop.header>
7777
; CHECK-NEXT: Group GRP1:
78-
; CHECK-NEXT: (Low: %A High: (2000 + %A))
78+
; CHECK-NEXT: (Low: %A High: inttoptr (i64 -1 to ptr))
7979
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop.header>
8080
; CHECK-EMPTY:
8181
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
@@ -189,9 +189,9 @@ define i32 @all_exits_dominate_latch_countable_exits_at_most_1000_iterations_not
189189
; CHECK-NEXT: Grouped accesses:
190190
; CHECK-NEXT: Group GRP0:
191191
; CHECK-NEXT: (Low: %B High: (4004 + %B))
192-
; CHECK-NEXT: Member: {%B,+,4}<nuw><%loop.header>
192+
; CHECK-NEXT: (Low: %B High: inttoptr (i64 -1 to ptr))
193193
; CHECK-NEXT: Group GRP1:
194-
; CHECK-NEXT: (Low: %A High: (4004 + %A))
194+
; CHECK-NEXT: (Low: %A High: inttoptr (i64 -1 to ptr))
195195
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop.header>
196196
; CHECK-EMPTY:
197197
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
@@ -233,6 +233,7 @@ e.2:
233233
ret i32 2
234234
}
235235

236+
236237
define i32 @not_all_exits_dominate_latch(ptr %A, ptr %B) {
237238
; CHECK-LABEL: 'not_all_exits_dominate_latch'
238239
; CHECK-NEXT: loop.header:
@@ -345,10 +346,10 @@ define i32 @b3_does_not_dominate_latch_not_known_deref(ptr %A, ptr %B) {
345346
; CHECK-NEXT: %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
346347
; CHECK-NEXT: Grouped accesses:
347348
; CHECK-NEXT: Group GRP0:
348-
; CHECK-NEXT: (Low: %B High: (4004 + %B))
349+
; CHECK-NEXT: (Low: %B High: inttoptr (i64 -1 to ptr))
349350
; CHECK-NEXT: Member: {%B,+,4}<nuw><%loop.header>
350351
; CHECK-NEXT: Group GRP1:
351-
; CHECK-NEXT: (Low: %A High: (4004 + %A))
352+
; CHECK-NEXT: (Low: %A High: inttoptr (i64 -1 to ptr))
352353
; CHECK-NEXT: Member: {%A,+,4}<nuw><%loop.header>
353354
; CHECK-EMPTY:
354355
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.

‎llvm/test/Analysis/LoopAccessAnalysis/evaluate-at-symbolic-max-backedge-taken-count-may-wrap.ll

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
55

6+
; FIXME: Start == End for access group with AddRec.
67
define void @runtime_checks_with_symbolic_max_btc_neg_1(ptr %P, ptr %S, i32 %x, i32 %y) {
78
; CHECK-LABEL: 'runtime_checks_with_symbolic_max_btc_neg_1'
89
; CHECK-NEXT: loop:
@@ -16,7 +17,7 @@ define void @runtime_checks_with_symbolic_max_btc_neg_1(ptr %P, ptr %S, i32 %x,
1617
; CHECK-NEXT: ptr %S
1718
; CHECK-NEXT: Grouped accesses:
1819
; CHECK-NEXT: Group GRP0:
19-
; CHECK-NEXT: (Low: ((4 * %y) + %P) High: -1)
20+
; CHECK-NEXT: (Low: ((4 * %y) + %P) High: inttoptr (i32 -1 to ptr))
2021
; CHECK-NEXT: Member: {((4 * %y) + %P),+,4}<%loop>
2122
; CHECK-NEXT: Group GRP1:
2223
; CHECK-NEXT: (Low: %S High: (4 + %S))
@@ -43,6 +44,7 @@ exit:
4344
ret void
4445
}
4546

47+
; FIXME: Start > End for access group with AddRec.
4648
define void @runtime_check_with_symbolic_max_btc_neg_2(ptr %P, ptr %S, i32 %x, i32 %y) {
4749
; CHECK-LABEL: 'runtime_check_with_symbolic_max_btc_neg_2'
4850
; CHECK-NEXT: loop:
@@ -56,7 +58,7 @@ define void @runtime_check_with_symbolic_max_btc_neg_2(ptr %P, ptr %S, i32 %x, i
5658
; CHECK-NEXT: ptr %S
5759
; CHECK-NEXT: Grouped accesses:
5860
; CHECK-NEXT: Group GRP0:
59-
; CHECK-NEXT: (Low: ((4 * %y) + %P) High: -1)
61+
; CHECK-NEXT: (Low: ((4 * %y) + %P) High: inttoptr (i32 -1 to ptr))
6062
; CHECK-NEXT: Member: {((4 * %y) + %P),+,4}<%loop>
6163
; CHECK-NEXT: Group GRP1:
6264
; CHECK-NEXT: (Low: %S High: (4 + %S))
@@ -135,8 +137,8 @@ exit:
135137
ret i32 %res
136138
}
137139

138-
; FIXME: evaluating at symbolic max BTC wraps around to a positive
139-
; offset: (2 + (2 * %y) + %P)
140+
; Evaluating at symbolic max BTC wraps around to a positive
141+
; offset: (2 + (2 * %y) + %P).
140142
define void @runtime_check_with_symbolic_max_wraps_to_positive_offset(ptr %P, ptr %S, i32 %x, i32 %y) {
141143
; CHECK-LABEL: 'runtime_check_with_symbolic_max_wraps_to_positive_offset'
142144
; CHECK-NEXT: loop:
@@ -150,7 +152,7 @@ define void @runtime_check_with_symbolic_max_wraps_to_positive_offset(ptr %P, pt
150152
; CHECK-NEXT: ptr %S
151153
; CHECK-NEXT: Grouped accesses:
152154
; CHECK-NEXT: Group GRP0:
153-
; CHECK-NEXT: (Low: ((2 * %y) + %P) High: (2 + (2 * %y) + %P))
155+
; CHECK-NEXT: (Low: ((2 * %y) + %P) High: inttoptr (i32 -1 to ptr))
154156
; CHECK-NEXT: Member: {((2 * %y) + %P),+,2}<%loop>
155157
; CHECK-NEXT: Group GRP1:
156158
; CHECK-NEXT: (Low: %S High: (4 + %S))

‎llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll

Lines changed: 1 addition & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,35 +1971,6 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
19711971
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
19721972
; CHECK: vector.body:
19731973
; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT4:%.*]], [[VECTOR_BODY]] ]
1974-
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX1]]
1975-
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
1976-
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
1977-
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 -3
1978-
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
1979-
; CHECK-NEXT: [[REVERSE:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
1980-
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
1981-
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
1982-
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 -3
1983-
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
1984-
; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD2]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
1985-
; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <4 x i8> [[REVERSE]], [[REVERSE3]]
1986-
; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4
1987-
; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
1988-
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 1020
1989-
; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
1990-
; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
1991-
; CHECK: middle.split:
1992-
; CHECK-NEXT: br i1 [[TMP9]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
1993-
; CHECK: middle.block:
1994-
; CHECK-NEXT: br i1 false, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
1995-
; CHECK: vector.early.exit:
1996-
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 true)
1997-
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]
1998-
; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = sub i64 1023, [[TMP12]]
1999-
; CHECK-NEXT: br label [[LOOP_END]]
2000-
; CHECK: scalar.ph:
2001-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[MIDDLE_BLOCK]] ], [ 1023, [[ENTRY:%.*]] ]
2002-
; CHECK-NEXT: br label [[LOOP1:%.*]]
20031974
; CHECK: loop:
20041975
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
20051976
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
@@ -2011,9 +1982,6 @@ define i64 @same_exit_block_pre_inc_use1_reverse() {
20111982
; CHECK: loop.inc:
20121983
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], -1
20131984
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDEX_NEXT]], 0
2014-
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP_END]], label [[LOOP1]], !llvm.loop [[LOOP51:![0-9]+]]
2015-
; CHECK: loop.end:
2016-
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ 1024, [[LOOP_INC]] ], [ 1024, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
20171985
; CHECK-NEXT: ret i64 [[RETVAL]]
20181986
;
20191987
entry:
@@ -2114,31 +2082,6 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
21142082
; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
21152083
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
21162084
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
2117-
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
2118-
; CHECK: middle.split:
2119-
; CHECK-NEXT: br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]]
2120-
; CHECK: middle.block:
2121-
; CHECK-NEXT: br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]]
2122-
; CHECK: vector.early.exit:
2123-
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true)
2124-
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX1]], [[FIRST_ACTIVE_LANE]]
2125-
; CHECK-NEXT: [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
2126-
; CHECK-NEXT: br label [[LOOP_END]]
2127-
; CHECK: scalar.ph:
2128-
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 67, [[MIDDLE_BLOCK]] ], [ 3, [[ENTRY:%.*]] ]
2129-
; CHECK-NEXT: br label [[LOOP:%.*]]
2130-
; CHECK: loop:
2131-
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
2132-
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
2133-
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
2134-
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
2135-
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
2136-
; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
2137-
; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
2138-
; CHECK: loop.inc:
2139-
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
2140-
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
2141-
; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP53:![0-9]+]]
21422085
; CHECK: loop.end:
21432086
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ 67, [[LOOP_INC]] ], [ 67, [[MIDDLE_BLOCK]] ], [ [[EARLY_EXIT_VALUE]], [[VECTOR_EARLY_EXIT]] ]
21442087
; CHECK-NEXT: ret i64 [[RETVAL]]
@@ -2208,3 +2151,4 @@ attributes #0 = { "vector-function-abi-variant"="_ZGVsNxv_foo(foo_vec)" }
22082151
; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[META2]], [[META1]]}
22092152
; CHECK: [[LOOP36]] = distinct !{[[LOOP36]], [[META1]], [[META2]]}
22102153
; CHECK: [[LOOP37]] = distinct !{[[LOOP37]], [[META2]], [[META1]]}
2154+

0 commit comments

Comments
 (0)