Skip to content

Commit 559eb47

Browse files
fhahnfhahn
committed
[LAA] Perform checks for no-wrap separately from getPtrStride.
Reorganize the code in isNoWrap to perform the no-wrap checks without relying on getPtrStride directly. getPtrStride now uses isNoWrap. The new structure allows deriving no-wrap in more cases in LAA, because there are some cases where getPtrStride bails out early because it cannot return a constant stride, but we can still prove no-wrap for the pointer. An example are AddRecs with non-ConstantInt strides with inbound GEPs, in the improved test cases. This enables vectorization with runtime checks in a few more cases.
1 parent 8260528 commit 559eb47

File tree

2 files changed

+178
-80
lines changed

2 files changed

+178
-80
lines changed

‎llvm/lib/Analysis/LoopAccessAnalysis.cpp

Lines changed: 93 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -813,16 +813,102 @@ static bool hasComputableBounds(PredicatedScalarEvolution &PSE, Value *Ptr,
813813
return AR->isAffine();
814814
}
815815

816+
/// Try to compute the stride for \p AR. Used by getPtrStride.
817+
static std::optional<int64_t>
818+
getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
819+
Value *Ptr, PredicatedScalarEvolution &PSE) {
820+
// The access function must stride over the innermost loop.
821+
if (Lp != AR->getLoop()) {
822+
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop "
823+
<< *Ptr << " SCEV: " << *AR << "\n");
824+
return std::nullopt;
825+
}
826+
827+
// Check the step is constant.
828+
const SCEV *Step = AR->getStepRecurrence(*PSE.getSE());
829+
830+
// Calculate the pointer stride and check if it is constant.
831+
const SCEVConstant *C = dyn_cast<SCEVConstant>(Step);
832+
if (!C) {
833+
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not a constant strided " << *Ptr
834+
<< " SCEV: " << *AR << "\n");
835+
return std::nullopt;
836+
}
837+
838+
const auto &DL = Lp->getHeader()->getDataLayout();
839+
TypeSize AllocSize = DL.getTypeAllocSize(AccessTy);
840+
int64_t Size = AllocSize.getFixedValue();
841+
const APInt &APStepVal = C->getAPInt();
842+
843+
// Huge step value - give up.
844+
if (APStepVal.getBitWidth() > 64)
845+
return std::nullopt;
846+
847+
int64_t StepVal = APStepVal.getSExtValue();
848+
849+
// Strided access.
850+
int64_t Stride = StepVal / Size;
851+
int64_t Rem = StepVal % Size;
852+
if (Rem)
853+
return std::nullopt;
854+
855+
return Stride;
856+
}
857+
858+
static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR,
859+
PredicatedScalarEvolution &PSE, const Loop *L);
860+
816861
/// Check whether a pointer address cannot wrap.
817862
static bool isNoWrap(PredicatedScalarEvolution &PSE,
818863
const DenseMap<Value *, const SCEV *> &Strides, Value *Ptr,
819-
Type *AccessTy, Loop *L, bool Assume) {
820-
const SCEV *PtrScev = PSE.getSCEV(Ptr);
864+
Type *AccessTy, const Loop *L, bool Assume,
865+
std::optional<int64_t> Stride = std::nullopt) {
866+
const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
821867
if (PSE.getSE()->isLoopInvariant(PtrScev, L))
822868
return true;
823869

824-
return getPtrStride(PSE, AccessTy, Ptr, L, Strides, Assume).has_value() ||
825-
PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
870+
const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
871+
if (Assume && !AR)
872+
AR = PSE.getAsAddRec(Ptr);
873+
if (!AR)
874+
return false;
875+
876+
// The address calculation must not wrap. Otherwise, a dependence could be
877+
// inverted.
878+
if (isNoWrapAddRec(Ptr, AR, PSE, L))
879+
return true;
880+
881+
// An nusw getelementptr that is an AddRec cannot wrap. If it would wrap,
882+
// the distance between the previously accessed location and the wrapped
883+
// location will be larger than half the pointer index type space. In that
884+
// case, the GEP would be poison and any memory access dependent on it would
885+
// be immediate UB when executed.
886+
if (auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
887+
GEP && GEP->hasNoUnsignedSignedWrap())
888+
return true;
889+
890+
// If the null pointer is undefined, then a access sequence which would
891+
// otherwise access it can be assumed not to unsigned wrap. Note that this
892+
// assumes the object in memory is aligned to the natural alignment.
893+
if (!Stride)
894+
Stride = getStrideFromAddRec(AR, L, AccessTy, Ptr, PSE);
895+
if (Stride) {
896+
unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
897+
if (!NullPointerIsDefined(L->getHeader()->getParent(), AddrSpace) &&
898+
(*Stride == 1 || *Stride == -1))
899+
return true;
900+
}
901+
902+
if (Assume) {
903+
PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
904+
LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n"
905+
<< "LAA: Pointer: " << *Ptr << "\n"
906+
<< "LAA: SCEV: " << *AR << "\n"
907+
<< "LAA: Added an overflow assumption\n");
908+
return true;
909+
}
910+
911+
return PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
826912
}
827913

828914
static void visitPointers(Value *StartPtr, const Loop &InnermostLoop,
@@ -1458,74 +1544,13 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
14581544
return std::nullopt;
14591545
}
14601546

1461-
// The access function must stride over the innermost loop.
1462-
if (Lp != AR->getLoop()) {
1463-
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop "
1464-
<< *Ptr << " SCEV: " << *AR << "\n");
1465-
return std::nullopt;
1466-
}
1467-
1468-
// Check the step is constant.
1469-
const SCEV *Step = AR->getStepRecurrence(*PSE.getSE());
1470-
1471-
// Calculate the pointer stride and check if it is constant.
1472-
const SCEVConstant *C = dyn_cast<SCEVConstant>(Step);
1473-
if (!C) {
1474-
LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not a constant strided " << *Ptr
1475-
<< " SCEV: " << *AR << "\n");
1476-
return std::nullopt;
1477-
}
1478-
1479-
const auto &DL = Lp->getHeader()->getDataLayout();
1480-
TypeSize AllocSize = DL.getTypeAllocSize(AccessTy);
1481-
int64_t Size = AllocSize.getFixedValue();
1482-
const APInt &APStepVal = C->getAPInt();
1483-
1484-
// Huge step value - give up.
1485-
if (APStepVal.getBitWidth() > 64)
1486-
return std::nullopt;
1487-
1488-
int64_t StepVal = APStepVal.getSExtValue();
1489-
1490-
// Strided access.
1491-
int64_t Stride = StepVal / Size;
1492-
int64_t Rem = StepVal % Size;
1493-
if (Rem)
1494-
return std::nullopt;
1495-
1496-
if (!ShouldCheckWrap)
1497-
return Stride;
1498-
1499-
// The address calculation must not wrap. Otherwise, a dependence could be
1500-
// inverted.
1501-
if (isNoWrapAddRec(Ptr, AR, PSE, Lp))
1502-
return Stride;
1503-
1504-
// An nusw getelementptr that is an AddRec cannot wrap. If it would wrap,
1505-
// the distance between the previously accessed location and the wrapped
1506-
// location will be larger than half the pointer index type space. In that
1507-
// case, the GEP would be poison and any memory access dependent on it would
1508-
// be immediate UB when executed.
1509-
if (auto *GEP = dyn_cast<GetElementPtrInst>(Ptr);
1510-
GEP && GEP->hasNoUnsignedSignedWrap())
1547+
auto Stride = getStrideFromAddRec(AR, Lp, AccessTy, Ptr, PSE);
1548+
if (!ShouldCheckWrap || !Stride)
15111549
return Stride;
15121550

1513-
// If the null pointer is undefined, then a access sequence which would
1514-
// otherwise access it can be assumed not to unsigned wrap. Note that this
1515-
// assumes the object in memory is aligned to the natural alignment.
1516-
unsigned AddrSpace = Ty->getPointerAddressSpace();
1517-
if (!NullPointerIsDefined(Lp->getHeader()->getParent(), AddrSpace) &&
1518-
(Stride == 1 || Stride == -1))
1551+
if (isNoWrap(PSE, StridesMap, Ptr, AccessTy, Lp, Assume, Stride))
15191552
return Stride;
15201553

1521-
if (Assume) {
1522-
PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW);
1523-
LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n"
1524-
<< "LAA: Pointer: " << *Ptr << "\n"
1525-
<< "LAA: SCEV: " << *AR << "\n"
1526-
<< "LAA: Added an overflow assumption\n");
1527-
return Stride;
1528-
}
15291554
LLVM_DEBUG(
15301555
dbgs() << "LAA: Bad stride - Pointer may wrap in the address space "
15311556
<< *Ptr << " SCEV: " << *AR << "\n");

‎llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll

Lines changed: 85 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,13 +65,38 @@ exit:
6565
define void @dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_step5(ptr %a, ptr %b, i64 %offset, i64 %n) {
6666
; CHECK-LABEL: 'dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_step5'
6767
; CHECK-NEXT: loop:
68-
; CHECK-NEXT: Report: cannot check memory dependencies at runtime
68+
; CHECK-NEXT: Memory dependences are safe with run-time checks
6969
; CHECK-NEXT: Dependences:
7070
; CHECK-NEXT: Run-time memory checks:
71+
; CHECK-NEXT: Check 0:
72+
; CHECK-NEXT: Comparing group ([[GRP4:0x[0-9a-f]+]]):
73+
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
74+
; CHECK-NEXT: Against group ([[GRP5:0x[0-9a-f]+]]):
75+
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
76+
; CHECK-NEXT: Check 1:
77+
; CHECK-NEXT: Comparing group ([[GRP4]]):
78+
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
79+
; CHECK-NEXT: Against group ([[GRP6:0x[0-9a-f]+]]):
80+
; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
81+
; CHECK-NEXT: Check 2:
82+
; CHECK-NEXT: Comparing group ([[GRP5]]):
83+
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
84+
; CHECK-NEXT: Against group ([[GRP6]]):
85+
; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2
7186
; CHECK-NEXT: Grouped accesses:
87+
; CHECK-NEXT: Group [[GRP4]]:
88+
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
89+
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
90+
; CHECK-NEXT: Group [[GRP5]]:
91+
; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
92+
; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
93+
; CHECK-NEXT: Group [[GRP6]]:
94+
; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b))
95+
; CHECK-NEXT: Member: {%b,+,5}<%loop>
7296
; CHECK-EMPTY:
7397
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
7498
; CHECK-NEXT: SCEV assumptions:
99+
; CHECK-NEXT: {%b,+,5}<%loop> Added Flags: <nusw>
75100
; CHECK-EMPTY:
76101
; CHECK-NEXT: Expressions re-written:
77102
;
@@ -102,10 +127,34 @@ exit:
102127
define void @dependency_check_and_runtime_checks_needed_gepb_is_inbounds_iv2_step_not_constant(ptr %a, ptr %b, i64 %offset, i64 %n, i64 %s) {
103128
; CHECK-LABEL: 'dependency_check_and_runtime_checks_needed_gepb_is_inbounds_iv2_step_not_constant'
104129
; CHECK-NEXT: loop:
105-
; CHECK-NEXT: Report: cannot check memory dependencies at runtime
130+
; CHECK-NEXT: Memory dependences are safe with run-time checks
106131
; CHECK-NEXT: Dependences:
107132
; CHECK-NEXT: Run-time memory checks:
133+
; CHECK-NEXT: Check 0:
134+
; CHECK-NEXT: Comparing group ([[GRP7:0x[0-9a-f]+]]):
135+
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
136+
; CHECK-NEXT: Against group ([[GRP8:0x[0-9a-f]+]]):
137+
; CHECK-NEXT: %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv2
138+
; CHECK-NEXT: Check 1:
139+
; CHECK-NEXT: Comparing group ([[GRP7]]):
140+
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
141+
; CHECK-NEXT: Against group ([[GRP9:0x[0-9a-f]+]]):
142+
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
143+
; CHECK-NEXT: Check 2:
144+
; CHECK-NEXT: Comparing group ([[GRP8]]):
145+
; CHECK-NEXT: %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv2
146+
; CHECK-NEXT: Against group ([[GRP9]]):
147+
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
108148
; CHECK-NEXT: Grouped accesses:
149+
; CHECK-NEXT: Group [[GRP7]]:
150+
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
151+
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
152+
; CHECK-NEXT: Group [[GRP8]]:
153+
; CHECK-NEXT: (Low: %b High: (3 + %n + %b))
154+
; CHECK-NEXT: Member: {%b,+,1}<%loop>
155+
; CHECK-NEXT: Group [[GRP9]]:
156+
; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
157+
; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
109158
; CHECK-EMPTY:
110159
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
111160
; CHECK-NEXT: SCEV assumptions:
@@ -144,10 +193,34 @@ exit:
144193
define void @dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_step_not_constant(ptr %a, ptr %b, i64 %offset, i64 %n, i64 %s) {
145194
; CHECK-LABEL: 'dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_step_not_constant'
146195
; CHECK-NEXT: loop:
147-
; CHECK-NEXT: Report: cannot check memory dependencies at runtime
196+
; CHECK-NEXT: Memory dependences are safe with run-time checks
148197
; CHECK-NEXT: Dependences:
149198
; CHECK-NEXT: Run-time memory checks:
199+
; CHECK-NEXT: Check 0:
200+
; CHECK-NEXT: Comparing group ([[GRP10:0x[0-9a-f]+]]):
201+
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
202+
; CHECK-NEXT: Against group ([[GRP11:0x[0-9a-f]+]]):
203+
; CHECK-NEXT: %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv2
204+
; CHECK-NEXT: Check 1:
205+
; CHECK-NEXT: Comparing group ([[GRP10]]):
206+
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
207+
; CHECK-NEXT: Against group ([[GRP12:0x[0-9a-f]+]]):
208+
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
209+
; CHECK-NEXT: Check 2:
210+
; CHECK-NEXT: Comparing group ([[GRP11]]):
211+
; CHECK-NEXT: %gep.b = getelementptr inbounds i8, ptr %b, i64 %iv2
212+
; CHECK-NEXT: Against group ([[GRP12]]):
213+
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
150214
; CHECK-NEXT: Grouped accesses:
215+
; CHECK-NEXT: Group [[GRP10]]:
216+
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
217+
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
218+
; CHECK-NEXT: Group [[GRP11]]:
219+
; CHECK-NEXT: (Low: %b High: (3 + %n + %b))
220+
; CHECK-NEXT: Member: {%b,+,1}<%loop>
221+
; CHECK-NEXT: Group [[GRP12]]:
222+
; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
223+
; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
151224
; CHECK-EMPTY:
152225
; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop.
153226
; CHECK-NEXT: SCEV assumptions:
@@ -189,28 +262,28 @@ define void @dependency_check_and_runtime_checks_needed_gepb_may_wrap(ptr %a, pt
189262
; CHECK-NEXT: Dependences:
190263
; CHECK-NEXT: Run-time memory checks:
191264
; CHECK-NEXT: Check 0:
192-
; CHECK-NEXT: Comparing group ([[GRP4:0x[0-9a-f]+]]):
265+
; CHECK-NEXT: Comparing group ([[GRP13:0x[0-9a-f]+]]):
193266
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
194-
; CHECK-NEXT: Against group ([[GRP5:0x[0-9a-f]+]]):
267+
; CHECK-NEXT: Against group ([[GRP14:0x[0-9a-f]+]]):
195268
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
196269
; CHECK-NEXT: Check 1:
197-
; CHECK-NEXT: Comparing group ([[GRP4]]):
270+
; CHECK-NEXT: Comparing group ([[GRP13]]):
198271
; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv
199-
; CHECK-NEXT: Against group ([[GRP6:0x[0-9a-f]+]]):
272+
; CHECK-NEXT: Against group ([[GRP15:0x[0-9a-f]+]]):
200273
; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
201274
; CHECK-NEXT: Check 2:
202-
; CHECK-NEXT: Comparing group ([[GRP5]]):
275+
; CHECK-NEXT: Comparing group ([[GRP14]]):
203276
; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset
204-
; CHECK-NEXT: Against group ([[GRP6]]):
277+
; CHECK-NEXT: Against group ([[GRP15]]):
205278
; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2
206279
; CHECK-NEXT: Grouped accesses:
207-
; CHECK-NEXT: Group [[GRP4]]:
280+
; CHECK-NEXT: Group [[GRP13]]:
208281
; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a))
209282
; CHECK-NEXT: Member: {%a,+,4}<nuw><%loop>
210-
; CHECK-NEXT: Group [[GRP5]]:
283+
; CHECK-NEXT: Group [[GRP14]]:
211284
; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a))
212285
; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop>
213-
; CHECK-NEXT: Group [[GRP6]]:
286+
; CHECK-NEXT: Group [[GRP15]]:
214287
; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b))
215288
; CHECK-NEXT: Member: {%b,+,8}<%loop>
216289
; CHECK-EMPTY:

0 commit comments

Comments
 (0)