Skip to content

Commit b9aef84

Browse files
fhahnfhahn
committed
[Loads] Support dereferenceable assumption with variable size.
Update isDereferenceableAndAlignedPointer to make use of dereferenceable assumptions with variable sizes via SCEV. To do so, factor out the logic to check via an assumption to a helper, and use SE to check if the access size is less than the dereferenceable size.
1 parent ab17ff0 commit b9aef84

File tree

4 files changed

+90
-13
lines changed

4 files changed

+90
-13
lines changed

‎llvm/include/llvm/Analysis/AssumeBundleQueries.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ LLVM_ABI void fillMapFromAssume(AssumeInst &Assume,
102102
struct RetainedKnowledge {
103103
Attribute::AttrKind AttrKind = Attribute::None;
104104
uint64_t ArgValue = 0;
105+
Value *IRArgValue = nullptr;
105106
Value *WasOn = nullptr;
106107
bool operator==(RetainedKnowledge Other) const {
107108
return AttrKind == Other.AttrKind && WasOn == Other.WasOn &&

‎llvm/lib/Analysis/AssumeBundleQueries.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ llvm::getKnowledgeFromBundle(AssumeInst &Assume,
114114
};
115115
if (BOI.End - BOI.Begin > ABA_Argument)
116116
Result.ArgValue = GetArgOr1(0);
117+
Result.IRArgValue = getValueFromBundleOpInfo(Assume, BOI, ABA_Argument);
117118
if (Result.AttrKind == Attribute::Alignment)
118119
if (BOI.End - BOI.Begin > ABA_Argument + 1)
119120
Result.ArgValue = MinAlign(Result.ArgValue, GetArgOr1(1));

‎llvm/lib/Analysis/Loads.cpp

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,35 @@ static bool isAligned(const Value *Base, Align Alignment,
3131
return Base->getPointerAlignment(DL) >= Alignment;
3232
}
3333

34+
static bool isDereferenceableAndAlignedPointerViaAssumption(
35+
const Value *Ptr, Align Alignment,
36+
function_ref<bool(const RetainedKnowledge &RK)> CheckSize,
37+
const DataLayout &DL, const Instruction *CtxI, AssumptionCache *AC,
38+
const DominatorTree *DT) {
39+
if (!CtxI || Ptr->canBeFreed())
40+
return false;
41+
/// Look through assumes to see if both dereferencability and alignment can
42+
/// be proven by an assume if needed.
43+
RetainedKnowledge AlignRK;
44+
RetainedKnowledge DerefRK;
45+
bool IsAligned = Ptr->getPointerAlignment(DL) >= Alignment;
46+
return getKnowledgeForValue(
47+
Ptr, {Attribute::Dereferenceable, Attribute::Alignment}, AC,
48+
[&](RetainedKnowledge RK, Instruction *Assume, auto) {
49+
if (!isValidAssumeForContext(Assume, CtxI, DT))
50+
return false;
51+
if (RK.AttrKind == Attribute::Alignment)
52+
AlignRK = std::max(AlignRK, RK);
53+
if (RK.AttrKind == Attribute::Dereferenceable)
54+
DerefRK = std::max(DerefRK, RK);
55+
IsAligned |= AlignRK && AlignRK.ArgValue >= Alignment.value();
56+
if (IsAligned && DerefRK && CheckSize(DerefRK))
57+
return true; // We have found what we needed so we stop looking
58+
return false; // Other assumes may have better information. so
59+
// keep looking
60+
});
61+
}
62+
3463
/// Test if V is always a pointer to allocated and suitably aligned memory for
3564
/// a simple load or store.
3665
static bool isDereferenceableAndAlignedPointer(
@@ -199,8 +228,12 @@ static bool isDereferenceableAndAlignedPointer(
199228
return true;
200229
}
201230

202-
// If we don't know, assume the worst.
203-
return false;
231+
return isDereferenceableAndAlignedPointerViaAssumption(
232+
V, Alignment,
233+
[Size](const RetainedKnowledge &RK) {
234+
return RK.ArgValue >= Size.getZExtValue();
235+
},
236+
DL, CtxI, AC, DT);
204237
}
205238

206239
bool llvm::isDereferenceableAndAlignedPointer(
@@ -317,8 +350,8 @@ bool llvm::isDereferenceableAndAlignedInLoop(
317350
return false;
318351

319352
const SCEV *MaxBECount =
320-
Predicates ? SE.getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates)
321-
: SE.getConstantMaxBackedgeTakenCount(L);
353+
Predicates ? SE.getPredicatedSymbolicMaxBackedgeTakenCount(L, *Predicates)
354+
: SE.getSymbolicMaxBackedgeTakenCount(L);
322355
if (isa<SCEVCouldNotCompute>(MaxBECount))
323356
return false;
324357

@@ -334,9 +367,11 @@ bool llvm::isDereferenceableAndAlignedInLoop(
334367

335368
Value *Base = nullptr;
336369
APInt AccessSize;
370+
const SCEV *AccessSizeSCEV = nullptr;
337371
if (const SCEVUnknown *NewBase = dyn_cast<SCEVUnknown>(AccessStart)) {
338372
Base = NewBase->getValue();
339373
AccessSize = MaxPtrDiff;
374+
AccessSizeSCEV = PtrDiff;
340375
} else if (auto *MinAdd = dyn_cast<SCEVAddExpr>(AccessStart)) {
341376
if (MinAdd->getNumOperands() != 2)
342377
return false;
@@ -360,12 +395,20 @@ bool llvm::isDereferenceableAndAlignedInLoop(
360395
return false;
361396

362397
AccessSize = MaxPtrDiff + Offset->getAPInt();
398+
AccessSizeSCEV = SE.getAddExpr(PtrDiff, Offset);
363399
Base = NewBase->getValue();
364400
} else
365401
return false;
366402

367403
Instruction *HeaderFirstNonPHI = &*L->getHeader()->getFirstNonPHIIt();
368-
return isDereferenceableAndAlignedPointer(Base, Alignment, AccessSize, DL,
404+
return isDereferenceableAndAlignedPointerViaAssumption(
405+
Base, Alignment,
406+
[&SE, PtrDiff](const RetainedKnowledge &RK) {
407+
return SE.isKnownPredicate(CmpInst::ICMP_ULE, PtrDiff,
408+
SE.getSCEV(RK.IRArgValue));
409+
},
410+
DL, HeaderFirstNonPHI, AC, &DT) ||
411+
isDereferenceableAndAlignedPointer(Base, Alignment, AccessSize, DL,
369412
HeaderFirstNonPHI, AC, &DT);
370413
}
371414

‎llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -183,14 +183,30 @@ define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_acc
183183
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
184184
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
185185
; CHECK: [[VECTOR_BODY]]:
186-
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
187-
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
188186
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
189187
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
190188
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
191189
; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
192-
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[TMP1]], i32 0
193-
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP5]], align 1
190+
; CHECK-NEXT: [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
191+
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
192+
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
193+
; CHECK: [[PRED_LOAD_IF]]:
194+
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
195+
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 1
196+
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 0
197+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
198+
; CHECK: [[PRED_LOAD_CONTINUE]]:
199+
; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]
200+
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP15]], i32 1
201+
; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
202+
; CHECK: [[PRED_LOAD_IF1]]:
203+
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
204+
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
205+
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 1
206+
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
207+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
208+
; CHECK: [[PRED_LOAD_CONTINUE2]]:
209+
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
194210
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
195211
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
196212
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
@@ -265,14 +281,30 @@ define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_ac
265281
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
266282
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
267283
; CHECK: [[VECTOR_BODY]]:
268-
; CHECK-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
269-
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
270284
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
271285
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0
272286
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 1
273287
; CHECK-NEXT: [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
274-
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[TMP1]], i32 0
275-
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP5]], align 1
288+
; CHECK-NEXT: [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
289+
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
290+
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
291+
; CHECK: [[PRED_LOAD_IF]]:
292+
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
293+
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 1
294+
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> poison, i32 [[TMP17]], i32 0
295+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
296+
; CHECK: [[PRED_LOAD_CONTINUE]]:
297+
; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP18]], %[[PRED_LOAD_IF]] ]
298+
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP15]], i32 1
299+
; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
300+
; CHECK: [[PRED_LOAD_IF1]]:
301+
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1
302+
; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP11]]
303+
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 1
304+
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
305+
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
306+
; CHECK: [[PRED_LOAD_CONTINUE2]]:
307+
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
276308
; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
277309
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
278310
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0

0 commit comments

Comments
 (0)