Skip to content

LAA: improve code in a couple of routines (NFC) #108092

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 28, 2024
Merged

LAA: improve code in a couple of routines (NFC) #108092

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aa7f6b9
LAA: improve code in a couple of routines (NFC)
artagnon Sep 10, 2024
9db56bd
LAA: fix nit
artagnon Nov 28, 2024
dc7cd0e
LAA: address review
artagnon Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 14 additions & 19 deletions llvm/lib/Analysis/LoopAccessAnalysis.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1978,13 +1978,12 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize(
<< " Sink induction step: " << StrideBPtrInt << "\n");
// At least Src or Sink are loop invariant and the other is strided or
// invariant. We can generate a runtime check to disambiguate the accesses.
if (StrideAPtrInt == 0 || StrideBPtrInt == 0)
if (!StrideAPtrInt || !StrideBPtrInt)
return MemoryDepChecker::Dependence::Unknown;

// Both Src and Sink have a constant stride, check if they are in the same
// direction.
if ((StrideAPtrInt > 0 && StrideBPtrInt < 0) ||
(StrideAPtrInt < 0 && StrideBPtrInt > 0)) {
if ((StrideAPtrInt > 0) != (StrideBPtrInt > 0)) {
LLVM_DEBUG(
dbgs() << "Pointer access with strides in different directions\n");
return MemoryDepChecker::Dependence::Unknown;
Expand Down Expand Up @@ -2040,19 +2039,16 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
*Dist, MaxStride, TypeByteSize))
return Dependence::NoDep;

const SCEVConstant *C = dyn_cast<SCEVConstant>(Dist);
const SCEVConstant *ConstDist = dyn_cast<SCEVConstant>(Dist);

// Attempt to prove strided accesses independent.
if (C) {
const APInt &Val = C->getAPInt();
int64_t Distance = Val.getSExtValue();
if (ConstDist) {
uint64_t Distance = ConstDist->getAPInt().abs().getZExtValue();

// If the distance between accesses and their strides are known constants,
// check whether the accesses interlace each other.
if (std::abs(Distance) > 0 && CommonStride && *CommonStride > 1 &&
HasSameSize &&
areStridedAccessesIndependent(std::abs(Distance), *CommonStride,
TypeByteSize)) {
if (Distance > 0 && CommonStride && CommonStride > 1 && HasSameSize &&
areStridedAccessesIndependent(Distance, *CommonStride, TypeByteSize)) {
LLVM_DEBUG(dbgs() << "LAA: Strided accesses are independent\n");
return Dependence::NoDep;
}
Expand Down Expand Up @@ -2085,7 +2081,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
// forward dependency will allow vectorization using any width.

if (IsTrueDataDependence && EnableForwardingConflictDetection) {
if (!C) {
if (!ConstDist) {
// TODO: FoundNonConstantDistanceDependence is used as a necessary
// condition to consider retrying with runtime checks. Historically, we
// did not set it when strides were different but there is no inherent
Expand All @@ -2094,8 +2090,8 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
return Dependence::Unknown;
}
if (!HasSameSize ||
couldPreventStoreLoadForward(C->getAPInt().abs().getZExtValue(),
TypeByteSize)) {
couldPreventStoreLoadForward(
ConstDist->getAPInt().abs().getZExtValue(), TypeByteSize)) {
LLVM_DEBUG(
dbgs() << "LAA: Forward but may prevent st->ld forwarding\n");
return Dependence::ForwardButPreventsForwarding;
Expand All @@ -2113,7 +2109,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
return Dependence::Unknown;
}

if (!isa<SCEVConstant>(Dist)) {
if (!ConstDist) {
// Previously this case would be treated as Unknown, possibly setting
// FoundNonConstantDistanceDependence to force re-trying with runtime
// checks. Until the TODO below is addressed, set it here to preserve
Expand Down Expand Up @@ -2175,7 +2171,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
uint64_t MinDistanceNeeded =
TypeByteSize * *CommonStride * (MinNumIter - 1) + TypeByteSize;
if (MinDistanceNeeded > static_cast<uint64_t>(MinDistance)) {
if (!isa<SCEVConstant>(Dist)) {
if (!ConstDist) {
// For non-constant distances, we checked the lower bound of the
// dependence distance and the distance may be larger at runtime (and safe
// for vectorization). Classify it as Unknown, so we re-try with runtime
Expand Down Expand Up @@ -2216,8 +2212,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,

bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
uint64_t MinDepDistBytesOld = MinDepDistBytes;
if (IsTrueDataDependence && EnableForwardingConflictDetection &&
isa<SCEVConstant>(Dist) &&
if (IsTrueDataDependence && EnableForwardingConflictDetection && ConstDist &&
couldPreventStoreLoadForward(MinDistance, TypeByteSize)) {
// Sanity check that we didn't update MinDepDistBytes when calling
// couldPreventStoreLoadForward
Expand All @@ -2235,7 +2230,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
<< " with max VF = " << MaxVF << '\n');

uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
if (!isa<SCEVConstant>(Dist) && MaxVFInBits < MaxTargetVectorWidthInBits) {
if (!ConstDist && MaxVFInBits < MaxTargetVectorWidthInBits) {
// For non-constant distances, we checked the lower bound of the dependence
// distance and the distance may be larger at runtime (and safe for
// vectorization). Classify it as Unknown, so we re-try with runtime checks.
Expand Down