Skip to content

[ConstraintElimination] Use SCEV ranges information for Loop counter #91457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion llvm/include/llvm/Analysis/ConstraintSystem.h
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ class ConstraintSystem {
return 0;
}

// WARNING: it represents rather the maximum number of coefficients in the
// constraints which is actually the number of variables PLUS one (for the
// constant part).
size_t NumVariables = 0;

/// Current linear constraints in the system.
Expand All @@ -54,7 +57,13 @@ class ConstraintSystem {
/// constraint system.
DenseMap<Value *, unsigned> Value2Index;

// Eliminate constraints from the system using Fourier–Motzkin elimination.
// Eliminate the last variable from the system using Fourier–Motzkin
// elimination, while possibly relaxing it if it is beyond
// acceptable means (too many created constraints, overflow when
// computing the coefficients)
//
// return true if the updated system is equivalent, otherwise return
// false if it is relaxed.
bool eliminateUsingFM();

/// Returns true if there may be a solution for the constraints in the system.
Expand Down
62 changes: 33 additions & 29 deletions llvm/lib/Analysis/ConstraintSystem.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,67 +47,67 @@ bool ConstraintSystem::eliminateUsingFM() {
}
}

// Track if an overflow occurred when computing coefficents
bool Overflow = false;

// Process rows where the variable is != 0.
unsigned NumRemainingConstraints = RemainingRows.size();
for (unsigned R1 = 0; R1 < NumRemainingConstraints; R1++) {
// FIXME do not use copy
for (unsigned R2 = R1 + 1; R2 < NumRemainingConstraints; R2++) {
if (R1 == R2)
continue;

int64_t UpperLast = getLastCoefficient(RemainingRows[R2], LastIdx);
int64_t LowerLast = getLastCoefficient(RemainingRows[R1], LastIdx);
assert(
UpperLast != 0 && LowerLast != 0 &&
"RemainingRows should only contain rows where the variable is != 0");

// ensure signs are different then swap if necessary to only
// deal with UpperLast > 0
if ((LowerLast < 0 && UpperLast < 0) || (LowerLast > 0 && UpperLast > 0))
continue;

unsigned LowerR = R1;
unsigned UpperR = R2;
if (UpperLast < 0) {
std::swap(LowerR, UpperR);
std::swap(LowerLast, UpperLast);
}

// The following loop does the element-wise operation on sparse
// vectors:
//
// NR = - UpperRow * LowerLast + LowerRow * UpperLast
SmallVector<Entry, 8> NR;
unsigned IdxUpper = 0;
unsigned IdxLower = 0;
auto &LowerRow = RemainingRows[LowerR];
auto &UpperRow = RemainingRows[UpperR];
while (true) {
if (IdxUpper >= UpperRow.size() || IdxLower >= LowerRow.size())
break;

while (IdxUpper < UpperRow.size() && IdxLower < LowerRow.size()) {
int64_t M1, M2, N;
int64_t UpperV = 0;
int64_t LowerV = 0;
uint16_t CurrentId = std::numeric_limits<uint16_t>::max();
if (IdxUpper < UpperRow.size()) {
CurrentId = std::min(UpperRow[IdxUpper].Id, CurrentId);
}
if (IdxLower < LowerRow.size()) {
CurrentId = std::min(LowerRow[IdxLower].Id, CurrentId);
}
uint16_t CurrentId =
std::min(UpperRow[IdxUpper].Id, LowerRow[IdxLower].Id);

if (IdxUpper < UpperRow.size() && UpperRow[IdxUpper].Id == CurrentId) {
if (UpperRow[IdxUpper].Id == CurrentId) {
UpperV = UpperRow[IdxUpper].Coefficient;
IdxUpper++;
IdxUpper += 1;
}

if (MulOverflow(UpperV, -1 * LowerLast, M1))
return false;
if (IdxLower < LowerRow.size() && LowerRow[IdxLower].Id == CurrentId) {
if (LowerRow[IdxLower].Id == CurrentId) {
LowerV = LowerRow[IdxLower].Coefficient;
IdxLower++;
IdxLower += 1;
}

if (MulOverflow(UpperV, -1 * LowerLast, M1) ||
MulOverflow(LowerV, UpperLast, M2) || AddOverflow(M1, M2, N)) {
Overflow = true;
continue;
}

if (MulOverflow(LowerV, UpperLast, M2))
return false;
if (AddOverflow(M1, M2, N))
return false;
// useless to add a 0 to a sparse vector
if (N == 0)
continue;

NR.emplace_back(N, CurrentId);
}
if (NR.empty())
Expand All @@ -120,13 +120,17 @@ bool ConstraintSystem::eliminateUsingFM() {
}
NumVariables -= 1;

return true;
return !Overflow;
}

bool ConstraintSystem::mayHaveSolutionImpl() {
while (!Constraints.empty() && NumVariables > 1) {
if (!eliminateUsingFM())
return true;
while (!Constraints.empty() && Constraints.size() <= 500 &&
NumVariables > 1) {
if (!eliminateUsingFM()) {
LLVM_DEBUG(
dbgs()
<< "Some new constraints has been ignored during elimination.\n");
};
}

if (Constraints.empty() || NumVariables > 1)
Expand Down
44 changes: 39 additions & 5 deletions llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -916,6 +916,45 @@ void State::addInfoForInductions(BasicBlock &BB) {
!SE.isSCEVable(PN->getType()))
return;

auto *AR = dyn_cast_or_null<SCEVAddRecExpr>(SE.getSCEV(PN));
BasicBlock *LoopPred = L->getLoopPredecessor();
if (!AR || AR->getLoop() != L || !LoopPred)
return;

// If SCEV provides relevant range information, we push those facts
// to the worklist relatively to the header node itself (and not its
// successor).
auto UnsignedRange = SE.getUnsignedRange(AR);
auto SignedRange = SE.getSignedRange(AR);
DomTreeNode *DTNHeader = DT.getNode(&BB);

// The range can wrap thus we take Min/Max instead of Lower/Upper
auto UnsignedMin = UnsignedRange.getUnsignedMin();
auto UnsignedMax = UnsignedRange.getUnsignedMax();
if (!UnsignedMax.isMaxValue()) {
WorkList.push_back(FactOrCheck::getConditionFact(
DTNHeader, CmpInst::ICMP_ULE, PN,
Constant::getIntegerValue(PN->getType(), UnsignedMax)));
}
if (!UnsignedMin.isMinValue()) {
WorkList.push_back(FactOrCheck::getConditionFact(
DTNHeader, CmpInst::ICMP_UGE, PN,
Constant::getIntegerValue(PN->getType(), UnsignedMin)));
}

auto SignedMin = SignedRange.getSignedMin();
auto SignedMax = SignedRange.getSignedMax();
if (!SignedMax.isMaxSignedValue()) {
WorkList.push_back(FactOrCheck::getConditionFact(
DTNHeader, CmpInst::ICMP_SLE, PN,
Constant::getIntegerValue(PN->getType(), SignedMax)));
}
if (!SignedMin.isMinSignedValue()) {
WorkList.push_back(FactOrCheck::getConditionFact(
DTNHeader, CmpInst::ICMP_SGE, PN,
Constant::getIntegerValue(PN->getType(), SignedMin)));
}

BasicBlock *InLoopSucc = nullptr;
if (Pred == CmpInst::ICMP_NE)
InLoopSucc = cast<BranchInst>(BB.getTerminator())->getSuccessor(0);
Expand All @@ -927,11 +966,6 @@ void State::addInfoForInductions(BasicBlock &BB) {
if (!L->contains(InLoopSucc) || !L->isLoopExiting(&BB) || InLoopSucc == &BB)
return;

auto *AR = dyn_cast_or_null<SCEVAddRecExpr>(SE.getSCEV(PN));
BasicBlock *LoopPred = L->getLoopPredecessor();
if (!AR || AR->getLoop() != L || !LoopPred)
return;

const SCEV *StartSCEV = AR->getStart();
Value *StartValue = nullptr;
if (auto *C = dyn_cast<SCEVConstant>(StartSCEV)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@ define i32 @f(i64 %a3, i64 %numElements) {
; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[A1]], [[A3]]
; CHECK-NEXT: br i1 [[CMP]], label [[IF_END_I:%.*]], label [[ABORT:%.*]]
; CHECK: if.end.i:
; CHECK-NEXT: [[CMP2_NOT_I:%.*]] = icmp ult i64 [[A1]], [[A3]]
; CHECK-NEXT: br i1 [[CMP2_NOT_I]], label [[ABORT]], label [[EXIT:%.*]]
; CHECK-NEXT: br i1 false, label [[ABORT]], label [[EXIT:%.*]]
; CHECK: abort:
; CHECK-NEXT: ret i32 -1
; CHECK: exit:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,8 @@ define void @loop_iv_cond_variable_bound(i32 %n) {
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[T_1:%.*]] = icmp ule i32 [[IV]], [[N:%.*]]
; CHECK-NEXT: call void @use(i1 [[T_1]])
; CHECK-NEXT: [[T_2:%.*]] = icmp sge i32 [[IV]], 0
; CHECK-NEXT: call void @use(i1 [[T_2]])
; CHECK-NEXT: [[T_3:%.*]] = icmp sge i32 [[IV]], -1
; CHECK-NEXT: call void @use(i1 [[T_3]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_1:%.*]] = icmp ult i32 [[IV]], [[N]]
; CHECK-NEXT: call void @use(i1 [[C_1]])
; CHECK-NEXT: [[C_2:%.*]] = icmp ugt i32 [[IV]], 1
Expand Down Expand Up @@ -56,12 +54,9 @@ define void @loop_iv_cond_constant_bound() {
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[T_1:%.*]] = icmp ule i32 [[IV]], 2
; CHECK-NEXT: call void @use(i1 [[T_1]])
; CHECK-NEXT: [[T_2:%.*]] = icmp sge i32 [[IV]], 0
; CHECK-NEXT: call void @use(i1 [[T_2]])
; CHECK-NEXT: [[T_3:%.*]] = icmp sge i32 [[IV]], -1
; CHECK-NEXT: call void @use(i1 [[T_3]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_1:%.*]] = icmp ult i32 [[IV]], 2
; CHECK-NEXT: call void @use(i1 [[C_1]])
; CHECK-NEXT: [[C_2:%.*]] = icmp ugt i32 [[IV]], 1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,11 @@ define void @loop_phi_pos_start_value(i32 %y, i1 %c, i32 %n) {
; CHECK: loop.latch:
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 false)
; CHECK-NEXT: [[T_2:%.*]] = icmp sge i32 [[X]], 10
; CHECK-NEXT: call void @use(i1 [[T_2]])
; CHECK-NEXT: [[C_2:%.*]] = icmp sle i32 [[X]], 9
; CHECK-NEXT: call void @use(i1 [[C_2]])
; CHECK-NEXT: [[C_3:%.*]] = icmp sgt i32 [[X]], 9
; CHECK-NEXT: call void @use(i1 [[C_3]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_5:%.*]] = icmp sge i32 [[X]], 9
; CHECK-NEXT: call void @use(i1 [[C_5]])
; CHECK-NEXT: call void @use(i1 false)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[X_NEXT]] = add nsw i32 [[X]], 1
; CHECK-NEXT: br label [[LOOP_HEADER]]
; CHECK: exit:
Expand Down Expand Up @@ -75,8 +71,7 @@ define void @loop_phi_neg_start_value(i32 %y, i1 %c, i32 %n) {
; CHECK: loop.latch:
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 false)
; CHECK-NEXT: [[T_2:%.*]] = icmp sge i32 [[X]], -10
; CHECK-NEXT: call void @use(i1 [[T_2]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_2:%.*]] = icmp sle i32 [[X]], 9
; CHECK-NEXT: call void @use(i1 [[C_2]])
; CHECK-NEXT: [[C_3:%.*]] = icmp sgt i32 [[X]], 9
Expand Down Expand Up @@ -248,12 +243,11 @@ define void @loop_latch_not_executed_constant_bound(i32 %y, i1 %c) {
; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_HEADER:%.*]], label [[EXIT:%.*]]
; CHECK: loop.header:
; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[C_1:%.*]] = icmp ugt i32 [[X]], 10
; CHECK-NEXT: br i1 [[C_1]], label [[LOOP_LATCH]], label [[EXIT]]
; CHECK-NEXT: br i1 false, label [[LOOP_LATCH]], label [[EXIT]]
; CHECK: loop.latch:
; CHECK-NEXT: call void @use(i1 false)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 false)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[X_NEXT]] = add i32 [[X]], 1
; CHECK-NEXT: br label [[LOOP_HEADER]]
Expand Down Expand Up @@ -299,10 +293,8 @@ define void @loop_iv_cond_variable_bound(i32 %n) {
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[T_1:%.*]] = icmp ule i32 [[IV]], [[N:%.*]]
; CHECK-NEXT: call void @use(i1 [[T_1]])
; CHECK-NEXT: [[T_2:%.*]] = icmp sge i32 [[IV]], 0
; CHECK-NEXT: call void @use(i1 [[T_2]])
; CHECK-NEXT: [[T_3:%.*]] = icmp sge i32 [[IV]], -1
; CHECK-NEXT: call void @use(i1 [[T_3]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_1:%.*]] = icmp ult i32 [[IV]], [[N]]
; CHECK-NEXT: call void @use(i1 [[C_1]])
; CHECK-NEXT: [[C_2:%.*]] = icmp ugt i32 [[IV]], 1
Expand Down Expand Up @@ -362,12 +354,9 @@ define void @loop_iv_cond_constant_bound() {
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[T_1:%.*]] = icmp ule i32 [[IV]], 2
; CHECK-NEXT: call void @use(i1 [[T_1]])
; CHECK-NEXT: [[T_2:%.*]] = icmp sge i32 [[IV]], 0
; CHECK-NEXT: call void @use(i1 [[T_2]])
; CHECK-NEXT: [[T_3:%.*]] = icmp sge i32 [[IV]], -1
; CHECK-NEXT: call void @use(i1 [[T_3]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_1:%.*]] = icmp ult i32 [[IV]], 2
; CHECK-NEXT: call void @use(i1 [[C_1]])
; CHECK-NEXT: [[C_2:%.*]] = icmp ugt i32 [[IV]], 1
Expand Down
11 changes: 4 additions & 7 deletions llvm/test/Transforms/ConstraintElimination/loops.ll
Original file line number Diff line number Diff line change
Expand Up @@ -107,15 +107,12 @@ define i32 @loop_header_dom_successors_flipped(i32 %y, i1 %c) {
; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP_HEADER:%.*]], label [[EXIT:%.*]]
; CHECK: loop.header:
; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[X_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[C_1:%.*]] = icmp ule i32 [[X]], 10
; CHECK-NEXT: br i1 [[C_1]], label [[EXIT]], label [[LOOP_LATCH]]
; CHECK-NEXT: br i1 true, label [[EXIT]], label [[LOOP_LATCH]]
; CHECK: loop.latch:
; CHECK-NEXT: call void @use(i1 false)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[C_2:%.*]] = icmp ugt i32 [[X]], 11
; CHECK-NEXT: call void @use(i1 [[C_2]])
; CHECK-NEXT: [[C_3:%.*]] = icmp ule i32 [[X]], 11
; CHECK-NEXT: call void @use(i1 [[C_3]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[X_NEXT]] = add i32 [[X]], 1
; CHECK-NEXT: br label [[LOOP_HEADER]]
; CHECK: exit:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,8 +197,7 @@ define void @add_rec_decreasing_2_cond_true_constant(i8 noundef %len) {
; CHECK-NEXT: [[CMP2_NOT:%.*]] = icmp eq i8 [[K_0]], 0
; CHECK-NEXT: br i1 [[CMP2_NOT]], label [[EXIT:%.*]], label [[LOOP_LATCH]]
; CHECK: loop.latch:
; CHECK-NEXT: [[CMP_NOT_I:%.*]] = icmp ult i8 [[K_0]], 5
; CHECK-NEXT: call void @use(i1 [[CMP_NOT_I]])
; CHECK-NEXT: call void @use(i1 true)
; CHECK-NEXT: [[K_DEC]] = add i8 [[K_0]], -2
; CHECK-NEXT: br label [[LOOP_HEADER]]
; CHECK: exit:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@ define void @test_iv_nuw_nsw_1_uge_start(i8 %len.n, i8 %a) {
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
; CHECK-NEXT: [[IV:%.*]] = phi i8 [ -1, [[LOOP_PH:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[IV]], 1
; CHECK-NEXT: br i1 [[C]], label [[EXIT:%.*]], label [[FOR_BODY:%.*]]
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[C_2:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[C_2]], label [[LOOP_LATCH]], label [[EXIT]]
Expand Down Expand Up @@ -90,8 +89,7 @@ define void @test_iv_nuw_nsw_2_uge_start(i8 %len.n, i8 %a) {
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
; CHECK-NEXT: [[IV:%.*]] = phi i8 [ [[START]], [[LOOP_PH:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[IV]], 1
; CHECK-NEXT: br i1 [[C]], label [[EXIT:%.*]], label [[FOR_BODY:%.*]]
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[C_2:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[C_2]], label [[LOOP_LATCH]], label [[EXIT]]
Expand Down Expand Up @@ -131,8 +129,7 @@ define void @test_iv_nsw_nuw_1_ult_end(i8 %len.n, i8 %a) {
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
; CHECK-NEXT: [[IV:%.*]] = phi i8 [ -2, [[LOOP_PH:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[IV]], 1
; CHECK-NEXT: br i1 [[C]], label [[EXIT:%.*]], label [[FOR_BODY:%.*]]
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[C_2:%.*]] = call i1 @cond()
; CHECK-NEXT: br i1 [[C_2]], label [[LOOP_LATCH]], label [[EXIT]]
Expand Down
Loading