[InstCombine] Improve infinite loop detection

Summary:
This patch limits the default number of iterations performed by InstCombine. It also exposes a new option that allows to specify how many iterations is considered getting stuck in an infinite loop.

Based on experiments performed on real-world C++ programs, InstCombine seems to perform at most ~8-20 iterations, so treating 1000 iterations as an infinite loop seems like a safe choice. See D71145 for details.

The two limits can be specified via command line options.

Reviewers: spatel, lebedev.ri, nikic, xbolva00, grosser

Reviewed By: spatel

Subscribers: hiraditya, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D71673

Author: kuhar

llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

@@ -122,7 +122,8 @@ STATISTIC(NumReassoc  , "Number of reassociations");
 DEBUG_COUNTER(VisitCounter, "instcombine-visit",
               "Controls which instructions are visited");
 
-static constexpr unsigned InstCombineDefaultMaxIterations = UINT_MAX - 1;
+static constexpr unsigned InstCombineDefaultMaxIterations = 1000;
+static constexpr unsigned InstCombineDefaultInfiniteLoopThreshold = 1000;
 
 static cl::opt<bool>
 EnableCodeSinking("instcombine-code-sinking", cl::desc("Enable code sinking"),
@@ -137,6 +138,12 @@ static cl::opt<unsigned> LimitMaxIterations(
     cl::desc("Limit the maximum number of instruction combining iterations"),
     cl::init(InstCombineDefaultMaxIterations));
 
+static cl::opt<unsigned> InfiniteLoopDetectionThreshold(
+    "instcombine-infinite-loop-threshold",
+    cl::desc("Number of instruction combining iterations considered an "
+             "infinite loop"),
+    cl::init(InstCombineDefaultInfiniteLoopThreshold), cl::Hidden);
+
 static cl::opt<unsigned>
 MaxArraySize("instcombine-maxarray-size", cl::init(1024),
              cl::desc("Maximum array size considered when doing a combine"));
@@ -3571,13 +3578,17 @@ static bool combineInstructionsOverFunction(
   unsigned Iteration = 0;
   while (true) {
     ++Iteration;
+
+    if (Iteration > InfiniteLoopDetectionThreshold) {
+      report_fatal_error(
+          "Instruction Combining seems stuck in an infinite loop after " +
+          Twine(InfiniteLoopDetectionThreshold) + " iterations.");
+    }
+
     if (Iteration > MaxIterations) {
       LLVM_DEBUG(dbgs() << "\n\n[IC] Iteration limit #" << MaxIterations
                         << " on " << F.getName()
                         << " reached; stopping before reaching a fixpoint\n");
-      LLVM_DEBUG(dbgs().flush());
-      assert(Iteration <= InstCombineDefaultMaxIterations &&
-             "InstCombine stuck in an infinite loop?");
       break;
     }
 

llvm/test/Transforms/InstCombine/limit-max-iterations.ll

@@ -2,6 +2,7 @@
 ; RUN: opt < %s -instcombine --instcombine-max-iterations=0 -S | FileCheck %s --check-prefix=ZERO
 ; RUN: opt < %s -instcombine --instcombine-max-iterations=1 -S | FileCheck %s --check-prefix=ONE
 ; RUN: opt < %s -instcombine -S | FileCheck %s --check-prefix=FIXPOINT
+; RUN: not opt < %s -instcombine -S --instcombine-infinite-loop-threshold=3 2>&1 | FileCheck %s --check-prefix=LOOP
 
 ; Based on xor-of-icmps-with-extra-uses.ll. This requires multiple iterations of
 ; InstCombine to reach a fixpoint.
@@ -32,6 +33,8 @@ define i1 @v0_select_of_consts(i32 %X, i32* %selected) {
 ; FIXPOINT-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 65535
 ; FIXPOINT-NEXT:    ret i1 [[TMP1]]
 
+; LOOP: LLVM ERROR: Instruction Combining seems stuck in an infinite loop after 3 iterations.
+
   %cond0 = icmp sgt i32 %X, 32767
   %cond1 = icmp sgt i32 %X, -32768
   %select = select i1 %cond0, i32 32767, i32 -32768