llvm-reduce: Do not assert if the input is no longer interesting (#133386)

If the interestingness script is flaky, we should not assert. Print
a warning, and continue. This could still happen as a result of an
llvm-reduce bug, so make a note of that.

Add a --skip-verify-interesting-after-counting-chunks option to
avoid the extra run of the reduction script, and to silence the
warning.

Author: arsenm

llvm/docs/CommandGuide/llvm-reduce.rst

@@ -71,6 +71,12 @@ GENERIC OPTIONS
 
  Delta passes to not run, separated by commas. By default, run all delta passes.
 
+.. option::--skip-verify-interesting-after-counting-chunks
+
+ Do not validate testcase is interesting after counting chunks. This
+ will save time by avoiding extra executions of the interestingness
+ test, but a warning will no longer be printed on flaky reproducers.
+
 .. option:: --starting-granularity-level=<uint>
 
   Number of times to divide chunks prior to first test.

llvm/test/tools/llvm-reduce/Inputs/flaky-test.py

@@ -0,0 +1,8 @@
+"""Script to exit 0 on the first run, and non-0 on subsequent
+runs. This demonstrates a flaky interestingness test.
+"""
+import sys
+import pathlib
+
+# This will exit 0 the first time the script is run, and fail the second time
+pathlib.Path(sys.argv[1]).touch(exist_ok=False)

llvm/test/tools/llvm-reduce/flaky-interestingness.ll

@@ -0,0 +1,27 @@
+; Test that there is no assertion if the reproducer is flaky
+; RUN: rm -f %t
+; RUN: llvm-reduce -j=1 --delta-passes=instructions --test %python --test-arg %p/Inputs/flaky-test.py --test-arg %t %s -o /dev/null 2>&1 | FileCheck %s
+
+; Check no error with -skip-verify-interesting-after-counting-chunks
+; RUN: rm -f %t
+; RUN: llvm-reduce -j=1 -skip-verify-interesting-after-counting-chunks --delta-passes=instructions --test %python --test-arg %p/Inputs/flaky-test.py --test-arg %t %s -o /dev/null 2>&1 | FileCheck --allow-empty -check-prefix=QUIET %s
+
+; CHECK: warning: input module no longer interesting after counting chunks
+; CHECK-NEXT: note: the interestingness test may be flaky, or there may be an llvm-reduce bug
+; CHECK-NEXT: note: use -skip-verify-interesting-after-counting-chunks to suppress this warning
+
+; QUIET-NOT: warning
+; QUIET-NOT: note
+; QUIET-NOT: error
+
+declare void @foo(i32)
+
+define void @func() {
+  call void @foo(i32 0)
+  call void @foo(i32 1)
+  call void @foo(i32 2)
+  call void @foo(i32 3)
+  call void @foo(i32 4)
+  call void @foo(i32 5)
+  ret void
+}

llvm/tools/llvm-reduce/deltas/Delta.cpp

@@ -29,6 +29,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/ThreadPool.h"
+#include "llvm/Support/WithColor.h"
 #include <fstream>
 
 using namespace llvm;
@@ -40,6 +41,12 @@ static cl::opt<bool> AbortOnInvalidReduction(
     cl::desc("Abort if any reduction results in invalid IR"),
     cl::cat(LLVMReduceOptions));
 
+static cl::opt<bool> SkipVerifyAfterCountingChunks(
+    "skip-verify-interesting-after-counting-chunks",
+    cl::desc("Do not validate testcase is interesting after counting chunks "
+             "(may speed up reduction)"),
+    cl::cat(LLVMReduceOptions));
+
 static cl::opt<unsigned int> StartingGranularityLevel(
     "starting-granularity-level",
     cl::desc("Number of times to divide chunks prior to first test"),
@@ -191,8 +198,16 @@ void llvm::runDeltaPass(TestRunner &Test, ReductionFunc ExtractChunksFromModule,
 
     assert(!Test.getProgram().verify(&errs()) &&
            "input module is broken after counting chunks");
-    assert(Test.getProgram().isReduced(Test) &&
-           "input module no longer interesting after counting chunks");
+
+    if (!SkipVerifyAfterCountingChunks && !Test.getProgram().isReduced(Test)) {
+      WithColor::warning()
+          << "input module no longer interesting after counting chunks\n";
+      WithColor::note() << "the interestingness test may be flaky, or there "
+                           "may be an llvm-reduce bug\n";
+      WithColor::note()
+          << "use -skip-verify-interesting-after-counting-chunks to "
+             "suppress this warning\n";
+    }
 
 #ifndef NDEBUG
     // Make sure that the number of chunks does not change as we reduce.