[lit] introduce --show-attempts-count

This introduces `--show-attempts-count` output option to `lit`.

It shows the number of attempts needed for a test to pass. It also shows
the maximum number of attempts that were allowed for this test. This option
is useful when you want to understand how many attempts took for a flaky
test (``FLAKYPASS``) to pass.

This is how the test output looks like *without* `--show-attempts-count`:

```
PASS: your-test-suite :: your-first-test.py (1 of 3)
FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
```

This is the output *with* `--show-attempts-count`:

```
PASS: your-test-suite :: your-first-test.py (1 of 2)
FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
```

In this case `your-second-test.py` was executed three times and it
succeeded after the third time (`attempts=3`). Technically another run would have
been possible (`max_allowed_attempts=4`).

We will only append the extra information when a test was allowed more
than one attempt to succeed (i.e. see :option:`--max-retries-per-test`).

NOTE: Additionally this is a fixup for #141851 where the tests were not
quite right. To correlate better between the test output and the test script
I've used higher numbers of max allowed retries.

Author: kwk

llvm/docs/CommandGuide/lit.rst

@@ -144,6 +144,34 @@ OUTPUT OPTIONS
 
  Show expectedly failed tests.
 
+.. option:: --show-attempts-count
+
+ Show the number of attempts needed for a test to pass. It also shows the
+ maximum number of attempts that were allowed for this test. This option is
+ useful when you want to understand how many attempts took for a flaky test
+ (``FLAKYPASS``) to pass.
+
+ This is how the test output looks like *without* :option:`--show-attempts-count`
+
+ .. code-block:: none
+
+  PASS: your-test-suite :: your-first-test.py (1 of 3)
+  FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2)
+
+ This is the output *with* :option:`--show-attempts-count`
+
+ .. code-block:: none
+
+  PASS: your-test-suite :: your-first-test.py (1 of 2)
+  FLAKYPASS: your-test-suite :: your-second-test.py (2 of 2) [attempts=3,max_allowed_attempts=4]
+
+ In this case ``your-second-test.py`` was executed three times and it succeeded
+ after the third time (``attempts=3``). Technically another run would have been
+ possible (``max_allowed_attempts=4``).
+
+ We will only append the extra information when a test was allowed more than one
+ attempt to succeed (i.e. see :option:`--max-retries-per-test`).
+
 .. _execution-options:
 
 EXECUTION OPTIONS

llvm/utils/lit/lit/Test.py

@@ -151,7 +151,7 @@ def toMetricValue(value):
 class Result(object):
     """Wrapper for the results of executing an individual test."""
 
-    def __init__(self, code, output="", elapsed=None):
+    def __init__(self, code, output="", elapsed=None, attempts=None, max_allowed_attempts=None):
         # The result code.
         self.code = code
         # The test output.
@@ -164,6 +164,10 @@ def __init__(self, code, output="", elapsed=None):
         self.metrics = {}
         # The micro-test results reported by this test.
         self.microResults = {}
+        # How often was the test run?
+        self.attempts = attempts
+        # How many attempts were allowed for this test
+        self.max_allowed_attempts = max_allowed_attempts
 
     def addMetric(self, name, value):
         """

llvm/utils/lit/lit/TestRunner.py

@@ -2292,7 +2292,7 @@ def runOnce(
     if err:
         output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
 
-    return lit.Test.Result(status, output)
+    return lit.Test.Result(status, output, attempts=i+1, max_allowed_attempts=attempts)
 
 
 def executeShTest(

llvm/utils/lit/lit/cl_arguments.py

@@ -104,6 +104,12 @@ def parse_args():
         help="Do not use curses based progress bar",
         action="store_false",
     )
+    format_group.add_argument(
+        "--show-attempts-count",
+        dest="showAttemptsCount",
+        help="Show number of attempts and maximum attempts for flaky tests.",
+        action="store_true",
+    )
 
     # Note: this does not generate flags for user-defined result codes.
     success_codes = [c for c in lit.Test.ResultCode.all_codes() if not c.isFailure]

llvm/utils/lit/lit/display.py

@@ -117,10 +117,23 @@ def clear(self, interrupted):
     def print_result(self, test):
         # Show the test result line.
         test_name = test.getFullName()
+
+        extra_info = ""
+        extra_attrs = []
+
+        if self.opts.showAttemptsCount:
+            if test.result.max_allowed_attempts is not None and test.result.max_allowed_attempts > 1:
+                extra_attrs.append(f"attempts={test.result.attempts}")
+                extra_attrs.append(f"max_allowed_attempts={test.result.max_allowed_attempts}")
+
+        if len(extra_attrs) > 0:
+            extra_info = " [" + ",".join(extra_attrs) + "]"
+
         print(
-            "%s: %s (%d of %d)"
-            % (test.result.code.name, test_name, self.completed, self.num_tests)
+            "%s: %s (%d of %d)%s"
+            % (test.result.code.name, test_name, self.completed, self.num_tests, extra_info)
         )
+        
 
         # Show the test failure output, if requested.
         if (test.isFailure() and self.opts.showOutput) or self.opts.showAllOutput:

llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py

@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 8
 # RUN: "%python" "%s" "%counter"
 
 import sys

llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py

@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 10
 # RUN: "%python" "%s" "%counter"
 
 import sys

llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg

@@ -9,4 +9,4 @@ config.test_exec_root = None
 config.substitutions.append(("%python", lit_config.params.get("python", "")))
 config.substitutions.append(("%counter", lit_config.params.get("counter", "")))
 
-config.test_retry_attempts = 3
+config.test_retry_attempts = 9

llvm/utils/lit/tests/allow-retries.py

@@ -72,49 +72,58 @@
 #     CHECK-TEST7: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
-#  * The "--max-retries-per-test" is specified high enough (3).
+#  * The "--max-retries-per-test" is specified high enough (7).
 #  * No ALLOW_RETRIES keyword is used in the test script.
 #  * No config.test_retry_attempts is adjusted in the test suite config file.
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
-# RUN:   --max-retries-per-test=3 \
+# RUN:   --max-retries-per-test=7 \
+# RUN:   --show-attempts-count \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=8]
 # CHECK-TEST8: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (2).
-#  * ALLOW_RETRIES is specified high enough (3)
+#  * ALLOW_RETRIES is specified high enough (8)
 #  * No config.test_retry_attempts is adjusted in the test suite config file.
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=2 \
+# RUN:   --show-attempts-count \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=9]
 # CHECK-TEST9: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (2).
 #  * No ALLOW_RETRIES keyword is used in the test script.
-#  * config.test_retry_attempts is set high enough (3).
+#  * config.test_retry_attempts is set high enough (9).
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=2 \
+# RUN:   --show-attempts-count \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=10]
 # CHECK-TEST10: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (1).
-#  * ALLOW_RETRIES keyword set high enough (3).
-#  * config.test_retry_attempts is set too low enough (2).
+#  * ALLOW_RETRIES keyword is set high enough (10)
+#  * config.test_retry_attempts is set too low (2).
 # RUN: rm -f %t.counter
-# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
+# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=1 \
+# RUN:   --show-attempts-count \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1) [attempts=4,max_allowed_attempts=11]
 # CHECK-TEST11: Passed With Retry: 1
+