[lit] show retry attempts (#142413)

If a test took more than one attempt to complete, show the number of attempts and the maximum allowed attempts as `2 of 4 attempts` inside the `<progress info>` (see [TEST RUN OUTPUT FORMAT](https://llvm.org/docs/CommandGuide/lit.html#test-run-output-format)).

NOTE: Additionally this is a fixup for #141851 where the tests were not quite right. `max-retries-per-test/allow-retries-test_retry_attempts/test.py` was added but never used there. Now we're calling it. To correlate better between the test output and the test script I've used higher numbers of max allowed retries.

Author: kwk

llvm/docs/CommandGuide/lit.rst

@@ -688,6 +688,14 @@ newline.
 The ``<progress info>`` field can be used to report progress information such
 as (1/300) or can be empty, but even when empty the parentheses are required.
 
+Should a test be allowed retries (see ``ALLOW_RETRIES:`` annotation) and it
+needed more than one attempt to succeed, then ``<progress info>`` is extended
+by this information:
+
+.. code-block:: none
+
+  , <num_attempts_made> of <max_allowed_attempts> attempts
+
 Each test result may include additional (multiline) log information in the
 following format:
 

llvm/utils/lit/lit/Test.py

@@ -151,7 +151,9 @@ def toMetricValue(value):
 class Result(object):
     """Wrapper for the results of executing an individual test."""
 
-    def __init__(self, code, output="", elapsed=None):
+    def __init__(
+        self, code, output="", elapsed=None, attempts=1, max_allowed_attempts=None
+    ):
         # The result code.
         self.code = code
         # The test output.
@@ -164,6 +166,10 @@ def __init__(self, code, output="", elapsed=None):
         self.metrics = {}
         # The micro-test results reported by this test.
         self.microResults = {}
+        # How often was the test run?
+        self.attempts = attempts
+        # How many attempts were allowed for this test
+        self.max_allowed_attempts = max_allowed_attempts
 
     def addMetric(self, name, value):
         """

llvm/utils/lit/lit/TestRunner.py

@@ -2292,7 +2292,9 @@ def runOnce(
     if err:
         output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
 
-    return lit.Test.Result(status, output)
+    return lit.Test.Result(
+        status, output, attempts=i + 1, max_allowed_attempts=attempts
+    )
 
 
 def executeShTest(

llvm/utils/lit/lit/display.py

@@ -117,9 +117,20 @@ def clear(self, interrupted):
     def print_result(self, test):
         # Show the test result line.
         test_name = test.getFullName()
+
+        extra_info = ""
+        if test.result.attempts > 1:
+            extra_info = f", {test.result.attempts} of {test.result.max_allowed_attempts} attempts"
+
         print(
-            "%s: %s (%d of %d)"
-            % (test.result.code.name, test_name, self.completed, self.num_tests)
+            "%s: %s (%d of %d%s)"
+            % (
+                test.result.code.name,
+                test_name,
+                self.completed,
+                self.num_tests,
+                extra_info,
+            )
         )
 
         # Show the test failure output, if requested.

llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py

@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 8
 # RUN: "%python" "%s" "%counter"
 
 import sys

llvm/utils/lit/tests/Inputs/max-retries-per-test/allow-retries-test_retry_attempts/test.py

@@ -1,4 +1,4 @@
-# ALLOW_RETRIES: 3
+# ALLOW_RETRIES: 10
 # RUN: "%python" "%s" "%counter"
 
 import sys

llvm/utils/lit/tests/Inputs/max-retries-per-test/no-allow-retries-test_retry_attempts/lit.cfg

@@ -9,4 +9,4 @@ config.test_exec_root = None
 config.substitutions.append(("%python", lit_config.params.get("python", "")))
 config.substitutions.append(("%counter", lit_config.params.get("counter", "")))
 
-config.test_retry_attempts = 3
+config.test_retry_attempts = 9

llvm/utils/lit/tests/allow-retries.py

@@ -22,7 +22,7 @@
 # RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py -v |\
 # RUN:   FileCheck --check-prefix=CHECK-TEST3 -match-full-lines %s
 #
-#       CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1)
+#       CHECK-TEST3: FAIL: allow-retries :: does-not-succeed-within-limit.py (1 of 1, 4 of 4 attempts)
 #  CHECK-TEST3-NEXT: {{\**}} TEST 'allow-retries :: does-not-succeed-within-limit.py' FAILED {{\**}}
 #  CHECK-TEST3-NEXT: Exit Code: 1
 # CHECK-TEST3-EMPTY:
@@ -72,49 +72,53 @@
 #     CHECK-TEST7: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
-#  * The "--max-retries-per-test" is specified high enough (3).
+#  * The "--max-retries-per-test" is specified high enough (7).
 #  * No ALLOW_RETRIES keyword is used in the test script.
 #  * No config.test_retry_attempts is adjusted in the test suite config file.
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-no-test_retry_attempts/test.py \
-# RUN:   --max-retries-per-test=3 \
+# RUN:   --max-retries-per-test=7 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST8 %s
+# CHECK-TEST8: FLAKYPASS: no-allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 8 attempts)
 # CHECK-TEST8: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (2).
-#  * ALLOW_RETRIES is specified high enough (3)
+#  * ALLOW_RETRIES is specified high enough (8)
 #  * No config.test_retry_attempts is adjusted in the test suite config file.
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-no-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=2 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST9 %s
+# CHECK-TEST9: FLAKYPASS: allow-retries-no-test_retry_attempts :: test.py (1 of 1, 4 of 9 attempts)
 # CHECK-TEST9: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (2).
 #  * No ALLOW_RETRIES keyword is used in the test script.
-#  * config.test_retry_attempts is set high enough (3).
+#  * config.test_retry_attempts is set high enough (9).
 # RUN: rm -f %t.counter
 # RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=2 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST10 %s
+# CHECK-TEST10: FLAKYPASS: no-allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 10 attempts)
 # CHECK-TEST10: Passed With Retry: 1
 
 # This test only passes on the 4th try. Here we check that a test can be re-run when:
 #  * The "--max-retries-per-test" is specified too low (1).
-#  * ALLOW_RETRIES keyword set high enough (3).
-#  * config.test_retry_attempts is set too low enough (2).
+#  * ALLOW_RETRIES keyword is set high enough (10)
+#  * config.test_retry_attempts is set too low (2).
 # RUN: rm -f %t.counter
-# RUN: %{lit} %{inputs}/max-retries-per-test/no-allow-retries-test_retry_attempts/test.py \
+# RUN: %{lit} %{inputs}/max-retries-per-test/allow-retries-test_retry_attempts/test.py \
 # RUN:   --max-retries-per-test=1 \
 # RUN:   -Dcounter=%t.counter \
 # RUN:   -Dpython=%{python} \
 # RUN: | FileCheck --check-prefix=CHECK-TEST11 %s
+# CHECK-TEST11: FLAKYPASS: allow-retries-test_retry_attempts :: test.py (1 of 1, 4 of 11 attempts)
 # CHECK-TEST11: Passed With Retry: 1