[llvm][llvm-lit] Add option to create unique result file names if results already exist (#112729)

When running a build like:
```
ninja check-clang check-llvm
```
Prior to my changes you ended up with one results file, in this specific case Junit XML:
```
results.xml
```
This would only include the last set of tests lit ran, which were for
llvm. To get around this, many CI systems will run one check target,
move the file away, then run another, somehow propgating the return code
as well.
```
rectode=0
for target in targets:
  ninja target
  retcode=$?
  mv results.xml results-${target}.xml
<report the overall return code>
```
I want to use something like this Buildkite reporting plugin in CI, which needs to have all the results available:
https://buildkite.com/docs/agent/v3/cli-annotate#using-annotations-to-report-test-results

Modifying CI's build scripts for Windows and Linux is a lot of work. So
my changes instead make lit detect an existing result file and modify
the file name to find a new file to write to. Now you will get:
```
results.xml results.<tempfile generated value>.xml
```
This will work for all result file types since I'm doing it in the base
Report class. Now you've got separate files, it's easy to collect them
with `<path>/*.xml`.

Note that the `<tempfile generated value>` is not ordered.

Author: DavidSpickett

llvm/utils/lit/lit/cl_arguments.py

@@ -175,6 +175,15 @@ def parse_args():
         type=lit.reports.TimeTraceReport,
         help="Write Chrome tracing compatible JSON to the specified file",
     )
+    execution_group.add_argument(
+        "--use-unique-output-file-name",
+        help="When enabled, lit will not overwrite existing test report files. "
+        "Instead it will write to a new file named the same as the output file "
+        "name but with an extra part before the file extension. For example "
+        "if results.xml already exists, results.<something>.xml will be written "
+        "to. The <something> is not ordered in any way. [Default: Off]",
+        action="store_true",
+    )
     execution_group.add_argument(
         "--timeout",
         dest="maxIndividualTestTime",
@@ -332,16 +341,21 @@ def parse_args():
     else:
         opts.shard = None
 
-    opts.reports = filter(
-        None,
-        [
-            opts.output,
-            opts.xunit_xml_output,
-            opts.resultdb_output,
-            opts.time_trace_output,
-        ],
+    opts.reports = list(
+        filter(
+            None,
+            [
+                opts.output,
+                opts.xunit_xml_output,
+                opts.resultdb_output,
+                opts.time_trace_output,
+            ],
+        )
     )
 
+    for report in opts.reports:
+        report.use_unique_output_file_name = opts.use_unique_output_file_name
+
     return opts
 
 

llvm/utils/lit/lit/reports.py

@@ -1,7 +1,10 @@
+import abc
 import base64
 import datetime
 import itertools
 import json
+import os
+import tempfile
 
 from xml.sax.saxutils import quoteattr as quo
 
@@ -14,11 +17,34 @@ def by_suite_and_test_path(test):
     return (test.suite.name, id(test.suite), test.path_in_suite)
 
 
-class JsonReport(object):
+class Report(object):
     def __init__(self, output_file):
         self.output_file = output_file
+        # Set by the option parser later.
+        self.use_unique_output_file_name = False
 
     def write_results(self, tests, elapsed):
+        if self.use_unique_output_file_name:
+            filename, ext = os.path.splitext(os.path.basename(self.output_file))
+            fd, _ = tempfile.mkstemp(
+                suffix=ext, prefix=f"{filename}.", dir=os.path.dirname(self.output_file)
+            )
+            report_file = os.fdopen(fd, "w")
+        else:
+            # Overwrite if the results already exist.
+            report_file = open(self.output_file, "w")
+
+        with report_file:
+            self._write_results_to_file(tests, elapsed, report_file)
+
+    @abc.abstractmethod
+    def _write_results_to_file(self, tests, elapsed, file):
+        """Write test results to the file object "file"."""
+        pass
+
+
+class JsonReport(Report):
+    def _write_results_to_file(self, tests, elapsed, file):
         unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
         tests = [t for t in tests if t.result.code not in unexecuted_codes]
         # Construct the data we will write.
@@ -67,9 +93,8 @@ def write_results(self, tests, elapsed):
 
             tests_data.append(test_data)
 
-        with open(self.output_file, "w") as file:
-            json.dump(data, file, indent=2, sort_keys=True)
-            file.write("\n")
+        json.dump(data, file, indent=2, sort_keys=True)
+        file.write("\n")
 
 
 _invalid_xml_chars_dict = {
@@ -88,21 +113,18 @@ def remove_invalid_xml_chars(s):
     return s.translate(_invalid_xml_chars_dict)
 
 
-class XunitReport(object):
-    def __init__(self, output_file):
-        self.output_file = output_file
-        self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+class XunitReport(Report):
+    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    def write_results(self, tests, elapsed):
+    def _write_results_to_file(self, tests, elapsed, file):
         tests.sort(key=by_suite_and_test_path)
         tests_by_suite = itertools.groupby(tests, lambda t: t.suite)
 
-        with open(self.output_file, "w") as file:
-            file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
-            file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
-            for suite, test_iter in tests_by_suite:
-                self._write_testsuite(file, suite, list(test_iter))
-            file.write("</testsuites>\n")
+        file.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+        file.write('<testsuites time="{time:.2f}">\n'.format(time=elapsed))
+        for suite, test_iter in tests_by_suite:
+            self._write_testsuite(file, suite, list(test_iter))
+        file.write("</testsuites>\n")
 
     def _write_testsuite(self, file, suite, tests):
         skipped = 0
@@ -206,11 +228,8 @@ def gen_resultdb_test_entry(
     return test_data
 
 
-class ResultDBReport(object):
-    def __init__(self, output_file):
-        self.output_file = output_file
-
-    def write_results(self, tests, elapsed):
+class ResultDBReport(Report):
+    def _write_results_to_file(self, tests, elapsed, file):
         unexecuted_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED}
         tests = [t for t in tests if t.result.code not in unexecuted_codes]
         data = {}
@@ -249,17 +268,14 @@ def write_results(self, tests, elapsed):
                         )
                     )
 
-        with open(self.output_file, "w") as file:
-            json.dump(data, file, indent=2, sort_keys=True)
-            file.write("\n")
+        json.dump(data, file, indent=2, sort_keys=True)
+        file.write("\n")
 
 
-class TimeTraceReport(object):
-    def __init__(self, output_file):
-        self.output_file = output_file
-        self.skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
+class TimeTraceReport(Report):
+    skipped_codes = {lit.Test.EXCLUDED, lit.Test.SKIPPED, lit.Test.UNSUPPORTED}
 
-    def write_results(self, tests, elapsed):
+    def _write_results_to_file(self, tests, elapsed, file):
         # Find when first test started so we can make start times relative.
         first_start_time = min([t.result.start for t in tests])
         events = [
@@ -270,8 +286,7 @@ def write_results(self, tests, elapsed):
 
         json_data = {"traceEvents": events}
 
-        with open(self.output_file, "w") as time_trace_file:
-            json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
+        json.dump(json_data, time_trace_file, indent=2, sort_keys=True)
 
     def _get_test_event(self, test, first_start_time):
         test_name = test.getFullName()

llvm/utils/lit/tests/unique-output-file.py

@@ -0,0 +1,22 @@
+## Check that lit will not overwrite existing result files when given
+## --use-unique-output-file-name.
+
+## Files are overwritten without the option.
+# RUN: rm -f %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=NEW
+# NEW:      <?xml version="1.0" encoding="UTF-8"?>
+# NEW-NEXT: <testsuites time="{{[0-9.]+}}">
+## (other tests will check the contents of the whole file)
+
+# RUN: rm -f %t.xunit*.xml
+# RUN: echo "test" > %t.xunit.xml
+## Files should not be overwritten with the option.
+# RUN: not %{lit} --xunit-xml-output %t.xunit.xml --use-unique-output-file-name %{inputs}/xunit-output
+# RUN: FileCheck < %t.xunit.xml %s --check-prefix=EXISTING
+# EXISTING: test
+## Results in a new file with some discriminator added.
+# RUN: ls -l %t.xunit*.xml | wc -l | FileCheck %s --check-prefix=NUMFILES
+# NUMFILES: 2
+# RUN: FileCheck < %t.xunit.*.xml %s --check-prefix=NEW