[Utils] Add UTC support for lit's --update-tests

Adds support for invoking the appropriate update_*_test_checks.py script
from lit. Checks the header comment for which script was used to
generate it in the first place, so only test cases that were already
generated are affected.

To support this the interface for test updater functions is expanded to
not only take a ShellCommandResult, but also the Test object. This makes
it easy to get the file path of the current test.

Also adds a --path flag to update_any_test_checks.py as a convenience
to avoid having to manually set the PATH variable.

Author: hnrklssn

clang/test/lit.cfg.py

@@ -362,3 +362,12 @@ def calculate_arch_features(arch_string):
 # possibly be present in system and user configuration files, so disable
 # default configs for the test runs.
 config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    utilspath = os.path.join(config.llvm_src_root, "utils")
+    sys.path.append(utilspath)
+    from update_any_test_checks import utc_lit_plugin
+    lit_config.test_updaters.append(utc_lit_plugin)

clang/utils/UpdateVerifyTests/litplugin.py

@@ -22,7 +22,7 @@ def get_default(prefix):
         prefixes.add(prefix)
     return prefixes
 
-def verify_test_updater(result):
+def verify_test_updater(result, test):
     if not result.stderr:
         return None
     prefixes = get_verify_prefixes(result.command)

llvm/test/lit.cfg.py

@@ -622,3 +622,12 @@ def have_ld64_plugin_support():
 
 if config.has_logf128:
     config.available_features.add("has_logf128")
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    utilspath = os.path.join(config.llvm_src_root, "utils")
+    sys.path.append(utilspath)
+    from update_any_test_checks import utc_lit_plugin
+    lit_config.test_updaters.append(utc_lit_plugin)

llvm/utils/lit/lit/DiffUpdater.py

@@ -21,7 +21,7 @@ def get_source_and_target(a, b):
 def filter_flags(args):
     return [arg for arg in args if not arg.startswith("-")]
 
-def diff_test_updater(result):
+def diff_test_updater(result, test):
     args = filter_flags(result.command.args)
     if len(args) != 3:
         return None

llvm/utils/lit/lit/TestRunner.py

@@ -1046,7 +1046,6 @@ def formatOutput(title, data, limit=None):
 def executeScriptInternal(
     test, litConfig, tmpBase, commands, cwd, debug=True
 ) -> Tuple[str, str, int, Optional[str]]:
-    print("executeScriptInternal")
     cmds = []
     for i, ln in enumerate(commands):
         # Within lit, we try to always add '%dbg(...)' to command lines in order
@@ -1170,7 +1169,7 @@ def executeScriptInternal(
         if litConfig.update_tests:
             for test_updater in litConfig.test_updaters:
                 try:
-                    update_output = test_updater(result)
+                    update_output = test_updater(result, test)
                 except Exception as e:
                     out += f"Exception occurred in test updater: {e}"
                     continue
@@ -1183,7 +1182,6 @@ def executeScriptInternal(
 
 
 def executeScript(test, litConfig, tmpBase, commands, cwd):
-    print("executeScript")
     bashPath = litConfig.getBashPath()
     isWin32CMDEXE = litConfig.isWindows and not bashPath
     script = tmpBase + ".script"

llvm/utils/update_any_test_checks.py

@@ -34,9 +34,9 @@ def find_utc_tool(search_path, utc_name):
     return None
 
 
-def run_utc_tool(utc_name, utc_tool, testname):
+def run_utc_tool(utc_name, utc_tool, testname, environment):
     result = subprocess.run(
-        [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment
     )
     return (result.returncode, result.stdout, result.stderr)
 
@@ -60,6 +60,40 @@ def expand_listfile_args(arg_list):
     return exp_arg_list
 
 
+def utc_lit_plugin(result, test):
+    testname = test.getFilePath()
+    if not testname:
+        return None
+
+    script_name = os.path.abspath(__file__)
+    utc_search_path = os.path.join(os.path.dirname(script_name), os.path.pardir)
+
+    with open(testname, "r") as f:
+        header = f.readline().strip()
+
+    m = RE_ASSERTIONS.search(header)
+    if m is None:
+        return None
+
+    utc_name = m.group(1)
+    utc_tool = find_utc_tool([utc_search_path], utc_name)
+    if not utc_tool:
+        return f"update-utc-tests: {utc_name} not found"
+
+    return_code, stdout, stderr = run_utc_tool(utc_name, utc_tool, testname, test.config.environment)
+
+    stderr = stderr.decode(errors="replace")
+    if return_code != 0:
+        if stderr:
+            return f"update-utc-tests: {utc_name} exited with return code {return_code}\n{stderr.rstrip()}"
+        return f"update-utc-tests: {utc_name} exited with return code {return_code}"
+
+    stdout = stdout.decode(errors="replace")
+    if stdout:
+        return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}"
+    return f"update-utc-tests: updated {testname}"
+
+
 def main():
     from argparse import RawTextHelpFormatter
 
@@ -78,6 +112,11 @@ def main():
         nargs="*",
         help="Additional directories to scan for update_*_test_checks scripts",
     )
+    parser.add_argument(
+        "--path",
+        help="""Additional directories to scan for executables invoked by the update_*_test_checks scripts,
+separated by the platform path separator""",
+    )
     parser.add_argument("tests", nargs="+")
     config = parser.parse_args()
 
@@ -88,6 +127,10 @@ def main():
     script_name = os.path.abspath(__file__)
     utc_search_path.append(os.path.join(os.path.dirname(script_name), os.path.pardir))
 
+    local_env = os.environ.copy()
+    if config.path:
+        local_env["PATH"] = config.path + os.pathsep + local_env["PATH"]
+
     not_autogenerated = []
     utc_tools = {}
     have_error = False
@@ -117,7 +160,7 @@ def main():
                         continue
 
                 future = executor.submit(
-                    run_utc_tool, utc_name, utc_tools[utc_name], testname
+                    run_utc_tool, utc_name, utc_tools[utc_name], testname, local_env
                 )
                 jobs.append((testname, future))