[UpdateTestChecks] Auto-generate stub bodies for unused prefixes

This is scoped to autogenerated tests.

The goal is to support having each RUN line specify a list of
check-prefixes where one can specify potentially redundant prefixes. For example,
for X86, if one specified prefixes for  both AVX1 and AVX2, and the codegen happened to
match today, one of the prefixes would be used and the onther one not.
If the unused prefix were dropped, and later, codegen differences were
introduced, one would have to go figure out where to add what prefix
(paraphrasing
https://lists.llvm.org/pipermail/llvm-dev/2021-February/148326.html)

To avoid getting errors due to unused prefixes, whole directories can be
opted out (as discussed on that thread), but that means that tests that
aren't autogenerated in such directories could have undetected unused
prefix bugs.

This patch proposes an alternative that both avoids the above, dir-level
optout, and supports the main autogen scenario discussed first. The autogen
tool appends at the end of the test file the list of unused prefixes,
together with a note explaining that is the case. Each prefix is set up
to always pass.

This way, unexpected unused prefixes are easily discoverable, and
expected cases "just work".

Differential Revision: https://reviews.llvm.org/D124306

Author: mtrofin

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/amdgpu_no_merge_comments.ll.expected

@@ -28,3 +28,5 @@ define hidden i32 @main(i32 %a) {
   %sub = sub i32 %mul, %add
   ret i32 %sub
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GCN: {{.*}}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/common-label-different-bodies-1-next.ll

@@ -0,0 +1,30 @@
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefixes=A,B
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --allow-unused-prefixes=true --check-prefixes=C,A,UNUSED
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --allow-unused-prefixes=true --check-prefixe=A
+
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+
+define <2 x i64> @fold_v2i64() {
+; B-LABEL: fold_v2i64:
+; B:       # %bb.0: # %entry
+; B-NEXT:    movaps {{.*#+}} xmm0 = [0,4278190080,4294967295,4294967295]
+; B-NEXT:    retl
+;
+; C-LABEL: fold_v2i64:
+; C:       # %bb.0: # %entry
+; C-NEXT:    movaps {{.*#+}} xmm0 = [18374686479671623680,18446744073709551615]
+; C-NEXT:    retq
+entry:
+  %r = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> <i64 255, i64 -1>)
+  ret <2 x i64> %r
+}
+
+declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
+
+define <4 x i32> @test2(<4 x i32> %v) {
+  %r = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %v)
+  ret <4 x i32> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; A: {{.*}}
+; UNUSED: {{.*}}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/common-label-different-bodies-1.ll

@@ -2,7 +2,6 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --allow-unused-prefixes=true --check-prefixes=C,A,UNUSED
 
 declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
-; A: declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
 
 define <2 x i64> @fold_v2i64() {
 entry:

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/common-label-different-bodies-2.ll

@@ -2,7 +2,6 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=C,A
 
 declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
-; A: declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
 
 define <2 x i64> @fold_v2i64() {
 entry:

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/common-label-different-bodies-3.ll

@@ -2,7 +2,6 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=A,C
 
 declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
-; A: declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
 
 define <2 x i64> @fold_v2i64() {
 entry:

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/redundant-and-unmatching-prefixes.ll

@@ -0,0 +1,18 @@
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefixes=A,B,REDUNDANT
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=C,A,UNUSED
+
+; prefix 'A' has conflicting outputs, while the REDUNDANT and UNUSED ones are
+; unused
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+
+define <2 x i64> @function_1() {
+entry:
+  %r = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> <i64 255, i64 -1>)
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @function_2() {
+entry:
+  %r = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> <i64 16, i64 -1>)
+  ret <2 x i64> %r
+}

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/common-label-different-bodies.test

@@ -4,11 +4,29 @@
 # RUN: cp -f %S/Inputs/common-label-different-bodies-2.ll %t-2.ll
 # RUN: cp -f %S/Inputs/common-label-different-bodies-3.ll %t-3.ll
 # RUN: %update_llc_test_checks %t-1.ll
+
+# re-running update_llc_test_checks leaves the file as-is.
+# RUN: cp -f %t-1.ll %t-1-copy.ll
+# RUN: %update_llc_test_checks %t-1.ll
+# RUN: diff %t-1.ll %t-1-copy.ll
 # RUN: %update_llc_test_checks %t-2.ll
 # RUN: %update_llc_test_checks %t-3.ll
-# RUN: FileCheck --input-file=%t-1.ll %s
+# RUN: FileCheck --input-file=%t-1.ll %s --check-prefixes=CHECK,CHECK-UNUSED
 # RUN: FileCheck --input-file=%t-2.ll %s
 # RUN: FileCheck --input-file=%t-3.ll %s
 
 # CHECK: B-LABEL: fold_v2i64
-# CHECK-NOT: A-LABEL: fold_v2i64
+# CHECK-NOT: A-LABEL: fold_v2i64
+# CHECK: NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# CHECK-NEXT: A:
+# CHECK-UNUSED-NEXT: UNUSED:
+
+# adding a test removes that label from "unused" prefixes list
+# the input file is like "1" after the tool was run, and then we added a new test
+# RUN: cp -f %S/Inputs/common-label-different-bodies-1-next.ll %t-1-next.ll
+# RUN: %update_llc_test_checks %t-1-next.ll
+# RUN: FileCheck --input-file=%t-1-next.ll %s --check-prefixes=AFTER-CHANGE
+
+# AFTER-CHANGE: NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# AFTER-CHANGE-NEXT: UNUSED:
+# AFTER-CHANGE-NOT: A:

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/prefix-never-matches.test

@@ -1,8 +1,8 @@
 # REQUIRES: x86-registered-target
 
 # RUN: cp -f %S/Inputs/prefix-never-matches.ll %t.ll
-# RUN: %update_llc_test_checks %t.ll 2>&1 | FileCheck %s
+# RUN: %update_llc_test_checks --no-generate-body-for-unused-prefixes %t.ll 2>&1 | FileCheck %s
 # RUN: FileCheck --input-file=%t.ll %s --check-prefix=OUTPUT
 
 # CHECK: WARNING: Prefix A had conflicting output
-# OUTPUT-NOT: A:
+# OUTPUT-NOT: A:

llvm/test/tools/UpdateTestChecks/update_llc_test_checks/redundant-and-unmatched-prefixes.test

@@ -0,0 +1,18 @@
+# REQUIRES: x86-registered-target
+
+# RUN: cp -f %S/Inputs/redundant-and-unmatching-prefixes.ll %t-1.ll
+# RUN: %update_llc_test_checks %t-1.ll
+# RUN: FileCheck --input-file=%t-1.ll %s
+
+# CHECK: B-LABEL: function_1:
+# CHECK-NOT: A-LABEL: function_1
+# CHECK-NOT: REDUNDANT-LABEL: function_1:
+
+# CHECK: B-LABEL: function_2:
+# CHECK-NOT: A-LABEL: function_2:
+# CHECK-NOT: UNUSED-LABEL: function_2:
+
+# CHECK: NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# CHECK-NEXT: A:
+# CHECK-NEXT: REDUNDANT:
+# CHECK-NEXT: UNUSED:

llvm/utils/UpdateTestChecks/asm.py

@@ -484,6 +484,6 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict,
                func_name, global_vars_seen_dict, is_filtered):
   # Label format is based on ASM string.
   check_label_format = '{} %s-LABEL: %s%s%s'.format(comment_marker)
-  common.add_checks(output_lines, comment_marker, prefix_list, func_dict,
-                    func_name, check_label_format, True, False,
-                    global_vars_seen_dict, is_filtered=is_filtered)
+  return common.add_checks(output_lines, comment_marker, prefix_list, func_dict,
+                           func_name, check_label_format, True, False,
+                           global_vars_seen_dict, is_filtered=is_filtered)

llvm/utils/UpdateTestChecks/common.py

@@ -10,6 +10,8 @@
 import sys
 import shlex
 
+from typing import List
+
 ##### Common utilities for update_*test_checks.py
 
 
@@ -129,6 +131,11 @@ def __call__(self, parser, namespace, values, option_string=None):
                       help='List of regular expressions that a global value declaration must match to generate a check (has no effect if checking globals is not enabled)')
   parser.add_argument('--global-hex-value-regex', nargs='+', default=[],
                       help='List of regular expressions such that, for matching global value declarations, literal integer values should be encoded in hex in the associated FileCheck directives')
+  parser.add_argument('--generate-body-for-unused-prefixes',
+                      action=argparse.BooleanOptionalAction,
+                      dest='gen_unused_prefix_body',
+                      default=True,
+                      help='Generate a function body that always matches for unused prefixes. This is useful when unused prefixes are desired, and it avoids needing to annotate each FileCheck as allowing them.')
   args = parser.parse_args()
   global _verbose, _global_value_regex, _global_hex_value_regex
   _verbose = args.verbose
@@ -167,6 +174,7 @@ def __init__(self, test, parser, script_name, input_lines, args, argv,
     self.autogenerated_note_prefix = self.comment_prefix + ' ' + UTC_ADVERT
     self.test_autogenerated_note = self.autogenerated_note_prefix + script_name
     self.test_autogenerated_note += get_autogennote_suffix(parser, self.args)
+    self.test_unused_note = self.comment_prefix + self.comment_prefix + ' ' + UNUSED_NOTE
 
   def ro_iterlines(self):
     for line_num, input_line in enumerate(self.input_lines):
@@ -188,6 +196,22 @@ def iterlines(self, output_lines):
         continue
       yield line_info
 
+  def get_checks_for_unused_prefixes(self, run_list, used_prefixes: List[str]) -> List[str]:
+    unused_prefixes = set(
+        [prefix for sublist in run_list for prefix in sublist[0]]).difference(set(used_prefixes))
+
+    ret = []
+    if not unused_prefixes:
+      return ret
+    ret.append(self.test_unused_note)
+    for unused in sorted(unused_prefixes):
+      ret.append('{comment} {prefix}: {match_everything}'.format(
+        comment=self.comment_prefix,
+        prefix=unused,
+        match_everything=r"""{{.*}}"""
+      ))
+    return ret
+
 def itertests(test_patterns, parser, script_name, comment_prefix=None, argparse_callback=None):
   for pattern in test_patterns:
     # On Windows we must expand the patterns ourselves.
@@ -212,7 +236,12 @@ def itertests(test_patterns, parser, script_name, comment_prefix=None, argparse_
         assert UTC_ADVERT not in first_line
         warn("Skipping test which isn't autogenerated: " + test)
         continue
-      yield TestInfo(test, parser, script_name, input_lines, args, argv,
+      final_input_lines = []
+      for l in input_lines:
+        if UNUSED_NOTE in l:
+          break
+        final_input_lines.append(l)
+      yield TestInfo(test, parser, script_name, final_input_lines, args, argv,
                      comment_prefix, argparse_callback)
 
 
@@ -293,6 +322,7 @@ def invoke_tool(exe, cmd_args, ir, preprocess_cmd=None, verbose=False):
 UTC_ARGS_KEY = 'UTC_ARGS:'
 UTC_ARGS_CMD = re.compile(r'.*' + UTC_ARGS_KEY + '\s*(?P<cmd>.*)\s*$')
 UTC_ADVERT = 'NOTE: Assertions have been autogenerated by '
+UNUSED_NOTE = 'NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:'
 
 OPT_FUNCTION_RE = re.compile(
     r'^(\s*;\s*Function\sAttrs:\s(?P<attrs>[\w\s]+?))?\s*define\s+(?:internal\s+)?[^@]*@(?P<func>[\w.$-]+?)\s*'
@@ -471,7 +501,7 @@ def __init__(self, run_list, flags, scrubber_args, path):
         self._global_var_dict.update({prefix:dict()})
 
   def finish_and_get_func_dict(self):
-    for prefix in self._get_failed_prefixes():
+    for prefix in self.get_failed_prefixes():
       warn('Prefix %s had conflicting output from different RUN lines for all functions in test %s' % (prefix,self._path,))
     return self._func_dict
 
@@ -577,11 +607,10 @@ def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes, is_
             scrubbed_body, scrubbed_extra, args_and_sig, attrs, func_name_separator)
         self._func_order[prefix].append(func)
 
-  def _get_failed_prefixes(self):
+  def get_failed_prefixes(self):
     # This returns the list of those prefixes that failed to match any function,
     # because there were conflicting bodies produced by different RUN lines, in
-    # all instances of the prefix. Effectively, this prefix is unused and should
-    # be removed.
+    # all instances of the prefix.
     for prefix in self._func_dict:
       if (self._func_dict[prefix] and
           (not [fct for fct in self._func_dict[prefix]
@@ -974,23 +1003,24 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
         if key not in global_vars_seen_before:
           global_vars_seen_dict[checkprefix][key] = global_vars_seen[key]
       break
+  return printed_prefixes
 
 def add_ir_checks(output_lines, comment_marker, prefix_list, func_dict,
                   func_name, preserve_names, function_sig,
                   global_vars_seen_dict, is_filtered):
   # Label format is based on IR string.
   function_def_regex = 'define {{[^@]+}}' if function_sig else ''
   check_label_format = '{} %s-LABEL: {}@%s%s%s'.format(comment_marker, function_def_regex)
-  add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
-             check_label_format, False, preserve_names, global_vars_seen_dict,
-             is_filtered)
+  return add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
+                    check_label_format, False, preserve_names, global_vars_seen_dict,
+                    is_filtered)
 
 def add_analyze_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, is_filtered):
   check_label_format = '{} %s-LABEL: \'%s%s%s\''.format(comment_marker)
   global_vars_seen_dict = {}
-  add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
-             check_label_format, False, True, global_vars_seen_dict,
-             is_filtered)
+  return add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
+                    check_label_format, False, True, global_vars_seen_dict,
+                    is_filtered)
 
 def build_global_values_dictionary(glob_val_dict, raw_tool_output, prefixes):
   for nameless_value in itertools.chain(ir_nameless_values, asm_nameless_values):
@@ -1189,6 +1219,7 @@ def dump_input_lines(output_lines, test_info, prefix_set, comment_string):
 def add_checks_at_end(output_lines, prefix_list, func_order,
                       comment_string, check_generator):
   added = set()
+  generated_prefixes = []
   for prefix in prefix_list:
     prefixes = prefix[0]
     tool_args = prefix[1]
@@ -1212,6 +1243,7 @@ def add_checks_at_end(output_lines, prefix_list, func_order,
         # single prefix before moving on to the next prefix.  So checks
         # are ordered by prefix instead of by function as in "normal"
         # mode.
-        check_generator(output_lines,
+        generated_prefixes.extend(check_generator(output_lines,
                         [([prefix], tool_args)],
-                        func)
+                        func))
+  return generated_prefixes

llvm/utils/UpdateTestChecks/isel.py

@@ -52,6 +52,6 @@ def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
                global_vars_seen_dict, is_filtered):
   # Label format is based on iSel string.
   check_label_format = '{} %s-LABEL: %s%s%s'.format(comment_marker)
-  common.add_checks(output_lines, comment_marker, prefix_list, func_dict,
-                    func_name, check_label_format, True, False,
-                    global_vars_seen_dict, is_filtered = is_filtered)
+  return common.add_checks(output_lines, comment_marker, prefix_list, func_dict,
+                           func_name, check_label_format, True, False,
+                           global_vars_seen_dict, is_filtered=is_filtered)

llvm/utils/update_cc_test_checks.py

@@ -340,17 +340,17 @@ def main():
       # Now generate all the checks.
       def check_generator(my_output_lines, prefixes, func):
         if '-emit-llvm' in clang_args:
-          common.add_ir_checks(my_output_lines, '//',
-                               prefixes,
-                               func_dict, func, False,
-                               ti.args.function_signature,
-                               global_vars_seen_dict,
-                               is_filtered=builder.is_filtered())
+          return common.add_ir_checks(my_output_lines, '//',
+                                      prefixes,
+                                      func_dict, func, False,
+                                      ti.args.function_signature,
+                                      global_vars_seen_dict,
+                                      is_filtered=builder.is_filtered())
         else:
-          asm.add_checks(my_output_lines, '//',
-                             prefixes,
-                             func_dict, func, global_vars_seen_dict,
-                             is_filtered=builder.is_filtered())
+          return asm.add_checks(my_output_lines, '//',
+                                prefixes,
+                                func_dict, func, global_vars_seen_dict,
+                                is_filtered=builder.is_filtered())
 
       if ti.args.check_globals:
         common.add_global_checks(builder.global_var_dict(), '//', run_list,

llvm/utils/update_llc_test_checks.py

@@ -153,6 +153,7 @@ def main():
                                                       '--include-generated-funcs',
                                                       True)
 
+    generated_prefixes = []
     if include_generated_funcs:
       # Generate the appropriate checks for each function.  We need to emit
       # these in the order according to the generated output so that CHECK-LABEL
@@ -164,14 +165,15 @@ def main():
       common.dump_input_lines(output_lines, ti, prefix_set, ';')
 
       # Now generate all the checks.
-      common.add_checks_at_end(output_lines, run_list, builder.func_order(),
-                               check_indent + ';',
-                               lambda my_output_lines, prefixes, func:
-                               output_type.add_checks(my_output_lines,
-                                                  check_indent + ';',
-                                                  prefixes, func_dict, func,
-                                                  global_vars_seen_dict,
-                                                  is_filtered=builder.is_filtered()))
+      generated_prefixes = common.add_checks_at_end(
+          output_lines, run_list, builder.func_order(),
+          check_indent + ';',
+          lambda my_output_lines, prefixes, func:
+          output_type.add_checks(my_output_lines,
+                                 check_indent + ';',
+                                 prefixes, func_dict, func,
+                                 global_vars_seen_dict,
+                                 is_filtered=builder.is_filtered()))
     else:
       for input_info in ti.iterlines(output_lines):
         input_line = input_info.line
@@ -186,9 +188,10 @@ def main():
               continue
 
           # Print out the various check lines here.
-          output_type.add_checks(output_lines, check_indent + ';', run_list,
-                                 func_dict, func_name, global_vars_seen_dict,
-                                 is_filtered=builder.is_filtered())
+          generated_prefixes.extend(
+              output_type.add_checks(output_lines, check_indent + ';', run_list,
+                                     func_dict, func_name, global_vars_seen_dict,
+                                     is_filtered=builder.is_filtered()))
           is_in_function_start = False
 
         if is_in_function:
@@ -213,8 +216,11 @@ def main():
           continue
         is_in_function = is_in_function_start = True
 
+    if ti.args.gen_unused_prefix_body:
+      output_lines.extend(ti.get_checks_for_unused_prefixes(
+          run_list, generated_prefixes))
+    
     common.debug('Writing %d lines to %s...' % (len(output_lines), ti.path))
-
     with open(ti.path, 'wb') as f:
       f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])