Merge commit '5b55eb1e8884' from llvm.org/main into next

git apple-llvm automerger · git apple-llvm automerger · commit 78f2e02878de · 2023-06-27T16:27:08.000-04:00
diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py
@@ -182,7 +182,7 @@ def parseScript(test, preamble):
     return script
 
 
-class CxxStandardLibraryTest(lit.formats.TestFormat):
+class CxxStandardLibraryTest(lit.formats.FileBasedTest):
     """
     Lit test format for the C++ Standard Library conformance test suite.
 
@@ -278,7 +278,7 @@ class CxxStandardLibraryTest(lit.formats.TestFormat):
             in conjunction with the %{build} substitution.
     """
 
-    def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
+    def getTestsForPath(self, testSuite, pathInSuite, litConfig, localConfig):
         SUPPORTED_SUFFIXES = [
             "[.]pass[.]cpp$",
             "[.]pass[.]mm$",
@@ -293,22 +293,22 @@ def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
             "[.]verify[.]cpp$",
             "[.]fail[.]cpp$",
         ]
+
         sourcePath = testSuite.getSourcePath(pathInSuite)
-        for filename in os.listdir(sourcePath):
-            # Ignore dot files and excluded tests.
-            if filename.startswith(".") or filename in localConfig.excludes:
-                continue
-
-            filepath = os.path.join(sourcePath, filename)
-            if not os.path.isdir(filepath):
-                if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
-                    # If this is a generated test, run the generation step and add
-                    # as many Lit tests as necessary.
-                    if re.search('[.]gen[.][^.]+$', filename):
-                        for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig):
-                            yield test
-                    else:
-                        yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
+        filename = os.path.basename(sourcePath)
+
+        # Ignore dot files, excluded tests and tests with an unsupported suffix
+        hasSupportedSuffix = lambda f: any([re.search(ext, f) for ext in SUPPORTED_SUFFIXES])
+        if filename.startswith(".") or filename in localConfig.excludes or not hasSupportedSuffix(filename):
+            return
+
+        # If this is a generated test, run the generation step and add
+        # as many Lit tests as necessary.
+        if re.search('[.]gen[.][^.]+$', filename):
+            for test in self._generateGenTest(testSuite, pathInSuite, litConfig, localConfig):
+                yield test
+        else:
+            yield lit.Test.Test(testSuite, pathInSuite, localConfig)
 
     def execute(self, test, litConfig):
         VERIFY_FLAGS = (
diff --git a/llvm/utils/lit/examples/many-tests/ManyTests.py b/llvm/utils/lit/examples/many-tests/ManyTests.py
@@ -1,7 +1,7 @@
-from lit import Test
+from lit import Test, TestFormat
 
 
-class ManyTests(object):
+class ManyTests(TestFormat):
     def __init__(self, N=10000):
         self.N = N
 
diff --git a/llvm/utils/lit/lit/discovery.py b/llvm/utils/lit/lit/discovery.py
@@ -163,35 +163,43 @@ def getTestsInSuite(
     if not os.path.isdir(source_path):
         test_dir_in_suite = path_in_suite[:-1]
         lc = getLocalConfig(ts, test_dir_in_suite, litConfig, localConfigCache)
-        test = Test.Test(ts, path_in_suite, lc)
-
-        # Issue a error if the specified test would not be run if
-        # the user had specified the containing directory instead of
-        # of naming the test directly. This helps to avoid writing
-        # tests which are not executed. The check adds some performance
-        # overhead which might be important if a large number of tests
-        # are being run directly.
-        # This check can be disabled by using --no-indirectly-run-check or
-        # setting the standalone_tests variable in the suite's configuration.
-        if (
-            indirectlyRunCheck
-            and lc.test_format is not None
-            and not lc.standalone_tests
-        ):
-            found = False
-            for res in lc.test_format.getTestsInDirectory(
-                ts, test_dir_in_suite, litConfig, lc
+
+        # TODO: Stop checking for indirectlyRunCheck and lc.standalone_tests here
+        #       once we remove --no-indirectly-run-check, which is not needed anymore
+        #       now that we error out when trying to run a test that wouldn't be
+        #       discovered in the directory.
+        fallbackOnSingleTest = lc.test_format is None or not indirectlyRunCheck or lc.standalone_tests
+        tests = [Test.Test(ts, path_in_suite, lc)] if fallbackOnSingleTest else \
+                lc.test_format.getTestsForPath(ts, path_in_suite, litConfig, lc)
+
+        for test in tests:
+            # Issue a error if the specified test would not be run if
+            # the user had specified the containing directory instead of
+            # of naming the test directly. This helps to avoid writing
+            # tests which are not executed. The check adds some performance
+            # overhead which might be important if a large number of tests
+            # are being run directly.
+            # This check can be disabled by using --no-indirectly-run-check or
+            # setting the standalone_tests variable in the suite's configuration.
+            if (
+                indirectlyRunCheck
+                and lc.test_format is not None
+                and not lc.standalone_tests
             ):
-                if test.getFullName() == res.getFullName():
-                    found = True
-                    break
-            if not found:
-                litConfig.error(
-                    "%r would not be run indirectly: change name or LIT config"
-                    "(e.g. suffixes or standalone_tests variables)" % test.getFullName()
-                )
-
-        yield test
+                found = False
+                for res in lc.test_format.getTestsInDirectory(
+                    ts, test_dir_in_suite, litConfig, lc
+                ):
+                    if test.getFullName() == res.getFullName():
+                        found = True
+                        break
+                if not found:
+                    litConfig.error(
+                        "%r would not be run indirectly: change name or LIT config"
+                        "(e.g. suffixes or standalone_tests variables)" % test.getFullName()
+                    )
+
+            yield test
         return
 
     # Otherwise we have a directory to search for tests, start by getting the
diff --git a/llvm/utils/lit/lit/formats/base.py b/llvm/utils/lit/lit/formats/base.py
@@ -6,27 +6,42 @@
 
 
 class TestFormat(object):
-    pass
-
+    def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+        """
+        Given the path to a test in the test suite, generates the Lit tests associated
+        to that path. There can be zero, one or more tests. For example, some testing
+        formats allow expanding a single path in the test suite into multiple Lit tests
+        (e.g. they are generated on the fly).
+        """
+        yield lit.Test.Test(testSuite, path_in_suite, localConfig)
 
 ###
 
 
 class FileBasedTest(TestFormat):
+    def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+        """
+        Expand each path in a test suite to a Lit test using that path and assuming
+        it is a file containing the test. File extensions excluded by the configuration
+        or not contained in the allowed extensions are ignored.
+        """
+        filename = path_in_suite[-1]
+
+        # Ignore dot files and excluded tests.
+        if filename.startswith(".") or filename in localConfig.excludes:
+            return
+
+        base, ext = os.path.splitext(filename)
+        if ext in localConfig.suffixes:
+            yield lit.Test.Test(testSuite, path_in_suite, localConfig)
+
     def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
         source_path = testSuite.getSourcePath(path_in_suite)
         for filename in os.listdir(source_path):
-            # Ignore dot files and excluded tests.
-            if filename.startswith(".") or filename in localConfig.excludes:
-                continue
-
             filepath = os.path.join(source_path, filename)
             if not os.path.isdir(filepath):
-                base, ext = os.path.splitext(filename)
-                if ext in localConfig.suffixes:
-                    yield lit.Test.Test(
-                        testSuite, path_in_suite + (filename,), localConfig
-                    )
+                for t in self.getTestsForPath(testSuite, path_in_suite + (filename,), litConfig, localConfig):
+                    yield t
 
 
 ###
diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/custom_format.py
@@ -0,0 +1,12 @@
+import os
+import lit.formats
+
+class CustomFormat(lit.formats.ShTest):
+    def getTestsForPath(self, testSuite, path_in_suite, litConfig, localConfig):
+        for sub in ['one.test', 'two.test']:
+            basePath = os.path.dirname(testSuite.getExecPath(path_in_suite))
+            os.makedirs(basePath, exist_ok=True)
+            generatedFile = os.path.join(basePath, sub)
+            with open(generatedFile, 'w') as f:
+                f.write('RUN: true')
+            yield lit.Test.Test(testSuite, (generatedFile, ), localConfig)
diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/lit.cfg
@@ -0,0 +1,9 @@
+import sys, os
+sys.path.append(os.path.dirname(__file__))
+from custom_format import CustomFormat
+
+config.name = "discovery-getTestsForPath-suite"
+config.suffixes = [".test"]
+config.test_format = CustomFormat()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test b/llvm/utils/lit/tests/Inputs/discovery-getTestsForPath/x.test
diff --git a/llvm/utils/lit/tests/discovery.py b/llvm/utils/lit/tests/discovery.py
@@ -134,14 +134,15 @@
 # CHECK-ASEXEC-DIRECT-TEST: -- Available Tests --
 # CHECK-ASEXEC-DIRECT-TEST: top-level-suite :: subdir/test-three
 
-# Check an error is emitted when the directly named test would not be run
-# indirectly (e.g. when the directory containing the test is specified).
+# Check that an error is emitted when the directly named test does not satisfy
+# the test config's requirements.
 #
 # RUN: not %{lit} \
 # RUN:     %{inputs}/discovery/test.not-txt 2>%t.err
-# RUN: FileCheck --check-prefix=CHECK-ERROR-INDIRECT-RUN-CHECK < %t.err %s
+# RUN: FileCheck --check-prefix=CHECK-ERROR-INPUT-CONTAINED-NO-TESTS < %t.err %s
 #
-# CHECK-ERROR-INDIRECT-RUN-CHECK: error: 'top-level-suite :: test.not-txt' would not be run indirectly
+# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: warning: input 'Inputs/discovery/test.not-txt' contained no tests
+# CHECK-ERROR-INPUT-CONTAINED-NO-TESTS: error: did not discover any tests for provided path(s)
 
 # Check that no error is emitted with --no-indirectly-run-check.
 #
@@ -178,6 +179,15 @@
 #
 # CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
 
+# Check that a single file path can result in multiple tests being discovered if
+# the test format implements those semantics.
+#
+# RUN: %{lit} %{inputs}/discovery-getTestsForPath/x.test > %t.out
+# RUN: FileCheck --check-prefix=CHECK-getTestsForPath < %t.out %s
+#
+# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}one.test
+# CHECK-getTestsForPath: PASS: discovery-getTestsForPath-suite :: {{.+}}two.test
+
 # Check that we don't recurse infinitely when loading an site specific test
 # suite located inside the test source root.
 #
