[lldb][test] Add pexpect category for tests that import pexpect (#84860)

Instead of directly annotating pexpect-based tests with
`@skipIfWindows`, we can tag them with a new `pexpect` category. We
still automatically skip windows behavior by adding `pexpect` to the
skip category list if the platform is windows, but also allow
non-Windows users to skip them by configuring cmake with
`-DLLDB_TEST_USER_ARGS=--skip-category=pexpect`

As a prerequisite, remove the restriction that `@add_test_categories`
can only apply to test cases, and we make the test runner look for
categories on both the class and the test method.

Author: rupprecht

lldb/packages/Python/lldbsuite/test/decorators.py

@@ -409,10 +409,6 @@ def add_test_categories(cat):
     cat = test_categories.validate(cat, True)
 
     def impl(func):
-        if isinstance(func, type) and issubclass(func, unittest.TestCase):
-            raise Exception(
-                "@add_test_categories can only be used to decorate a test method"
-            )
         try:
             if hasattr(func, "categories"):
                 cat.extend(func.categories)

lldb/packages/Python/lldbsuite/test/dotest.py

@@ -914,6 +914,18 @@ def checkForkVForkSupport():
         configuration.skip_categories.append("fork")
 
 
+def checkPexpectSupport():
+    from lldbsuite.test import lldbplatformutil
+
+    platform = lldbplatformutil.getPlatform()
+
+    # llvm.org/pr22274: need a pexpect replacement for windows
+    if platform in ["windows"]:
+        if configuration.verbose:
+            print("pexpect tests will be skipped because of unsupported platform")
+        configuration.skip_categories.append("pexpect")
+
+
 def run_suite():
     # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
     # does not exist before proceeding to running the test suite.
@@ -1013,6 +1025,7 @@ def run_suite():
     checkDebugServerSupport()
     checkObjcSupport()
     checkForkVForkSupport()
+    checkPexpectSupport()
 
     skipped_categories_list = ", ".join(configuration.skip_categories)
     print(

lldb/packages/Python/lldbsuite/test/lldbpexpect.py

@@ -10,7 +10,7 @@
 
 
 @skipIfRemote
-@skipIfWindows  # llvm.org/pr22274: need a pexpect replacement for windows
+@add_test_categories(["pexpect"])
 class PExpectTest(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
     PROMPT = "(lldb) "

lldb/packages/Python/lldbsuite/test/test_categories.py

@@ -33,6 +33,7 @@
     "lldb-server": "Tests related to lldb-server",
     "lldb-dap": "Tests for the Debug Adaptor Protocol with lldb-dap",
     "llgs": "Tests for the gdb-server functionality of lldb-server",
+    "pexpect": "Tests requiring the pexpect library to be available",
     "objc": "Tests related to the Objective-C programming language support",
     "pyapi": "Tests related to the Python API",
     "std-module": "Tests related to importing the std module",

lldb/packages/Python/lldbsuite/test/test_result.py

@@ -148,9 +148,11 @@ def getCategoriesForTest(self, test):
         Gets all the categories for the currently running test method in test case
         """
         test_categories = []
+        test_categories.extend(getattr(test, "categories", []))
+
         test_method = getattr(test, test._testMethodName)
-        if test_method is not None and hasattr(test_method, "categories"):
-            test_categories.extend(test_method.categories)
+        if test_method is not None:
+            test_categories.extend(getattr(test_method, "categories", []))
 
         test_categories.extend(self._getFileBasedCategories(test))
 

lldb/test/API/benchmarks/expression/TestExpressionCmd.py

@@ -17,10 +17,7 @@ def setUp(self):
         self.count = 25
 
     @benchmarks_test
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows",
-    )
+    @add_test_categories(["pexpect"])
     def test_expr_cmd(self):
         """Test lldb's expression commands and collect statistics."""
         self.build()

lldb/test/API/benchmarks/expression/TestRepeatedExprs.py

@@ -19,10 +19,7 @@ def setUp(self):
         self.count = 100
 
     @benchmarks_test
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows",
-    )
+    @add_test_categories(["pexpect"])
     def test_compare_lldb_to_gdb(self):
         """Test repeated expressions with lldb vs. gdb."""
         self.build()

lldb/test/API/benchmarks/frame_variable/TestFrameVariableResponse.py

@@ -17,10 +17,7 @@ def setUp(self):
 
     @benchmarks_test
     @no_debug_info_test
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows",
-    )
+    @add_test_categories(["pexpect"])
     def test_startup_delay(self):
         """Test response time for the 'frame variable' command."""
         print()

lldb/test/API/benchmarks/startup/TestStartupDelays.py

@@ -22,10 +22,7 @@ def setUp(self):
 
     @benchmarks_test
     @no_debug_info_test
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows",
-    )
+    @add_test_categories(["pexpect"])
     def test_startup_delay(self):
         """Test start up delays creating a target, setting a breakpoint, and run to breakpoint stop."""
         print()

lldb/test/API/benchmarks/stepping/TestSteppingSpeed.py

@@ -22,10 +22,7 @@ def setUp(self):
 
     @benchmarks_test
     @no_debug_info_test
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows",
-    )
+    @add_test_categories(["pexpect"])
     def test_run_lldb_steppings(self):
         """Test lldb steppings on a large executable."""
         print()

lldb/test/API/benchmarks/turnaround/TestCompileRunToBreakpointTurnaround.py

@@ -21,10 +21,7 @@ def setUp(self):
 
     @benchmarks_test
     @no_debug_info_test
-    @expectedFailureAll(
-        oslist=["windows"],
-        bugnumber="llvm.org/pr22274: need a pexpect replacement for windows",
-    )
+    @add_test_categories(["pexpect"])
     def test_run_lldb_then_gdb(self):
         """Benchmark turnaround time with lldb vs. gdb."""
         print()

lldb/test/API/terminal/TestSTTYBeforeAndAfter.py

@@ -19,7 +19,7 @@ def classCleanup(cls):
         cls.RemoveTempFile("child_send2.txt")
         cls.RemoveTempFile("child_read2.txt")
 
-    @skipIfWindows  # llvm.org/pr22274: need a pexpect replacement for windows
+    @add_test_categories(["pexpect"])
     @no_debug_info_test
     def test_stty_dash_a_before_and_afetr_invoking_lldb_command(self):
         """Test that 'stty -a' displays the same output before and after running the lldb command."""