[lldb][test][NFC] Add option to exclude third_party packages (#83191)

The goal here is to remove the third_party/Python/module tree, which
LLDB tests only use to `import pexpect`. This package is available on
`pip`, and I believe should not be hard to obtain.

However, in case it isn't easily available, deleting the tree right now
could cause disruption. This introduces a
`LLDB_TEST_USE_VENDOR_PACKAGES` cmake param that can be enabled, and the
tests will continue loading that tree. By default, it is enabled,
meaning there's really no change here. A followup change will disable it
by default once all known build bots are updated to include this
package. When disabled, an eager cmake check runs that makes sure
`pexpect` is available before waiting for the test to fail in an obscure
way.

Later, this option will go away, and when it does, we can delete the
tree too. Ideally this is not disruptive, and we can remove it in a week
or two.

Author: rupprecht

lldb/cmake/modules/LLDBConfig.cmake

@@ -67,6 +67,8 @@ option(LLDB_SKIP_STRIP "Whether to skip stripping of binaries when installing ll
 option(LLDB_SKIP_DSYM "Whether to skip generating a dSYM when installing lldb." OFF)
 option(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS
   "Fail to configure if certain requirements are not met for testing." OFF)
+option(LLDB_TEST_USE_VENDOR_PACKAGES
+  "Use packages from lldb/third_party/Python/module instead of system deps." ON)
 
 set(LLDB_GLOBAL_INIT_DIRECTORY "" CACHE STRING
   "Path to the global lldbinit directory. Relative paths are resolved relative to the

lldb/test/API/lit.cfg.py

@@ -309,3 +309,6 @@ def delete_module_cache(path):
 # Propagate XDG_CACHE_HOME
 if "XDG_CACHE_HOME" in os.environ:
     config.environment["XDG_CACHE_HOME"] = os.environ["XDG_CACHE_HOME"]
+
+if is_configured("use_vendor_packages"):
+    config.environment["LLDB_TEST_USE_VENDOR_PACKAGES"] = "1"

lldb/test/API/lit.site.cfg.py.in

@@ -38,6 +38,7 @@ config.libcxx_include_target_dir = "@LIBCXX_GENERATED_INCLUDE_TARGET_DIR@"
 # The API tests use their own module caches.
 config.lldb_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_LLDB@", "lldb-api")
 config.clang_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_CLANG@", "lldb-api")
+config.use_vendor_packages = @LLDB_TEST_USE_VENDOR_PACKAGES@
 
 # Plugins
 lldb_build_intel_pt = '@LLDB_BUILD_INTEL_PT@'

lldb/test/CMakeLists.txt

@@ -26,6 +26,20 @@ if(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS)
   endforeach()
 endif()
 
+# The "pexpect" package should come from the system environment, not from the
+# LLDB tree. However, we delay the deletion of it from the tree in case
+# users/buildbots don't have the package yet and need some time to install it.
+if (NOT LLDB_TEST_USE_VENDOR_PACKAGES)
+  lldb_find_python_module(pexpect)
+  if (NOT PY_pexpect_FOUND)
+    message(FATAL_ERROR
+      "Python module 'pexpect' not found. Please install it via pip or via "
+      "your operating system's package manager. For a temporary workaround, "
+      "use a version from the LLDB tree with "
+      "`LLDB_TEST_USE_VENDOR_PACKAGES=ON`")
+  endif()
+endif()
+
 if(LLDB_BUILT_STANDALONE)
   # In order to run check-lldb-* we need the correct map_config directives in
   # llvm-lit. Because this is a standalone build, LLVM doesn't know about LLDB,
@@ -240,7 +254,8 @@ llvm_canonicalize_cmake_booleans(
   LLDB_HAS_LIBCXX
   LLDB_TOOL_LLDB_SERVER_BUILD
   LLDB_USE_SYSTEM_DEBUGSERVER
-  LLDB_IS_64_BITS)
+  LLDB_IS_64_BITS
+  LLDB_TEST_USE_VENDOR_PACKAGES)
 
 # Configure the individual test suites.
 add_subdirectory(API)

lldb/use_lldb_suite_root.py

@@ -21,5 +21,7 @@ def add_lldbsuite_packages_dir(lldb_root):
 
 lldb_root = os.path.dirname(inspect.getfile(inspect.currentframe()))
 
-add_third_party_module_dirs(lldb_root)
+# Use environment variables to avoid plumbing flags, lit configs, etc.
+if os.getenv("LLDB_TEST_USE_VENDOR_PACKAGES"):
+    add_third_party_module_dirs(lldb_root)
 add_lldbsuite_packages_dir(lldb_root)

lldb/utils/lldb-dotest/CMakeLists.txt

@@ -10,6 +10,7 @@ set(LLDB_LIBS_DIR "${LLVM_LIBRARY_OUTPUT_INTDIR}")
 llvm_canonicalize_cmake_booleans(
   LLDB_BUILD_INTEL_PT
   LLDB_HAS_LIBCXX
+  LLDB_TEST_USE_VENDOR_PACKAGES
 )
 
 if ("libcxx" IN_LIST LLVM_ENABLE_RUNTIMES)

lldb/utils/lldb-dotest/lldb-dotest.in

@@ -1,4 +1,5 @@
 #!@Python3_EXECUTABLE@
+import os
 import subprocess
 import sys
 
@@ -17,8 +18,12 @@ has_libcxx = @LLDB_HAS_LIBCXX@
 libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
 libcxx_include_dir = "@LIBCXX_GENERATED_INCLUDE_DIR@"
 libcxx_include_target_dir = "@LIBCXX_GENERATED_INCLUDE_TARGET_DIR@"
+use_vendor_packages = @LLDB_TEST_USE_VENDOR_PACKAGES@
 
 if __name__ == '__main__':
+    if use_vendor_packages:
+        os.putenv("LLDB_TEST_USE_VENDOR_PACKAGES", "1")
+
     wrapper_args = sys.argv[1:]
     dotest_args = []
     # split on an empty string will produce [''] and if you