Merge pull request #6906 from Michael137/bugfix/lldb-buildbots-fixes

[cherry-pick][stable/20221013] [lldb][cmake] Allow specifying custom libcxx for tests in standalone builds

Author: adrian-prantl

lldb/cmake/modules/LLDBStandalone.cmake

@@ -25,6 +25,9 @@ set(LLVM_MAIN_SRC_DIR ${LLVM_BUILD_MAIN_SRC_DIR} CACHE PATH "Path to LLVM source
 set(LLVM_MAIN_INCLUDE_DIR ${LLVM_MAIN_INCLUDE_DIR} CACHE PATH "Path to llvm/include")
 set(LLVM_BINARY_DIR ${LLVM_BINARY_DIR} CACHE PATH "Path to LLVM build tree")
 
+set(LLDB_TEST_LIBCXX_ROOT_DIR "${LLVM_BINARY_DIR}" CACHE PATH
+    "The build root for libcxx. Used in standalone builds to point the API tests to a custom build of libcxx.")
+
 set(LLVM_LIT_ARGS "-sv" CACHE STRING "Default options for lit")
 
 set(lit_file_name "llvm-lit")

lldb/packages/Python/lldbsuite/test/builders/builder.py

@@ -135,8 +135,12 @@ def getModuleCacheSpec(self):
 
     def getLibCxxArgs(self):
         if configuration.libcxx_include_dir and configuration.libcxx_library_dir:
-            return ["LIBCPP_INCLUDE_DIR={}".format(configuration.libcxx_include_dir),
-                    "LIBCPP_LIBRARY_DIR={}".format(configuration.libcxx_library_dir)]
+            libcpp_args = ["LIBCPP_INCLUDE_DIR={}".format(configuration.libcxx_include_dir),
+                           "LIBCPP_LIBRARY_DIR={}".format(configuration.libcxx_library_dir)]
+            if configuration.libcxx_include_target_dir:
+                libcpp_args.append("LIBCPP_INCLUDE_TARGET_DIR={}".format(
+                    configuration.libcxx_include_target_dir))
+            return libcpp_args
         return []
 
     def getLLDBSwiftLibs(self):

lldb/packages/Python/lldbsuite/test/configuration.py

@@ -131,6 +131,7 @@
 lldb_libs_dir = None
 
 libcxx_include_dir = None
+libcxx_include_target_dir = None
 libcxx_library_dir = None
 
 # A plugin whose tests will be enabled, like intel-pt.

lldb/packages/Python/lldbsuite/test/dotest.py

@@ -280,17 +280,15 @@ def parseOptionsAndInitTestdirs():
         logging.warning('No valid FileCheck executable; some tests may fail...')
         logging.warning('(Double-check the --llvm-tools-dir argument to dotest.py)')
 
-    configuration.libcxx_include_dir = args.libcxx_include_dir
-    configuration.libcxx_library_dir = args.libcxx_library_dir
     if args.libcxx_include_dir or args.libcxx_library_dir:
         if args.lldb_platform_name:
             logging.warning('Custom libc++ is not supported for remote runs: ignoring --libcxx arguments')
-        elif args.libcxx_include_dir and args.libcxx_library_dir:
-            configuration.libcxx_include_dir = args.libcxx_include_dir
-            configuration.libcxx_library_dir = args.libcxx_library_dir
-        else:
+        elif not (args.libcxx_include_dir and args.libcxx_library_dir):
             logging.error('Custom libc++ requires both --libcxx-include-dir and --libcxx-library-dir')
             sys.exit(-1)
+    configuration.libcxx_include_dir = args.libcxx_include_dir
+    configuration.libcxx_include_target_dir = args.libcxx_include_target_dir
+    configuration.libcxx_library_dir = args.libcxx_library_dir
 
     if args.channels:
         lldbtest_config.channels = args.channels

lldb/packages/Python/lldbsuite/test/dotest_args.py

@@ -47,8 +47,12 @@ def create_parser():
     if sys.platform == 'darwin':
         group.add_argument('--apple-sdk', metavar='apple_sdk', dest='apple_sdk', default="", help=textwrap.dedent(
             '''Specify the name of the Apple SDK (macosx, macosx.internal, iphoneos, iphoneos.internal, or path to SDK) and use the appropriate tools from that SDK's toolchain.'''))
-    group.add_argument('--libcxx-include-dir', help=textwrap.dedent('Specify the path to a custom libc++ include directory. Must be used in conjunction with --libcxx-library-dir.'))
-    group.add_argument('--libcxx-library-dir', help=textwrap.dedent('Specify the path to a custom libc++ library directory. Must be used in conjunction with --libcxx-include-dir.'))
+    group.add_argument('--libcxx-include-dir', help=textwrap.dedent(
+        'Specify the path to a custom libc++ include directory. Must be used in conjunction with --libcxx-library-dir.'))
+    group.add_argument('--libcxx-include-target-dir', help=textwrap.dedent(
+        'Specify the path to a custom libc++ include target directory to use in addition to --libcxx-include-dir. Optional.'))
+    group.add_argument('--libcxx-library-dir', help=textwrap.dedent(
+        'Specify the path to a custom libc++ library directory. Must be used in conjunction with --libcxx-include-dir.'))
     # FIXME? This won't work for different extra flags according to each arch.
     group.add_argument(
         '-E',

lldb/packages/Python/lldbsuite/test/make/Makefile.rules

@@ -482,7 +482,10 @@ endif
 ifeq (1,$(USE_LIBCPP))
 	ifneq ($(and $(LIBCPP_INCLUDE_DIR), $(LIBCPP_LIBRARY_DIR)),)
 		CXXFLAGS += -nostdlib++ -nostdinc++ -cxx-isystem $(LIBCPP_INCLUDE_DIR)
-		LDFLAGS += -L$(LLVM_LIBS_DIR) -Wl,-rpath,$(LIBCPP_LIBRARY_DIR) -lc++
+		ifneq "$(LIBCPP_INCLUDE_TARGET_DIR)" ""
+				CXXFLAGS += -cxx-isystem $(LIBCPP_INCLUDE_TARGET_DIR)
+		endif
+		LDFLAGS += -L$(LIBCPP_LIBRARY_DIR) -Wl,-rpath,$(LIBCPP_LIBRARY_DIR) -lc++
 	else
 		ifeq "$(OS)" "Android"
 				# Nothing to do, this is already handled in
@@ -504,7 +507,10 @@ endif
 ifeq ($(or $(USE_LIBSTDCPP), $(USE_LIBCPP), $(USE_SYSTEM_STDLIB)),)
 	ifneq ($(and $(LIBCPP_INCLUDE_DIR), $(LIBCPP_LIBRARY_DIR)),)
 		CXXFLAGS += -nostdlib++ -nostdinc++ -cxx-isystem $(LIBCPP_INCLUDE_DIR)
-		LDFLAGS += -L$(LLVM_LIBS_DIR) -Wl,-rpath,$(LIBCPP_LIBRARY_DIR) -lc++
+		ifneq "$(LIBCPP_INCLUDE_TARGET_DIR)" ""
+				CXXFLAGS += -cxx-isystem $(LIBCPP_INCLUDE_TARGET_DIR)
+		endif
+		LDFLAGS += -L$(LIBCPP_LIBRARY_DIR) -Wl,-rpath,$(LIBCPP_LIBRARY_DIR) -lc++
 	endif
 endif
 

lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/ranges/ref_view/TestDataFormatterLibcxxRangesRefView.py

@@ -28,7 +28,6 @@ def check_foo(self):
                 children=self.check_string_vec_children())
 
     @add_test_categories(["libc++"])
-    @skipIf(bugnumber="rdar://109455906") # The standalone build of LLDB doesn't test against custom libcxx builds
     def test_with_run_command(self):
         """Test that std::ranges::ref_view is formatted correctly when printed.
         """

lldb/test/API/lang/cpp/std-function-step-into-callable/TestStdFunctionStepIntoCallable.py

@@ -14,6 +14,7 @@ class LibCxxFunctionSteppingIntoCallableTestCase(TestBase):
 
     NO_DEBUG_INFO_TESTCASE = True
 
+    @skipIfDarwin
     @add_test_categories(["libc++"])
     def test(self):
         """Test that std::function as defined by libc++ is correctly printed by LLDB"""

lldb/test/API/lit.cfg.py

@@ -181,9 +181,11 @@ def delete_module_cache(path):
 # If we have a just-built libcxx, prefer it over the system one.
 if is_configured('has_libcxx') and config.has_libcxx:
   if platform.system() != 'Windows':
-    if is_configured('llvm_include_dir') and is_configured('llvm_libs_dir'):
-      dotest_cmd += ['--libcxx-include-dir', os.path.join(config.llvm_include_dir, 'c++', 'v1')]
-      dotest_cmd += ['--libcxx-library-dir', config.llvm_libs_dir]
+    if is_configured('libcxx_include_dir') and is_configured('libcxx_libs_dir'):
+      dotest_cmd += ['--libcxx-include-dir', config.libcxx_include_dir]
+      if is_configured('libcxx_include_target_dir'):
+        dotest_cmd += ['--libcxx-include-target-dir', config.libcxx_include_target_dir]
+      dotest_cmd += ['--libcxx-library-dir', config.libcxx_libs_dir]
 
 # Forward ASan-specific environment variables to tests, as a test may load an
 # ASan-ified dylib.

lldb/test/API/lit.site.cfg.py.in

@@ -33,6 +33,9 @@ config.test_compiler = lit_config.substitute('@LLDB_TEST_COMPILER@')
 config.test_swift_compiler = lit_config.substitute('@LLDB_SWIFTC@')
 config.dsymutil = lit_config.substitute('@LLDB_TEST_DSYMUTIL@')
 config.has_libcxx = @LLDB_HAS_LIBCXX@
+config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
+config.libcxx_include_dir = "@LIBCXX_GENERATED_INCLUDE_DIR@"
+config.libcxx_include_target_dir = "@LIBCXX_GENERATED_INCLUDE_TARGET_DIR@"
 # The API tests use their own module caches.
 config.lldb_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_LLDB@", "lldb-api")
 config.clang_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_CLANG@", "lldb-api")

lldb/test/CMakeLists.txt

@@ -122,6 +122,14 @@ if(TARGET clang)
 
   if (TARGET libcxx OR ("libcxx" IN_LIST LLVM_ENABLE_RUNTIMES))
     set(LLDB_HAS_LIBCXX ON)
+    if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
+      set(LIBCXX_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE})
+      set(LIBCXX_GENERATED_INCLUDE_DIR "${LLVM_BINARY_DIR}/include/c++/v1")
+      set(LIBCXX_GENERATED_INCLUDE_TARGET_DIR "${LLVM_BINARY_DIR}/include/${LLVM_DEFAULT_TARGET_TRIPLE}/c++/v1")
+    else()
+      set(LIBCXX_LIBRARY_DIR ${CMAKE_BINARY_DIR}/lib${LIBCXX_LIBDIR_SUFFIX})
+      set(LIBCXX_GENERATED_INCLUDE_DIR "${CMAKE_BINARY_DIR}/include/c++/v1")
+    endif()
     add_lldb_test_dependency(cxx)
   endif()
 
@@ -134,9 +142,18 @@ if(TARGET clang)
     # FIXME: Standalone builds should import the cxx target as well.
     if(LLDB_BUILT_STANDALONE)
       # For now check that the include directory exists.
-      set(cxx_dir "${LLVM_BINARY_DIR}/include/c++")
-      if(NOT EXISTS ${cxx_dir})
-        message(WARNING "LLDB test suite requires libc++ in llvm/projects/libcxx or an existing build symlinked to ${cxx_dir}")
+      set(cxx_dir "${LLDB_TEST_LIBCXX_ROOT_DIR}/include/c++")
+      if(EXISTS ${cxx_dir})
+        # These variables make sure the API tests can run against a custom
+        # build of libcxx even for standalone builds.
+        set(LLDB_HAS_LIBCXX ON)
+        set(LIBCXX_LIBRARY_DIR "${LLDB_TEST_LIBCXX_ROOT_DIR}/lib${LIBCXX_LIBDIR_SUFFIX}")
+        set(LIBCXX_GENERATED_INCLUDE_DIR "${LLDB_TEST_LIBCXX_ROOT_DIR}/include/c++/v1")
+      else()
+        message(FATAL_ERROR
+            "Couldn't find libcxx build in '${LLDB_TEST_LIBCXX_ROOT_DIR}'. To run the "
+            "test-suite for a standalone LLDB build please build libcxx and point "
+            "LLDB_TEST_LIBCXX_ROOT_DIR to it.")
       endif()
     else()
       # We require libcxx for the test suite, so if we aren't building it,

lldb/utils/lldb-dotest/CMakeLists.txt

@@ -9,8 +9,21 @@ set(LLDB_LIBS_DIR "${LLVM_LIBRARY_OUTPUT_INTDIR}")
 
 llvm_canonicalize_cmake_booleans(
   LLDB_BUILD_INTEL_PT
+  LLDB_HAS_LIBCXX
 )
 
+if ("libcxx" IN_LIST LLVM_ENABLE_RUNTIMES)
+  set(LLDB_HAS_LIBCXX ON)
+  if(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR AND NOT APPLE)
+    set(LIBCXX_LIBRARY_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${LLVM_DEFAULT_TARGET_TRIPLE})
+    set(LIBCXX_GENERATED_INCLUDE_DIR "${LLVM_BINARY_DIR}/include/c++/v1")
+    set(LIBCXX_GENERATED_INCLUDE_TARGET_DIR "${LLVM_BINARY_DIR}/include/${LLVM_DEFAULT_TARGET_TRIPLE}/c++/v1")
+  else()
+    set(LIBCXX_LIBRARY_DIR ${CMAKE_BINARY_DIR}/lib${LIBCXX_LIBDIR_SUFFIX})
+    set(LIBCXX_GENERATED_INCLUDE_DIR "${CMAKE_BINARY_DIR}/include/c++/v1")
+  endif()
+endif()
+
 set(LLVM_TOOLS_DIR "${LLVM_TOOLS_BINARY_DIR}")
 set(vars
   LLDB_TEST_COMMON_ARGS
@@ -24,8 +37,13 @@ set(vars
   LLDB_LIBS_DIR
   LLVM_TOOLS_DIR
   LLDB_SWIFTC
+  LIBCXX_LIBRARY_DIR
+  LIBCXX_GENERATED_INCLUDE_DIR
+  LIBCXX_GENERATED_INCLUDE_TARGET_DIR
   )
 
+llvm_canonicalize_cmake_booleans(LLDB_HAS_LIBCXX)
+
 # Generate lldb-dotest Python driver script for each build mode.
 if(LLDB_BUILT_STANDALONE)
   set(config_types ".")

lldb/utils/lldb-dotest/lldb-dotest.in

@@ -15,6 +15,10 @@ lldb_framework_dir = "@LLDB_FRAMEWORK_DIR_CONFIGURED@"
 lldb_libs_dir = "@LLDB_LIBS_DIR_CONFIGURED@"
 llvm_tools_dir = "@LLVM_TOOLS_DIR_CONFIGURED@"
 swift_libs_dir = '@LLDB_SWIFT_LIBS@' 
+has_libcxx = @LLDB_HAS_LIBCXX@
+libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
+libcxx_include_dir = "@LIBCXX_GENERATED_INCLUDE_DIR@"
+libcxx_include_target_dir = "@LIBCXX_GENERATED_INCLUDE_TARGET_DIR@"
 
 if __name__ == '__main__':
     wrapper_args = sys.argv[1:]
@@ -35,6 +39,11 @@ if __name__ == '__main__':
     cmd.extend(['--llvm-tools-dir', llvm_tools_dir])
     if swift_compiler:
         cmd.extend(['--swift-compiler', swift_compiler])
+    if has_libcxx:
+        cmd.extend(['--libcxx-include-dir', libcxx_include_dir])
+        if libcxx_include_target_dir:
+            cmd.extend(['--libcxx-include-target-dir', libcxx_include_target_dir])
+        cmd.extend(['--libcxx-library-dir', libcxx_libs_dir])
     if lldb_framework_dir:
         cmd.extend(['--framework', lldb_framework_dir])
     if lldb_build_intel_pt == "1":