Add support for cross-compiling Swift stdlib with an external sysroot

cmake/modules/Libdispatch.cmake

@@ -72,9 +72,17 @@ foreach(sdk ${DISPATCH_SDKS})
 
   foreach(arch ${ARCHS})
     set(LIBDISPATCH_VARIANT_NAME "libdispatch-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-${arch}")
+    set(SWIFT_LIBDISPATCH_C_FLAGS ${CMAKE_C_FLAGS})
+    set(SWIFT_LIBDISPATCH_CXX_FLAGS ${CMAKE_CXX_FLAGS})
 
     if(sdk MATCHES WINDOWS)
       set(SWIFT_LIBDISPATCH_COMPILER_TRIPLE_CMAKE_ARGS -DCMAKE_C_COMPILER_TARGET=${SWIFT_SDK_WINDOWS_ARCH_${arch}_TRIPLE};-DCMAKE_CXX_COMPILER_TARGET=${SWIFT_SDK_WINDOWS_ARCH_${arch}_TRIPLE})
+    elseif(sdk MATCHES LINUX)
+      set(SWIFT_LIBDISPATCH_COMPILER_CMAKE_ARGS ${SWIFT_LIBDISPATCH_COMPILER_CMAKE_ARGS};-DCMAKE_SYSROOT=${SWIFT_SDK_${sdk}_ARCH_${arch}_PATH})
+      set(SWIFT_LIBDISPATCH_COMPILER_TRIPLE_CMAKE_ARGS -DCMAKE_C_COMPILER_TARGET=${SWIFT_SDK_${sdk}_ARCH_${arch}_TRIPLE};-DCMAKE_CXX_COMPILER_TARGET=${SWIFT_SDK_${sdk}_ARCH_${arch}_TRIPLE})
+      # Use linker used to build Swift
+      set(SWIFT_LIBDISPATCH_C_FLAGS "${SWIFT_LIBDISPATCH_C_FLAGS} -w -fuse-ld=${SWIFT_USE_LINKER}")
+      set(SWIFT_LIBDISPATCH_CXX_FLAGS "${SWIFT_LIBDISPATCH_CXX_FLAGS} -w -fuse-ld=${SWIFT_USE_LINKER}")
     endif()
 
     if("${sdk}" STREQUAL "ANDROID")
@@ -93,8 +101,8 @@ foreach(sdk ${DISPATCH_SDKS})
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
         ${SWIFT_LIBDISPATCH_COMPILER_CMAKE_ARGS}
         ${SWIFT_LIBDISPATCH_COMPILER_TRIPLE_CMAKE_ARGS}
-        -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-        -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+        -DCMAKE_C_FLAGS=${SWIFT_LIBDISPATCH_C_FLAGS}
+        -DCMAKE_CXX_FLAGS=${SWIFT_LIBDISPATCH_CXX_FLAGS}
         -DCMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS}
         -DCMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS}
         -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
@@ -193,8 +201,8 @@ foreach(sdk ${DISPATCH_SDKS})
                             -DCMAKE_AR=${CMAKE_AR}
                             -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                             ${SWIFT_LIBDISPATCH_COMPILER_CMAKE_ARGS}
-                            -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-                            -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+                            -DCMAKE_C_FLAGS=${SWIFT_LIBDISPATCH_C_FLAGS}
+                            -DCMAKE_CXX_FLAGS=${SWIFT_LIBDISPATCH_CXX_FLAGS}
                             -DCMAKE_MAKE_PROGRAM=${CMAKE_MAKE_PROGRAM}
                             -DCMAKE_INSTALL_LIBDIR=lib
                             -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>

cmake/modules/SwiftConfigureSDK.cmake

@@ -416,6 +416,14 @@ macro(configure_sdk_unix name architectures)
         else()
           message(FATAL_ERROR "unknown arch for ${prefix}: ${arch}")
         endif()
+
+        # If we are using an external sysroot, update path and CXX compile flags to point to it
+        if(SWIFT_CROSS_COMPILE_SYSROOTS)
+          set(SWIFT_SDK_${prefix}_ARCH_${arch}_PATH ${SWIFT_CROSS_COMPILE_SYSROOTS})
+
+          # Clear hardcoded --gcc-toolchain to let the external sysroot be used to find libstdc++
+          set(SWIFT_SDK_${prefix}_CXX_OVERLAY_SWIFT_COMPILE_FLAGS "")
+        endif()
       elseif("${prefix}" STREQUAL "FREEBSD")
         if(NOT arch MATCHES "(arm64|x86_64)")
           message(FATAL_ERROR "unsupported arch for FreeBSD: ${arch}")

utils/build-script-impl

@@ -288,6 +288,7 @@ KNOWN_SETTINGS=(
     cross-compile-with-host-tools                 ""                "set to use the clang we build for the host to then build the cross-compile hosts"
     cross-compile-install-prefixes                ""                "semicolon-separated list of install prefixes to use for the cross-compiled hosts. The list expands, so if there are more cross-compile hosts than prefixes, unmatched hosts use the last prefix in the list"
     cross-compile-deps-path                       ""                "path for CMake to look for cross-compiled library dependencies, such as libXML2"
+    cross-compile-sysroots                        ""                "path(s) for CMake to look for cross-compilation sysroots for different platforms or architectures"
     cross-compile-append-host-target-to-destdir   "1"               "turns on appending the host target name of each cross-compiled toolchain to its install-destdir, to keep them separate from the natively-built toolchain"
     skip-merge-lipo-cross-compile-tools           ""                "set to skip running merge-lipo after installing cross-compiled host Swift tools"
     coverage-db                                   ""                "If set, coverage database to use when prioritizing testing"
@@ -1103,7 +1104,7 @@ function false_true() {
 CROSS_COMPILE_HOSTS=($CROSS_COMPILE_HOSTS)
 for t in "${CROSS_COMPILE_HOSTS[@]}"; do
     case ${t} in
-        macosx* | iphone* | appletv* | watch* | linux-armv5 | linux-armv6 | linux-armv7 | android-* | openbsd-* | linux-static-* )
+        macosx* | iphone* | appletv* | watch* | linux-* | android-* | openbsd-* )
             ;;
         *)
             echo "Unknown host to cross-compile for: ${t}"
@@ -1704,6 +1705,16 @@ for host in "${ALL_HOSTS[@]}"; do
                         -DLLVM_TABLEGEN=$(build_directory "${LOCAL_HOST}" llvm)/bin/llvm-tblgen
                         -DSWIFT_INCLUDE_TEST_BINARIES:BOOL=FALSE
                     )
+
+                    # This allows passing a sysroot other than "/" to compile Swift and libraries
+                    # It also assumes the sysroot contains arch libraries matching --stdlib-deployment-targets
+                    if [ ! -z "${CROSS_COMPILE_SYSROOTS}" ]; then
+                        cmake_options=(
+                            "${cmake_options[@]}"
+                            -DSWIFT_CROSS_COMPILE_SYSROOTS:STRING="${CROSS_COMPILE_SYSROOTS}"
+                            -DSWIFT_USE_LINKER:STRING="lld"
+                        )
+                    fi
                 fi
 
                 # Command-line parameters override any autodetection that we

utils/build_swift/build_swift/driver_arguments.py

@@ -668,6 +668,12 @@ def create_argument_parser():
                 'library dependencies of the corelibs and other Swift repos, '
                 'such as the libcurl dependency of FoundationNetworking')
 
+    option('--cross-compile-sysroots', store_path,
+           help='Path(s) to one or more directories that contain C/C++ sysroots for '
+                'different platforms and/or architectures for cross-compiling Swift. '
+                'The sysroots must contain C/C++ libraries that match the '
+                'architectures that are being built for.')
+
     option('--cross-compile-append-host-target-to-destdir', toggle_true,
            default=True,
            help="Append each cross-compilation host target's name as a subdirectory "

utils/build_swift/tests/expected_options.py

@@ -157,6 +157,7 @@
     'coverage_db': None,
     'cross_compile_append_host_target_to_destdir': True,
     'cross_compile_deps_path': None,
+    'cross_compile_sysroots': None,
     'cross_compile_hosts': [],
     'infer_cross_compile_hosts_on_darwin': False,
     'darwin_deployment_version_ios':
@@ -816,6 +817,7 @@ class BuildScriptImplOption(_BaseOption):
     PathOption('--cmake'),
     PathOption('--coverage-db'),
     PathOption('--cross-compile-deps-path'),
+    PathOption('--cross-compile-sysroots'),
     PathOption('--host-cc'),
     PathOption('--host-cxx'),
     PathOption('--host-libtool'),

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -199,6 +199,10 @@ def convert_to_impl_arguments(self):
             impl_args += [
                 "--cross-compile-deps-path=%s" % args.cross_compile_deps_path
             ]
+        if args.cross_compile_sysroots is not None:
+            impl_args += [
+                "--cross-compile-sysroots=%s" % args.cross_compile_sysroots
+            ]
 
         if args.test_paths:
             impl_args += ["--test-paths", " ".join(args.test_paths)]

utils/swift_build_support/swift_build_support/products/product.py

@@ -379,6 +379,8 @@ def get_linux_target_components(self, arch):
     def get_linux_sysroot(self, platform, arch):
         if not self.is_cross_compile_target('{}-{}'.format(platform, arch)):
             return None
+        if self.args.cross_compile_sysroots:
+            return self.args.cross_compile_sysroots
         sysroot_arch, _, abi = self.get_linux_target_components(arch)
         # $ARCH-$PLATFORM-$ABI
         # E.x.: aarch64-linux-gnu
@@ -479,6 +481,11 @@ def common_cross_c_flags(self, platform, arch, include_arch=False):
         if self.is_release():
             cross_flags.append('-fno-stack-protector')
 
+        # Use lld if external sysroot is provided
+        if (self.is_cross_compile_target('{}-{}'.format(platform, arch))
+                and self.args.cross_compile_sysroots):
+            cross_flags.append('-w -fuse-ld=lld')
+
         return self.common_c_flags + cross_flags
 
 

utils/swift_build_support/tests/products/test_llvm_linux_cross_compile.py

@@ -60,11 +60,60 @@ def tearDown(self):
         self.toolchain = None
         self.args = None
 
-    def test_llvm_get_linux_sysroot(self):
+    def test_llvm_get_linux_sysroot_default(self):
+        args = argparse.Namespace(
+            llvm_targets_to_build='X86;ARM;AArch64',
+            llvm_assertions='true',
+            compiler_vendor='none',
+            clang_compiler_version=None,
+            clang_user_visible_version=None,
+            cross_compile_hosts='linux-aarch64',
+            cross_compile_sysroots=None
+        )
+
         llvm = LLVM(
-            args=self.args,
+            args=args,
             toolchain=self.toolchain,
             source_dir='/path/to/src',
             build_dir='/path/to/build')
         expected_arg = '/usr/aarch64-linux-gnu'
         self.assertIn(expected_arg, llvm.get_linux_sysroot("linux", "aarch64"))
+
+    def test_llvm_get_linux_sysroot_external(self):
+        args = argparse.Namespace(
+            llvm_targets_to_build='X86;ARM;AArch64',
+            llvm_assertions='true',
+            compiler_vendor='none',
+            clang_compiler_version=None,
+            clang_user_visible_version=None,
+            cross_compile_hosts='linux-armv7',
+            cross_compile_sysroots='sysroot'
+        )
+
+        llvm = LLVM(
+            args=args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        expected_arg = 'sysroot'
+        self.assertIn(expected_arg, llvm.get_linux_sysroot("linux", "armv7"))
+
+    def test_llvm_common_cross_c_flags_external_sysroot(self):
+        args = argparse.Namespace(
+            llvm_targets_to_build='X86;ARM;AArch64',
+            llvm_assertions='true',
+            compiler_vendor='none',
+            clang_compiler_version=None,
+            build_variant='Debug',
+            clang_user_visible_version=None,
+            cross_compile_hosts='linux-armv7',
+            cross_compile_sysroots='sysroot'
+        )
+
+        llvm = LLVM(
+            args=args,
+            toolchain=self.toolchain,
+            source_dir='/path/to/src',
+            build_dir='/path/to/build')
+        expected_arg = '-w -fuse-ld=lld'
+        self.assertIn(expected_arg, llvm.common_cross_c_flags("linux", "armv7"))