[build] Add the flags to enable cross-compiling the corelibs

Pass the Swift and CMake flags needed to cross-compile Foundation and so on, with
the first example of Android. Add a new flag, --cross-compile-deps-path, which is
used to search for cross-compiled libraries, like libcurl, that the corelibs
depend on. Also add a new flag, --common-swift-flags, to pass additional Swift
flags to the corelibs.

Author: finagolfin

CMakeLists.txt

@@ -461,7 +461,9 @@ endif()
 
 set(SWIFT_BUILD_HOST_DISPATCH FALSE)
 if(SWIFT_ENABLE_DISPATCH AND NOT CMAKE_SYSTEM_NAME STREQUAL Darwin)
-  if(SWIFT_BUILD_SYNTAXPARSERLIB OR SWIFT_BUILD_SOURCEKIT)
+  # Only build libdispatch for the host if the host tools are being built and
+  # specifically if these two libraries that depend on it are built.
+  if(SWIFT_INCLUDE_TOOLS AND (SWIFT_BUILD_SYNTAXPARSERLIB OR SWIFT_BUILD_SOURCEKIT))
     set(SWIFT_BUILD_HOST_DISPATCH TRUE)
   endif()
 

utils/build-script

@@ -836,6 +836,7 @@ class BuildScriptInvocation(object):
         """
 
         args = self.args
+        args.build_root = self.workspace.build_root
 
         options = {}
         for host_target in [args.host_target] + args.cross_compile_hosts:
@@ -857,6 +858,8 @@ class BuildScriptInvocation(object):
                     config.swift_benchmark_run_targets),
                 "SWIFT_TEST_TARGETS": " ".join(
                     config.swift_test_run_targets),
+                "SWIFT_FLAGS": config.swift_flags,
+                "SWIFT_TARGET_CMAKE_OPTIONS": config.cmake_options,
             }
 
         return options

utils/build-script-impl

@@ -137,7 +137,6 @@ KNOWN_SETTINGS=(
     common-cmake-options                          ""                "CMake options used for all targets, including LLVM/Clang"
     extra-cmake-options                           ""                "Extra options to pass to CMake for all targets"
     ninja-cmake-options                           ""                "CMake options used for all ninja targets"
-    ninja-cmake-options                           ""                "CMake options used for all ninja targets"
 
     ## Build ...
     build-llvm                                    "1"               "set to 1 to build LLVM and Clang"
@@ -203,6 +202,7 @@ KNOWN_SETTINGS=(
     swift-stdlib-os-versioning                    "1"               "whether to build stdlib with availability based on OS versions (Darwin only)"
     swift-stdlib-stable-abi                       ""                "should stdlib be built with stable ABI, if not set defaults to true on Darwin, false otherwise"
     swift-disable-dead-stripping                  "0"               "turns off Darwin-specific dead stripping for Swift host tools"
+    common-swift-flags                            ""                "Flags used for Swift targets other than the stdlib, like the corelibs"
 
     ## FREESTANDING Stdlib Options
     swift-freestanding-sdk                      ""                "which SDK to use when building the FREESTANDING stdlib"
@@ -227,6 +227,7 @@ KNOWN_SETTINGS=(
     cross-compile-hosts                           ""                "space-separated list of targets to cross-compile host Swift tools for"
     cross-compile-with-host-tools                 ""                "set to use the clang we build for the host to then build the cross-compile hosts"
     cross-compile-install-prefixes                ""                "semicolon-separated list of install prefixes to use for the cross-compiled hosts. The list expands, so if there are more cross-compile hosts than prefixes, unmatched hosts use the last prefix in the list"
+    cross-compile-deps-path                       ""                "path for CMake to look for cross-compiled library dependencies, such as libXML2"
     skip-merge-lipo-cross-compile-tools           ""                "set to skip running merge-lipo after installing cross-compiled host Swift tools"
     coverage-db                                   ""                "If set, coverage database to use when prioritizing testing"
     skip-local-host-install                       ""                "If we are cross-compiling multiple targets, skip an install pass locally if the hosts match"
@@ -1239,6 +1240,8 @@ function calculate_targets_for_host() {
     SWIFT_BENCHMARK_TARGETS=($(get_host_specific_variable ${host} SWIFT_BENCHMARK_TARGETS))
     SWIFT_RUN_BENCHMARK_TARGETS=($(get_host_specific_variable ${host} SWIFT_RUN_BENCHMARK_TARGETS))
     SWIFT_TEST_TARGETS=($(get_host_specific_variable ${host} SWIFT_TEST_TARGETS))
+    SWIFT_FLAGS=($(get_host_specific_variable ${host} SWIFT_FLAGS))
+    SWIFT_TARGET_CMAKE_OPTIONS=($(get_host_specific_variable ${host} SWIFT_TARGET_CMAKE_OPTIONS))
 }
 
 
@@ -1380,6 +1383,14 @@ function swift_c_flags() {
     fi
 }
 
+function common_swift_flags() {
+    if [ "${module_cache}" == "" ] ; then
+        echo "error: a module cache path has not been set"
+        exit 1
+    fi
+    echo -n "${SWIFT_FLAGS[@]} ${COMMON_SWIFT_FLAGS} -module-cache-path \"${module_cache}\" "
+}
+
 function cmake_config_opt() {
     product=$1
     if [[ "${CMAKE_GENERATOR}" == "Xcode" ]] ; then
@@ -1685,6 +1696,9 @@ for host in "${ALL_HOSTS[@]}"; do
                   -DCMAKE_BUILD_TYPE:STRING="${CMARK_BUILD_TYPE}"
                   "${cmark_cmake_options[@]}"
                 )
+                if [[ $(is_cross_tools_host ${host}) ]] ; then
+                    cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
+                fi
                 build_targets=(all)
                 ;;
 
@@ -1812,6 +1826,7 @@ for host in "${ALL_HOSTS[@]}"; do
                         -DCLANG_TABLEGEN=$(build_directory "${LOCAL_HOST}" llvm)/bin/clang-tblgen
                         -DLLVM_NATIVE_BUILD=$(build_directory "${LOCAL_HOST}" llvm)
                     )
+                    cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
                 fi
 
                 ;;
@@ -2222,7 +2237,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DCMAKE_CXX_COMPILER:PATH="${CLANG_BIN}/clang++"
                     -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
                     -DCMAKE_Swift_COMPILER:PATH=${SWIFTC_BIN}
-                    -DCMAKE_Swift_FLAGS:STRING="-module-cache-path \"${module_cache}\""
+                    -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)"
 
                     -DLLBUILD_ENABLE_ASSERTIONS:BOOL=$(true_false "${LLBUILD_ENABLE_ASSERTIONS}")
                     -DLLBUILD_SUPPORT_BINDINGS:=Swift
@@ -2238,6 +2253,22 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DFoundation_DIR:PATH=$(build_directory ${host} foundation)/cmake/modules
                 )
 
+                if [[ $(is_cross_tools_host ${host}) ]] ; then
+                    cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
+
+                    # llbuild looks for the SQlite3 library using find_package(),
+                    # so search for it in CROSS_COMPILE_DEPS_PATH using the CMake
+                    # process for doing so and don't use cross-compiled
+                    # executables, see the linked CMake docs for more info:
+                    #
+                    # https://cmake.org/cmake/help/latest/command/find_package.html
+                    # https://cmake.org/cmake/help/latest/command/find_program.html
+                    cmake_options+=(
+                        -DCMAKE_FIND_ROOT_PATH:PATH="${CROSS_COMPILE_DEPS_PATH}"
+                        -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER
+                    )
+                fi
+
                 # Ensure on Darwin platforms that we consider only the SQLite headers
                 # from the SDK instead of picking ones found elsewhere
                 # (e.g. in /usr/include )
@@ -2304,7 +2335,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang"
                     -DCMAKE_CXX_COMPILER:PATH="${CLANG_BIN}/clang++"
                     -DCMAKE_Swift_COMPILER:PATH=${SWIFTC_BIN}
-                    -DCMAKE_Swift_FLAGS:STRING="-module-cache-path \"${module_cache}\""
+                    -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)"
                     -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
                     -DCMAKE_INSTALL_LIBDIR:PATH="lib"
 
@@ -2320,6 +2351,9 @@ for host in "${ALL_HOSTS[@]}"; do
 
                     -DENABLE_TESTING=YES
                   )
+                  if [[ $(is_cross_tools_host ${host}) ]] ; then
+                      cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
+                  fi
                 ;;
                 esac
 
@@ -2373,7 +2407,7 @@ for host in "${ALL_HOSTS[@]}"; do
                   -DCMAKE_CXX_COMPILER:PATH=${CLANG_BIN}/clang++
                   -DCMAKE_SWIFT_COMPILER:PATH=${SWIFTC_BIN}
                   -DCMAKE_Swift_COMPILER:PATH=${SWIFTC_BIN}
-                  -DCMAKE_Swift_FLAGS:STRING="-module-cache-path \"${module_cache}\""
+                  -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)"
                   -DCMAKE_INSTALL_PREFIX:PATH=$(get_host_install_prefix ${host})
 
                   ${LIBICU_BUILD_ARGS[@]}
@@ -2389,6 +2423,24 @@ for host in "${ALL_HOSTS[@]}"; do
                   -DBUILD_SHARED_LIBS=$([[ ${product} == foundation_static ]] && echo "NO" || echo "YES")
                 )
 
+                if [[ $(is_cross_tools_host ${host}) ]] ; then
+                    cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
+
+                    # Foundation looks for the ICU, libXML2 and libcurl libraries
+                    # using find_package(), so search for them in
+                    # CROSS_COMPILE_DEPS_PATH using the CMake process for doing
+                    # so, see the linked CMake docs for more info:
+                    #
+                    # https://cmake.org/cmake/help/latest/command/find_package.html
+                    cmake_options+=(
+                        -DCMAKE_FIND_ROOT_PATH:PATH="${CROSS_COMPILE_DEPS_PATH}"
+                    )
+                fi
+                if [[ "${host}" == "android-"* ]]; then
+                    cmake_options+=(
+                        -DCMAKE_HAVE_LIBC_PTHREAD=True
+                    )
+                fi
                 ;;
             libdispatch|libdispatch_static)
                 LIBDISPATCH_BUILD_DIR=$(build_directory ${host} ${product})
@@ -2415,7 +2467,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DCMAKE_CXX_COMPILER:PATH="${CLANG_BIN}/clang++"
                     -DCMAKE_SWIFT_COMPILER:PATH="${SWIFTC_BIN}"
                     -DCMAKE_Swift_COMPILER:PATH="${SWIFTC_BIN}"
-                    -DCMAKE_Swift_FLAGS:STRING="-module-cache-path \"${module_cache}\""
+                    -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)"
                     -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
                     -DCMAKE_INSTALL_LIBDIR:PATH="lib"
 
@@ -2424,6 +2476,9 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DENABLE_TESTING=YES
                     -DBUILD_SHARED_LIBS=$([[ ${product} == libdispatch_static ]] && echo "NO" || echo "YES")
                   )
+                  if [[ $(is_cross_tools_host ${host}) ]] ; then
+                      cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
+                  fi
                 ;;
                 esac
 

utils/swift_build_support/swift_build_support/host_specific_configuration.py

@@ -72,6 +72,8 @@ def __init__(self, host_target, args):
         self.swift_test_run_targets = []
         self.swift_benchmark_build_targets = []
         self.swift_benchmark_run_targets = []
+        self.swift_flags = ''
+        self.cmake_options = ''
         for deployment_target_name in stdlib_targets_to_configure:
             # Get the target object.
             deployment_target = StdlibDeploymentTarget.get_target_for_name(
@@ -194,6 +196,16 @@ def __init__(self, host_target, args):
                         "check-swift{}-optimize_none_with_implicit_dynamic-{}"
                         .format(subset_suffix, name))
 
+            # Only pull in these flags when cross-compiling with
+            # --cross-compile-hosts.
+            if deployment_target_name != args.host_target and \
+               host_target != args.host_target:
+                self.add_flags_for_cross_compilation(args, deployment_target)
+
+    def add_flags_for_cross_compilation(self, args, deployment_target):
+        self.swift_flags = deployment_target.platform.swift_flags(args)
+        self.cmake_options = deployment_target.platform.cmake_options(args)
+
     def __platforms_to_skip_build(self, args):
         platforms_to_skip_build = set()
         if not args.build_linux:

utils/swift_build_support/swift_build_support/targets.py

@@ -68,6 +68,18 @@ def contains(self, target_name):
                 return True
         return False
 
+    def swift_flags(self, args):
+        """
+        Swift compiler flags for a platform, useful for cross-compiling
+        """
+        return ''
+
+    def cmake_options(self, args):
+        """
+        CMake flags to build for a platform, useful for cross-compiling
+        """
+        return ''
+
 
 class DarwinPlatform(Platform):
     def __init__(self, name, archs, sdk_name=None, is_simulator=False):
@@ -136,6 +148,29 @@ def uses_host_tests(self):
         """
         return True
 
+    def swift_flags(self, args):
+        flags = '-target %s-unknown-linux-android%s ' % (args.android_arch,
+                                                         args.android_api_level)
+
+        flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % (
+                 args.build_root, self.name, args.android_arch)
+
+        android_toolchain_path = '%s/toolchains/llvm/prebuilt/%s' % (
+            args.android_ndk, StdlibDeploymentTarget.host_target().name)
+
+        flags += '-sdk %s/sysroot ' % (android_toolchain_path)
+        flags += '-tools-directory %s/bin' % (android_toolchain_path)
+        return flags
+
+    def cmake_options(self, args):
+        options = '-DCMAKE_SYSTEM_NAME=Android '
+        options += '-DCMAKE_SYSTEM_VERSION=%s ' % (args.android_api_level)
+        options += '-DCMAKE_SYSTEM_PROCESSOR=%s ' % (args.android_arch if not
+                                                     args.android_arch == 'armv7'
+                                                     else 'armv7-a')
+        options += '-DCMAKE_ANDROID_NDK:PATH=%s' % (args.android_ndk)
+        return options
+
 
 class Target(object):
     """

validation-test/BuildSystem/android_cross_compile.test

@@ -0,0 +1,8 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=linux-gnu
+
+# RUN: %empty-directory(%t)
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --libdispatch --cross-compile-hosts=android-aarch64 --skip-local-build --android --android-ndk %t/ndk/ --android-arch aarch64 --android-icu-uc %t/lib/libicuuc.so --android-icu-uc-include %t/include/ --android-icu-i18n %t/lib/libicui18n.so --android-icu-i18n-include %t/include/ --android-icu-data %t/lib/libicudata.so 2>&1 | %FileCheck %s
+
+# CHECK: -DCMAKE_Swift_FLAGS{{.*}}-target {{.*}}unknown-linux-android{{.*}} -sdk
+# CHECK: -DCMAKE_SYSTEM_NAME=Android {{.*}} -DCMAKE_ANDROID_NDK