[android] A few tweaks for native compilation and to get more tests working

Now that CMAKE_HOST_SYSTEM_NAME and CMAKE_SYSTEM_NAME are set by default to
Android in the Termux app, make the needed tweaks. Some tests were adapted
to work natively on Android too, adds sys/cdefs.h to the Bionic modulemap,
and includes the start of native Android platform support in the build-script.

Author: finagolfin

CMakeLists.txt

@@ -601,13 +601,7 @@ if(SWIFT_HOST_VARIANT_SDK)
   set(SWIFT_HOST_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}")
 else()
   if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
-    # CMake on an Android host sets this to Linux, so check for the ANDROID_DATA
-    # environment variable to see if we're building on Android.
-    if(NOT "$ENV{ANDROID_DATA}" STREQUAL "")
-      set(SWIFT_HOST_VARIANT_SDK_default "ANDROID")
-    else()
-      set(SWIFT_HOST_VARIANT_SDK_default "LINUX")
-    endif()
+    set(SWIFT_HOST_VARIANT_SDK_default "LINUX")
   elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD")
     set(SWIFT_HOST_VARIANT_SDK_default "FREEBSD")
   elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "CYGWIN")
@@ -617,7 +611,6 @@ else()
   elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Haiku")
     set(SWIFT_HOST_VARIANT_SDK_default "HAIKU")
   elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
-    # CMAKE_SYSTEM_NAME might be set this way when cross-compiling to Android.
     set(SWIFT_HOST_VARIANT_SDK_default "ANDROID")
   elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
     set(SWIFT_HOST_VARIANT_SDK_default "OSX")

cmake/modules/AddSwift.cmake

@@ -1105,9 +1105,13 @@ function(_add_swift_library_single target name)
       PROPERTIES
       INSTALL_RPATH "$ORIGIN:/usr/lib/swift/cygwin")
   elseif("${SWIFTLIB_SINGLE_SDK}" STREQUAL "ANDROID")
-    # CMake generates incorrect rule `$SONAME_FLAG $INSTALLNAME_DIR$SONAME` for Android build on macOS cross-compile host.
-    # Proper linker flags constructed manually. See below variable `swiftlib_link_flags_all`.
-    set_target_properties("${target}" PROPERTIES NO_SONAME TRUE)
+    # CMake generates an incorrect rule `$SONAME_FLAG $INSTALLNAME_DIR$SONAME`
+    # for an Android cross-build from a macOS host. Construct the proper linker
+    # flags manually in add_swift_target_library instead, see there with
+    # variable `swiftlib_link_flags_all`.
+    if(SWIFTLIB_SINGLE_TARGET_LIBRARY)
+      set_target_properties("${target}" PROPERTIES NO_SONAME TRUE)
+    endif()
     # Only set the install RPATH if cross-compiling the host tools, in which
     # case both the NDK and Sysroot paths must be set.
     if(NOT "${SWIFT_ANDROID_NDK_PATH}" STREQUAL "" AND

cmake/modules/SwiftConfigureSDK.cmake

@@ -262,6 +262,8 @@ macro(configure_sdk_unix name architectures)
         set(_swift_android_prebuilt_build linux-x86_64)
       elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL Windows)
         set(_swift_android_prebuilt_build Windows-x86_64)
+      elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL Android)
+        # When building natively on an Android host, there's no NDK or prebuilt suffix.
       else()
         message(SEND_ERROR "cannot cross-compile to android from ${CMAKE_HOST_SYSTEM_NAME}")
       endif()

lib/Driver/UnixToolChains.cpp

@@ -42,7 +42,8 @@ std::string
 toolchains::GenericUnix::sanitizerRuntimeLibName(StringRef Sanitizer,
                                                  bool shared) const {
   return (Twine("libclang_rt.") + Sanitizer + "-" +
-          this->getTriple().getArchName() + ".a")
+          this->getTriple().getArchName() +
+          (this->getTriple().isAndroid() ? "-android" : "") + ".a")
       .str();
 }
 

stdlib/public/Platform/bionic.modulemap.gyb

@@ -282,6 +282,10 @@ module SwiftGlibc [system] {
     module sys {
       export *
 
+      module cdefs {
+        header "${GLIBC_ARCH_INCLUDE_PATH}/sys/cdefs.h"
+        export *
+      }
       module file {
         header "${GLIBC_ARCH_INCLUDE_PATH}/sys/file.h"
         export *

test/Driver/multi-threaded.swift

@@ -5,7 +5,8 @@
 // RUN: %target-swiftc_driver -driver-print-jobs -module-name=ThisModule -wmo -num-threads 4 %S/Inputs/main.swift %s -c | %FileCheck -check-prefix=OBJECT %s
 // RUN: cd %t && %target-swiftc_driver -parseable-output -module-name=ThisModule -wmo -num-threads 4 %S/Inputs/main.swift %s -c 2> %t/parseable-output
 // RUN: cat %t/parseable-output | %FileCheck -check-prefix=PARSEABLE %s
-// RUN: cd %t && env TMPDIR=/tmp %swiftc_driver -driver-print-jobs -module-name=ThisModule -wmo -num-threads 4 %S/Inputs/main.swift %s -o a.out | %FileCheck -check-prefix=EXEC %s
+// RUN: %empty-directory(%t/tmp)
+// RUN: cd %t && env TMPDIR=%t/tmp/ %swiftc_driver -driver-print-jobs -module-name=ThisModule -wmo -num-threads 4 %S/Inputs/main.swift %s -o a.out | %FileCheck -check-prefix=EXEC %s
 // RUN: echo "{\"%/s\": {\"llvm-bc\": \"%/t/multi-threaded.bc\", \"object\": \"%/t/multi-threaded.o\"}, \"%/S/Inputs/main.swift\": {\"llvm-bc\": \"%/t/main.bc\", \"object\": \"%/t/main.o\"}}" > %t/ofmo.json
 // RUN: %target-swiftc_driver -module-name=ThisModule -wmo -num-threads 4 %S/Inputs/main.swift %s  -emit-dependencies -output-file-map %t/ofmo.json -c
 // RUN: cat %t/*.d | %FileCheck -check-prefix=DEPENDENCIES %s

test/lit.cfg

@@ -1021,6 +1021,9 @@ elif (run_os in ['linux-gnu', 'linux-gnueabihf', 'freebsd', 'windows-cygnus', 'w
       config.target_shared_library_prefix = 'lib'
       config.target_shared_library_suffix = ".so"
       config.target_sdk_name = "android"
+      # Needed by several ParseableInterface/swift_build_sdk_interfaces tests on
+      # Android
+      config.environment['ANDROID_DATA'] = os.environ['ANDROID_DATA']
     else:
       lit_config.note("Testing Linux " + config.variant_triple)
       config.target_object_format = "elf"
@@ -1239,7 +1242,7 @@ ENV_VAR_PREFIXES = {
     'appletvsimulator': SIMULATOR_ENV_PREFIX,
     'android': 'ANDROID_CHILD_'
 }
-TARGET_ENV_PREFIX = ENV_VAR_PREFIXES.get(config.target_sdk_name, "")
+TARGET_ENV_PREFIX = ENV_VAR_PREFIXES.get(config.target_sdk_name, "") if not kIsAndroid else ""
 
 if 'remote_run_host' in lit_config.params:
     if 'remote_run_tmpdir' not in lit_config.params:
@@ -1351,7 +1354,7 @@ def source_compiler_rt_libs(path):
                                         and config.compiler_rt_platform in lib])
 
 compiler_rt_dir = make_path(test_resource_dir, 'clang', 'lib',
-                            platform.system().lower())
+                            platform.system().lower() if not kIsAndroid else 'android')
 source_compiler_rt_libs(compiler_rt_dir)
 
 def check_runtime_libs(features_to_check):

tools/SourceKit/cmake/modules/AddSwiftSourceKit.cmake

@@ -168,6 +168,13 @@ macro(add_sourcekit_library name)
     endif()
   endif()
 
+  if("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
+    if(SOURCEKITLIB_SHARED)
+      set_target_properties(${name} PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE)
+      set_target_properties(${name} PROPERTIES INSTALL_RPATH "$ORIGIN/../lib/swift/android")
+    endif()
+  endif()
+
   if("${SOURCEKITLIB_INSTALL_IN_COMPONENT}" STREQUAL "")
     if(SOURCEKITLIB_SHARED)
       set(SOURCEKITLIB_INSTALL_IN_COMPONENT tools)

utils/build-script-impl

@@ -681,6 +681,18 @@ function set_build_options_for_host() {
                 -DSWIFT_DARWIN_DEPLOYMENT_VERSION_WATCHOS="${DARWIN_DEPLOYMENT_VERSION_WATCHOS}"
             )
             ;;
+        android-*)
+            SWIFT_HOST_VARIANT="android"
+            SWIFT_HOST_VARIANT_SDK="ANDROID"
+            case ${host} in
+                android-armv7)
+                    SWIFT_HOST_VARIANT_ARCH="armv7"
+                    ;;
+                android-aarch64)
+                    SWIFT_HOST_VARIANT_ARCH="aarch64"
+                    ;;
+                esac
+        ;;
         *)
             echo "Unknown host tools target: ${host}"
             exit 1
@@ -2933,7 +2945,7 @@ for host in "${ALL_HOSTS[@]}"; do
                 fi
 
                 case ${host} in
-                  linux-*|freebsd-*|cygwin-*|haiku-*) ;;
+                  linux-*|freebsd-*|cygwin-*|haiku-*|android-*) ;;
                   *)
                     echo "error: --install-xctest is not supported on this platform"
                     exit 1

utils/swift_build_sdk_interfaces.py

@@ -402,9 +402,18 @@ def main():
             xfails = json.load(xfails_file)
 
     make_dirs_if_needed(args.output_dir, args.dry_run)
-    shared_output_lock = multiprocessing.Lock()
-    pool = multiprocessing.Pool(args.jobs, set_up_child,
-                                (args, shared_output_lock))
+    if 'ANDROID_DATA' not in os.environ:
+        shared_output_lock = multiprocessing.Lock()
+        pool = multiprocessing.Pool(args.jobs, set_up_child,
+                                    (args, shared_output_lock))
+    else:
+        # Android doesn't support Python's multiprocessing as it doesn't have
+        # sem_open, so switch to a ThreadPool instead.
+        import threading
+        shared_output_lock = threading.Lock()
+        from multiprocessing.pool import ThreadPool
+        pool = ThreadPool(args.jobs, set_up_child,
+                          (args, shared_output_lock))
 
     interface_framework_dirs = (args.interface_framework_dirs or
                                 DEFAULT_FRAMEWORK_INTERFACE_SEARCH_PATHS)

utils/swift_build_support/swift_build_support/targets.py

@@ -186,6 +186,14 @@ def host_target():
         machine = platform.machine()
 
         if system == 'Linux':
+            if 'ANDROID_DATA' in os.environ:
+                if machine.startswith('armv7'):
+                    return StdlibDeploymentTarget.Android.armv7
+                elif machine == 'aarch64':
+                    return StdlibDeploymentTarget.Android.aarch64
+                raise NotImplementedError('Android System with architecture '
+                                          '"%s" is not supported' % machine)
+
             if machine == 'x86_64':
                 return StdlibDeploymentTarget.Linux.x86_64
             elif machine == 'i686':