Merge pull request #73455 from al45tair/eng/PR-123503470-6.0

[Build] Add the new fully-static Linux SDK.

Author: al45tair

CMakeLists.txt

@@ -477,6 +477,21 @@ set(SWIFT_MIN_RUNTIME_VERSION "${DEFAULT_SWIFT_MIN_RUNTIME_VERSION}" CACHE STRIN
 the compiler itself. This is used on non-Darwin platforms to ensure \
 that it's possible to build the compiler using host tools.")
 
+#
+# User-configurable Linux specific options.
+#
+
+set(SWIFT_MUSL_PATH "/usr/local/musl" CACHE STRING
+  "Path to the directory that contains the Musl headers and libraries. \
+This is only required if we have been asked to build the Musl SDK, and \
+defaults to the default install location for Musl.")
+
+set(SWIFT_SDK_LINUX_STATIC_ARCHITECTURES "" CACHE STRING
+  "The architectures to configure when using the static Linux SDK.")
+
+set(SWIFT_SDK_LINUX_ARCHITECTURES "" CACHE STRING
+  "The architectures to configure when using the Linux SDK.")
+
 #
 # User-configurable Android specific options.
 #
@@ -1078,12 +1093,26 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX")
   set(SWIFT_HOST_VARIANT "linux" CACHE STRING
       "Deployment OS for Swift host tools (the compiler) [linux].")
 
-  # Should we build the standard library for the host?
   is_sdk_requested(LINUX swift_build_linux)
   if(swift_build_linux)
-    configure_sdk_unix("Linux" "${SWIFT_HOST_VARIANT_ARCH}")
-    set(SWIFT_PRIMARY_VARIANT_SDK_default  "${SWIFT_HOST_VARIANT_SDK}")
-    set(SWIFT_PRIMARY_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}")
+    if("${SWIFT_SDK_LINUX_ARCHITECTURES}" STREQUAL "")
+      set(SWIFT_SDK_LINUX_ARCHITECTURES "${SWIFT_HOST_VARIANT_ARCH}")
+    endif()
+
+    configure_sdk_unix("Linux" "${SWIFT_SDK_LINUX_ARCHITECTURES}")
+  endif()
+
+  is_sdk_requested(LINUX_STATIC swift_build_linux_static)
+  if(swift_build_linux_static)
+    if("${SWIFT_MUSL_PATH}" STREQUAL "")
+      message(FATAL_ERROR "You must set SWIFT_MUSL_PATH to point to the Musl libraries and headers.  Specifically, we expect to find Musl at <SWIFT_MUSL_PATH>/<arch> for each requested architecture.")
+    endif()
+
+    if("${SWIFT_SDK_LINUX_STATIC_ARCHITECTURES}" STREQUAL "")
+      set(SWIFT_SDK_LINUX_STATIC_ARCHITECTURES "aarch64;x86_64")
+    endif()
+
+    configure_sdk_unix("Linux_Static" "${SWIFT_SDK_LINUX_STATIC_ARCHITECTURES}")
   endif()
 
   is_sdk_requested(FREESTANDING swift_build_freestanding)
@@ -1092,6 +1121,10 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX")
     # configure_sdk_unix("FREESTANDING" "${SWIFT_HOST_VARIANT_ARCH}")
   endif()
 
+  # Default is Linux SDK for host
+  set(SWIFT_PRIMARY_VARIANT_SDK_default  "${SWIFT_HOST_VARIANT_SDK}")
+  set(SWIFT_PRIMARY_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}")
+
 elseif("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "FREEBSD")
 
   set(SWIFT_HOST_VARIANT "freebsd" CACHE STRING

cmake/modules/SwiftConfigureSDK.cmake

@@ -321,12 +321,20 @@ macro(configure_sdk_unix name architectures)
 
   # Static linking is suported on Linux and WASI
   if("${prefix}" STREQUAL "LINUX"
+      OR "${prefix}" STREQUAL "LINUX_STATIC"
       OR "${prefix}" STREQUAL "WASI")
     set(SWIFT_SDK_${prefix}_STATIC_LINKING_SUPPORTED TRUE)
   else()
     set(SWIFT_SDK_${prefix}_STATIC_LINKING_SUPPORTED FALSE)
   endif()
 
+  # For LINUX_STATIC, build static only
+  if("${prefix}" STREQUAL "LINUX_STATIC")
+    set(SWIFT_SDK_${prefix}_STATIC_ONLY TRUE)
+  else()
+    set(SWIFT_SDK_${prefix}_STATIC_ONLY FALSE)
+  endif()
+
   # GCC on Linux is usually located under `/usr`.
   # However, Ubuntu 20.04 ships with another GCC installation under `/`, which
   # does not include libstdc++. Swift build scripts pass `--sysroot=/` to
@@ -343,7 +351,7 @@ macro(configure_sdk_unix name architectures)
     CACHE STRING "Extra flags for compiling the C++ overlay")
 
   set(_default_threading_package "pthreads")
-  if("${prefix}" STREQUAL "LINUX")
+  if("${prefix}" STREQUAL "LINUX" OR "${prefix}" STREQUAL "LINUX_STATIC")
     set(_default_threading_package "linux")
   elseif("${prefix}" STREQUAL "WASI")
     if(SWIFT_ENABLE_WASI_THREADS)
@@ -444,6 +452,9 @@ macro(configure_sdk_unix name architectures)
         else()
           set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi")
         endif()
+      elseif("${prefix}" STREQUAL "LINUX_STATIC")
+        set(SWIFT_SDK_LINUX_STATIC_ARCH_${arch}_TRIPLE "${arch}-swift-linux-musl")
+        set(SWIFT_SDK_LINUX_STATIC_ARCH_${arch}_PATH "${SWIFT_MUSL_PATH}/${arch}")
       else()
         message(FATAL_ERROR "unknown Unix OS: ${prefix}")
       endif()

stdlib/cmake/modules/AddSwiftStdlib.cmake

@@ -119,6 +119,12 @@ function(_add_target_variant_c_compile_link_flags)
     list(APPEND result "--sysroot=${_sysroot}")
   endif()
 
+  if("${CFLAGS_SDK}" STREQUAL "LINUX_STATIC")
+    list(APPEND result "-isystem" "${SWIFT_MUSL_PATH}/${CFLAGS_ARCH}/usr/include/c++/v1")
+    list(APPEND result "-DSWIFT_LIBC_IS_MUSL")
+  endif()
+
+
   if("${CFLAGS_SDK}" STREQUAL "ANDROID")
     # Make sure the Android NDK lld is used.
     swift_android_tools_path(${CFLAGS_ARCH} tools_path)
@@ -512,6 +518,8 @@ function(_add_target_variant_link_flags)
     if("${LFLAGS_ARCH}" MATCHES "armv5|armv6|armv7|i686")
       list(APPEND link_libraries "atomic")
     endif()
+  elseif("${LFLAGS_SDK}" STREQUAL "LINUX_STATIC")
+    list(APPEND link_libraries "pthread" "dl")
   elseif("${LFLAGS_SDK}" STREQUAL "FREEBSD")
     list(APPEND link_libraries "pthread")
   elseif("${LFLAGS_SDK}" STREQUAL "OPENBSD")
@@ -1352,10 +1360,14 @@ function(add_swift_target_library_single target name)
   endif()
 
   if(target_static)
-    _list_add_string_suffix(
-        "${SWIFTLIB_SINGLE_LINK_LIBRARIES}"
-        "-static"
-        target_static_depends)
+    set(target_static_depends)
+    foreach(dep ${SWIFTLIB_SINGLE_LINK_LIBRARIES})
+      if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$")
+        list(APPEND target_static_depends "${dep}-static")
+      endif()
+    endforeach()
+
+
     # FIXME: should this be target_link_libraries?
     add_dependencies_multiple_targets(
         TARGETS "${target_static}"
@@ -1500,6 +1512,7 @@ function(add_swift_target_library_single target name)
         "SHELL:-Xclang --dependent-lib=msvcrt$<$<CONFIG:Debug>:d>")
     endif()
   endif()
+
   target_compile_options(${target} PRIVATE
     ${c_compile_flags})
   target_link_options(${target} PRIVATE
@@ -1869,6 +1882,7 @@ function(add_swift_target_library name)
         SWIFT_MODULE_DEPENDS_HAIKU
         SWIFT_MODULE_DEPENDS_IOS
         SWIFT_MODULE_DEPENDS_LINUX
+        SWIFT_MODULE_DEPENDS_LINUX_STATIC
         SWIFT_MODULE_DEPENDS_OSX
         SWIFT_MODULE_DEPENDS_TVOS
         SWIFT_MODULE_DEPENDS_WASI
@@ -1915,6 +1929,17 @@ function(add_swift_target_library name)
   endif()
   list_replace(SWIFTLIB_TARGET_SDKS ALL_APPLE_PLATFORMS "${SWIFT_DARWIN_PLATFORMS}")
 
+  # Support adding a "NOT" on the front to mean all SDKs except the following
+  list(GET SWIFTLIB_TARGET_SDKS 0 first_sdk)
+  if("${first_sdk}" STREQUAL "NOT")
+    list(REMOVE_AT SWIFTLIB_TARGET_SDKS 0)
+    list_subtract("${SWIFT_SDKS}" "${SWIFTLIB_TARGET_SDKS}"
+      "SWIFTLIB_TARGET_SDKS")
+  endif()
+
+  list_intersect(
+    "${SWIFTLIB_TARGET_SDKS}" "${SWIFT_SDKS}" SWIFTLIB_TARGET_SDKS)
+
   # All Swift code depends on the standard library, except for the standard
   # library itself.
   if(SWIFTLIB_HAS_SWIFT_CONTENT AND NOT SWIFTLIB_IS_STDLIB_CORE)
@@ -2064,6 +2089,9 @@ function(add_swift_target_library name)
     elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID")
       list(APPEND swiftlib_module_depends_flattened
            ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX})
+    elseif(sdk STREQUAL "LINUX_STATIC")
+      list(APPEND swiftlib_module_depends_flattened
+          ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX_STATIC})
     elseif(sdk STREQUAL "CYGWIN")
       list(APPEND swiftlib_module_depends_flattened
            ${SWIFTLIB_SWIFT_MODULE_DEPENDS_CYGWIN})
@@ -2319,12 +2347,21 @@ function(add_swift_target_library name)
         set(back_deployment_library_option)
       endif()
 
+      # If the SDK is static only, always build static instead of dynamic
+      if(SWIFT_SDK_${sdk}_STATIC_ONLY AND SWIFTLIB_SHARED)
+        set(shared_keyword)
+        set(static_keyword STATIC)
+      else()
+        set(shared_keyword ${SWIFTLIB_SHARED_keyword})
+        set(static_keyword ${SWIFTLIB_STATIC_keyword})
+      endif()
+
       # Add this library variant.
       add_swift_target_library_single(
         ${variant_name}
         ${name}
-        ${SWIFTLIB_SHARED_keyword}
-        ${SWIFTLIB_STATIC_keyword}
+        ${shared_keyword}
+        ${static_keyword}
         ${SWIFTLIB_NO_LINK_NAME_keyword}
         ${SWIFTLIB_OBJECT_LIBRARY_keyword}
         ${SWIFTLIB_INSTALL_WITH_SHARED_keyword}
@@ -2384,9 +2421,9 @@ function(add_swift_target_library name)
       if(NOT SWIFTLIB_OBJECT_LIBRARY)
         # Add dependencies on the (not-yet-created) custom lipo target.
         foreach(DEP ${SWIFTLIB_LINK_LIBRARIES})
-          if (NOT "${DEP}" STREQUAL "icucore" AND
-              NOT "${DEP}" STREQUAL "dispatch" AND
-              NOT "${DEP}" STREQUAL "BlocksRuntime")
+          if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND
+              NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND
+              NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$")
             add_dependencies(${VARIANT_NAME}
               "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}")
           endif()
@@ -2395,9 +2432,9 @@ function(add_swift_target_library name)
         if (SWIFTLIB_IS_STDLIB AND SWIFTLIB_STATIC)
           # Add dependencies on the (not-yet-created) custom lipo target.
           foreach(DEP ${SWIFTLIB_LINK_LIBRARIES})
-            if (NOT "${DEP}" STREQUAL "icucore" AND
-                NOT "${DEP}" STREQUAL "dispatch" AND
-                NOT "${DEP}" STREQUAL "BlocksRuntime")
+            if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND
+                NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND
+                NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$")
               add_dependencies("${VARIANT_NAME}-static"
                 "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static")
             endif()
@@ -2443,7 +2480,7 @@ function(add_swift_target_library name)
 
     if(NOT SWIFTLIB_OBJECT_LIBRARY)
       # Determine the name of the universal library.
-      if(SWIFTLIB_SHARED)
+      if(SWIFTLIB_SHARED AND NOT SWIFT_SDK_${sdk}_STATIC_ONLY)
         if("${sdk}" STREQUAL "WINDOWS")
           set(UNIVERSAL_LIBRARY_NAME
             "${SWIFTLIB_DIR}/${library_subdir}/${name}.dll")
@@ -2455,12 +2492,18 @@ function(add_swift_target_library name)
             "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_SHARED_LIBRARY_PREFIX}${name}${CMAKE_SHARED_LIBRARY_SUFFIX}")
         endif()
       else()
+        if(SWIFT_SDK_${sdk}_STATIC_ONLY)
+          set(lib_dir "${SWIFTSTATICLIB_DIR}")
+        else()
+          set(lib_dir "${SWIFTLIB_DIR}")
+        endif()
+
         if("${sdk}" STREQUAL "WINDOWS")
           set(UNIVERSAL_LIBRARY_NAME
-            "${SWIFTLIB_DIR}/${library_subdir}/${name}.lib")
+            "${lib_dir}/${library_subdir}/${name}.lib")
         else()
           set(UNIVERSAL_LIBRARY_NAME
-            "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_STATIC_LIBRARY_PREFIX}${name}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+            "${lib_dir}/${library_subdir}/${CMAKE_STATIC_LIBRARY_PREFIX}${name}${CMAKE_STATIC_LIBRARY_SUFFIX}")
         endif()
       endif()
 
@@ -2492,7 +2535,8 @@ function(add_swift_target_library name)
 
       precondition(resource_dir_sdk_subdir)
 
-      if(SWIFTLIB_SHARED OR SWIFTLIB_INSTALL_WITH_SHARED)
+      if((SWIFTLIB_SHARED AND NOT SWIFT_SDK_${sdk}_STATIC_ONLY)
+          OR SWIFTLIB_INSTALL_WITH_SHARED)
         set(resource_dir "swift")
         set(file_permissions
             OWNER_READ OWNER_WRITE OWNER_EXECUTE
@@ -2868,6 +2912,7 @@ function(add_swift_target_executable name)
     SWIFT_MODULE_DEPENDS_HAIKU
     SWIFT_MODULE_DEPENDS_IOS
     SWIFT_MODULE_DEPENDS_LINUX
+    SWIFT_MODULE_DEPENDS_LINUX_STATIC
     SWIFT_MODULE_DEPENDS_OSX
     SWIFT_MODULE_DEPENDS_TVOS
     SWIFT_MODULE_DEPENDS_WASI
@@ -2924,6 +2969,14 @@ function(add_swift_target_executable name)
   endif()
   list_replace(SWIFTEXE_TARGET_TARGET_SDKS ALL_APPLE_PLATFORMS "${SWIFT_DARWIN_PLATFORMS}")
 
+  # Support adding a "NOT" on the front to mean all SDKs except the following
+  list(GET SWIFTEXE_TARGET_TARGET_SDKS 0 first_sdk)
+  if("${first_sdk}" STREQUAL "NOT")
+    list(REMOVE_AT SWIFTEXE_TARGET_TARGET_SDKS 0)
+    list_subtract("${SWIFT_SDKS}" "${SWIFTEXE_TARGET_TARGET_SDKS}"
+      "SWIFTEXE_TARGET_TARGET_SDKS")
+  endif()
+
   list_intersect(
     "${SWIFTEXE_TARGET_TARGET_SDKS}" "${SWIFT_SDKS}" SWIFTEXE_TARGET_TARGET_SDKS)
 
@@ -2965,6 +3018,9 @@ function(add_swift_target_executable name)
     elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID")
       list(APPEND swiftexe_module_depends_flattened
         ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_LINUX})
+    elseif(sdk STREQUAL "LINUX_STATIC")
+      list(APPEND swiftexe_module_depends_flattened
+        ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_LINUX_STATIC})
     elseif(sdk STREQUAL "CYGWIN")
       list(APPEND swiftexe_module_depends_flattened
         ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_CYGWIN})

stdlib/public/Resources/linux-static/static-executable-args.lnk

@@ -0,0 +1,15 @@
+-static
+-lswiftCore
+-lswift_RegexParser
+-Xlinker
+-undefined=pthread_self
+-Xlinker
+-undefined=pthread_once
+-Xlinker
+-undefined=pthread_key_create
+-ldispatch
+-lBlocksRuntime
+-lpthread
+-ldl
+-lc++
+-lm

stdlib/public/Resources/linux-static/static-stdlib-args.lnk

@@ -0,0 +1,9 @@
+-ldl
+-lpthread
+-lswiftCore
+-ldispatch -lBlocksRuntime
+-lc++
+-lm
+-Xlinker -export-dynamic
+-Xlinker --exclude-libs
+-Xlinker ALL

test/CMakeLists.txt

@@ -47,6 +47,10 @@ function(get_test_dependencies SDK result_var_name)
     list(APPEND deps sdk-overlay)
   endif()
 
+  if(SWIFT_BUILD_REMOTE_MIRROR)
+    list(APPEND deps swiftRemoteInspection)
+  endif()
+
   set(deps_binaries)
 
   if (SWIFT_INCLUDE_TOOLS)
@@ -109,6 +113,7 @@ function(get_test_dependencies SDK result_var_name)
      ("${SDK}" STREQUAL "XROS_SIMULATOR") OR
      ("${SDK}" STREQUAL "FREESTANDING") OR
      ("${SDK}" STREQUAL "LINUX") OR
+     ("${SDK}" STREQUAL "LINUX_STATIC") OR
      ("${SDK}" STREQUAL "CYGWIN") OR
      ("${SDK}" STREQUAL "FREEBSD") OR
      ("${SDK}" STREQUAL "OPENBSD") OR
@@ -214,6 +219,7 @@ normalize_boolean_spelling(SWIFT_STDLIB_ENABLE_OBJC_INTEROP)
 normalize_boolean_spelling(SWIFT_ENABLE_BACKTRACING)
 normalize_boolean_spelling(SWIFT_BUILD_SWIFT_SYNTAX)
 normalize_boolean_spelling(SWIFT_ENABLE_SYNCHRONIZATION)
+normalize_boolean_spelling(SWIFT_BUILD_REMOTE_MIRROR)
 is_build_type_optimized("${SWIFT_STDLIB_BUILD_TYPE}" SWIFT_OPTIMIZED)
 
 # Get 'SWIFT_HOST_SDKROOT' for lit.site.cfg.in
@@ -423,6 +429,10 @@ foreach(SDK ${SWIFT_SDKS})
           list(APPEND LIT_ARGS "--param" "synchronization")
         endif()
 
+        if(SWIFT_BUILD_REMOTE_MIRROR)
+          list(APPEND LIT_ARGS "--param" "remote_mirror")
+        endif()
+
         list(APPEND LIT_ARGS "--param" "threading=${SWIFT_SDK_${SDK}_THREADING_PACKAGE}")
 
         # Enable on-crash backtracing if supported

test/LinkerSections/function_sections.swift

@@ -1,8 +1,9 @@
 // REQUIRES: OS=linux-gnu || OS=freebsd
 // RUN: %empty-directory(%t)
-// RUN: %target-build-swift -Xfrontend -function-sections -emit-module -emit-library -static -parse-stdlib %S/Inputs/FunctionSections.swift
-// RUN: %target-build-swift -Xlinker -v -Xlinker --gc-sections -Xlinker -Map=%t/../../FunctionSections.map -I%t/../.. -L%t/../.. -lFunctionSections %S/Inputs/FunctionSectionsUse.swift 2>&1 | sed 's/.*\(gold\|LLD\).*/\1/g' | tr "[:lower:]" "[:upper:]" > %t/../../Linker.txt
-// RUN: %FileCheck --check-prefix $(cat %t/../../Linker.txt) %s < %t/../../FunctionSections.map
+// RUN: %target-build-swift -Xfrontend -function-sections -emit-module -emit-library -static -parse-stdlib %S/Inputs/FunctionSections.swift -o %t/libFunctionSections.a
+// RUN: %target-build-swift -Xlinker --gc-sections -Xlinker -Map=%t/FunctionSections.map -I%t -L%t -lFunctionSections %S/Inputs/FunctionSectionsUse.swift
+// RUN: if head -1 %t/FunctionSections.map | grep "Archive members"; then  %FileCheck --check-prefix GOLD %s < %t/FunctionSections.map ; fi
+// RUN: if head -1 %t/FunctionSections.map | grep "VMA"; then %FileCheck --check-prefix LLD %s < %t/FunctionSections.map ; fi
 
 // GOLD: Discarded input sections
 // GOLD: .text.$s16FunctionSections5func2yyF