SR-8876: Always build libicu on Linux

lib/Driver/CMakeLists.txt

@@ -29,7 +29,7 @@ if(SWIFT_BUILD_STATIC_STDLIB)
   foreach(sdk ${SWIFT_CONFIGURED_SDKS})
     if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF")
       string(TOLOWER "${sdk}" lowercase_sdk)
-      if(SWIFT_LINUX_ICU_STATICLIB)
+      if(SWIFT_${SWIFT_HOST_VARIANT_SDK}_${SWIFT_HOST_VARIANT_ARCH}_ICU_STATICLIB)
         set(ICU_STATICLIB "TRUE")
       else()
         set(ICU_STATICLIB "FALSE")

stdlib/public/core/CMakeLists.txt

@@ -231,7 +231,7 @@ else()
     list(APPEND swift_core_private_link_libraries
       ICU_UC ICU_I18N)
   else()
-    list(APPEND swift_core_private_link_libraries -licui18n -licuuc -licudata)
+    list(APPEND swift_core_private_link_libraries -licui18nswift -licuucswift -licudataswift)
   endif()
 endif()
 

utils/build-presets.ini

@@ -716,6 +716,8 @@ mixin-preset=mixin_lightweight_assertions
 llbuild
 swiftpm
 xctest
+libicu
+
 dash-dash
 
 build-ninja
@@ -724,6 +726,7 @@ install-lldb
 install-llbuild
 install-swiftpm
 install-xctest
+install-libicu
 install-prefix=/usr
 swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;swift-remote-mirror;sdk-overlay;license;sourcekit-inproc
 llvm-install-components=llvm-cov;llvm-profdata;IndexStore
@@ -941,6 +944,7 @@ reconfigure
 mixin-preset=buildbot_incremental_linux_base
 build-subdir=buildbot_incremental
 
+libicu
 llbuild
 swiftpm
 xctest

utils/build-script-impl

@@ -2550,6 +2550,20 @@ for host in "${ALL_HOSTS[@]}"; do
                     continue
                 fi
 
+                if [[ ! "${SKIP_BUILD_LIBICU}" ]] ; then
+                    ICU_ROOT=$(build_directory ${host} libicu)/tmp_install
+                    LIBICU_BUILD_ARGS=(
+                        -DICU_ROOT:PATH=${ICU_ROOT}
+                        -DICU_INCLUDE_DIR:PATH=${ICU_ROOT}/include
+                        -DICU_UC_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_UC_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_I18N_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                        -DICU_I18N_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                    )
+                else
+                    LIBICU_BUILD_ARGS=()
+                fi
+
                 # Staging: require opt-in for building with dispatch
                 if [[ ! "${SKIP_BUILD_LIBDISPATCH}" ]] ; then
                     LIBDISPATCH_BUILD_DIR="$(build_directory ${host} libdispatch)"
@@ -2574,6 +2588,7 @@ for host in "${ALL_HOSTS[@]}"; do
                   -DCMAKE_SWIFT_COMPILER:PATH=${SWIFTC_BIN}
                   -DCMAKE_INSTALL_PREFIX:PATH=$(get_host_install_prefix ${host})
 
+                  ${LIBICU_BUILD_ARGS[@]}
                   ${LIBDISPATCH_BUILD_ARGS[@]}
 
                   # NOTE(compnerd) we disable tests because XCTest is not ready
@@ -2658,47 +2673,62 @@ for host in "${ALL_HOSTS[@]}"; do
                 ICU_TMPLIBDIR="${SWIFT_BUILD_PATH}/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
                 if [[ "${RECONFIGURE}" || ! -f "${LIBICU_BUILD_DIR}"/config.status ]]; then
                     echo "Reconfiguring libicu"
-                    if [[ "$LIBICU_BUILD_TYPE" == "Release" ]] ; then
-                        icu_build_variant_arg="--enable-release"
-                    elif [[ "$LIBICU_BUILD_TYPE" == "RelWithDebInfo" ]]; then
-                        icu_build_variant_arg="--enable-release"
+                    if [[ "$LIBICU_BUILD_TYPE" != "Release" ]] ; then
+                        libicu_enable_debug="--enable-debug"
                     else
-                        icu_build_variant_arg="--enable-debug"
+                        libicu_enable_debug=""
                     fi
                     call mkdir -p "${LIBICU_BUILD_DIR}"
+
+                    if [ $(true_false "${BUILD_SWIFT_STATIC_STDLIB}") == "TRUE" ]; then
+                          libicu_enable_static="--enable-static"
+                    else
+                          libicu_enable_static=""
+                    fi
+
                     with_pushd "${LIBICU_BUILD_DIR}" \
-                        call "${LIBICU_SOURCE_DIR}"/source/configure \
+                        call env CXXFLAGS=-fPIC "${LIBICU_SOURCE_DIR}"/icu4c/source/runConfigureICU Linux \
                         ${icu_build_variant_arg} --prefix=${ICU_TMPINSTALL} \
+                        ${libicu_enable_debug} \
+                        --enable-renaming --with-library-suffix=swift \
                         --libdir=${ICU_TMPLIBDIR} \
                         --enable-shared --enable-static \
                         --enable-strict --disable-icuio \
                         --disable-plugins --disable-dyload --disable-extras \
-                        --disable-samples --with-data-packaging=auto
+                        --disable-samples --disable-layoutex --with-data-packaging=auto
                 else
                     echo "Skipping reconfiguration of libicu"
                 fi
                 with_pushd "${LIBICU_BUILD_DIR}" \
-                    call make install
+                    call make -j ${BUILD_JOBS} install
                 ICU_LIBDIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
                 ICU_LIBDIR_STATIC="$(build_directory ${host} swift)/lib/swift_static/${SWIFT_HOST_VARIANT}"
                 ICU_LIBDIR_STATIC_ARCH="$(build_directory ${host} swift)/lib/swift_static/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
-                mkdir -p "${ICU_LIBDIR_STATIC_ARCH}"
-                # Copy the static libs into the swift_static directory
-                for l in uc i18n data
-                do
-                    lib="${ICU_LIBDIR}/libicu${l}.a"
-                    cp "${lib}" "${ICU_LIBDIR_STATIC}"
-                    cp "${lib}" "${ICU_LIBDIR_STATIC_ARCH}"
-                done
+                # Add in the ICU renaming config into uconfig.h
+                sed -e "/^#define __UCONFIG_H__/ r ${LIBICU_BUILD_DIR}/uconfig.h.prepend" -i ${ICU_TMPINSTALL}/include/unicode/uconfig.h
+
+                if [ $(true_false "${BUILD_SWIFT_STATIC_STDLIB}") == "TRUE" ]; then
+                    # Copy the static libs into the swift_static directory
+                    mkdir -p "${ICU_LIBDIR_STATIC_ARCH}"
+                    for l in uc i18n data
+                    do
+                        lib="${ICU_LIBDIR}/libicu${l}swift.a"
+                        cp "${lib}" "${ICU_LIBDIR_STATIC}"
+                        cp "${lib}" "${ICU_LIBDIR_STATIC_ARCH}"
+                    done
+                fi
 
                 # Set the PKG_CONFIG_PATH so that core-foundation can find the libraries and
                 # header files
-                export PKG_CONFIG_PATH="${ICU_TMPLIBDIR}/pkgconfig:${PKG_CONFIG_PATH}"
+                export PKG_CONFIG_PATH="${LIBICU_BUILD_DIR}/config:${PKG_CONFIG_PATH}"
+                export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}":"${LIBICU_BUILD_DIR}/lib"
                 swift_cmake_options=(
                     "${swift_cmake_options[@]}"
                     -DSWIFT_${SWIFT_HOST_VARIANT_SDK}_${SWIFT_HOST_VARIANT_ARCH}_ICU_UC_INCLUDE:STRING="${ICU_TMPINSTALL}/include"
                     -DSWIFT_${SWIFT_HOST_VARIANT_SDK}_${SWIFT_HOST_VARIANT_ARCH}_ICU_I18N_INCLUDE:STRING="${ICU_TMPINSTALL}/include"
                     -DSWIFT_${SWIFT_HOST_VARIANT_SDK}_${SWIFT_HOST_VARIANT_ARCH}_ICU_STATICLIB:BOOL=TRUE
+                    -DICU_UC_LIBDIR:PATH="${LIBICU_BUILD_DIR}/lib"
+                    -DICU_I18N_LIBDIR:PATH="${LIBICU_BUILD_DIR}/lib"
                 )
                 # libicu builds itself and doesn't use cmake
                 continue
@@ -3110,6 +3140,20 @@ for host in "${ALL_HOSTS[@]}"; do
                     continue
                 fi
 
+                if [[ ! "${SKIP_BUILD_LIBICU}" ]] ; then
+                    ICU_ROOT=$(build_directory ${host} libicu)/tmp_install
+                    LIBICU_BUILD_ARGS=(
+                        -DICU_ROOT:PATH=${ICU_ROOT}
+                        -DICU_INCLUDE_DIR:PATH=${ICU_ROOT}/include
+                        -DICU_UC_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_UC_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_I18N_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                        -DICU_I18N_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                    )
+                else
+                    LIBICU_BUILD_ARGS=()
+                fi
+
                 if [[ ! "${SKIP_BUILD_LIBDISPATCH}" ]] ; then
                     LIBDISPATCH_BUILD_DIR="$(build_directory ${host} libdispatch)"
                     LIBDISPATCH_BUILD_ARGS=(
@@ -3128,6 +3172,7 @@ for host in "${ALL_HOSTS[@]}"; do
                   -DCMAKE_SWIFT_COMPILER:PATH=${SWIFTC_BIN}
                   -DCMAKE_INSTALL_PREFIX:PATH=$(get_host_install_prefix ${host})
 
+                  ${LIBICU_BUILD_ARGS[@]}
                   ${LIBDISPATCH_BUILD_ARGS[@]}
 
                   # NOTE(compnerd) the time has come to enable tests now
@@ -3169,7 +3214,7 @@ for host in "${ALL_HOSTS[@]}"; do
                 LIBICU_BUILD_DIR=$(build_directory ${host} ${product})
                 echo "--- Running tests for ${product} ---"
                 with_pushd "${LIBICU_BUILD_DIR}/test" \
-                    call make
+                    call make -j ${BUILD_JOBS}
                 echo "--- Finished tests for ${product} ---"
                 continue
                 ;;
@@ -3411,21 +3456,35 @@ for host in "${ALL_HOSTS[@]}"; do
                     exit 1
                 fi
                 echo "--- Installing ${product} ---"
-                LIBICU_BUILD_DIR=$(build_directory ${host} ${product})
+                ICU_BUILD_DIR=$(build_directory ${host} ${product})
+                ICU_INSTALL_DIR="$(get_host_install_destdir ${host})$(get_host_install_prefix ${host})"
                 ICU_LIBDIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
-                LIBICU_DEST_DIR="$(get_host_install_destdir ${host})$(get_host_install_prefix ${host})lib/swift/${SWIFT_HOST_VARIANT}"
-                LIBICU_DEST_DIR_STATIC="$(get_host_install_destdir ${host})$(get_host_install_prefix ${host})lib/swift_static/${SWIFT_HOST_VARIANT}"
+                LIBICU_DEST_DIR="${ICU_INSTALL_DIR}lib/swift/${SWIFT_HOST_VARIANT}"
                 mkdir -p ${LIBICU_DEST_DIR}
-                mkdir -p ${LIBICU_DEST_DIR_STATIC}
+
                 for l in uc i18n data
                 do
-                    lib=${ICU_LIBDIR}/libicu${l}
+                    lib=${ICU_LIBDIR}/libicu${l}swift
                     echo "${lib} => ${LIBICU_DEST_DIR}"
                     cp -d ${lib}.so ${lib}.so.* ${LIBICU_DEST_DIR}
-                    cp -d ${lib}.so ${lib}.so.* ${LIBICU_DEST_DIR}
-                    cp -d ${lib}.a ${LIBICU_DEST_DIR_STATIC}
-                    cp -d ${lib}.a ${LIBICU_DEST_DIR_STATIC}
                 done
+
+                if [ $(true_false "${BUILD_SWIFT_STATIC_STDLIB}") == "TRUE" ]; then
+                    LIBICU_DEST_DIR_STATIC="${ICU_INSTALL_DIR}lib/swift_static/${SWIFT_HOST_VARIANT}"
+                    mkdir -p ${LIBICU_DEST_DIR_STATIC}
+                    for l in uc i18n data
+                    do
+                        lib=${ICU_LIBDIR}/libicu${l}swift
+                        echo "${lib} => ${LIBICU_DEST_DIR_STATIC}"
+                        cp -d ${lib}.a ${LIBICU_DEST_DIR_STATIC}
+                    done
+                fi
+
+                ICU_TMP_INSTALL_DIR="${ICU_BUILD_DIR}/tmp_install"
+                mkdir -p  "${ICU_INSTALL_DIR}include"
+                cp -a "${ICU_TMP_INSTALL_DIR}/include/unicode" "${ICU_INSTALL_DIR}include"
+                mkdir -p "${ICU_INSTALL_DIR}share/icuswift"
+                cp -a "${ICU_TMP_INSTALL_DIR}/share/icuswift" "${ICU_INSTALL_DIR}share"
                 continue
                 ;;
             playgroundsupport)

utils/gen-static-stdlib-link-args

@@ -46,9 +46,9 @@ EOF
 # If libicu was compiled as part of the swift build then link the libs as normal
 function use_local_iculibs {
    read -d '' ICU_LIBS <<EOF
--licui18n
--licuuc
--licudata
+-licui18nswift
+-licuucswift
+-licudataswift
 EOF
 }
 

utils/update_checkout/update-checkout-config.json

@@ -31,7 +31,11 @@
         "swift-xcode-playground-support": {
             "remote": { "id": "apple/swift-xcode-playground-support" } },
         "ninja": {
-            "remote": { "id": "ninja-build/ninja" } }
+            "remote": { "id": "ninja-build/ninja" } },
+        "icu": {
+            "remote": { "id": "unicode-org/icu" },
+            "platforms": [ "Linux" ]
+        }
     },
     "default-branch-scheme": "master",
     "branch-schemes": {
@@ -52,7 +56,8 @@
                 "swift-corelibs-libdispatch": "master",
                 "swift-integration-tests": "master",
                 "swift-xcode-playground-support": "master",
-                "ninja": "release"
+                "ninja": "release",
+                "icu": "release-61-1"
             }
         },
         "next" : {
@@ -74,7 +79,8 @@
                 "swift-corelibs-libdispatch": "master",
                 "swift-integration-tests": "master",
                 "swift-xcode-playground-support": "master",
-                "ninja": "release"
+                "ninja": "release",
+                "icu": "release-61-1"
             }
         },
         "swift-3.0-branch" : {
@@ -194,7 +200,8 @@
                 "swift-corelibs-libdispatch": "master",
                 "swift-integration-tests": "master",
                 "swift-xcode-playground-support": "master",
-                "ninja": "release"
+                "ninja": "release",
+                "icu": "release-61-1"
             }
         }
     }

utils/update_checkout/update_checkout/update_checkout.py

@@ -13,6 +13,7 @@
 import argparse
 import json
 import os
+import platform
 import re
 import sys
 import traceback
@@ -131,8 +132,8 @@ def update_single_repository(args):
             # If we were asked to reset to the specified branch, do the hard
             # reset and return.
             if checkout_target and reset_to_remote and not cross_repo:
-                shell.run(['git', 'reset', '--hard',
-                           "origin/%s" % checkout_target], echo=True)
+                full_target = full_target_name('origin', checkout_target)
+                shell.run(['git', 'reset', '--hard', full_target], echo=True)
                 return
 
             # Query whether we have a "detached HEAD", which will mean that
@@ -371,6 +372,37 @@ def validate_config(config):
                        'aliases?!')
 
 
+def full_target_name(repository, target):
+    tag = shell.capture(["git", "tag", "-l", target], echo=True).strip()
+    if tag == target:
+        return tag
+
+    branch = shell.capture(["git", "branch", "--list", target],
+                           echo=True).strip().replace("* ", "")
+    if branch == target:
+        name = "%s/%s" % (repository, target)
+        return name
+
+    raise RuntimeError('Cannot determine if %s is a branch or a tag' % target)
+
+
+def skip_list_for_platform(config):
+    # If there is a platforms key only include the repo if the
+    # plaform is in the list
+    skip_list = []
+    platform_name = platform.system()
+
+    for repo_name, repo_info in config['repos'].items():
+        if 'platforms' in repo_info:
+            if platform_name not in repo_info['platforms']:
+                print("Skipping", repo_name, "on", platform_name)
+                skip_list.append(repo_name)
+            else:
+                print("Including", repo_name, "on", platform_name)
+
+    return skip_list
+
+
 def main():
     freeze_support()
     parser = argparse.ArgumentParser(
@@ -494,7 +526,8 @@ def main():
         if scheme is None:
             scheme = config['default-branch-scheme']
 
-        skip_repo_list = args.skip_repository_list
+        skip_repo_list = skip_list_for_platform(config)
+        skip_repo_list.extend(args.skip_repository_list)
         clone_results = obtain_all_additional_swift_sources(args, config,
                                                             clone_with_ssh,
                                                             scheme,