Revert "Remove ICU from the toolchain (#75262)"

This reverts commit f3acbb0.

Author: MaxDesiatov

docs/HowToGuides/GettingStarted.md

@@ -135,6 +135,9 @@ toolchain as a one-off, there are a couple of differences:
 - Before running `update-checkout`, double-check that `swift` is the only
   repository inside the `swift-project` directory. Otherwise,
   `update-checkout` may not clone the necessary dependencies.
+- Running `update-checkout` may fail if the `git-lfs` dependency is not
+  installed. This may report as an error related to `icu`. A workaround is
+  passing `--skip-repository icu` to `update-checkout`.
 
 ## Installing dependencies
 

utils/build-presets.ini

@@ -840,6 +840,7 @@ llbuild
 swiftpm
 swift-driver
 xctest
+libicu
 swiftdocc
 swiftformat
 
@@ -853,6 +854,7 @@ install-swiftpm
 install-swift-driver
 install-swiftsyntax
 install-xctest
+install-libicu
 install-prefix=/usr
 install-sourcekit-lsp
 install-swiftformat
@@ -949,6 +951,7 @@ skip-test-xctest
 skip-test-foundation
 skip-test-libdispatch
 skip-test-playgroundsupport
+skip-test-libicu
 skip-test-indexstore-db
 skip-test-sourcekit-lsp
 skip-test-swiftdocc
@@ -1176,6 +1179,7 @@ mixin-preset=
     mixin_linux_install_components_with_clang
 build-subdir=buildbot_incremental
 
+libicu
 llbuild
 swiftpm
 swift-driver
@@ -1190,6 +1194,7 @@ install-llvm
 install-static-linux-config
 install-swift
 install-llbuild
+install-libicu
 install-swiftpm
 install-swift-driver
 install-swiftsyntax
@@ -1783,6 +1788,7 @@ skip-test-osx
 llvm-cmake-options=-DCLANG_DEFAULT_LINKER=gold
 
 foundation
+libicu
 libdispatch
 test
 
@@ -1826,14 +1832,17 @@ mixin-preset=
     mixin_linux_install_components_with_clang
 
 libdispatch
+libicu
 foundation
 xctest
 libcxx=false
 
+install-libicu
 install-foundation
 install-libdispatch
 install-xctest
 
+skip-test-libicu
 skip-test-foundation
 skip-test-libdispatch
 skip-test-xctest
@@ -2104,6 +2113,7 @@ mixin-preset=buildbot_indexstoredb_linux,sanitize
 mixin-preset=
     mixin_linux_install_components_with_clang
 
+libicu
 llbuild
 swiftpm
 xctest
@@ -2119,6 +2129,7 @@ reconfigure
 
 install-foundation
 install-libdispatch
+install-libicu
 install-libcxx
 install-llvm
 install-static-linux-config
@@ -3045,6 +3056,7 @@ mixin-preset=
 [preset: linux_lldb]
 lldb
 foundation
+libicu
 libdispatch
 build-ninja
 

utils/build-script-impl

@@ -298,6 +298,7 @@ components=(
   foundation
   libcxx
   libdispatch
+  libicu
   libxml2
   zlib
   curl
@@ -1248,13 +1249,15 @@ FOUNDATION_SWIFTFOUNDATION_SOURCE_DIR="${WORKSPACE}/swift-foundation"
 FOUNDATION_SWIFTFOUNDATIONICU_SOURCE_DIR="${WORKSPACE}/swift-foundation-icu"
 LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
+LIBICU_SOURCE_DIR="${WORKSPACE}/icu"
 LIBCXX_SOURCE_DIR="${WORKSPACE}/llvm-project/runtimes"
 SWIFT_COLLECTIONS_SOURCE_DIR="${WORKSPACE}/swift-collections"
 SWIFT_PATH_TO_STRING_PROCESSING_SOURCE="${WORKSPACE}/swift-experimental-string-processing"
 SWIFTSYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax"
 SWIFT_SYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax"
 
 [[ "${SKIP_BUILD_LIBCXX}" ]] || PRODUCTS+=(libcxx)
+[[ "${SKIP_BUILD_LIBICU}" ]] || PRODUCTS+=(libicu)
 [[ "${SKIP_BUILD_SWIFT}" ]] || PRODUCTS+=(swift)
 [[ "${SKIP_BUILD_LLDB}" ]] || PRODUCTS+=(lldb)
 [[ "${SKIP_BUILD_LIBDISPATCH}" ]] || PRODUCTS+=(libdispatch)
@@ -1342,6 +1345,8 @@ function build_directory_bin() {
             libdispatch|libdispatch_static)
                 echo "${root}/${LIBDISPATCH_BUILD_TYPE}/bin"
                 ;;
+            libicu)
+                ;;
             *)
                 echo "error: unknown product: ${product}"
                 exit 1
@@ -1480,6 +1485,8 @@ function cmake_config_opt() {
             libdispatch|libdispatch_static)
                 echo "--config ${LIBDISPATCH_BUILD_TYPE}"
                 ;;
+            libicu)
+                ;;
             *)
                 echo "error: unknown product: ${product}"
                 exit 1
@@ -2413,6 +2420,29 @@ for host in "${ALL_HOSTS[@]}"; do
                     continue
                 fi
 
+                if [[ ! "${SKIP_BUILD_LIBICU}" ]] ; then
+                    ICU_ROOT=$(build_directory ${host} libicu)/tmp_install
+                    ICU_LIBDIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
+                    LIBICU_BUILD_ARGS=(
+                        -DICU_ROOT:PATH=${ICU_ROOT}
+                        -DICU_INCLUDE_DIR:PATH=${ICU_ROOT}/include
+                        -DICU_DATA_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicudataswift.so
+                        -DICU_DATA_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicudataswift.so
+                        -DICU_DATA_LIBRARY_DEBUG:FILEPATH=${ICU_LIBDIR}/libicudataswift.so
+                        -DICU_DATA_LIBRARY_RELEASE:FILEPATH=${ICU_LIBDIR}/libicudataswift.so
+                        -DICU_UC_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_UC_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_UC_LIBRARY_DEBUG:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_UC_LIBRARY_RELEASE:FILEPATH=${ICU_LIBDIR}/libicuucswift.so
+                        -DICU_I18N_LIBRARIES:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                        -DICU_I18N_LIBRARY:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                        -DICU_I18N_LIBRARY_DEBUG:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                        -DICU_I18N_LIBRARY_RELEASE:FILEPATH=${ICU_LIBDIR}/libicui18nswift.so
+                    )
+                else
+                    LIBICU_BUILD_ARGS=()
+                fi
+
                 if [[ ! "${SKIP_BUILD_LIBXML2}" ]]; then
                     BASE_INSTALL_DIR="$(get_host_install_destdir ${host})"
                     LIBXML2_HEADERS="${BASE_INSTALL_DIR}/usr/include/libxml2"
@@ -2476,6 +2506,12 @@ for host in "${ALL_HOSTS[@]}"; do
                   call rm -rf "${build_dir}"
                 fi
 
+                # Set the PKG_CONFIG_PATH so that core-foundation can find the libraries and
+                # header files
+                LIBICU_BUILD_DIR="$(build_directory ${host} libicu)"
+                export PKG_CONFIG_PATH="${LIBICU_BUILD_DIR}/config:${PKG_CONFIG_PATH}"
+                export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}":"${LIBICU_BUILD_DIR}/lib"
+
                 cmake_options=(
                   ${cmake_options[@]}
                   -DCMAKE_BUILD_TYPE:STRING=${FOUNDATION_BUILD_TYPE}
@@ -2486,6 +2522,8 @@ for host in "${ALL_HOSTS[@]}"; do
                   -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)"
                   -DCMAKE_INSTALL_PREFIX:PATH=$(get_host_install_prefix ${host})
 
+                  ${LIBICU_BUILD_ARGS[@]}
+
                   ${LIBXML2_BUILD_ARGS[@]}
 
                   ${ZLIB_BUILD_ARGS[@]}
@@ -2513,7 +2551,7 @@ for host in "${ALL_HOSTS[@]}"; do
                 if [[ $(is_cross_tools_host ${host}) ]] ; then
                     cmake_options+=("${SWIFT_TARGET_CMAKE_OPTIONS[@]}")
 
-                    # Foundation looks for the libXML2 and libcurl libraries
+                    # Foundation looks for the ICU, libXML2 and libcurl libraries
                     # using find_package(), so search for them in
                     # CROSS_COMPILE_DEPS_PATH using the CMake process for doing
                     # so, see the linked CMake docs for more info:
@@ -2573,6 +2611,62 @@ for host in "${ALL_HOSTS[@]}"; do
                 ;;
                 esac
 
+                ;;
+            libicu)
+                SWIFT_BUILD_PATH=$(build_directory ${host} swift)
+                LIBICU_BUILD_DIR=$(build_directory ${host} ${product})
+                ICU_TMPINSTALL=$LIBICU_BUILD_DIR/tmp_install
+                ICU_TMPLIBDIR="${SWIFT_BUILD_PATH}/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
+                if [[ "${RECONFIGURE}" || ! -f "${LIBICU_BUILD_DIR}"/config.status ]]; then
+                    echo "Reconfiguring libicu"
+                    if [[ "$LIBICU_BUILD_TYPE" != "Release" ]] ; then
+                        libicu_enable_debug="--enable-debug"
+                    else
+                        libicu_enable_debug=""
+                    fi
+                    call mkdir -p "${LIBICU_BUILD_DIR}"
+
+                    if [ $(true_false "${BUILD_SWIFT_STATIC_STDLIB}") == "TRUE" ]; then
+                          libicu_enable_static="--enable-static"
+                    else
+                          libicu_enable_static=""
+                    fi
+
+                    with_pushd "${LIBICU_BUILD_DIR}" \
+                        call env CXXFLAGS=-fPIC LDFLAGS='-Wl,-rpath=\$$ORIGIN' \
+                        "${LIBICU_SOURCE_DIR}"/icu4c/source/runConfigureICU Linux \
+                        ${icu_build_variant_arg} --prefix=${ICU_TMPINSTALL} \
+                        ${libicu_enable_debug} \
+                        --enable-renaming --with-library-suffix=swift \
+                        --libdir=${ICU_TMPLIBDIR} \
+                        --enable-shared --enable-static \
+                        --enable-strict --disable-icuio \
+                        --disable-plugins --disable-dyload --disable-extras \
+                        --disable-samples --disable-layoutex --with-data-packaging=auto
+                else
+                    echo "Skipping reconfiguration of libicu"
+                fi
+                with_pushd "${LIBICU_BUILD_DIR}" \
+                    call make -j ${BUILD_JOBS} install
+                ICU_LIBDIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
+                ICU_LIBDIR_STATIC="$(build_directory ${host} swift)/lib/swift_static/${SWIFT_HOST_VARIANT}"
+                ICU_LIBDIR_STATIC_ARCH="$(build_directory ${host} swift)/lib/swift_static/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
+                # Add in the ICU renaming config into uconfig.h
+                call sed -e "/^#define __UCONFIG_H__/ r ${LIBICU_BUILD_DIR}/uconfig.h.prepend" -i ${ICU_TMPINSTALL}/include/unicode/uconfig.h
+
+                if [ $(true_false "${BUILD_SWIFT_STATIC_STDLIB}") == "TRUE" ]; then
+                    # Copy the static libs into the swift_static directory
+                    call mkdir -p "${ICU_LIBDIR_STATIC_ARCH}"
+                    for l in uc i18n data
+                    do
+                        lib="${ICU_LIBDIR}/libicu${l}swift.a"
+                        call cp "${lib}" "${ICU_LIBDIR_STATIC}"
+                        call cp "${lib}" "${ICU_LIBDIR_STATIC_ARCH}"
+                    done
+                fi
+
+                # libicu builds itself and doesn't use cmake
+                continue
                 ;;
             *)
                 echo "error: unknown product: ${product}"
@@ -2853,6 +2947,17 @@ for host in "${ALL_HOSTS[@]}"; do
               # libdispatch_static
               continue
             ;;
+            libicu)
+                if [[ "${SKIP_TEST_LIBICU}" ]]; then
+                    continue
+                fi
+                LIBICU_BUILD_DIR=$(build_directory ${host} ${product})
+                echo "--- Running tests for ${product} ---"
+                with_pushd "${LIBICU_BUILD_DIR}/test" \
+                    call make -j ${BUILD_JOBS}
+                echo "--- Finished tests for ${product} ---"
+                continue
+                ;;
             *)
                 echo "error: unknown product: ${product}"
                 exit 1
@@ -3031,6 +3136,42 @@ for host in "${ALL_HOSTS[@]}"; do
                 ;;
                 esac
                 ;;
+            libicu)
+                if [[ -z "${INSTALL_LIBICU}" ]]; then
+                    continue
+                fi
+                echo "--- Installing ${product} ---"
+                ICU_BUILD_DIR=$(build_directory ${host} ${product})
+                ICU_INSTALL_DIR="$(get_host_install_destdir ${host})$(get_host_install_prefix ${host})"
+                ICU_LIBDIR="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
+                LIBICU_DEST_DIR="${ICU_INSTALL_DIR}lib/swift/${SWIFT_HOST_VARIANT}"
+                call mkdir -p ${LIBICU_DEST_DIR}
+
+                for l in uc i18n data
+                do
+                    lib=${ICU_LIBDIR}/libicu${l}swift
+                    echo "${lib} => ${LIBICU_DEST_DIR}"
+                    call cp -d ${lib}.so ${lib}.so.* ${LIBICU_DEST_DIR}
+                done
+
+                if [ $(true_false "${BUILD_SWIFT_STATIC_STDLIB}") == "TRUE" ]; then
+                    LIBICU_DEST_DIR_STATIC="${ICU_INSTALL_DIR}lib/swift_static/${SWIFT_HOST_VARIANT}"
+                    call mkdir -p ${LIBICU_DEST_DIR_STATIC}
+                    for l in uc i18n data
+                    do
+                        lib=${ICU_LIBDIR}/libicu${l}swift
+                        echo "${lib} => ${LIBICU_DEST_DIR_STATIC}"
+                        call cp -d ${lib}.a ${LIBICU_DEST_DIR_STATIC}
+                    done
+                fi
+
+                ICU_TMP_INSTALL_DIR="${ICU_BUILD_DIR}/tmp_install"
+                call mkdir -p  "${ICU_INSTALL_DIR}include"
+                call cp -a "${ICU_TMP_INSTALL_DIR}/include/unicode" "${ICU_INSTALL_DIR}include"
+                call mkdir -p "${ICU_INSTALL_DIR}share/icuswift"
+                call cp -a "${ICU_TMP_INSTALL_DIR}/share/icuswift" "${ICU_INSTALL_DIR}share"
+                continue
+                ;;
             *)
                 echo "error: unknown product: ${product}"
                 exit 1