Simplify build script (#2733)

* [WASM] Simplify Foundation build script

* [WASM] Remove WASI ICU related code from CMake build system

* Refactoring build-toolchain script

* [WASM] Fix prebuilt libraries path

* [WASM] Always install ubuntu wasi-sdk

* Fix unarchive command

Author: kateinoigakukun

CMakeLists.txt

@@ -921,36 +921,6 @@ function(swift_icu_variables_set sdk arch result)
   endif()
 endfunction()
 
-if(SWIFT_PRIMARY_VARIANT_SDK STREQUAL "WASI")
-  set(WASI_ICU_URL "" CACHE STRING
-    "Download URL for the WASI ICU distribution")
-  set(WASI_ICU_MD5 "" CACHE STRING
-    "The expected MD5 hash of the WASI ICU distribution archive")
-
-  file(DOWNLOAD "${WASI_ICU_URL}" "${SWIFT_SOURCE_DIR}/../icu.tar.xz"
-    EXPECTED_HASH MD5=${WASI_ICU_MD5})
-
-  get_filename_component(SWIFT_SOURCE_BASEDIR "${SWIFT_SOURCE_DIR}" DIRECTORY)
-  set(WASI_ICU_OUT_LIB "${SWIFT_SOURCE_BASEDIR}/icu_out/lib")
-  set(SWIFT_WASI_wasm32_ICU_UC_INCLUDE "${SWIFT_SOURCE_BASEDIR}/icu_out/include")
-  set(SWIFT_WASI_wasm32_ICU_I18N_INCLUDE "${SWIFT_SOURCE_BASEDIR}/icu_out/include")
-  set(SWIFT_WASI_wasm32_ICU_UC "${WASI_ICU_OUT_LIB}/libicuuc.a")
-  set(SWIFT_WASI_wasm32_ICU_I18N "${WASI_ICU_OUT_LIB}/libicui18n.a")
-  set(SWIFT_WASI_wasm32_ICU_DATA "${WASI_ICU_OUT_LIB}/libicudata.a")
-
-  set(WASI_ICU_DISTRIBUTION_TARGET)
-  add_custom_command_target(WASI_ICU_DISTRIBUTION_TARGET
-    COMMAND 
-      ${CMAKE_COMMAND} -E 
-        tar xfv "${SWIFT_SOURCE_BASEDIR}/icu.tar.xz"
-    WORKING_DIRECTORY 
-      "${SWIFT_SOURCE_BASEDIR}"
-    OUTPUT
-      "${SWIFT_WASI_wasm32_ICU_DATA}"
-      "${SWIFT_WASI_wasm32_ICU_I18N}"
-      "${SWIFT_WASI_wasm32_ICU_UC}")
-endif()
-
 # ICU is provided through CoreFoundation on Darwin.  On other hosts, if the ICU
 # unicode and i18n include and library paths are not defined, perform a standard
 # package lookup.  Otherwise, rely on the paths specified by the user.  These

cmake/caches/Runtime-WASI-wasm32.cmake

@@ -27,8 +27,4 @@ set(SWIFT_STDLIB_ENABLE_THINCMO YES CACHE BOOL "")
 # build with the host compiler
 set(SWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER YES CACHE BOOL "")
 
-set(WASI_ICU_URL "https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz" CACHE STRING "")
-set(WASI_ICU_MD5 "25943864ebbfff15cf5aee8d9d5cc4d7" CACHE STRING "")
-
-
 set(SWIFT_STDLIB_SINGLE_THREADED_RUNTIME YES CACHE BOOL "")

stdlib/public/stubs/CMakeLists.txt

@@ -27,14 +27,7 @@ set(swift_stubs_c_compile_flags ${SWIFT_RUNTIME_CORE_CXX_FLAGS})
 list(APPEND swift_stubs_c_compile_flags -DswiftCore_EXPORTS)
 list(APPEND swift_stubs_c_compile_flags -I${SWIFT_SOURCE_DIR}/include)
 
-set(swift_stubs_dependencies)
-if(SWIFT_PRIMARY_VARIANT_SDK STREQUAL "WASI")
-  list(APPEND swift_stubs_dependencies ${WASI_ICU_DISTRIBUTION_TARGET})
-endif()
-
 add_swift_target_library(swiftStdlibStubs
-                  DEPENDS
-                    ${swift_stubs_dependencies}
                   OBJECT_LIBRARY
                     ${swift_stubs_sources}
                     ${swift_stubs_objc_sources}

utils/webassembly/amazonlinux/Dockerfile

@@ -40,6 +40,6 @@ WORKDIR /home/ec2-user/swift
 RUN git fetch origin $SWIFT_GIT_SHA
 RUN git checkout $SWIFT_GIT_SHA
 RUN ./utils/update-checkout --clone --scheme $SWIFT_CHECKOUT_SCHEME --skip-repository swift
-RUN ./utils/webassembly/install-wasi-sdk.sh linux ubuntu-18.04
+RUN ./utils/webassembly/install-build-sdk.sh
 
 RUN ./utils/webassembly/build-toolchain.sh

utils/webassembly/build-foundation.sh

@@ -3,6 +3,8 @@ set -ex
 DESTINATION_TOOLCHAIN=$1
 WASI_SYSROOT_PATH=$2
 SOURCE_PATH="$(cd "$(dirname $0)/../../.." && pwd)"
+BUILD_SDK_PATH="$SOURCE_PATH/build-sdk"
+LIBXML2_PATH="$BUILD_SDK_PATH/libxml2"
 
 FOUNDATION_BUILD="$SOURCE_PATH/target-build/foundation-wasi-wasm32"
 
@@ -15,11 +17,15 @@ cmake -G Ninja \
   -DCMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \
   -DCMAKE_TOOLCHAIN_FILE="$SOURCE_PATH/swift/utils/webassembly/toolchain-wasi.cmake" \
   -DLLVM_BIN="$DESTINATION_TOOLCHAIN/usr/bin" \
-  -DICU_ROOT="$SOURCE_PATH/icu_out" \
+  -DICU_ROOT="$BUILD_SDK_PATH/icu" \
+  -DLIBXML2_INCLUDE_DIR="$LIBXML2_PATH/include/libxml2" \
+  -DLIBXML2_LIBRARY="$LIBXML2_PATH/lib" \
   -DBUILD_SHARED_LIBS=OFF \
+  -DBUILD_NETWORKING=OFF \
+  -DBUILD_TOOLS=OFF \
+  -DHAS_LIBDISPATCH_API=OFF \
   -DCMAKE_Swift_COMPILER_FORCED=ON \
-  -DCMAKE_ASM_FLAGS="-target wasm32-unknown-wasi" \
   "${SOURCE_PATH}/swift-corelibs-foundation"
 
-ninja -v
-ninja -v install
+ninja
+ninja install

utils/webassembly/build-presets.ini

@@ -3,8 +3,7 @@
 #===----------------------------------------------------------------------===#
 
 [preset: webassembly]
-cmake-c-launcher=%(C_CXX_LAUNCHER)s
-cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
+sccache
 skip-build-benchmarks
 llvm-targets-to-build=X86;AArch64;WebAssembly
 swift-darwin-supported-archs=%(HOST_ARCHITECTURE)s

utils/webassembly/build-toolchain.sh

@@ -4,7 +4,8 @@ set -ex
 SOURCE_PATH="$(cd "$(dirname "$0")/../../.." && pwd)"
 UTILS_PATH="$(cd "$(dirname "$0")" && pwd)"
 
-WASI_SDK_PATH="$SOURCE_PATH/wasi-sdk"
+BUILD_SDK_PATH="$SOURCE_PATH/build-sdk"
+WASI_SDK_PATH="$BUILD_SDK_PATH/wasi-sdk"
 WASI_SYSROOT_PATH="$WASI_SDK_PATH/share/wasi-sysroot"
 
 case $(uname -s) in
@@ -33,6 +34,21 @@ case $(uname -s) in
   ;;
 esac
 
+BUILD_HOST_TOOLCHAIN=1
+
+while [ $# -ne 0 ]; do
+  case "$1" in
+    --skip-build-host-toolchain)
+    BUILD_HOST_TOOLCHAIN=0
+  ;;
+  *)
+    echo "Unrecognised argument \"$1\""
+    exit 1
+  ;;
+  esac
+  shift
+done
+
 YEAR=$(date +"%Y")
 MONTH=$(date +"%m")
 DAY=$(date +"%d")
@@ -56,15 +72,12 @@ build_host_toolchain() {
     --preset-file="$UTILS_PATH/build-presets.ini" \
     --preset=$HOST_PRESET \
     --build-dir="$HOST_BUILD_DIR" \
-    HOST_ARCHITECTURE=$(uname -m) \
+    HOST_ARCHITECTURE="$(uname -m)" \
     INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
-    TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
-    C_CXX_LAUNCHER="$(which sccache)"
+    TOOLCHAIN_NAME="$TOOLCHAIN_NAME"
 }
 
 build_target_toolchain() {
-  rm -rf "$DIST_TOOLCHAIN_DESTDIR"
-  rsync -a "$HOST_TOOLCHAIN_DESTDIR/" "$DIST_TOOLCHAIN_DESTDIR"
 
   COMPILER_RT_BUILD_DIR="$TARGET_BUILD_ROOT/compiler-rt-wasi-wasm32"
   cmake -B "$COMPILER_RT_BUILD_DIR" \
@@ -97,6 +110,11 @@ build_target_toolchain() {
     -D LLVM_DIR="$HOST_BUILD_DIR/llvm-$HOST_SUFFIX/lib/cmake/llvm/" \
     -D SWIFT_NATIVE_SWIFT_TOOLS_PATH="$HOST_BUILD_DIR/swift-$HOST_SUFFIX/bin" \
     -D SWIFT_WASI_SYSROOT_PATH="$WASI_SYSROOT_PATH" \
+    -D SWIFT_WASI_wasm32_ICU_UC_INCLUDE="$BUILD_SDK_PATH/icu/include" \
+    -D SWIFT_WASI_wasm32_ICU_UC="$BUILD_SDK_PATH/icu/lib/libicuuc.a" \
+    -D SWIFT_WASI_wasm32_ICU_I18N_INCLUDE="$BUILD_SDK_PATH/icu/include" \
+    -D SWIFT_WASI_wasm32_ICU_I18N="$BUILD_SDK_PATH/icu/lib/libicui18n.a" \
+    -D SWIFT_WASI_wasm32_ICU_DATA="$BUILD_SDK_PATH/icu/lib/libicudata.a" \
     -G Ninja \
     -S "$SOURCE_PATH/swift"
 
@@ -169,7 +187,12 @@ create_darwin_info_plist() {
   chmod a+r "${DARWIN_TOOLCHAIN_INFO_PLIST}"
 }
 
-build_host_toolchain
+if [ ${BUILD_HOST_TOOLCHAIN} -eq 1 ]; then
+  build_host_toolchain
+  rm -rf "$DIST_TOOLCHAIN_DESTDIR"
+  rsync -a "$HOST_TOOLCHAIN_DESTDIR/" "$DIST_TOOLCHAIN_DESTDIR"
+fi
+
 build_target_toolchain
 
 embed_wasi_sysroot

utils/webassembly/ci.sh

@@ -26,6 +26,7 @@ RUN_TEST_BIN=$SWIFT_PATH/utils/run-test
 TARGET_STDLIB_BUILD_DIR=$SOURCE_PATH/target-build/swift-stdlib-wasi-wasm32
 
 $DEPENDENCIES_SCRIPT
+$UTILS_PATH/install-build-sdk.sh
 
 export PATH="$HOME/.wasmer/bin:$PATH"
 

utils/webassembly/install-build-sdk.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -eux
+
+SOURCE_PATH="$( cd "$(dirname "$0")/../../../" && pwd  )"
+BUILD_SDK_PATH="$SOURCE_PATH/build-sdk"
+
+install_libxml2() {
+  LIBXML2_URL="https://github.com/swiftwasm/libxml2-wasm/releases/download/1.0.0/libxml2-wasm32-unknown-wasi.tar.gz"
+  curl -L "$LIBXML2_URL" | tar xz
+  rm -rf "$BUILD_SDK_PATH/libxml2"
+  mv libxml2-wasm32-unknown-wasi "$BUILD_SDK_PATH/libxml2"
+}
+
+install_icu() {
+  ICU_URL="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
+  curl -L "$ICU_URL" | tar Jx
+  rm -rf "$BUILD_SDK_PATH/icu"
+  mv icu_out "$BUILD_SDK_PATH/icu"
+}
+
+install_wasi-sdk() {
+  # We only use wasi-sysroot and do not use binaries in wasi-sdk,
+  # so build machine's os and wasi-sdk's host os don't have to be matched
+  WASI_SDK_URL="https://github.com/swiftwasm/wasi-sdk/releases/download/0.2.2-swiftwasm/dist-ubuntu-18.04.zip"
+
+  curl -L -o dist-wasi-sdk.zip "$WASI_SDK_URL"
+  unzip -u dist-wasi-sdk.zip -d .
+
+  WASI_SDK_TAR_PATH=$(find . -type f -name "wasi-sdk-*")
+  WASI_SDK_FULL_NAME=$(basename "$WASI_SDK_TAR_PATH" -linux.tar.gz)
+  tar xfz "$WASI_SDK_TAR_PATH"
+
+  rm -rf "$BUILD_SDK_PATH/wasi-sdk"
+  mv "$WASI_SDK_FULL_NAME" "$BUILD_SDK_PATH/wasi-sdk"
+}
+
+workdir=$(mktemp -d)
+pushd "$workdir"
+
+mkdir -p "$BUILD_SDK_PATH"
+
+install_libxml2
+install_icu
+install_wasi-sdk

utils/webassembly/install-wasi-sdk.sh

@@ -52,12 +52,6 @@ fi
 
 cmake --version
 
-$SWIFT_PATH/utils/webassembly/install-wasi-sdk.sh linux ubuntu-18.04
-
-# Link wasm32-wasi-unknown to wasm32-wasi because clang finds crt1.o from sysroot
-# with os and environment name `getMultiarchTriple`.
-ln -s wasm32-wasi wasi-sdk/share/wasi-sysroot/lib/wasm32-wasi-unknown
-
 # Install sccache
 
 if [ -z $(which sccache) ]; then

utils/webassembly/linux/install-dependencies.sh

@@ -26,12 +26,3 @@ cd $SWIFT_PATH
 
 cd $SOURCE_PATH
 
-$SWIFT_PATH/utils/webassembly/install-wasi-sdk.sh macos macos-10.15
-
-# Link sysroot/usr/include to sysroot/include because Darwin sysroot doesn't 
-# find header files in sysroot/include but sysroot/usr/include
-mkdir wasi-sdk/share/wasi-sysroot/usr/
-ln -s ../include wasi-sdk/share/wasi-sysroot/usr/include
-# Link wasm32-wasi-unknown to wasm32-wasi because clang finds crt1.o from sysroot
-# with os and environment name `getMultiarchTriple`.
-ln -s wasm32-wasi wasi-sdk/share/wasi-sysroot/lib/wasm32-wasi-unknown

utils/webassembly/macos/install-dependencies.sh

@@ -9,6 +9,9 @@ set(CMAKE_AR "${LLVM_BIN}/llvm-ar" CACHE STRING "LLVM Archiver for wasm32")
 set(CMAKE_RANLIB "${LLVM_BIN}/llvm-ranlib" CACHE STRING "LLVM Ranlib for wasm32")
 set(CMAKE_C_COMPILER_TARGET ${triple} CACHE STRING "")
 set(CMAKE_CXX_COMPILER_TARGET ${triple} CACHE STRING "")
+set(CMAKE_Swift_COMPILER_TARGET ${triple} CACHE STRING "")
+set(CMAKE_ASM_COMPILER_TARGET ${triple} CACHE STRING "")
+
 set(CMAKE_EXE_LINKER_FLAGS "-Wl,--no-threads" CACHE STRING "Single thread options")
 
 # Don't look in the sysroot for executables to run during the build