[build-script] Introduce -n, --dry-run feature

If -n or --dry-run is specified in command line arguments, print the commands
that would be executed to stdout, but do not execute them.

Supported in build-script and build-toolchain.

utils/build-script -n -Rt
utils/build-script -n --preset=buildbot_incremental,tools=RA,stdlib=RA
utils/build-toolchain -n local.swift

Author: rintaro

utils/build-script

@@ -57,6 +57,12 @@ def main_preset():
     parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,
         description="""Builds Swift using a preset.""")
+    parser.add_argument(
+        "-n", "--dry-run",
+        help="print the commands that would be executed, but do not execute "
+             "them",
+        action="store_true",
+        default=False)
     parser.add_argument(
         "--preset-file",
         help="load presets from the specified file",
@@ -110,6 +116,8 @@ def main_preset():
         args.preset_substitutions, args.preset_file_names, args.preset)
 
     build_script_args = [sys.argv[0]]
+    if args.dry_run:
+        build_script_args += ["--dry-run"]
     build_script_args += preset_args
     if args.distcc:
         build_script_args += ["--distcc"]
@@ -306,6 +314,13 @@ It is a policy decision aimed at making the builds uniform across all
 environments and easily reproducible by engineers who are not familiar with the
 details of the setups of other systems or automated environments.""")
 
+    parser.add_argument(
+        "-n", "--dry-run",
+        help="print the commands that would be executed, but do not execute "
+             "them",
+        action="store_true",
+        default=False)
+
     targets_group = parser.add_argument_group(
         title="Host and cross-compilation targets")
     targets_group.add_argument(
@@ -968,6 +983,8 @@ details of the setups of other systems or automated environments.""")
         if '--check-args-only' in args.build_script_impl_args:
             return 0
 
+    shell.dry_run = args.dry_run
+
     # Prepare and validate toolchain
     toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
 
@@ -1419,6 +1436,9 @@ details of the setups of other systems or automated environments.""")
 
     build_script_impl_args += args.build_script_impl_args
 
+    if args.dry_run:
+        build_script_impl_args += ["--dry-run"]
+
     check_call([build_script_impl] + build_script_impl_args,
                disable_sleep=True)
 

utils/build-script-impl

@@ -55,6 +55,7 @@ LLVM_TARGETS_TO_BUILD="X86;ARM;AArch64;PowerPC;SystemZ"
 # referred to as `SWIFT_INSTALL_COMPONENTS` in the remainder of this script.
 KNOWN_SETTINGS=(
     # name                      default          description
+    dry-run                     ""               "print the commands that would be executed, but do not execute them"
     build-args                  ""               "arguments to the build tool; defaults to -j8 when CMake generator is \"Unix Makefiles\""
     build-dir                   ""               "out-of-tree build directory; default is in-tree. **This argument is required**"
     host-cc                     ""               "the path to CC, the 'clang' compiler for the host platform. **This argument is required**"
@@ -222,23 +223,42 @@ KNOWN_SETTINGS=(
 # Centalized access point for traced command invocation.
 # Every operation that might mutates file system should be called via
 # these functions.
+
 function call() {
-    { set -x; } 2>/dev/null
-    "$@"
-    { set +x; } 2>/dev/null
+    if [[ ${DRY_RUN} ]]; then
+        echo "${PS4}"$(quoted_print "$@")
+    else
+        { set -x; } 2>/dev/null
+        "$@"
+        { set +x; } 2>/dev/null
+    fi
 }
 
 function with_pushd() {
     local dir=$1
     shift
-    set -x
-    pushd "${dir}"
-    "$@"
-    { set -x; } 2>/dev/null # because $@ might includes { set +x; }
-    popd
-    { set +x; } 2>/dev/null
+    if [[ "$1" == "call" ]]; then
+        shift
+    fi
+    if [[ ${DRY_RUN} ]]; then
+        echo ${PS4}pushd "${dir}"
+        echo "${PS4}"$(quoted_print "$@")
+        echo ${PS4}popd
+    else
+        set -x
+        pushd "${dir}"
+        "$@"
+        { set -x; } 2>/dev/null # because $@ might includes { set +x; }
+        popd
+        { set +x; } 2>/dev/null
+    fi
 }
 
+function quoted_print() {
+    python -c 'import pipes; import sys; print(" ".join(pipes.quote(arg) for arg in sys.argv[1:]))' "$@"
+}
+
+
 function toupper() {
     echo "$@" | tr '[:lower:]' '[:upper:]'
 }
@@ -1434,7 +1454,7 @@ function set_swiftpm_bootstrap_command() {
             XCTEST_BUILD_DIR=$(build_directory ${host} xctest)
         fi
     fi
-    if [ ! -e "${LLBUILD_BIN}" ]; then
+    if [ "${SKIP_BUILD_LLBUILD}" ]; then
         echo "Error: Cannot build swiftpm without llbuild (swift-build-tool)."
         exit 1
     fi
@@ -2334,7 +2354,11 @@ for host in "${ALL_HOSTS[@]}"; do
             if [[ "${target}" != "" ]]; then
                 echo "--- ${target} ---"
                 trap "tests_busted ${product} '(${target})'" ERR
-                if [[ "${CMAKE_GENERATOR}" == Ninja ]] && !( "${build_cmd[@]}" --version 2>&1 | grep -i -q llbuild ); then
+
+                # NOTE: In dry-run mode, build_dir might not exist yet. In that
+                #       case, -n query will fail. So, in dry-run mode,
+                #       we don't expand test script.
+                if [[ ! "${DRY_RUN}" && "${CMAKE_GENERATOR}" == Ninja ]] && !( "${build_cmd[@]}" --version 2>&1 | grep -i -q llbuild ); then
                     # Ninja buffers command output to avoid scrambling the output
                     # of parallel jobs, which is awesome... except that it
                     # interferes with the progress meter when testing.  Instead of
@@ -2530,29 +2554,38 @@ for host in "${ALL_HOSTS[@]}"; do
     done
 
     if [[ "${DARWIN_INSTALL_EXTRACT_SYMBOLS}" ]] && [[ $(host_has_darwin_symbols ${host}) ]]; then
-        set -x
-        # Copy executables and shared libraries from the `host_install_destdir` to
-        # INSTALL_SYMROOT and run dsymutil on them.
-        (cd "${host_install_destdir}" &&
-         find ./"${TOOLCHAIN_PREFIX}" -perm -0111 -type f -print | cpio -pdm "${INSTALL_SYMROOT}")
+        echo "--- Extracting symbols ---"
 
-        # Run dsymutil on executables and shared libraries.
-        #
-        # Exclude shell scripts.
-        (cd "${INSTALL_SYMROOT}" &&
-         find ./"${TOOLCHAIN_PREFIX}" -perm -0111 -type f -print | \
-           grep -v swift-stdlib-tool | \
-           grep -v crashlog.py | \
-           grep -v symbolication.py | \
-           xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool dsymutil))
-
-        # Strip executables, shared libraries and static libraries in
-        # `host_install_destdir`.
-        find "${host_install_destdir}${TOOLCHAIN_PREFIX}/" \
-          -perm -0111 -or -name "*.a" -type f -print | \
-          xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool strip) -S
+        # FIXME: Since it's hard to trace output pipe call,
+        #        For now, We don't support dry-run trace for this block
+        #        Instead, just echo we do "darwin_intall_extract_symbols".
+        if [[ "${DRY_RUN}" ]]; then
+            call darwin_install_extract_symbols
+        else
+            set -x
+            # Copy executables and shared libraries from the `host_install_destdir` to
+            # INSTALL_SYMROOT and run dsymutil on them.
+            (cd "${host_install_destdir}" &&
+             find ./"${TOOLCHAIN_PREFIX}" -perm -0111 -type f -print | cpio -pdm "${INSTALL_SYMROOT}")
 
-        { set +x; } 2>/dev/null
+            # Run dsymutil on executables and shared libraries.
+            #
+            # Exclude shell scripts.
+            (cd "${INSTALL_SYMROOT}" &&
+             find ./"${TOOLCHAIN_PREFIX}" -perm -0111 -type f -print | \
+               grep -v swift-stdlib-tool | \
+               grep -v crashlog.py | \
+               grep -v symbolication.py | \
+               xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool dsymutil))
+
+            # Strip executables, shared libraries and static libraries in
+            # `host_install_destdir`.
+            find "${host_install_destdir}${TOOLCHAIN_PREFIX}/" \
+              -perm -0111 -or -name "*.a" -type f -print | \
+              xargs -n 1 -P ${BUILD_JOBS} $(xcrun_find_tool strip) -S
+
+            { set +x; } 2>/dev/null
+        fi
     fi
 done
 # Everything is 'installed', but some products may be awaiting lipo.

utils/build-toolchain

@@ -13,6 +13,12 @@
 cd "$(dirname $0)/.." || exit
 SRC_DIR=$PWD
 
+DRY_RUN=
+if [[ "$1" == "-n" || "$1" == "--dry-run" ]] ; then
+    DRY_RUN="-n"
+    shift
+fi
+
 YEAR=$(date +"%Y")
 MONTH=$(date +"%m")
 DAY=$(date +"%d")
@@ -37,7 +43,7 @@ else
     SWIFT_PACKAGE=buildbot_linux
 fi
 
-./utils/build-script --preset="${SWIFT_PACKAGE}" \
+./utils/build-script ${DRY_RUN} --preset="${SWIFT_PACKAGE}" \
         install_destdir="${SWIFT_INSTALL_DIR}" \
         installable_package="${SWIFT_INSTALLABLE_PACKAGE}" \
         install_toolchain_dir="${SWIFT_TOOLCHAIN_DIR}" \