Put symbol generation logic under test

...before adding the logic to filter paths

In particular:
* print the list of files that `cpio` copies, so we can test explicitly
  which files end up in the symroot (and also see those when asking for
  a toolchain in PR testing)
* use `find` instead of `grep` to filter files we want symbols generated
  for -- this is to avoid the script failing when there are no symbol to
  process, especially in lit tests
* remove an unnecessary check for `swift-api-digester` -- this is now a
  symlink to `swift-frontend` and we only process regular files.

Supports rdar://76865276

Author: edymtt

utils/build-script-impl

@@ -1015,11 +1015,19 @@ if [ -z "${CMAKE}" ] ; then
 fi
 
 function xcrun_find_tool() {
-  xcrun --sdk macosx --toolchain "${DARWIN_XCRUN_TOOLCHAIN}" --find "$@"
+  if [[ "${DRY_RUN}" ]]; then
+    echo echo "$@"
+  else
+    xcrun --sdk macosx --toolchain "${DARWIN_XCRUN_TOOLCHAIN}" --find "$@"
+  fi
 }
 
 function find_just_built_local_host_llvm_tool() {
-  find $(build_directory "${LOCAL_HOST}" llvm) -name "$1" -type f -print
+  if [[ "${DRY_RUN}" ]]; then
+    echo echo "$1"
+  else
+    find $(build_directory "${LOCAL_HOST}" llvm) -name "$1" -type f -print
+  fi
 }
 
 function not() {
@@ -3130,15 +3138,10 @@ for host in "${ALL_HOSTS[@]}"; do
         # descibes
         host_symroot="${INSTALL_SYMROOT}/${host}"
 
-        # FIXME: Since it's hard to trace output pipe call,
-        #        For now, We don't support dry-run trace for this block
-        #        Instead, just echo we do "darwin_intall_extract_symbols".
-        if [[ "${DRY_RUN}" ]]; then
-            call darwin_install_extract_symbols
-            printJSONStartTimestamp dsymutil
-            echo xargs -n 1 -P ${DSYMUTIL_JOBS} dsymutil
-            printJSONEndTimestamp dsymutil
-        else
+        # FIXME this if statement is a trick to have a more readable
+        # diff for the PR that made the following code
+        # amenable to dry run testing
+        if [ -n "${DRY_RUN}" -o -z "${DRY_RUN}" ]; then
             set -x
 
             CURRENT_INSTALL_DIR=${host_install_destdir}
@@ -3147,7 +3150,7 @@ for host in "${ALL_HOSTS[@]}"; do
             # Copy executables and shared libraries from the `host_install_destdir` to
             # INSTALL_SYMROOT and run dsymutil on them.
             (cd "${CURRENT_INSTALL_DIR}" &&
-             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | cpio --insecure -pdm "${host_symroot}")
+             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | cpio --insecure -pdm -v "${host_symroot}")
 
             dsymutil_path=
             if [[ -n "${DARWIN_INSTALL_EXTRACT_SYMBOLS_USE_JUST_BUILT_DSYMUTIL}" ]]; then
@@ -3159,15 +3162,11 @@ for host in "${ALL_HOSTS[@]}"; do
             # Run dsymutil on executables and shared libraries.
             #
             # Exclude shell scripts and static archives.
-            # Exclude swift-api-digester dSYM to reduce debug toolchain size.
             # Tweak carefully the amount of parallelism -- dsymutil can be memory intensive and
             # as such too many instance can exhaust the memory and slow down/panic the machine
             printJSONStartTimestamp dsymutil
             (cd "${host_symroot}" &&
-             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | \
-               grep -v '.py$' | \
-               grep -v '.a$' | \
-               grep -v 'swift-api-digester' | \
+             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -not -name "*.a" -not -name "*.py" -print | \
                xargs -n 1 -P ${DSYMUTIL_JOBS} ${dsymutil_path})
             printJSONEndTimestamp dsymutil
 

validation-test/BuildSystem/dsymutil_jobs.test

@@ -1,10 +1,12 @@
+# REQUIRES: standalone_build,OS=macosx
+
 # RUN: %empty-directory(%t)
 # RUN: mkdir -p %t
-# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --darwin-install-extract-symbols --dsymutil-jobs 5 --cmake %cmake 2>&1 | %FileCheck %s
-
-# REQUIRES: standalone_build,OS=macosx
+# RUN: mkdir -p %t/destdir
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --darwin-install-extract-symbols --dsymutil-jobs 5 --cmake %cmake --install-symroot=%t/symroot --install-destdir=%t/destdir --toolchain-prefix= 2>&1 | %FileCheck %s
 
 # CHECK: --- Extracting symbols ---
 # CHECK: { "command": "dsymutil", "start": "
-# CHECK-NEXT: xargs -n 1 -P 5 dsymutil
-# CHECK-NEXT: { "command": "dsymutil", "end": "
+# CHECK: xargs -n 1 -P 5 {{.*}}dsymutil
+# CHECK: { "command": "dsymutil", "end": "

validation-test/BuildSystem/extractsymbols-default-behaviour.test

@@ -0,0 +1,93 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=macosx
+
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+
+# Even though we are running build-script with dry-run,
+# symbol extraction runs real commands against the file system.
+# Thus we generate a series of files
+# to target each of the cases handled by the code
+# RUN: mkdir -p %t/destdir
+# RUN: %swiftc_driver %t/hello.swift -o %t/destdir/swift-demangle
+# RUN: ln -s %t/destdir/swift-demangle %t/destdir/swift-api-digester
+# RUN: cp %t/swift-util.py %t/destdir/
+# RUN: chmod a+x %t/destdir/swift-util.py
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -o %t/destdir/libswiftDemangle.dylib
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -static -o %t/destdir/libswiftASTSectionImporter.a 
+# Targets marked with INSTALL_WITH_SHARED are executable (e.g. compatibility libraries)
+# RUN: cp %t/destdir/libswiftASTSectionImporter.a %t/destdir/libswiftCompatibility51.a
+# RUN: chmod a+x %t/destdir/libswiftCompatibility51.a
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 > %t/build-script-impl-output.txt 2>&1
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt --check-prefixes CHECK-SKIPPED %s
+
+# CHECK: --- Extracting symbols ---
+
+# Ensure we copy executable regular files to the symroot
+# CHECK-LABEL: cpio
+# CHECK-DAG: swift-demangle
+# CHECK-DAG: swift-util.py
+# CHECK-DAG: libswiftDemangle.dylib
+# CHECK-DAG: libswiftCompatibility51.a
+
+# Ensure we extract symbols only for executables and
+# and dylibs
+# CHECK-LABEL: command": "dsymutil", "start"
+# CHECK-DAG: dsymutil {{.*}}swift-demangle
+# CHECK-DAG: dsymutil {{.*}}libswiftDemangle.dylib
+
+# Ensure we strip executables, shared libraries and static
+# libraries
+# CHECK-LABEL: xcrun_find_tool strip
+# CHECK-DAG: strip {{.*}}swift-demangle
+# CHECK-DAG: strip {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: strip {{.*}}libswiftASTSectionImporter.a
+# CHECK-DAG: strip {{.*}}libswiftCompatibility51.a
+# CHECK-DAG: strip {{.*}}swift-util.py
+
+# Ensure we codesign dylibds
+# CHECK-LABEL: xcrun_find_tool codesign
+# CHECK: codesign {{.*}}libswiftDemangle.dylib
+
+# CHECK-SKIPPED: --- Extracting symbols ---
+
+# Ensure symroot does not contain symlinks and static archives
+# that are not executable
+# CHECK-SKIPPED-LABEL: cpio
+# CHECK-SKIPPED-NOT: swift-api-digester
+# CHECK-SKIPPED-NOT: libswiftASTSectionImporter.a
+
+# Ensure we don't extract symbols for static archives, symlinks
+# and Python scripts
+# CHECK-SKIPPED-LABEL: command": "dsymutil", "start"
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}libswiftCompatibility51.a
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-api-digester
+
+# Ensure we don't strip symlinks
+# CHECK-SKIPPED-LABEL: xcrun_find_tool strip
+# CHECK-SKIPPED-NOT: strip {{.*}}swift-api-digester
+
+# Ensure we don't codesign executables, symlinks,
+# static archives and python scripts
+# CHECK-SKIPPED-LABEL: xcrun_find_tool codesign
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-demangle
+# CHECK-SKIPPED-NOT: codesign {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: codesign {{.*}}libswiftCompatibility51.a
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-api-digester
+
+#--- hello.swift
+print("hello")
+
+#--- dylib.swift
+func greet(person: String) -> String {
+    return "Hello \(person)"
+}
+#--- swift-util.py
+print("hello")

validation-test/BuildSystem/extractsymbols-dry-run-does-not-fail.test

@@ -0,0 +1,24 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=macosx
+
+# As a result of incremental changes,
+# in dry-run mode the symbol extraction
+# still runs some real commands against the installation
+# and symbol directories (e.g. find and cpio)
+# This test explictly checks that such commands
+# do not cause build-script to fail when run
+# against empty directories (which is the typical
+# scenario in dry-run)
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: mkdir -p %t/destdir
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 2>&1 | %FileCheck %s
+
+# CHECK: --- Extracting symbols ---
+
+# CHECK-NOT: {{^}}echo dsymutil
+
+# CHECK-NOT: {{^[^\+].*}}strip
+
+# CHECK-NOT: {{^[^\+].*}}codesign