[Build] allow to generate symbols for a subset of binaries

This would be needed to reduce overall build times in scenarios when
generating symbols for all binaries is too expensive and/or not needed.

Addresses rdar://76865276

Author: edymtt

utils/build-script

@@ -813,6 +813,12 @@ class BuildScriptInvocation(object):
                 ' '.join(args.llvm_ninja_targets_for_cross_compile_hosts)
             ]
 
+        if args.darwin_symroot_path_filters:
+            impl_args += [
+                "--darwin_symroot_path_filters=%s" %
+                ' '.join(args.darwin_symroot_path_filters)
+            ]
+
         # Compute the set of host-specific variables, which we pass through to
         # the build script via environment variables.
         host_specific_variables = self.compute_host_specific_variables()

utils/build-script-impl

@@ -103,6 +103,7 @@ KNOWN_SETTINGS=(
     darwin-deployment-version-watchos             "2.0"             "minimum deployment target version for watchOS"
     darwin-install-extract-symbols                ""                "whether to extract symbols with dsymutil during installations"
     darwin-install-extract-symbols-use-just-built-dsymutil "1"       "whether we should extract symbols using the just built dsymutil"
+    darwin-symroot-path-filters                   ""                "space-separated list of path patterns to consider for symbol generation"
     darwin-overlay-target                         ""                "single overlay target to build, dependencies are computed later"
     darwin-sdk-deployment-targets                 "xctest-ios-8.0"  "semicolon-separated list of triples like 'fookit-ios-9.0;barkit-watchos-9.0'"
     darwin-stdlib-install-name-dir                ""                "the directory of the install_name for standard library dylibs"
@@ -3111,6 +3112,26 @@ function printJSONEndTimestamp() {
     printJSONTimestamp ${command} "end"
 }
 
+function grep_that_allows_no_matches() {
+    # This will not cause the script to fail
+    # if no line in the input matches the pattern
+    grep "$@" || test $? = 1
+}
+
+function filter_paths() {
+    if [[ -n "$1" ]]; then
+        local -a filters
+        filters=($1)
+        local -a grep_arguments
+        for cfilter in "${filters[@]}"; do
+            grep_arguments+=('-e' "$cfilter")
+        done
+        grep_that_allows_no_matches "${grep_arguments[@]}"
+    else
+        cat
+    fi
+}
+
 for host in "${ALL_HOSTS[@]}"; do
     # Check if we should perform this action.
     if ! [[ $(should_execute_action "${host}-extractsymbols") ]]; then
@@ -3150,7 +3171,9 @@ for host in "${ALL_HOSTS[@]}"; do
             # Copy executables and shared libraries from the `host_install_destdir` to
             # INSTALL_SYMROOT and run dsymutil on them.
             (cd "${CURRENT_INSTALL_DIR}" &&
-             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | cpio --insecure -pdm -v "${host_symroot}")
+             find ./"${CURRENT_PREFIX}" -perm -0111 -type f -print | \
+             filter_paths "${DARWIN_SYMROOT_PATH_FILTERS}" | \
+             cpio --insecure -pdm -v "${host_symroot}")
 
             dsymutil_path=
             if [[ -n "${DARWIN_INSTALL_EXTRACT_SYMBOLS_USE_JUST_BUILT_DSYMUTIL}" ]]; then

utils/build_swift/build_swift/driver_arguments.py

@@ -684,6 +684,12 @@ def create_argument_parser():
     option('--symbols-package', store_path,
            help='if provided, an archive of the symbols directory will be '
                 'generated at this path')
+    option('--darwin-symroot-path-filters', append,
+           type=argparse.ShellSplitType(),
+           help='Space separated list of patterns used to match '
+                'a subset of files to generate symbols for. '
+                'Only supported on Darwin. Can be called multiple times '
+                'to add multiple such options.')
 
     # -------------------------------------------------------------------------
     in_group('Build variant')

utils/build_swift/tests/expected_options.py

@@ -136,6 +136,7 @@
         defaults.DARWIN_DEPLOYMENT_VERSION_TVOS,
     'darwin_deployment_version_watchos':
         defaults.DARWIN_DEPLOYMENT_VERSION_WATCHOS,
+    'darwin_symroot_path_filters': [],
     'darwin_xcrun_toolchain': None,
     'distcc': False,
     'sccache': False,
@@ -688,6 +689,7 @@ class BuildScriptImplOption(_BaseOption):
     AppendOption('--test-paths'),
     AppendOption('--llvm-ninja-targets'),
     AppendOption('--llvm-ninja-targets-for-cross-compile-hosts'),
+    AppendOption('--darwin-symroot-path-filters'),
 
     UnsupportedOption('--build-jobs'),
     UnsupportedOption('--common-cmake-options'),

validation-test/BuildSystem/extractsymbols-darwin-symroot-path-filters.test

@@ -0,0 +1,111 @@
+# REQUIRES: standalone_build
+# REQUIRES: OS=macosx
+
+# RUN: %empty-directory(%t)
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+
+# Even though we are running build-script with dry-run,
+# symbol extraction runs real commands against the file system.
+# Thus we generate a series of files
+# to exercise the filtering logic
+# RUN: mkdir -p %t/destdir/bin
+# RUN: mkdir -p %t/destdir/lib
+# RUN: %swiftc_driver %t/hello.swift -o %t/destdir/bin/swift-demangle
+# RUN: %swiftc_driver %t/hello.swift -o %t/destdir/bin/swift-def-to-yaml-converter
+# RUN: ln -s %t/destdir/swift-demangle %t/destdir/bin/swift-api-digester
+# RUN: cp %t/swift-util.py %t/destdir/bin
+# RUN: chmod a+x %t/destdir/bin/swift-util.py
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -o %t/destdir/lib/libswiftDemangle.dylib
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -o %t/destdir/lib/lib_InternalSwiftScan.dylib
+# RUN: %swiftc_driver %t/dylib.swift -emit-library -static -o %t/destdir/lib/libswiftASTSectionImporter.a 
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+
+# test build-script-impl on its own
+# RUN: SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script-impl --dry-run --build-dir=%t/build --workspace=%swift_src_root/.. --cmake %cmake --only-execute macosx-%target-cpu-extractsymbols --host-cc /usr/bin/true --darwin-install-extract-symbols=1 --host-target=macosx-%target-cpu --install-symroot=%t/symroot --install-destdir=%t/destdir --build-jobs=1 --darwin-symroot-path-filters="/lib/ /swift-demangle"  2>&1 | tee %t/build-script-impl-output.txt
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-impl-output.txt --check-prefixes CHECK-SKIPPED %s
+
+# ensure build-script pass the argument to build-script-impl
+# RUN: %empty-directory(%t/symroot)
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/ /swift-demangle" --jobs=1  2>&1 | tee %t/build-script-output.txt
+# RUN: %FileCheck --input-file=%t/build-script-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-output.txt --check-prefixes CHECK-SKIPPED %s
+
+# ensure we get all the values if we specify the flag multiple times
+# RUN: %empty-directory(%t/symroot)
+# RUN: mkdir -p %t/symroot/macosx-%target-cpu
+# RUN: SKIP_XCODE_VERSION_CHECK=1 SWIFT_BUILD_ROOT=%t %swift_src_root/utils/build-script --dry-run --cmake %cmake --darwin-install-extract-symbols=1 --install-destdir=%t/destdir --toolchain-prefix="" --install-symroot=%t/symroot --darwin-symroot-path-filters="/lib/" --darwin-symroot-path-filters="/swift-demangle" --jobs=1  2>&1 | tee %t/build-script-output.txt
+# RUN: %FileCheck --input-file=%t/build-script-output.txt %s
+# RUN: %FileCheck --input-file=%t/build-script-output.txt --check-prefixes CHECK-SKIPPED %s
+
+
+# CHECK: --- Extracting symbols ---
+
+# Ensure we copy all the files in lib and the swift-demangle executable
+# CHECK-LABEL: cpio
+# CHECK-DAG: swift-demangle
+# CHECK-DAG: libswiftDemangle.dylib
+# CHECK-DAG: lib_InternalSwiftScan.dylib
+
+# Ensure we generate symbols for the file in the symroot
+# CHECK-LABEL: "command": "dsymutil", "start"
+# CHECK-DAG: dsymutil {{.*}}swift-demangle
+# CHECK-DAG: dsymutil {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: dsymutil {{.*}}lib_InternalSwiftScan.dylib
+
+# Ensure we strip the files in the installation directory
+# (which are not subject to the filters)
+# CHECK-LABEL: xcrun_find_tool strip
+# CHECK-DAG: strip {{.*}}swift-demangle
+# CHECK-DAG: strip {{.*}}swift-def-to-yaml-converter
+# CHECK-DAG: strip {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: strip {{.*}}lib_InternalSwiftScan.dylib
+# CHECK-DAG: strip {{.*}}libswiftASTSectionImporter.a
+# CHECK-DAG: strip {{.*}}swift-util.py
+
+# Ensure we codesign dylibs
+# CHECK-LABEL: xcrun_find_tool codesign
+# CHECK-DAG: codesign {{.*}}libswiftDemangle.dylib
+# CHECK-DAG: codesign {{.*}}lib_InternalSwiftScan.dylib
+
+
+# CHECK-SKIPPED: --- Extracting symbols ---
+
+# Ensure we don't copy files not included by the filters
+# CHECK-SKIPPED-LABEL: cpio
+# CHECK-SKIPPED-NOT: swift-util.py
+# CHECK-SKIPPED-NOT: swift-def-to-yaml-converter
+# CHECK-SKIPPED-NOT: libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: swift-api-digester
+
+# Ensure we don't generate symbols for files we did not copy
+# CHECK-SKIPPED-LABEL: "command": "dsymutil", "start"
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-def-to-yaml-converter
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: dsymutil {{.*}}swift-api-digester
+
+# Ensure we don't strip symlinks
+# CHECK-SKIPPED-LABEL: xcrun_find_tool strip
+# CHECK-SKIPPED-NOT: strip {{.*}}swift-api-digester
+
+# Ensure we don't codesign executables, symlinks,
+# static archives and python scripts
+# CHECK-SKIPPED-LABEL: xcrun_find_tool codesign
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-demangle
+# CHECK-SKIPPED-NOT: codesign {{.*}}libswiftASTSectionImporter.a
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-util.py
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-api-digester
+# CHECK-SKIPPED-NOT: codesign {{.*}}swift-def-to-yaml-converter
+
+#--- hello.swift
+print("hello")
+
+#--- dylib.swift
+func greet(person: String) -> String {
+    return "Hello \(person)"
+}
+#--- swift-util.py
+print("hello")