[build-script] Archive symbols in Python

Rather than archiving symbols at the very end of the build-script-impl
shellscript, do so at the end of the Python build-script. A small step towards
achieving SR-237.

Author: modocache

utils/build-script

@@ -33,13 +33,15 @@ from SwiftBuildSupport import (
     get_preset_options,
     print_with_argv0,
     quote_shell_command,
+    WorkingDirectory,
 )
 
 sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
 import swift_build_support.clang
 import swift_build_support.cmake
 import swift_build_support.debug
 import swift_build_support.ninja
+import swift_build_support.tar
 import swift_build_support.targets
 from swift_build_support.migration import migrate_impl_args
 
@@ -336,6 +338,10 @@ details of the setups of other systems or automated environments.""")
     extra_actions_group.add_argument("--export-compile-commands",
         help="generate compilation databases in addition to building",
         action="store_true")
+    extra_actions_group.add_argument("--symbols-package",
+        metavar="PATH",
+        help="if provided, an archive of the symbols directory will be "
+             "generated at this path")
 
     build_variant_group = parser.add_mutually_exclusive_group(required=False)
     build_variant_group.add_argument("-d", "--debug",
@@ -538,6 +544,9 @@ placed""",
              "(like bin, lib, and include) will be installed.",
         metavar="PATH",
         default=swift_build_support.targets.install_prefix())
+    parser.add_argument("--install-symroot",
+        help="the path to install debug symbols into",
+        metavar="PATH")
 
     parser.add_argument("-j", "--jobs",
         help="""
@@ -573,12 +582,24 @@ the number of parallel build jobs to use""",
         '--skip-build',
         '--show-sdks',
         '--install-prefix',
+        '--install-symroot',
+        '--symbols-package',
     ]))
 
     if args.host_target is None:
         print_with_argv0("Unknown operating system.")
         return 1
 
+    if args.symbols_package:
+        if not os.path.isabs(args.symbols_package):
+            print('--symbols-package must be an absolute path '
+                  '(was \'{}\')'.format(args.symbols_package))
+            return 1
+        if not args.install_symroot:
+            print_with_argv0("--install-symroot is required when specifying "
+                             "--symbols-package.")
+            return 1
+
     # Build cmark if any cmark-related options were specified.
     if (args.cmark_build_variant is not None):
         args.build_cmark = True
@@ -891,6 +912,24 @@ the number of parallel build jobs to use""",
 
     check_call(build_script_impl_args)
 
+    if args.symbols_package:
+        print('--- Creating symbols package ---')
+        print('-- Package file: {} --'.format(args.symbols_package))
+
+        if platform.system() == 'Darwin':
+            prefix = swift_build_support.targets.darwin_toolchain_prefix(
+                args.install_prefix)
+        else:
+            prefix = args.install_prefix
+
+        # As a security measure, `tar` normally strips leading '/' from paths
+        # it is archiving. To stay safe, we change working directories, then
+        # run `tar` without the leading '/' (we remove it ourselves to keep
+        # `tar` from emitting a warning).
+        with WorkingDirectory(args.install_symroot):
+            swift_build_support.tar.tar(source=prefix.lstrip('/'),
+                                        destination=args.symbols_package)
+
     return 0
 
 

utils/build-script-impl

@@ -93,7 +93,6 @@ KNOWN_SETTINGS=(
     llvm-install-components    ""                "a semicolon-separated list of LLVM components to install"
     installable-package         ""               "the path to the archive of the installation directory"
     test-installable-package    ""               "whether to run post-packaging tests on the produced package"
-    symbols-package             ""               "the path to the archive of the symbols directory"
     reconfigure                 ""               "force a CMake configuration run even if CMakeCache.txt already exists"
     swift-sdks                  ""               "build target binaries only for specified SDKs (semicolon-separated list)"
     swift-primary-variant-sdk   ""               "default SDK for target binaries"
@@ -662,14 +661,6 @@ case "${INSTALLABLE_PACKAGE}" in
         exit 1
         ;;
 esac
-case "${SYMBOLS_PACKAGE}" in
-    /*) ;;
-    "") ;;
-    *)
-        echo "symbols-package must be an absolute path (was '${SYMBOLS_PACKAGE}')"
-        exit 1
-        ;;
-esac
 
 # WORKSPACE must exist
 if [ ! -e "${WORKSPACE}" ] ; then
@@ -2324,15 +2315,3 @@ if [[ "${INSTALLABLE_PACKAGE}" ]] ; then
         { set +x; } 2>/dev/null
     fi
 fi
-
-if [[ "${SYMBOLS_PACKAGE}" ]] ; then
-    echo "--- Creating symbols package ---"
-    echo "-- Package file: ${SYMBOLS_PACKAGE} --"
-    if [[ "$(uname -s)" == "Darwin" ]] ; then
-      (cd "${INSTALL_SYMROOT}" &&
-        tar -c -z -f "${SYMBOLS_PACKAGE}" "${TOOLCHAIN_PREFIX/#\/}")
-    else
-      (cd "${INSTALL_SYMROOT}" &&
-        tar -c -z -f "${SYMBOLS_PACKAGE}" --owner=0 --group=0 "${INSTALL_PREFIX/#\/}")
-    fi
-fi

utils/swift_build_support/swift_build_support/tar.py

@@ -0,0 +1,31 @@
+# swift_build_support/tar.py - Call tar from Python -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+import platform
+import subprocess
+
+
+def tar(source, destination):
+    """
+    Create a gzip archive of the file at 'source' at the given
+    'destination' path.
+    """
+    # We do not use `tarfile` here because:
+    #  - We wish to support LZMA2 compression while also supporting Python 2.7.
+    #  - We wish to explicitly set the owner and group of the archive.
+    args = ['tar', '-c', '-z', '-f', destination]
+
+    if platform.system() != 'Darwin':
+        args += ['--owner=0', '--group=0']
+
+    # Capture stderr output such as 'tar: Failed to open ...'. We'll detect
+    # these cases using the exit code, which should cause 'check_call' to
+    # raise.
+    subprocess.check_call(args + [source], stderr=subprocess.PIPE)

utils/swift_build_support/tests/test_tar.py

@@ -0,0 +1,33 @@
+# test_tar.py - Unit tests for swift_build_support.tar -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+import os
+import subprocess
+import tempfile
+import unittest
+
+from swift_build_support.tar import tar
+
+
+class TarTestCase(unittest.TestCase):
+	def test_tar_this_file_succeeds(self):
+		# `tar` complains about absolute paths, so use a relative path here.
+		source = os.path.relpath(__file__)
+		_, destination = tempfile.mkstemp()
+		tar(source=source, destination=destination)
+
+	def test_tar_nonexistent_file_raises(self):
+		with self.assertRaises(subprocess.CalledProcessError):
+			tar(source='/path/to/something/that/surely/doesnt/exist',
+				destination='/another/path/that/shouldnt/exist')
+
+
+if __name__ == '__main__':
+	unittest.main()