Merge pull request #2685 from rintaro/build-script-build-ninja

[SR-237][build-script] Migrate Ninja build to Python

Author: gribozavr

utils/build-script

@@ -18,7 +18,6 @@ import pipes
 import platform
 import re
 import shlex
-import shutil
 import sys
 
 # FIXME: Instead of modifying the system path in order to enable imports from
@@ -31,7 +30,6 @@ from SwiftBuildSupport import (
     HOME,
     SWIFT_BUILD_ROOT,
     SWIFT_SOURCE_ROOT,
-    WorkingDirectory,
     check_call,
     get_all_preset_names,
     get_preset_options,
@@ -45,6 +43,8 @@ sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
 from swift_build_support.toolchain import host_toolchain  # noqa (E402)
 import swift_build_support.debug      # noqa (E402)
 from swift_build_support import migration  # noqa (E402)
+from swift_build_support import products  # noqa (E402)
+from swift_build_support import shell  # noqa (E402)
 import swift_build_support.tar        # noqa (E402)
 import swift_build_support.targets    # noqa (E402)
 from swift_build_support.cmake import CMake  # noqa (E402)
@@ -393,6 +393,10 @@ details of the setups of other systems or automated environments.""")
         help="build libdispatch",
         action="store_true",
         dest="build_libdispatch")
+    projects_group.add_argument(
+        "--build-ninja",
+        help="build the Ninja tool",
+        action="store_true")
 
     extra_actions_group = parser.add_argument_group(
         title="Extra actions to perform before or in addition to building")
@@ -938,6 +942,27 @@ details of the setups of other systems or automated environments.""")
         type=argparse_clang_compiler_version,
         metavar="MAJOR.MINOR.PATCH")
 
+    parser.add_argument(
+        "--darwin-deployment-version-osx",
+        help="minimum deployment target version for OS X",
+        metavar="MAJOR.MINOR",
+        default="10.9")
+    parser.add_argument(
+        "--darwin-deployment-version-ios",
+        help="minimum deployment target version for iOS",
+        metavar="MAJOR.MINOR",
+        default="7.0")
+    parser.add_argument(
+        "--darwin-deployment-version-tvos",
+        help="minimum deployment target version for tvOS",
+        metavar="MAJOR.MINOR",
+        default="9.0")
+    parser.add_argument(
+        "--darwin-deployment-version-watchos",
+        help="minimum deployment target version for watchOS",
+        metavar="MAJOR.MINOR",
+        default="2.0")
+
     parser.add_argument(
         "--extra-cmake-options",
         help="Pass through extra options to CMake in the form of comma "
@@ -1098,6 +1123,11 @@ details of the setups of other systems or automated environments.""")
     if args.cmake_generator is None:
         args.cmake_generator = "Ninja"
 
+    ninja_required = (
+        args.cmake_generator == 'Ninja' or args.build_foundation)
+    if ninja_required and toolchain.ninja is None:
+        args.build_ninja = True
+
     # SwiftPM and XCTest have a dependency on Foundation.
     # On OS X, Foundation is built automatically using xcodebuild.
     # On Linux, we must ensure that it is built manually.
@@ -1200,8 +1230,41 @@ details of the setups of other systems or automated environments.""")
         source_root=SWIFT_SOURCE_ROOT,
         build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))
 
-    if args.clean and os.path.isdir(workspace.build_root):
-        shutil.rmtree(workspace.build_root)
+    if args.build_ninja:
+        if not os.path.exists(workspace.source_dir("ninja")):
+            print_with_argv0("Can't find source directory for ninja "
+                             "(tried %s)" % (workspace.source_dir("ninja")))
+            return 1
+
+    # Unset environment variables that might affect how tools behave.
+    for v in [
+            'MAKEFLAGS',
+            'SDKROOT',
+            'MACOSX_DEPLOYMENT_TARGET',
+            'IPHONEOS_DEPLOYMENT_TARGET',
+            'TVOS_DEPLOYMENT_TARGET',
+            'WATCHOS_DEPLOYMENT_TARGET']:
+        os.environ.pop(v, None)
+
+    if args.show_sdks:
+        swift_build_support.debug.print_xcodebuild_versions()
+
+    # Clean build direcotry if requested.
+    if args.clean:
+        shell.rmtree(workspace.build_root)
+
+    # Create build directory.
+    shell.makedirs(workspace.build_root)
+
+    # Build ninja if required.
+    if args.build_ninja:
+        ninja_build = products.Ninja(
+            args=args,
+            toolchain=toolchain,
+            source_dir=workspace.source_dir("ninja"),
+            build_dir=workspace.build_dir("build", "ninja"))
+        ninja_build.do_build()
+        toolchain.ninja = ninja_build.ninja_bin_path
 
     cmake = CMake(args=args,
                   toolchain=toolchain)
@@ -1216,6 +1279,14 @@ details of the setups of other systems or automated environments.""")
         "--host-cc", toolchain.cc,
         "--host-cxx", toolchain.cxx,
         "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
+        "--darwin-deployment-version-osx=%s" % (
+            args.darwin_deployment_version_osx),
+        "--darwin-deployment-version-ios=%s" % (
+            args.darwin_deployment_version_ios),
+        "--darwin-deployment-version-tvos=%s" % (
+            args.darwin_deployment_version_tvos),
+        "--darwin-deployment-version-watchos=%s" % (
+            args.darwin_deployment_version_watchos),
         "--cmake", toolchain.cmake,
         "--cmark-build-type", args.cmark_build_variant,
         "--llvm-build-type", args.llvm_build_variant,
@@ -1237,6 +1308,8 @@ details of the setups of other systems or automated environments.""")
             pipes.quote(arg) for arg in cmake.build_args()),
     ]
 
+    if toolchain.ninja:
+        build_script_impl_args += ["--ninja-bin=%s" % toolchain.ninja]
     if args.distcc:
         build_script_impl_args += [
             "--distcc",
@@ -1256,8 +1329,6 @@ details of the setups of other systems or automated environments.""")
         build_script_impl_args += [
             "--install-symroot", os.path.abspath(args.install_symroot)
         ]
-    if args.cmake_generator == 'Ninja' and toolchain.ninja is None:
-        build_script_impl_args += ["--build-ninja"]
 
     if args.skip_build:
         build_script_impl_args += ["--skip-build-cmark",
@@ -1381,19 +1452,6 @@ details of the setups of other systems or automated environments.""")
 
     build_script_impl_args += args.build_script_impl_args
 
-    # Unset environment variables that might affect how tools behave.
-    for v in [
-            'MAKEFLAGS',
-            'SDKROOT',
-            'MACOSX_DEPLOYMENT_TARGET',
-            'IPHONEOS_DEPLOYMENT_TARGET',
-            'TVOS_DEPLOYMENT_TARGET',
-            'WATCHOS_DEPLOYMENT_TARGET']:
-        os.environ.pop(v, None)
-
-    if args.show_sdks:
-        swift_build_support.debug.print_xcodebuild_versions()
-
     check_call([build_script_impl] + build_script_impl_args,
                disable_sleep=True)
 
@@ -1411,7 +1469,7 @@ details of the setups of other systems or automated environments.""")
         # it is archiving. To stay safe, we change working directories, then
         # run `tar` without the leading '/' (we remove it ourselves to keep
         # `tar` from emitting a warning).
-        with WorkingDirectory(args.install_symroot):
+        with shell.pushd(args.install_symroot):
             swift_build_support.tar.tar(source=prefix.lstrip('/'),
                                         destination=args.symbols_package)
 

utils/build-script-impl

@@ -60,7 +60,7 @@ KNOWN_SETTINGS=(
     host-cc                     ""               "the path to CC, the 'clang' compiler for the host platform. **This argument is required**"
     host-cxx                    ""               "the path to CXX, the 'clang++' compiler for the host platform. **This argument is required**"
     darwin-xcrun-toolchain      "default"        "the name of the toolchain to use on Darwin"
-    build-ninja                 ""               "build the Ninja tool"
+    ninja-bin                   ""               "the path to Ninja tool"
     cmark-build-type            "Debug"          "the CMake build variant for CommonMark (Debug, RelWithDebInfo, Release, MinSizeRel).  Defaults to Debug."
     lldb-extra-cmake-args       ""               "extra command line args to pass to lldb cmake"
     lldb-extra-xcodebuild-args  ""               "extra command line args to pass to lldb xcodebuild"
@@ -1258,43 +1258,6 @@ function set_swiftpm_bootstrap_command() {
     fi
 }
 
-mkdir -p "${BUILD_DIR}"
-
-#
-# Build Ninja
-#
-if [[ "${BUILD_NINJA}" ]] ; then
-    build_dir=$(build_directory build ninja)
-    if [ ! -f "${build_dir}/ninja" ] ; then
-        if [ ! -d "${NINJA_SOURCE_DIR}" ] ; then
-          echo "Can't find source directory for ninja (tried ${NINJA_SOURCE_DIR})"
-          exit 1
-        fi
-
-        # Ninja can only be built in-tree.  Copy the source tree to the build
-        # directory.
-        set -x
-        rm -rf "${build_dir}"
-        cp -r "${NINJA_SOURCE_DIR}" "${build_dir}"
-        if [[ $(uname -s) == "Darwin" ]]; then
-          (cd "${build_dir}" && \
-            env CXX=$(xcrun --sdk macosx -find clang++) \
-                CFLAGS="-isysroot $(xcrun --sdk macosx --show-sdk-path) -mmacosx-version-min=${DARWIN_DEPLOYMENT_VERSION_OSX}" \
-                LDFLAGS="-mmacosx-version-min=${DARWIN_DEPLOYMENT_VERSION_OSX}" \
-                python ./configure.py --bootstrap)
-          { set +x; } 2>/dev/null
-        else
-          (cd "${build_dir}" && python ./configure.py --bootstrap)
-          { set +x; } 2>/dev/null
-        fi
-    fi
-    NINJA_BIN="${build_dir}/ninja"
-    COMMON_CMAKE_OPTIONS=(
-        "${COMMON_CMAKE_OPTIONS[@]}"
-        -DCMAKE_MAKE_PROGRAM=${NINJA_BIN}
-    )
-fi
-
 #
 # Configure and build each product
 #
@@ -1809,7 +1772,6 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_COMPILE_TOOLS_DEPLOYMENT_TARG
                 SWIFT_BIN="$(build_directory_bin ${deployment_target} swift)/swift"
                 SWIFT_BUILD_PATH="$(build_directory ${deployment_target} swift)"
                 LLVM_BIN="$(build_directory_bin ${deployment_target} llvm)"
-                NINJA_BIN="ninja"
 
                 # Staging: require opt-in for building with dispatch
                 if [[ ! "${SKIP_BUILD_LIBDISPATCH}" ]] ; then
@@ -1821,11 +1783,6 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_COMPILE_TOOLS_DEPLOYMENT_TARG
                     SWIFT_USE_LINKER="-fuse-ld=gold"
                 fi
 
-                if [[ "${BUILD_NINJA}" ]]; then
-                    NINJA_BUILD_DIR=$(build_directory build ninja)
-                    NINJA_BIN="${NINJA_BUILD_DIR}/ninja"
-                fi
-                
                 set -x
                 pushd "${FOUNDATION_SOURCE_DIR}"
                 SWIFTC="${SWIFTC_BIN}" CLANG="${LLVM_BIN}"/clang SWIFT="${SWIFT_BIN}" \

utils/swift_build_support/swift_build_support/cmake.py

@@ -113,6 +113,9 @@ def common_options(self):
             define("LLVM_VERSION_MINOR:STRING", minor)
             define("LLVM_VERSION_PATCH:STRING", patch)
 
+        if args.build_ninja and args.cmake_generator == 'Ninja':
+            define('CMAKE_MAKE_PROGRAM', toolchain.ninja)
+
         return options
 
     def build_args(self):

utils/swift_build_support/swift_build_support/products/__init__.py

@@ -0,0 +1,17 @@
+# swift_build_support/products/__init__.py ----------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+from .ninja import Ninja
+
+__all__ = [
+    'Ninja',
+]

utils/swift_build_support/swift_build_support/products/ninja.py

@@ -0,0 +1,63 @@
+# swift_build_support/producsts/ninja.py ------------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+"""
+Ninja build
+"""
+# ----------------------------------------------------------------------------
+
+import os.path
+import platform
+import sys
+
+from .. import shell
+from .. import cache_util
+
+
+class Ninja(object):
+
+    def __init__(self, args, toolchain, source_dir, build_dir):
+        self.args = args
+        self.toolchain = toolchain
+        self.source_dir = source_dir
+        self.build_dir = build_dir
+
+    @cache_util.reify
+    def ninja_bin_path(self):
+        return os.path.join(self.build_dir, 'ninja')
+
+    def do_build(self):
+        if os.path.exists(self.ninja_bin_path):
+            return
+
+        env = None
+        if platform.system() == "Darwin":
+            from .. import xcrun
+            sysroot = xcrun.sdk_path("macosx")
+            osx_version_min = self.args.darwin_deployment_version_osx
+            assert sysroot is not None
+            env = [
+                ("CXX", self.toolchain.cxx),
+                ("CFLAGS", (
+                    "-isysroot {sysroot} -mmacosx-version-min={osx_version}"
+                ).format(sysroot=sysroot, osx_version=osx_version_min)),
+                ("LDFLAGS", (
+                    "-mmacosx-version-min={osx_version}"
+                ).format(osx_version=osx_version_min)),
+            ]
+
+        # Ninja can only be built in-tree.  Copy the source tree to the build
+        # directory.
+        shell.rmtree(self.build_dir)
+        shell.copytree(self.source_dir, self.build_dir)
+        with shell.pushd(self.build_dir):
+            shell.call([sys.executable, 'configure.py', '--bootstrap'],
+                       env=env)