Skip to content

[SR-237][build-script] Migrate Ninja build to Python #2685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 26, 2016
Merged

[SR-237][build-script] Migrate Ninja build to Python #2685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b1109c6
[build-script] Provide centralized command line and file system inter…
rintaro Apr 18, 2016
36898fc
[build-script][SR-237] Migrate Ninja build to Python
rintaro Apr 26, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 78 additions & 20 deletions utils/build-script
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import pipes
import platform
import re
import shlex
import shutil
import sys

# FIXME: Instead of modifying the system path in order to enable imports from
Expand All @@ -31,7 +30,6 @@ from SwiftBuildSupport import (
HOME,
SWIFT_BUILD_ROOT,
SWIFT_SOURCE_ROOT,
WorkingDirectory,
check_call,
get_all_preset_names,
get_preset_options,
Expand All @@ -45,6 +43,8 @@ sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
from swift_build_support.toolchain import host_toolchain # noqa (E402)
import swift_build_support.debug # noqa (E402)
from swift_build_support import migration # noqa (E402)
from swift_build_support import products # noqa (E402)
from swift_build_support import shell # noqa (E402)
import swift_build_support.tar # noqa (E402)
import swift_build_support.targets # noqa (E402)
from swift_build_support.cmake import CMake # noqa (E402)
Expand Down Expand Up @@ -393,6 +393,10 @@ details of the setups of other systems or automated environments.""")
help="build libdispatch",
action="store_true",
dest="build_libdispatch")
projects_group.add_argument(
"--build-ninja",
help="build the Ninja tool",
action="store_true")

extra_actions_group = parser.add_argument_group(
title="Extra actions to perform before or in addition to building")
Expand Down Expand Up @@ -938,6 +942,27 @@ details of the setups of other systems or automated environments.""")
type=argparse_clang_compiler_version,
metavar="MAJOR.MINOR.PATCH")

parser.add_argument(
"--darwin-deployment-version-osx",
help="minimum deployment target version for OS X",
metavar="MAJOR.MINOR",
default="10.9")
parser.add_argument(
"--darwin-deployment-version-ios",
help="minimum deployment target version for iOS",
metavar="MAJOR.MINOR",
default="7.0")
parser.add_argument(
"--darwin-deployment-version-tvos",
help="minimum deployment target version for tvOS",
metavar="MAJOR.MINOR",
default="9.0")
parser.add_argument(
"--darwin-deployment-version-watchos",
help="minimum deployment target version for watchOS",
metavar="MAJOR.MINOR",
default="2.0")

parser.add_argument(
"--extra-cmake-options",
help="Pass through extra options to CMake in the form of comma "
Expand Down Expand Up @@ -1098,6 +1123,11 @@ details of the setups of other systems or automated environments.""")
if args.cmake_generator is None:
args.cmake_generator = "Ninja"

ninja_required = (
args.cmake_generator == 'Ninja' or args.build_foundation)
if ninja_required and toolchain.ninja is None:
args.build_ninja = True

# SwiftPM and XCTest have a dependency on Foundation.
# On OS X, Foundation is built automatically using xcodebuild.
# On Linux, we must ensure that it is built manually.
Expand Down Expand Up @@ -1200,8 +1230,41 @@ details of the setups of other systems or automated environments.""")
source_root=SWIFT_SOURCE_ROOT,
build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))

if args.clean and os.path.isdir(workspace.build_root):
shutil.rmtree(workspace.build_root)
if args.build_ninja:
if not os.path.exists(workspace.source_dir("ninja")):
print_with_argv0("Can't find source directory for ninja "
"(tried %s)" % (workspace.source_dir("ninja")))
return 1

# Unset environment variables that might affect how tools behave.
for v in [
'MAKEFLAGS',
'SDKROOT',
'MACOSX_DEPLOYMENT_TARGET',
'IPHONEOS_DEPLOYMENT_TARGET',
'TVOS_DEPLOYMENT_TARGET',
'WATCHOS_DEPLOYMENT_TARGET']:
os.environ.pop(v, None)

if args.show_sdks:
swift_build_support.debug.print_xcodebuild_versions()

# Clean build direcotry if requested.
if args.clean:
shell.rmtree(workspace.build_root)
Copy link
Contributor

@gribozavr gribozavr May 25, 2016 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The if args.show_sdks: and if args.clean: statements seem to be interrupting the flow here. Is there an ordering dependency? If not, could you reorder this code so that first we show SDKs, then clean, create the build directory, then configure ninja and build it?


# Create build directory.
shell.makedirs(workspace.build_root)

# Build ninja if required.
if args.build_ninja:
ninja_build = products.Ninja(
args=args,
toolchain=toolchain,
source_dir=workspace.source_dir("ninja"),
build_dir=workspace.build_dir("build", "ninja"))
ninja_build.do_build()
toolchain.ninja = ninja_build.ninja_bin_path

cmake = CMake(args=args,
toolchain=toolchain)
Expand All @@ -1216,6 +1279,14 @@ details of the setups of other systems or automated environments.""")
"--host-cc", toolchain.cc,
"--host-cxx", toolchain.cxx,
"--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
"--darwin-deployment-version-osx=%s" % (
args.darwin_deployment_version_osx),
"--darwin-deployment-version-ios=%s" % (
args.darwin_deployment_version_ios),
"--darwin-deployment-version-tvos=%s" % (
args.darwin_deployment_version_tvos),
"--darwin-deployment-version-watchos=%s" % (
args.darwin_deployment_version_watchos),
"--cmake", toolchain.cmake,
"--cmark-build-type", args.cmark_build_variant,
"--llvm-build-type", args.llvm_build_variant,
Expand All @@ -1237,6 +1308,8 @@ details of the setups of other systems or automated environments.""")
pipes.quote(arg) for arg in cmake.build_args()),
]

if toolchain.ninja:
build_script_impl_args += ["--ninja-bin=%s" % toolchain.ninja]
if args.distcc:
build_script_impl_args += [
"--distcc",
Expand All @@ -1256,8 +1329,6 @@ details of the setups of other systems or automated environments.""")
build_script_impl_args += [
"--install-symroot", os.path.abspath(args.install_symroot)
]
if args.cmake_generator == 'Ninja' and toolchain.ninja is None:
build_script_impl_args += ["--build-ninja"]

if args.skip_build:
build_script_impl_args += ["--skip-build-cmark",
Expand Down Expand Up @@ -1381,19 +1452,6 @@ details of the setups of other systems or automated environments.""")

build_script_impl_args += args.build_script_impl_args

# Unset environment variables that might affect how tools behave.
for v in [
'MAKEFLAGS',
'SDKROOT',
'MACOSX_DEPLOYMENT_TARGET',
'IPHONEOS_DEPLOYMENT_TARGET',
'TVOS_DEPLOYMENT_TARGET',
'WATCHOS_DEPLOYMENT_TARGET']:
os.environ.pop(v, None)

if args.show_sdks:
swift_build_support.debug.print_xcodebuild_versions()

check_call([build_script_impl] + build_script_impl_args,
disable_sleep=True)

Expand All @@ -1411,7 +1469,7 @@ details of the setups of other systems or automated environments.""")
# it is archiving. To stay safe, we change working directories, then
# run `tar` without the leading '/' (we remove it ourselves to keep
# `tar` from emitting a warning).
with WorkingDirectory(args.install_symroot):
with shell.pushd(args.install_symroot):
swift_build_support.tar.tar(source=prefix.lstrip('/'),
destination=args.symbols_package)

Expand Down
45 changes: 1 addition & 44 deletions utils/build-script-impl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ KNOWN_SETTINGS=(
host-cc "" "the path to CC, the 'clang' compiler for the host platform. **This argument is required**"
host-cxx "" "the path to CXX, the 'clang++' compiler for the host platform. **This argument is required**"
darwin-xcrun-toolchain "default" "the name of the toolchain to use on Darwin"
build-ninja "" "build the Ninja tool"
ninja-bin "" "the path to Ninja tool"
cmark-build-type "Debug" "the CMake build variant for CommonMark (Debug, RelWithDebInfo, Release, MinSizeRel). Defaults to Debug."
lldb-extra-cmake-args "" "extra command line args to pass to lldb cmake"
lldb-extra-xcodebuild-args "" "extra command line args to pass to lldb xcodebuild"
Expand Down Expand Up @@ -1258,43 +1258,6 @@ function set_swiftpm_bootstrap_command() {
fi
}

mkdir -p "${BUILD_DIR}"

#
# Build Ninja
#
if [[ "${BUILD_NINJA}" ]] ; then
build_dir=$(build_directory build ninja)
if [ ! -f "${build_dir}/ninja" ] ; then
if [ ! -d "${NINJA_SOURCE_DIR}" ] ; then
echo "Can't find source directory for ninja (tried ${NINJA_SOURCE_DIR})"
exit 1
fi

# Ninja can only be built in-tree. Copy the source tree to the build
# directory.
set -x
rm -rf "${build_dir}"
cp -r "${NINJA_SOURCE_DIR}" "${build_dir}"
if [[ $(uname -s) == "Darwin" ]]; then
(cd "${build_dir}" && \
env CXX=$(xcrun --sdk macosx -find clang++) \
CFLAGS="-isysroot $(xcrun --sdk macosx --show-sdk-path) -mmacosx-version-min=${DARWIN_DEPLOYMENT_VERSION_OSX}" \
LDFLAGS="-mmacosx-version-min=${DARWIN_DEPLOYMENT_VERSION_OSX}" \
python ./configure.py --bootstrap)
{ set +x; } 2>/dev/null
else
(cd "${build_dir}" && python ./configure.py --bootstrap)
{ set +x; } 2>/dev/null
fi
fi
NINJA_BIN="${build_dir}/ninja"
COMMON_CMAKE_OPTIONS=(
"${COMMON_CMAKE_OPTIONS[@]}"
-DCMAKE_MAKE_PROGRAM=${NINJA_BIN}
)
fi

#
# Configure and build each product
#
Expand Down Expand Up @@ -1809,7 +1772,6 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_COMPILE_TOOLS_DEPLOYMENT_TARG
SWIFT_BIN="$(build_directory_bin ${deployment_target} swift)/swift"
SWIFT_BUILD_PATH="$(build_directory ${deployment_target} swift)"
LLVM_BIN="$(build_directory_bin ${deployment_target} llvm)"
NINJA_BIN="ninja"

# Staging: require opt-in for building with dispatch
if [[ ! "${SKIP_BUILD_LIBDISPATCH}" ]] ; then
Expand All @@ -1821,11 +1783,6 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_COMPILE_TOOLS_DEPLOYMENT_TARG
SWIFT_USE_LINKER="-fuse-ld=gold"
fi

if [[ "${BUILD_NINJA}" ]]; then
NINJA_BUILD_DIR=$(build_directory build ninja)
NINJA_BIN="${NINJA_BUILD_DIR}/ninja"
fi

set -x
pushd "${FOUNDATION_SOURCE_DIR}"
SWIFTC="${SWIFTC_BIN}" CLANG="${LLVM_BIN}"/clang SWIFT="${SWIFT_BIN}" \
Expand Down
3 changes: 3 additions & 0 deletions utils/swift_build_support/swift_build_support/cmake.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ def common_options(self):
define("LLVM_VERSION_MINOR:STRING", minor)
define("LLVM_VERSION_PATCH:STRING", patch)

if args.build_ninja and args.cmake_generator == 'Ninja':
define('CMAKE_MAKE_PROGRAM', toolchain.ninja)

return options

def build_args(self):
Expand Down
17 changes: 17 additions & 0 deletions utils/swift_build_support/swift_build_support/products/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# swift_build_support/products/__init__.py ----------------------*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See http://swift.org/LICENSE.txt for license information
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ----------------------------------------------------------------------------

from .ninja import Ninja

__all__ = [
'Ninja',
]
63 changes: 63 additions & 0 deletions utils/swift_build_support/swift_build_support/products/ninja.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# swift_build_support/producsts/ninja.py ------------------------*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See http://swift.org/LICENSE.txt for license information
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ----------------------------------------------------------------------------
"""
Ninja build
"""
# ----------------------------------------------------------------------------

import os.path
import platform
import sys

from .. import shell
from .. import cache_util


class Ninja(object):

def __init__(self, args, toolchain, source_dir, build_dir):
self.args = args
self.toolchain = toolchain
self.source_dir = source_dir
self.build_dir = build_dir

@cache_util.reify
def ninja_bin_path(self):
return os.path.join(self.build_dir, 'ninja')

def do_build(self):
if os.path.exists(self.ninja_bin_path):
return

env = None
if platform.system() == "Darwin":
from .. import xcrun
sysroot = xcrun.sdk_path("macosx")
osx_version_min = self.args.darwin_deployment_version_osx
assert sysroot is not None
env = [
("CXX", self.toolchain.cxx),
("CFLAGS", (
"-isysroot {sysroot} -mmacosx-version-min={osx_version}"
).format(sysroot=sysroot, osx_version=osx_version_min)),
("LDFLAGS", (
"-mmacosx-version-min={osx_version}"
).format(osx_version=osx_version_min)),
]

# Ninja can only be built in-tree. Copy the source tree to the build
# directory.
shell.rmtree(self.build_dir)
shell.copytree(self.source_dir, self.build_dir)
with shell.pushd(self.build_dir):
shell.call([sys.executable, 'configure.py', '--bootstrap'],
env=env)
Loading