swiftlang · gottesmm · Jun 8, 2016 · Jun 7, 2016 · Jun 8, 2016 · Jun 7, 2016
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -140,7 +140,7 @@ set(SWIFT_NATIVE_CLANG_TOOLS_PATH "" CACHE STRING
 set(SWIFT_NATIVE_SWIFT_TOOLS_PATH "" CACHE STRING
     "Path to the directory that contains Swift tools that are executable on the build machine")
 
-option(SWIFT_ENABLE_LTO
+option(SWIFT_TOOLS_ENABLE_LTO
   "If set to true, build the swift compiler with link time optimization enabled" FALSE)
 
 # The following only works with the Ninja generator in CMake >= 3.0.
@@ -743,7 +743,7 @@ endif()
 message(STATUS "Building host Swift tools for ${SWIFT_HOST_VARIANT_SDK} ${SWIFT_HOST_VARIANT_ARCH}")
 message(STATUS "  Build type: ${CMAKE_BUILD_TYPE}")
 message(STATUS "  Assertions: ${LLVM_ENABLE_ASSERTIONS}")
-message(STATUS "  LTO:        ${SWIFT_ENABLE_LTO}")
+message(STATUS "  LTO:        ${SWIFT_TOOLS_ENABLE_LTO}")
 message(STATUS "")
 
 message(STATUS "Building Swift standard library and SDK overlays for SDKs: ${SWIFT_SDKS}")

diff --git a/cmake/modules/AddSwift.cmake b/cmake/modules/AddSwift.cmake
@@ -131,7 +131,7 @@ function(_add_variant_c_compile_flags)
     ARCH "${CFLAGS_ARCH}"
     BUILD_TYPE "${CFLAGS_BUILD_TYPE}"
     ENABLE_ASSERTIONS "${CFLAGS_ENABLE_ASSERTIONS}"
-    ENABLE_LTO "${SWIFT_ENABLE_LTO}"
+    ENABLE_LTO "${SWIFT_TOOLS_ENABLE_LTO}"
     ANALYZE_CODE_COVERAGE FALSE
     DEPLOYMENT_VERSION_IOS "${CFLAGS_DEPLOYMENT_VERSION_IOS}"
     RESULT_VAR_NAME result)
@@ -150,7 +150,7 @@ function(_add_variant_c_compile_flags)
 
   is_build_type_with_debuginfo("${CFLAGS_BUILD_TYPE}" debuginfo)
   if(debuginfo)
-    if(SWIFT_ENABLE_LTO)
+    if(SWIFT_TOOLS_ENABLE_LTO)
       list(APPEND result "-gline-tables-only")
     else()
       list(APPEND result "-g")
@@ -236,7 +236,7 @@ function(_add_variant_link_flags)
     ARCH "${LFLAGS_ARCH}"
     BUILD_TYPE "${LFLAGS_BUILD_TYPE}"
     ENABLE_ASSERTIONS "${LFLAGS_ENABLE_ASSERTIONS}"
-    ENABLE_LTO "${SWIFT_ENABLE_LTO}"
+    ENABLE_LTO "${SWIFT_TOOLS_ENABLE_LTO}"
     ANALYZE_CODE_COVERAGE "${LFLAGS_ANALYZE_CODE_COVERAGE}"
     DEPLOYMENT_VERSION_IOS "${LFLAGS_DEPLOYMENT_VERSION_IOS}"
     RESULT_VAR_NAME result)

diff --git a/test/lit.site.cfg.in b/test/lit.site.cfg.in
@@ -26,7 +26,7 @@ if "@SWIFT_ASAN_BUILD@" == "TRUE":
 else:
     config.available_features.add('no_asan')
 
-if '@SWIFT_ENABLE_LTO@' == 'TRUE':
+if '@SWIFT_TOOLS_ENABLE_LTO@' == 'TRUE':
     config.available_features.add('lto')
 else:
     config.available_features.add('no_lto')

diff --git a/utils/build-presets.ini b/utils/build-presets.ini
@@ -182,15 +182,15 @@ test-optimized=0
 mixin-preset=
     mixin_buildbot_tools_RA_stdlib_RDA
 
+lto
+
 dash-dash
 
 # Don't run host tests for iOS, tvOS and watchOS platforms to make the build
 # faster.
 skip-test-ios-host
 skip-test-tvos-host
 skip-test-watchos-host
-swift-enable-lto
-llvm-enable-lto
 
 #===------------------------------------------------------------------------===#
 # Incremental buildbots for Darwin OSes
@@ -415,9 +415,9 @@ skip-build-benchmarks
 mixin-preset=buildbot_incremental,tools=RA,stdlib=RD,smoketest=macosx
 build-subdir=buildbot_incremental
 
-dash-dash
+lto
 
-swift-enable-lto
+dash-dash
 
 [preset: buildbot_incremental,tools=RA,llvm-only]
 build-subdir=buildbot_incremental_llvmonly

diff --git a/utils/build-script b/utils/build-script
@@ -37,6 +37,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
 # E402 means module level import not at top of file
 from swift_build_support import arguments  # noqa (E402)
 from swift_build_support import diagnostics  # noqa (E402)
+from swift_build_support import host  # noqa (E402)
 from swift_build_support.toolchain import host_toolchain  # noqa (E402)
 import swift_build_support.debug      # noqa (E402)
 from swift_build_support import migration  # noqa (E402)
@@ -976,6 +977,31 @@ details of the setups of other systems or automated environments.""")
         default=False,
         const=True)
 
+    parser.add_argument(
+        "--lto",
+        help="use lto optimization on llvm/swift tools. This does not "
+             "imply using lto on the swift standard library or runtime",
+        metavar="BOOL",
+        nargs='?',
+        type=arguments.type.bool,
+        default=False,
+        const=True)
+
+    default_max_lto_link_job_counts = host.max_lto_link_job_counts()
+    parser.add_argument(
+        "--llvm-max-parallel-lto-link-jobs",
+        help="the maximum number of parallel link jobs to use when compiling "
+             "llvm",
+        metavar="COUNT",
+        default=default_max_lto_link_job_counts['llvm'])
+
+    parser.add_argument(
+        "--swift-tools-max-parallel-lto-link-jobs",
+        help="the maximum number of parallel link jobs to use when compiling "
+             "swift tools.",
+        metavar="COUNT",
+        default=default_max_lto_link_job_counts['swift'])
+
     parser.add_argument(
         # Explicitly unavailable options here.
         "--build-jobs",
@@ -1442,6 +1468,22 @@ details of the setups of other systems or automated environments.""")
                 pipes.quote(opt) for opt in args.extra_cmake_options)
         ]
 
+    if args.lto:
+        build_script_impl_args += [
+            "--llvm-enable-lto",
+            "--swift-tools-enable-lto"
+        ]
+        if args.llvm_max_parallel_lto_link_jobs is not None:
+            build_script_impl_args += [
+                "--llvm-num-parallel-lto-link-jobs=%s" %
+                min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs)
+            ]
+        if args.swift_tools_max_parallel_lto_link_jobs is not None:
+            build_script_impl_args += [
+                "--swift-tools-num-parallel-lto-link-jobs=%s" %
+                min(args.swift_tools_max_parallel_lto_link_jobs,
+                    args.build_jobs)
+            ]
     build_script_impl_args += args.build_script_impl_args
 
     if args.dry_run:

diff --git a/utils/build-script-impl b/utils/build-script-impl
@@ -74,8 +74,10 @@ KNOWN_SETTINGS=(
     swift-build-type            "Debug"          "the CMake build variant for Swift"
     swift-enable-assertions     "1"              "enable assertions in Swift"
     swift-analyze-code-coverage "not-merged"     "Code coverage analysis mode for Swift (false, not-merged, merged). Defaults to false if the argument is not present, and not-merged if the argument is present without a modifier."
-    swift-enable-lto            "0"              "enable LTO compilation of just Swift."
+    swift-tools-enable-lto      "0"              "enable LTO compilation of Swift tools. *NOTE* This does not include the swift standard library and runtime."
     llvm-enable-lto             "0"              "enable LTO compilation of LLVM/Clang."
+    swift-tools-num-parallel-lto-link-jobs ""    "The number of parallel link jobs to use when compiling swift tools"
+    llvm-num-parallel-lto-link-jobs ""           "The number of parallel link jobs to use when compiling llvm"
     swift-stdlib-build-type     "Debug"          "the CMake build variant for Swift"
     swift-stdlib-enable-assertions "1"           "enable assertions in Swift"
     swift-stdlib-enable-resilience "0"           "build the Swift stdlib and overlays with resilience enabled"
@@ -287,28 +289,6 @@ function set_lldb_build_mode() {
     LLDB_BUILD_MODE="CustomSwift-${LLDB_BUILD_TYPE}"
 }
 
-function system_memory_in_bytes() {
-    case "$(uname -s -m)" in
-        Darwin\ x86_64)
-            sysctl hw.memsize | cut -f 2 -d " "
-            ;;
-        *)
-            echo "Unknown operating system"
-            exit 1
-            ;;
-    esac
-}
-
-function float_min() {
-    local LHS=$1
-    local RHS=$2
-    if (( $(echo "${LHS} < ${RHS}" | bc -l) )); then
-        echo ${LHS}
-    else
-        echo ${RHS}
-    fi
-}
-
 # Support for performing isolated actions.
 #
 # This is part of refactoring more work to be done or controllable via
@@ -354,48 +334,6 @@ function should_execute_host_actions_for_phase() {
     fi
 }
 
-function num_llvm_parallel_lto_link_jobs() {
-    case "$(uname -s -m)" in
-        Darwin\ x86_64)
-            # *WARNING! HEURISTIC!*
-            #
-            # Use the formula (GB Memory - 3)/6.0GB to get the number of
-            # parallel link threads we can support. This gives the OS 3 GB of
-            # room to work with.
-            #
-            # This is a bit conservative, but I have found that this number
-            # prevents me from swapping on my test machine.
-            local NUM_MEMORY_THREADS=$(echo \( $(system_memory_in_bytes)/1000000000.0 - 3.0 \) / 6.0 | bc)
-            echo $(float_min "${NUM_MEMORY_THREADS}" "${BUILD_JOBS}")
-            ;;
-        *)
-            echo "Unknown operating system"
-            exit 1
-            ;;
-    esac
-}
-
-function num_swift_parallel_lto_link_jobs() {
-    case "$(uname -s -m)" in
-        Darwin\ x86_64)
-            # *WARNING! HEURISTIC!*
-            #
-            # Use the formula (GB Memory - 3)/8.0GB to get the number of
-            # parallel link threads we can support. This gives the OS 3 GB of
-            # room to work with.
-            #
-            # This is a bit conservative, but I have found that this number
-            # prevents me from swapping on my test machine.
-            local NUM_MEMORY_THREADS=$(echo \( $(system_memory_in_bytes)/1000000000 - 3.0 \) / 8.0 | bc)
-            echo $(float_min "${NUM_MEMORY_THREADS}" "${BUILD_JOBS}")
-            ;;
-        *)
-            echo "Unknown operating system"
-            exit 1
-            ;;
-    esac
-}
-
 function set_build_options_for_host() {
     llvm_cmake_options=()
     swift_cmake_options=()
@@ -638,19 +576,28 @@ function set_build_options_for_host() {
                 fi
 
                 llvm_cmake_options+=(
-                    "-DLLVM_PARALLEL_LINK_JOBS=$(num_llvm_parallel_lto_link_jobs)"
+                    "-DLLVM_PARALLEL_LINK_JOBS=${LLVM_NUM_PARALLEL_LTO_LINK_JOBS}"
                 )
             fi
 
-            if [[ $(true_false "${SWIFT_ENABLE_LTO}") = "TRUE" ]]; then
+            if [[ $(true_false "${SWIFT_TOOLS_ENABLE_LTO}") = "TRUE" ]]; then
                 if [[ $(cmake_needs_to_specify_standard_computed_defaults) = "TRUE" ]]; then
                     swift_cmake_options+=(
                         "-DCMAKE_C_STANDARD_COMPUTED_DEFAULT=AppleClang"
                         "-DCMAKE_CXX_STANDARD_COMPUTED_DEFAULT=AppleClang"
                     )
                 fi
+
+                # We need to also pass in this flag so that parts of LLVM's
+                # build system that we use (for instance when compiling
+                # unittests) do not use too many threads when we are using lto.
+                if [[ $(true_false "${LLVM_ENABLE_LTO}") == "TRUE" ]]; then
+                    swift_cmake_options+=(
+                        "-DLLVM_PARALLEL_LINK_JOBS=${LLVM_NUM_PARALLEL_LTO_LINK_JOBS}"
+                    )
+                fi
                 swift_cmake_options+=(
-                    "-DSWIFT_PARALLEL_LINK_JOBS=$(num_swift_parallel_lto_link_jobs)"
+                    "-DSWIFT_PARALLEL_LINK_JOBS=${SWIFT_TOOLS_NUM_PARALLEL_LTO_LINK_JOBS}"
                 )
             fi
 
@@ -1891,7 +1838,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     -DSWIFT_INCLUDE_TESTS:BOOL=$(true_false "${build_tests_this_time}")
                     -DSWIFT_INSTALL_COMPONENTS:STRING="${SWIFT_INSTALL_COMPONENTS}"
                     -DSWIFT_EMBED_BITCODE_SECTION:BOOL=$(true_false "${EMBED_BITCODE_SECTION}")
-                    -DSWIFT_ENABLE_LTO:BOOL=$(true_false "${SWIFT_ENABLE_LTO}")
+                    -DSWIFT_TOOLS_ENABLE_LTO:BOOL=$(true_false "${SWIFT_TOOLS_ENABLE_LTO}")
                     -DSWIFT_BUILD_RUNTIME_WITH_HOST_COMPILER:BOOL=$(true_false "${BUILD_RUNTIME_WITH_HOST_COMPILER}")
                     "${swift_cmake_options[@]}"
                 )

diff --git a/utils/swift_build_support/swift_build_support/host.py b/utils/swift_build_support/swift_build_support/host.py
@@ -0,0 +1,89 @@
+# swift_build_support/host.py ----------- Migrating build-script -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# -----------------------------------------------------------------------------
+#
+# This file contains routines for determining information about the host for
+# use in utils/build-script.
+#
+# -----------------------------------------------------------------------------
+
+from __future__ import absolute_import
+
+import platform
+
+from . import shell
+
+
+# Utilities
+def _return_none_fun():
+    return None
+
+
+def _return_none_fun_pair():
+    return (_return_none_fun, _return_none_fun)
+
+
+def _compute_system_key():
+    return (platform.system(), platform.machine())
+
+
+# System Memory
+def _darwin_system_memory():
+    # Output looks like "hw.memsize: \d+\n"
+    return int(shell.capture(["sysctl", "hw.memsize"],
+                             dry_run=False, echo=False,
+                             optional=False).strip().split(" ")[1])
+
+_PER_PLATFORM_SYSTEM_MEMORY = {
+    ('Darwin', 'x86_64'): _darwin_system_memory
+}
+
+
+def system_memory():
+    return _PER_PLATFORM_SYSTEM_MEMORY.get(_compute_system_key(),
+                                           _return_none_fun)()
+
+
+# Max Num CPU Threads for use with LTO
+def _darwin_max_num_llvm_parallel_lto_link_jobs():
+    # *WARNING! HEURISTIC!*
+    #
+    # Use the formula (GB Memory - 3)/6.0GB to get the number of
+    # parallel link threads we can support. This gives the OS 3 GB of
+    # room to work with.
+    #
+    # This is a bit conservative, but I have found that this number
+    # prevents me from swapping on my test machine.
+    return int((_darwin_system_memory()/1000000000.0 - 3.0)/6.0)
+
+
+def _darwin_max_num_swift_parallel_lto_link_jobs():
+    # *WARNING! HEURISTIC!*
+    #
+    # Use the formula (GB Memory - 3)/8.0GB to get the number of
+    # parallel link threads we can support. This gives the OS 3 GB of
+    # room to work with.
+    #
+    # This is a bit conservative, but I have found that this number
+    # prevents me from swapping on my test machine.
+    return int((_darwin_system_memory()/1000000000.0 - 3.0)/8.0)
+
+_PER_PLATFORM_MAX_PARALLEL_LTO_JOBS = {
+    ('Darwin', 'x86_64'): (_darwin_max_num_llvm_parallel_lto_link_jobs,
+                           _darwin_max_num_swift_parallel_lto_link_jobs)
+}
+
+
+def max_lto_link_job_counts():
+    key = _compute_system_key()
+    info = _PER_PLATFORM_MAX_PARALLEL_LTO_JOBS.get(key,
+                                                   _return_none_fun_pair())
+    return {'llvm': info[0](), 'swift': info[1]()}