[Build System: build-script] Remove the host module from swift_build_support.

utils/build_swift/build_swift/defaults.py

@@ -3,8 +3,8 @@
 # Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
 # Licensed under Apache License v2.0 with Runtime Library Exception
 #
-# See http://swift.org/LICENSE.txt for license information
-# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 
 
 """
@@ -14,6 +14,9 @@
 
 from __future__ import absolute_import, unicode_literals
 
+import platform
+
+from . import shell
 from .versions import Version
 
 
@@ -32,6 +35,8 @@
     'DARWIN_DEPLOYMENT_VERSION_WATCHOS',
     'UNIX_INSTALL_PREFIX',
     'DARWIN_INSTALL_PREFIX',
+    'LLVM_MAX_PARALLEL_LTO_LINK_JOBS',
+    'SWIFT_MAX_PARALLEL_LTO_LINK_JOBS',
 
     # Constants
 ]
@@ -56,6 +61,58 @@
 DARWIN_INSTALL_PREFIX = ('/Applications/Xcode.app/Contents/Developer/'
                          'Toolchains/XcodeDefault.xctoolchain/usr')
 
+
+def _system_memory():
+    """Returns the system memory as an int. None if the system memory cannot
+    be determined.
+
+    TODO: Support Linux and Windows platforms.
+    """
+
+    if platform.platform() == 'Darwin':
+        try:
+            output = shell.check_output(['sysctl', 'hw.memsize']).strip()
+            return int(output.split(' ')[1])
+        except shell.CalledProcessError:
+            return None
+
+    return None
+
+
+def _default_llvm_lto_link_jobs():
+    """Use the formula (GB Memory - 3)/6.0GB to get the number of parallel
+    link threads we can support. This gives the OS 3 GB of room to work with.
+
+    This is a bit conservative, but I have found that this hueristic prevents
+    me from swapping on my test machine.
+    """
+
+    memory = _system_memory()
+    if memory is None:
+        return None
+
+    return int((memory / 1000000000.0 - 3.0) / 6.0)
+
+
+def _default_swift_lto_link_jobs():
+    """Use the formula (GB Memory - 3)/8.0GB to get the number of parallel
+    link threads we can support. This gives the OS 3 GB of room to work with.
+
+    This is a bit conservative, but I have found that this hueristic prevents
+    me from swapping on my test machine.
+    """
+
+    memory = _system_memory()
+    if memory is None:
+        return None
+
+    return int((memory / 1000000000.0 - 3.0) / 8.0)
+
+
+LLVM_MAX_PARALLEL_LTO_LINK_JOBS = _default_llvm_lto_link_jobs()
+SWIFT_MAX_PARALLEL_LTO_LINK_JOBS = _default_swift_lto_link_jobs()
+
+
 # Options that can only be "configured" by editing this file.
 #
 # These options are not exposed as command line options on purpose.  If you

utils/build_swift/build_swift/driver_arguments.py

@@ -14,7 +14,6 @@
 
 import android.adb.commands
 
-from swift_build_support.swift_build_support import host
 from swift_build_support.swift_build_support import targets
 from swift_build_support.swift_build_support.targets import \
     StdlibDeploymentTarget
@@ -464,15 +463,14 @@ def create_argument_parser():
     option('--clang-profile-instr-use', store_path,
            help='profile file to use for clang PGO')
 
-    default_max_lto_link_job_counts = host.max_lto_link_job_counts()
     option('--llvm-max-parallel-lto-link-jobs', store_int,
-           default=default_max_lto_link_job_counts['llvm'],
+           default=defaults.LLVM_MAX_PARALLEL_LTO_LINK_JOBS,
            metavar='COUNT',
            help='the maximum number of parallel link jobs to use when '
                 'compiling llvm')
 
     option('--swift-tools-max-parallel-lto-link-jobs', store_int,
-           default=default_max_lto_link_job_counts['swift'],
+           default=defaults.SWIFT_MAX_PARALLEL_LTO_LINK_JOBS,
            metavar='COUNT',
            help='the maximum number of parallel link jobs to use when '
                 'compiling swift tools.')

utils/build_swift/tests/build_swift/test_defaults.py

@@ -0,0 +1,121 @@
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+
+from __future__ import absolute_import, unicode_literals
+
+import unittest
+
+from build_swift import defaults
+from build_swift import shell
+
+from .. import utils
+
+
+try:
+    # Python 3.3
+    from unittest import mock
+    from unittest.mock import patch, MagicMock
+except ImportError:
+    mock = None
+
+    class MagicMock(object):
+        def __init__(self, *args, **kwargs):
+            pass
+
+    def patch(*args, **kwargs):
+        return lambda func: func
+
+
+# ----------------------------------------------------------------------------
+# Constants
+
+_SYSCTL_HW_MEMSIZE = 17179869184
+_SYSCTL_HW_MEMSIZE_OUTPUT = 'hw.memsize: {}'.format(_SYSCTL_HW_MEMSIZE)
+
+# Safe upper bound to sanity check the LTO link job heuristics.
+_LTO_LINK_JOBS_UPPER_BOUND = 100
+
+
+# ----------------------------------------------------------------------------
+
+class TestDefaults(unittest.TestCase):
+    """Unit tests for the defaults module in build_swift.
+    """
+
+    # ------------------------------------------------------------------------
+    # _system_memory
+
+    @utils.requires_module('unittest.mock')
+    @patch('platform.platform', MagicMock(return_value='Darwin'))
+    def test_system_memory_darwin_platform(self):
+        with mock.patch.object(shell, 'check_output') as mock_check_output:
+            mock_check_output.return_value = _SYSCTL_HW_MEMSIZE_OUTPUT
+
+            self.assertEqual(
+                defaults._system_memory(), _SYSCTL_HW_MEMSIZE)
+
+    @utils.requires_module('unittest.mock')
+    @patch('platform.platform', MagicMock(return_value='Darwin'))
+    def test_system_memory_darwin_platform_when_sysctl_fails(self):
+        with mock.patch.object(shell, 'check_output') as mock_check_output:
+            mock_check_output.side_effect = shell.CalledProcessError(
+                returncode=1,
+                cmd=['sysctl', 'hw.memsize'])
+
+            self.assertIsNone(defaults._system_memory())
+
+    @utils.requires_module('unittest.mock')
+    @patch('platform.platform', MagicMock(return_value='Linux'))
+    def test_system_memory_linux_platform(self):
+        self.assertIsNone(defaults._system_memory())
+
+    @utils.requires_module('unittest.mock')
+    @patch('platform.platform', MagicMock(return_value='Windows'))
+    def test_system_memory_windows_platform(self):
+        self.assertIsNone(defaults._system_memory())
+
+    # ------------------------------------------------------------------------
+    # _default_llvm_lto_link_jobs
+
+    @utils.requires_module('unittest.mock')
+    def test_default_llvm_lto_link_jobs(self):
+        with mock.patch.object(defaults, '_system_memory') as mock_memory:
+            mock_memory.return_value = _SYSCTL_HW_MEMSIZE
+
+            lto_link_jobs = defaults._default_llvm_lto_link_jobs()
+
+            self.assertIsNotNone(lto_link_jobs)
+            self.assertLess(lto_link_jobs, _LTO_LINK_JOBS_UPPER_BOUND)
+
+    @utils.requires_module('unittest.mock')
+    def test_default_llvm_lto_link_jobs_with_unknown_system_memory(self):
+        with mock.patch.object(defaults, '_system_memory') as mock_memory:
+            mock_memory.return_value = None
+
+            self.assertIsNone(defaults._default_llvm_lto_link_jobs())
+
+    # ------------------------------------------------------------------------
+    # _default_swift_lto_link_jobs
+
+    @utils.requires_module('unittest.mock')
+    def test_default_swift_lto_link_jobs(self):
+        with mock.patch.object(defaults, '_system_memory') as mock_memory:
+            mock_memory.return_value = _SYSCTL_HW_MEMSIZE
+
+            lto_link_jobs = defaults._default_swift_lto_link_jobs()
+
+            self.assertIsNotNone(lto_link_jobs)
+            self.assertLess(lto_link_jobs, _LTO_LINK_JOBS_UPPER_BOUND)
+
+    @utils.requires_module('unittest.mock')
+    def test_default_swift_lto_link_jobs_with_unknown_system_memory(self):
+        with mock.patch.object(defaults, '_system_memory') as mock_memory:
+            mock_memory.return_value = None
+
+            self.assertIsNone(defaults._default_llvm_lto_link_jobs())

utils/build_swift/tests/expected_options.py

@@ -14,7 +14,6 @@
 from build_swift import argparse
 from build_swift import defaults
 
-from swift_build_support.swift_build_support import host
 from swift_build_support.swift_build_support import targets
 
 
@@ -169,7 +168,7 @@
     'llvm_assertions': True,
     'llvm_build_variant': 'Debug',
     'llvm_max_parallel_lto_link_jobs':
-        host.max_lto_link_job_counts()['llvm'],
+        defaults.LLVM_MAX_PARALLEL_LTO_LINK_JOBS,
     'llvm_targets_to_build': 'X86;ARM;AArch64;PowerPC;SystemZ;Mips',
     'tsan_libdispatch_test': False,
     'long_test': False,
@@ -191,7 +190,7 @@
     'swift_stdlib_assertions': True,
     'swift_stdlib_build_variant': 'Debug',
     'swift_tools_max_parallel_lto_link_jobs':
-        host.max_lto_link_job_counts()['swift'],
+        defaults.SWIFT_MAX_PARALLEL_LTO_LINK_JOBS,
     'swift_user_visible_version': defaults.SWIFT_USER_VISIBLE_VERSION,
     'symbols_package': None,
     'test': None,