Skip to content

Commit ff49d97

Browse files
RostepherRostepher
authored
Merge pull request #29602 from Rostepher/lto-link-job-defaults
[Build System: build-script] Remove the host module from swift_build_support.
2 parents 3c582f4 + 52428ff commit ff49d97

File tree

6 files changed

+184
-161
lines changed

6 files changed

+184
-161
lines changed

‎utils/build_swift/build_swift/defaults.py

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
44
# Licensed under Apache License v2.0 with Runtime Library Exception
55
#
6-
# See http://swift.org/LICENSE.txt for license information
7-
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
6+
# See https://swift.org/LICENSE.txt for license information
7+
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
88

99

1010
"""
@@ -14,6 +14,9 @@
1414

1515
from __future__ import absolute_import, unicode_literals
1616

17+
import platform
18+
19+
from . import shell
1720
from .versions import Version
1821

1922

@@ -32,6 +35,8 @@
3235
'DARWIN_DEPLOYMENT_VERSION_WATCHOS',
3336
'UNIX_INSTALL_PREFIX',
3437
'DARWIN_INSTALL_PREFIX',
38+
'LLVM_MAX_PARALLEL_LTO_LINK_JOBS',
39+
'SWIFT_MAX_PARALLEL_LTO_LINK_JOBS',
3540

3641
# Constants
3742
]
@@ -56,6 +61,58 @@
5661
DARWIN_INSTALL_PREFIX = ('/Applications/Xcode.app/Contents/Developer/'
5762
'Toolchains/XcodeDefault.xctoolchain/usr')
5863

64+
65+
def _system_memory():
66+
"""Returns the system memory as an int. None if the system memory cannot
67+
be determined.
68+
69+
TODO: Support Linux and Windows platforms.
70+
"""
71+
72+
if platform.platform() == 'Darwin':
73+
try:
74+
output = shell.check_output(['sysctl', 'hw.memsize']).strip()
75+
return int(output.split(' ')[1])
76+
except shell.CalledProcessError:
77+
return None
78+
79+
return None
80+
81+
82+
def _default_llvm_lto_link_jobs():
83+
"""Use the formula (GB Memory - 3)/6.0GB to get the number of parallel
84+
link threads we can support. This gives the OS 3 GB of room to work with.
85+
86+
This is a bit conservative, but I have found that this hueristic prevents
87+
me from swapping on my test machine.
88+
"""
89+
90+
memory = _system_memory()
91+
if memory is None:
92+
return None
93+
94+
return int((memory / 1000000000.0 - 3.0) / 6.0)
95+
96+
97+
def _default_swift_lto_link_jobs():
98+
"""Use the formula (GB Memory - 3)/8.0GB to get the number of parallel
99+
link threads we can support. This gives the OS 3 GB of room to work with.
100+
101+
This is a bit conservative, but I have found that this hueristic prevents
102+
me from swapping on my test machine.
103+
"""
104+
105+
memory = _system_memory()
106+
if memory is None:
107+
return None
108+
109+
return int((memory / 1000000000.0 - 3.0) / 8.0)
110+
111+
112+
LLVM_MAX_PARALLEL_LTO_LINK_JOBS = _default_llvm_lto_link_jobs()
113+
SWIFT_MAX_PARALLEL_LTO_LINK_JOBS = _default_swift_lto_link_jobs()
114+
115+
59116
# Options that can only be "configured" by editing this file.
60117
#
61118
# These options are not exposed as command line options on purpose. If you

‎utils/build_swift/build_swift/driver_arguments.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
import android.adb.commands
1616

17-
from swift_build_support.swift_build_support import host
1817
from swift_build_support.swift_build_support import targets
1918
from swift_build_support.swift_build_support.targets import \
2019
StdlibDeploymentTarget
@@ -464,15 +463,14 @@ def create_argument_parser():
464463
option('--clang-profile-instr-use', store_path,
465464
help='profile file to use for clang PGO')
466465

467-
default_max_lto_link_job_counts = host.max_lto_link_job_counts()
468466
option('--llvm-max-parallel-lto-link-jobs', store_int,
469-
default=default_max_lto_link_job_counts['llvm'],
467+
default=defaults.LLVM_MAX_PARALLEL_LTO_LINK_JOBS,
470468
metavar='COUNT',
471469
help='the maximum number of parallel link jobs to use when '
472470
'compiling llvm')
473471

474472
option('--swift-tools-max-parallel-lto-link-jobs', store_int,
475-
default=default_max_lto_link_job_counts['swift'],
473+
default=defaults.SWIFT_MAX_PARALLEL_LTO_LINK_JOBS,
476474
metavar='COUNT',
477475
help='the maximum number of parallel link jobs to use when '
478476
'compiling swift tools.')

‎utils/build_swift/tests/build_swift/test_defaults.py

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# This source file is part of the Swift.org open source project
2+
#
3+
# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
4+
# Licensed under Apache License v2.0 with Runtime Library Exception
5+
#
6+
# See https://swift.org/LICENSE.txt for license information
7+
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
8+
9+
10+
from __future__ import absolute_import, unicode_literals
11+
12+
import unittest
13+
14+
from build_swift import defaults
15+
from build_swift import shell
16+
17+
from .. import utils
18+
19+
20+
try:
21+
# Python 3.3
22+
from unittest import mock
23+
from unittest.mock import patch, MagicMock
24+
except ImportError:
25+
mock = None
26+
27+
class MagicMock(object):
28+
def __init__(self, *args, **kwargs):
29+
pass
30+
31+
def patch(*args, **kwargs):
32+
return lambda func: func
33+
34+
35+
# ----------------------------------------------------------------------------
36+
# Constants
37+
38+
_SYSCTL_HW_MEMSIZE = 17179869184
39+
_SYSCTL_HW_MEMSIZE_OUTPUT = 'hw.memsize: {}'.format(_SYSCTL_HW_MEMSIZE)
40+
41+
# Safe upper bound to sanity check the LTO link job heuristics.
42+
_LTO_LINK_JOBS_UPPER_BOUND = 100
43+
44+
45+
# ----------------------------------------------------------------------------
46+
47+
class TestDefaults(unittest.TestCase):
48+
"""Unit tests for the defaults module in build_swift.
49+
"""
50+
51+
# ------------------------------------------------------------------------
52+
# _system_memory
53+
54+
@utils.requires_module('unittest.mock')
55+
@patch('platform.platform', MagicMock(return_value='Darwin'))
56+
def test_system_memory_darwin_platform(self):
57+
with mock.patch.object(shell, 'check_output') as mock_check_output:
58+
mock_check_output.return_value = _SYSCTL_HW_MEMSIZE_OUTPUT
59+
60+
self.assertEqual(
61+
defaults._system_memory(), _SYSCTL_HW_MEMSIZE)
62+
63+
@utils.requires_module('unittest.mock')
64+
@patch('platform.platform', MagicMock(return_value='Darwin'))
65+
def test_system_memory_darwin_platform_when_sysctl_fails(self):
66+
with mock.patch.object(shell, 'check_output') as mock_check_output:
67+
mock_check_output.side_effect = shell.CalledProcessError(
68+
returncode=1,
69+
cmd=['sysctl', 'hw.memsize'])
70+
71+
self.assertIsNone(defaults._system_memory())
72+
73+
@utils.requires_module('unittest.mock')
74+
@patch('platform.platform', MagicMock(return_value='Linux'))
75+
def test_system_memory_linux_platform(self):
76+
self.assertIsNone(defaults._system_memory())
77+
78+
@utils.requires_module('unittest.mock')
79+
@patch('platform.platform', MagicMock(return_value='Windows'))
80+
def test_system_memory_windows_platform(self):
81+
self.assertIsNone(defaults._system_memory())
82+
83+
# ------------------------------------------------------------------------
84+
# _default_llvm_lto_link_jobs
85+
86+
@utils.requires_module('unittest.mock')
87+
def test_default_llvm_lto_link_jobs(self):
88+
with mock.patch.object(defaults, '_system_memory') as mock_memory:
89+
mock_memory.return_value = _SYSCTL_HW_MEMSIZE
90+
91+
lto_link_jobs = defaults._default_llvm_lto_link_jobs()
92+
93+
self.assertIsNotNone(lto_link_jobs)
94+
self.assertLess(lto_link_jobs, _LTO_LINK_JOBS_UPPER_BOUND)
95+
96+
@utils.requires_module('unittest.mock')
97+
def test_default_llvm_lto_link_jobs_with_unknown_system_memory(self):
98+
with mock.patch.object(defaults, '_system_memory') as mock_memory:
99+
mock_memory.return_value = None
100+
101+
self.assertIsNone(defaults._default_llvm_lto_link_jobs())
102+
103+
# ------------------------------------------------------------------------
104+
# _default_swift_lto_link_jobs
105+
106+
@utils.requires_module('unittest.mock')
107+
def test_default_swift_lto_link_jobs(self):
108+
with mock.patch.object(defaults, '_system_memory') as mock_memory:
109+
mock_memory.return_value = _SYSCTL_HW_MEMSIZE
110+
111+
lto_link_jobs = defaults._default_swift_lto_link_jobs()
112+
113+
self.assertIsNotNone(lto_link_jobs)
114+
self.assertLess(lto_link_jobs, _LTO_LINK_JOBS_UPPER_BOUND)
115+
116+
@utils.requires_module('unittest.mock')
117+
def test_default_swift_lto_link_jobs_with_unknown_system_memory(self):
118+
with mock.patch.object(defaults, '_system_memory') as mock_memory:
119+
mock_memory.return_value = None
120+
121+
self.assertIsNone(defaults._default_llvm_lto_link_jobs())

‎utils/build_swift/tests/expected_options.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
from build_swift import argparse
1515
from build_swift import defaults
1616

17-
from swift_build_support.swift_build_support import host
1817
from swift_build_support.swift_build_support import targets
1918

2019

@@ -169,7 +168,7 @@
169168
'llvm_assertions': True,
170169
'llvm_build_variant': 'Debug',
171170
'llvm_max_parallel_lto_link_jobs':
172-
host.max_lto_link_job_counts()['llvm'],
171+
defaults.LLVM_MAX_PARALLEL_LTO_LINK_JOBS,
173172
'llvm_targets_to_build': 'X86;ARM;AArch64;PowerPC;SystemZ;Mips',
174173
'tsan_libdispatch_test': False,
175174
'long_test': False,
@@ -191,7 +190,7 @@
191190
'swift_stdlib_assertions': True,
192191
'swift_stdlib_build_variant': 'Debug',
193192
'swift_tools_max_parallel_lto_link_jobs':
194-
host.max_lto_link_job_counts()['swift'],
193+
defaults.SWIFT_MAX_PARALLEL_LTO_LINK_JOBS,
195194
'swift_user_visible_version': defaults.SWIFT_USER_VISIBLE_VERSION,
196195
'symbols_package': None,
197196
'test': None,

‎utils/swift_build_support/swift_build_support/host.py

Lines changed: 0 additions & 91 deletions
This file was deleted.

0 commit comments

Comments
 (0)