Skip to content

Commit c1fafae

Browse files
adriandoleadriandole
committed
Merge branch 'master' into retryable
2 parents fc99a75 + 8a85033 commit c1fafae

File tree

755 files changed

+12183
-6228
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

755 files changed

+12183
-6228
lines changed

‎.evergreen/config_generator/components/c_std_compile.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
from shrub.v3.evg_command import EvgCommandType
33
from shrub.v3.evg_task import EvgTaskRef
44

5+
from config_generator.components.funcs.find_cmake_latest import FindCMakeLatest
6+
57
from config_generator.etc.distros import find_large_distro
68
from config_generator.etc.distros import make_distro_str
79
from config_generator.etc.distros import to_cc
@@ -80,7 +82,10 @@ def tasks():
8082
name=task_name,
8183
run_on=distro.name,
8284
tags=tags + [f'std-c{std}'],
83-
commands=[StdCompile.call(vars=compile_vars | with_std)],
85+
commands=[
86+
FindCMakeLatest.call(),
87+
StdCompile.call(vars=compile_vars | with_std)
88+
],
8489
)
8590
)
8691

‎.evergreen/config_generator/components/cse/darwinssl.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ def tasks():
6262
def variants():
6363
expansions = {
6464
'CLIENT_SIDE_ENCRYPTION': 'on',
65-
'DEBUG': 'ON',
6665
}
6766

6867
return [

‎.evergreen/config_generator/components/cse/openssl.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ def tasks():
8585
def variants():
8686
expansions = {
8787
'CLIENT_SIDE_ENCRYPTION': 'on',
88-
'DEBUG': 'ON',
8988
}
9089

9190
return [

‎.evergreen/config_generator/components/cse/winssl.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ def tasks():
6363
def variants():
6464
expansions = {
6565
'CLIENT_SIDE_ENCRYPTION': 'on',
66-
'DEBUG': 'ON',
6766
}
6867

6968
return [

‎.evergreen/config_generator/components/earthly.py

Lines changed: 40 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar
66

77
from shrub.v3.evg_build_variant import BuildVariant
8-
from shrub.v3.evg_command import EvgCommandType, subprocess_exec
8+
from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, subprocess_exec
99
from shrub.v3.evg_task import EvgTaskRef
1010

1111
from ..etc.utils import Task, all_possible
@@ -164,6 +164,28 @@ def variants_for(config: Configuration) -> Iterable[EarthlyVariant]:
164164
return filter(allow_env_for_config, all_envs)
165165

166166

167+
def earthly_exec(
168+
*,
169+
kind: Literal["test", "setup", "system"],
170+
target: str,
171+
secrets: Mapping[str, str] | None = None,
172+
args: Mapping[str, str] | None = None,
173+
) -> BuiltInCommand:
174+
"""Create a subprocess_exec command that runs Earthly with the given arguments"""
175+
env: dict[str, str] = {k: v for k, v in (secrets or {}).items()}
176+
return subprocess_exec(
177+
"./tools/earthly.sh",
178+
args=[
179+
*(f"--secret={k}" for k in (secrets or ())),
180+
f"+{target}",
181+
*(f"--{arg}={val}" for arg, val in (args or {}).items()),
182+
],
183+
command_type=EvgCommandType(kind),
184+
env=env if env else None,
185+
working_dir="mongoc",
186+
)
187+
188+
167189
def earthly_task(
168190
*,
169191
name: str,
@@ -184,40 +206,30 @@ def earthly_task(
184206
return
185207
# Generate the build-arg arguments based on the configuration options. The
186208
# NamedTuple field names must match with the ARG keys in the Earthfile!
187-
earthly_args = [f"--{key}={val}" for key, val in config._asdict().items()]
188-
# Add arguments that come from parameter expansions defined in the build variant
189-
earthly_args += [
190-
f"--env=${{{_ENV_PARAM_NAME}}}",
191-
f"--c_compiler=${{{_CC_PARAM_NAME}}}",
192-
]
209+
earthly_args = config._asdict()
210+
earthly_args |= {
211+
# Add arguments that come from parameter expansions defined in the build variant
212+
"env": f"${{{_ENV_PARAM_NAME}}}",
213+
"c_compiler": f"${{{_CC_PARAM_NAME}}}",
214+
}
193215
return Task(
194216
name=name,
195217
commands=[
196218
# First, just build the "env-warmup" which will prepare the build environment.
197219
# This won't generate any output, but allows EVG to track it as a separate build step
198220
# for timing and logging purposes. The subequent build step will cache-hit the
199221
# warmed-up build environments.
200-
subprocess_exec(
201-
"bash",
202-
args=[
203-
"tools/earthly.sh",
204-
"+env-warmup",
205-
*earthly_args,
206-
],
207-
working_dir="mongoc",
208-
command_type=EvgCommandType.SETUP,
222+
earthly_exec(
223+
kind="setup",
224+
target="env-warmup",
225+
args=earthly_args,
209226
),
210227
# Now execute the main tasks:
211-
subprocess_exec(
212-
"bash",
213-
args=[
214-
"tools/earthly.sh",
215-
"+run",
216-
f"--targets={' '.join(targets)}",
217-
*earthly_args,
218-
],
219-
working_dir="mongoc",
220-
command_type=EvgCommandType.TEST,
228+
earthly_exec(
229+
kind="test",
230+
target="run",
231+
# The "targets" arg is for +run to specify which targets to run
232+
args={"targets": " ".join(targets)} | earthly_args,
221233
),
222234
],
223235
tags=[f"earthly", "pr-merge-gate", *env_tags],
@@ -230,7 +242,6 @@ def earthly_task(
230242
"ubuntu2204-large",
231243
"ubuntu2004-small",
232244
"ubuntu2004",
233-
"ubuntu1804",
234245
"ubuntu1804-medium",
235246
"debian10",
236247
"debian11",
@@ -249,5 +260,5 @@ def tasks() -> Iterable[Task]:
249260
yield task
250261

251262

252-
def variants() -> list[BuildVariant]:
253-
return [ev.as_evg_variant() for ev in all_possible(EarthlyVariant)]
263+
def variants() -> Iterable[BuildVariant]:
264+
yield from (ev.as_evg_variant() for ev in all_possible(EarthlyVariant))

‎.evergreen/config_generator/components/funcs/find_cmake_latest.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
from shrub.v3.evg_command import EvgCommandType
2+
3+
from config_generator.etc.function import Function
4+
from config_generator.etc.utils import bash_exec
5+
6+
7+
class FindCMakeLatest(Function):
8+
'''
9+
Call `find_cmake_latest` in an attempt to download-and-build the latest
10+
CMake version as a Setup task with `retry_on_failure: true` prior to
11+
subsequent use of `find-cmake-latest.sh` by compile and build scripts.
12+
'''
13+
14+
name = 'find-cmake-latest'
15+
command_type = EvgCommandType.SETUP
16+
commands = [
17+
bash_exec(
18+
command_type=command_type,
19+
retry_on_failure=True,
20+
working_dir='mongoc',
21+
script='. .evergreen/scripts/find-cmake-latest.sh && find_cmake_latest'
22+
),
23+
]
24+
25+
@classmethod
26+
def call(cls, **kwargs):
27+
return cls.default_call(**kwargs)
28+
29+
30+
def functions():
31+
return FindCMakeLatest.defn()

‎.evergreen/config_generator/components/loadbalanced.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from config_generator.components.funcs.bootstrap_mongo_orchestration import BootstrapMongoOrchestration
66
from config_generator.components.funcs.fetch_build import FetchBuild
77
from config_generator.components.funcs.fetch_det import FetchDET
8+
from config_generator.components.funcs.find_cmake_latest import FindCMakeLatest
89
from config_generator.components.funcs.run_simple_http_server import RunSimpleHTTPServer
910
from config_generator.components.funcs.run_tests import RunTests
1011
from config_generator.components.funcs.upload_build import UploadBuild
@@ -76,6 +77,7 @@ def tasks():
7677
run_on=find_large_distro(_DISTRO_NAME).name,
7778
tags=['loadbalanced', _DISTRO_NAME, _COMPILER],
7879
commands=[
80+
FindCMakeLatest.call(),
7981
bash_exec(
8082
command_type=EvgCommandType.TEST,
8183
env={

‎.evergreen/config_generator/components/make_docs.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
from shrub.v3.evg_command import s3_put
33
from shrub.v3.evg_task import EvgTask
44

5+
from config_generator.components.funcs.find_cmake_latest import FindCMakeLatest
6+
57
from config_generator.etc.function import Function
68
from config_generator.etc.function import merge_defns
79
from config_generator.etc.utils import bash_exec
@@ -16,9 +18,9 @@ class MakeDocs(Function):
1618
include_expansions_in_env=["distro_id"],
1719
script="""\
1820
set -o errexit
19-
bash tools/poetry.sh install --with=docs
21+
./tools/poetry.sh install --with=docs
2022
# See SphinxBuild.cmake for EVG_DOCS_BUILD reasoning
21-
bash tools/poetry.sh run env EVG_DOCS_BUILD=1 bash .evergreen/scripts/build-docs.sh
23+
./tools/poetry.sh run env EVG_DOCS_BUILD=1 .evergreen/scripts/build-docs.sh
2224
""",
2325
),
2426
]
@@ -132,6 +134,7 @@ def tasks():
132134
EvgTask(
133135
name="make-docs",
134136
commands=[
137+
FindCMakeLatest.call(),
135138
MakeDocs.call(),
136139
UploadDocs.call(),
137140
UploadManPages.call(),

‎.evergreen/config_generator/components/misc.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
from typing import Iterable
2+
3+
from shrub.v3.evg_build_variant import BuildVariant
4+
from shrub.v3.evg_task import EvgTaskRef
5+
6+
from ..etc.utils import Task
7+
from . import earthly
8+
9+
10+
def tasks() -> Iterable[Task]:
11+
yield Task(
12+
name="create-silk-asset-group",
13+
commands=[
14+
earthly.earthly_exec(
15+
kind="setup",
16+
target="create-silk-asset-group",
17+
args={
18+
"branch": r"${branch_name}",
19+
},
20+
secrets={
21+
"SILK_CLIENT_ID": r"${silk_client_id}",
22+
"SILK_CLIENT_SECRET": r"${silk_client_secret}",
23+
},
24+
)
25+
],
26+
run_on=earthly.CONTAINER_RUN_DISTROS,
27+
tags=["misc", "pr-merge-gate"],
28+
)
29+
30+
31+
def variants() -> Iterable[BuildVariant]:
32+
yield BuildVariant(
33+
name="misc",
34+
tasks=[EvgTaskRef(name=".misc")],
35+
display_name="Miscellaneous",
36+
)

‎.evergreen/config_generator/components/mock_server.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from shrub.v3.evg_task import EvgTaskRef
44

55
from config_generator.components.funcs.fetch_det import FetchDET
6+
from config_generator.components.funcs.find_cmake_latest import FindCMakeLatest
67
from config_generator.components.funcs.run_simple_http_server import RunSimpleHTTPServer
78
from config_generator.etc.utils import Task
89
from config_generator.etc.utils import bash_exec
@@ -16,6 +17,7 @@ def tasks():
1617
# Call fetch-det to define PYTHON3_BINARY expansion required for run-simple-http-server.
1718
FetchDET.call(),
1819
RunSimpleHTTPServer.call(),
20+
FindCMakeLatest.call(),
1921
bash_exec(
2022
command_type=EvgCommandType.TEST,
2123
add_expansions_to_env=True,

‎.evergreen/config_generator/components/openssl_static_compile.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
from shrub.v3.evg_command import EvgCommandType
33
from shrub.v3.evg_task import EvgTaskRef
44

5+
from config_generator.components.funcs.find_cmake_latest import FindCMakeLatest
6+
57
from config_generator.etc.distros import find_large_distro
68
from config_generator.etc.distros import make_distro_str
79
from config_generator.etc.distros import to_cc
@@ -69,7 +71,10 @@ def tasks():
6971
name=task_name,
7072
run_on=distro.name,
7173
tags=tags,
72-
commands=[StaticOpenSSLCompile.call(vars=compile_vars)],
74+
commands=[
75+
FindCMakeLatest.call(),
76+
StaticOpenSSLCompile.call(vars=compile_vars),
77+
],
7378
)
7479
)
7580

‎.evergreen/config_generator/components/sanitizers/asan_sasl.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@
1616
]
1717

1818
TEST_MATRIX = [
19-
('ubuntu1604', 'clang', None, 'cyrus', ['auth'], ['server', 'replica', 'sharded'], ['3.6', ]),
20-
('ubuntu1804', 'clang', None, 'cyrus', ['auth'], ['server', 'replica', 'sharded'], [ '4.0', '4.2', '4.4', '5.0', '6.0']),
19+
('ubuntu1804', 'clang', None, 'cyrus', ['auth'], ['server', 'replica', 'sharded'], ['4.0', '4.2', '4.4', '5.0', '6.0']),
2120

2221
# Test 7.0+ with Ubuntu 20.04+ since MongoDB 7.0 no longer ships binaries for Ubuntu 18.04.
2322
('ubuntu2004', 'clang', None, 'cyrus', ['auth'], ['server', 'replica', 'sharded'], ['7.0', '8.0', 'latest']),

‎.evergreen/config_generator/components/sasl/darwinssl.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
]
2121

2222
TEST_MATRIX = [
23-
('macos-1100', 'clang', None, 'cyrus', ['auth'], ['server'], ['3.6', '4.0', '4.2', '4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
23+
('macos-1100', 'clang', None, 'cyrus', ['auth'], ['server'], ['4.0', '4.2', '4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
2424
]
2525
# fmt: on
2626
# pylint: enable=line-too-long
@@ -55,9 +55,7 @@ def tasks():
5555

5656

5757
def variants():
58-
expansions = {
59-
'DEBUG': 'ON'
60-
}
58+
expansions = {}
6159

6260
return [
6361
BuildVariant(

‎.evergreen/config_generator/components/sasl/nossl.py

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,8 @@
2222
]
2323

2424
TEST_MATRIX = [
25-
('ubuntu1604', 'gcc', None, 'off', ['noauth'], ['server', 'replica', 'sharded'], ['3.6', ]),
26-
('ubuntu1804', 'gcc', None, 'off', ['noauth'], ['server', 'replica', 'sharded'], [ '4.0', '4.2', '4.4', '5.0', '6.0', ]),
27-
('ubuntu2004', 'gcc', None, 'off', ['noauth'], ['server', 'replica', 'sharded'], [ '7.0', '8.0', 'latest']),
25+
('ubuntu1804', 'gcc', None, 'off', ['noauth'], ['server', 'replica', 'sharded'], ['4.0', '4.2', '4.4', '5.0', '6.0', ]),
26+
('ubuntu2004', 'gcc', None, 'off', ['noauth'], ['server', 'replica', 'sharded'], [ '7.0', '8.0', 'latest']),
2827
]
2928
# fmt: on
3029
# pylint: enable=line-too-long
@@ -59,9 +58,7 @@ def tasks():
5958

6059

6160
def variants():
62-
expansions = {
63-
'DEBUG': 'ON'
64-
}
61+
expansions = {}
6562

6663
return [
6764
BuildVariant(

‎.evergreen/config_generator/components/sasl/openssl.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,7 @@ def tasks():
8181

8282

8383
def variants():
84-
expansions = {
85-
'DEBUG': 'ON'
86-
}
84+
expansions = {}
8785

8886
return [
8987
BuildVariant(

‎.evergreen/config_generator/components/sasl/winssl.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
]
2626

2727
TEST_MATRIX = [
28-
('windows-vsCurrent', 'vs2017x64', None, 'cyrus', ['auth'], ['server'], ['3.6', '4.0', '4.2', '4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
28+
('windows-vsCurrent', 'vs2017x64', None, 'cyrus', ['auth'], ['server'], ['4.0', '4.2', '4.4', '5.0', '6.0', '7.0', '8.0', 'latest']),
2929

3030
('windows-vsCurrent', 'mingw', None, 'sspi', ['auth'], ['server'], ['8.0', 'latest']),
3131
('windows-vsCurrent', 'vs2017x64', None, 'sspi', ['auth'], ['server'], ['8.0', 'latest']),
@@ -78,9 +78,7 @@ def tasks():
7878

7979

8080
def variants():
81-
expansions = {
82-
'DEBUG': 'ON'
83-
}
81+
expansions = {}
8482

8583
return [
8684
BuildVariant(

0 commit comments

Comments
 (0)