CXX-3126 convert EVG config to config_generator components

.evergreen/config.yml

@@ -0,0 +1,38 @@
+# Default timeout for all tasks is 1 hour.
+# May be overridden by individual tasks.
+exec_timeout_secs: 3600
+
+# Default command type is 'system' to indicate failures unrelated to tests or
+# their setup. Most commands should define their type as 'setup' or 'test'.
+#   - setup:  before test (lavendar).
+#   - test:   during test (red).
+#   - system: after test (purple).
+command_type: system
+
+# Ensure Evergreen tests earlier commits when a task fails.
+stepback: true
+
+# Ensure failure in pre commands is also considered task failure.
+pre_error_fails_task: true
+
+# Too many post commands are not designed to handle errors.
+# TODO: set to true once failing post commands have been moved into
+# teardown_group commands of appropriate task groups.
+post_error_fails_task: false
+
+# Commands run after all tasks (excluding those in task groups).
+# Use sparingly and ensure they are error-proof.
+# TODO: move into teardown_group commands of appropriate task groups.
+post:
+    - func: "stop_mongod"
+    - func: "backtrace"
+    # Workaround for CXX-2040
+    # - func: "upload working dir"
+    - func: "upload mongo orchestration artifacts"
+    - func: "upload code coverage"
+
+include:
+  - filename: .evergreen/generated_configs/functions.yml
+  - filename: .evergreen/generated_configs/task_groups.yml
+  - filename: .evergreen/generated_configs/tasks.yml
+  - filename: .evergreen/generated_configs/variants.yml

.evergreen/config_generator/components/abi_stability.py

@@ -0,0 +1,200 @@
+from config_generator.components.funcs.install_c_driver import InstallCDriver
+
+from config_generator.etc.distros import find_large_distro
+from config_generator.etc.function import Function, merge_defns
+from config_generator.etc.utils import TaskGroup, bash_exec
+
+from shrub.v3.evg_command import EvgCommandType, git_get_project, s3_put
+from shrub.v3.evg_task import EvgTask, EvgTaskRef
+from shrub.v3.evg_build_variant import BuildVariant, DisplayTask
+
+
+TAG = 'abi-stability'
+
+
+# pylint: disable=line-too-long
+# fmt: off
+MATRIX = [
+    ('polyfill', 11),
+    ('stdlib',   17),
+]
+# fmt: on
+# pylint: enable=line-too-long
+
+
+class AbiComplianceCheck(Function):
+    name = 'abi-compliance-check'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            script='mongo-cxx-driver/.evergreen/scripts/abi-compliance-check-setup.sh'
+        ),
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            script='mongo-cxx-driver/.evergreen/scripts/abi-compliance-check-test.sh'
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/html',
+            display_name='ABI Compliance Check (Stable): ',
+            local_files_include_filter='cxx-abi/compat_reports/**/compat_report.html',
+            permissions='public-read',
+            remote_file='mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/abi/',
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/plain',
+            display_name='ABI Compliance Check (Stable): ',
+            local_files_include_filter='cxx-abi/logs/**/log.txt',
+            permissions='public-read',
+            remote_file='mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/abi/',
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/html',
+            display_name='ABI Compliance Check (Unstable): ',
+            local_files_include_filter='cxx-noabi/compat_reports/**/compat_report.html',
+            permissions='public-read',
+            remote_file='mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/noabi/',
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/plain',
+            display_name='ABI Compliance Check (Unstable): ',
+            local_files_include_filter='cxx-noabi/logs/**/log.txt',
+            permissions='public-read',
+            remote_file='mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abi-compliance-check/noabi/',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+class Abidiff(Function):
+    name = 'abidiff'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            script='mongo-cxx-driver/.evergreen/scripts/abidiff-setup.sh'
+        ),
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            script='mongo-cxx-driver/.evergreen/scripts/abidiff-test.sh'
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/plain',
+            display_name='abidiff (Stable): ',
+            local_files_include_filter='cxx-abi/*.txt',
+            permissions='public-read',
+            remote_file='mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abidiff/abi/',
+        ),
+        s3_put(
+            command_type=EvgCommandType.SYSTEM,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/plain',
+            display_name='abidiff (Unstable): ',
+            local_files_include_filter='cxx-noabi/*.txt',
+            permissions='public-read',
+            remote_file='mongo-cxx-driver/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/abidiff/noabi/',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+class AbiProhibitedSymbols(Function):
+    name = 'abi-prohibited-symbols'
+    commands = bash_exec(
+        command_type=EvgCommandType.TEST,
+        script='mongo-cxx-driver/.evergreen/scripts/abi-prohibited-symbols-test.sh'
+    )
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return merge_defns(
+        AbiComplianceCheck.defn(),
+        Abidiff.defn(),
+        AbiProhibitedSymbols.defn(),
+    )
+
+
+def tasks():
+    distro_name = 'ubuntu2204'
+    distro = find_large_distro(distro_name)
+
+    return [
+        EvgTask(
+            name=func.name,
+            tags=[TAG, func.name, distro_name],
+            run_on=distro.name,
+            commands=[func.call()],
+        )
+        for func in [AbiComplianceCheck, Abidiff, AbiProhibitedSymbols]
+    ]
+
+
+def task_groups():
+    return [
+        TaskGroup(
+            name=f'tg-{TAG}',
+            max_hosts=-1,
+            setup_group_can_fail_task=True,
+            setup_task=[
+                git_get_project(directory='mongo-cxx-driver'),
+                InstallCDriver.call(),
+                bash_exec(
+                    include_expansions_in_env=['cxx_standard'],
+                    script='mongo-cxx-driver/.evergreen/scripts/abi-stability-setup.sh'
+                ),
+            ],
+            tasks=[f'.{TAG}'],
+            teardown_task_can_fail_task=True,
+            teardown_task=[bash_exec(script='rm -rf *'),],
+        )
+    ]
+
+
+def variants():
+    return [
+        BuildVariant(
+            name=f'abi-stability-{name}',
+            display_name=f'ABI Stability Checks ({name})',
+            expansions={
+                'cxx_standard': f'{cxx_standard}',  # Use a polyfill library.
+            },
+            tasks=[EvgTaskRef(name='tg-abi-stability')],
+            display_tasks=[
+                DisplayTask(
+                    name=f'ABI Stability Checks ({name})',
+                    execution_tasks=[f'.{TAG}'],
+                )
+            ],
+        )
+        for name, cxx_standard in MATRIX
+    ]

.evergreen/config_generator/components/add_subdirectory.py

@@ -0,0 +1,91 @@
+from config_generator.components.funcs.setup import Setup
+from config_generator.components.funcs.start_mongod import StartMongod
+
+from config_generator.etc.function import Function
+from config_generator.etc.distros import find_large_distro, make_distro_str
+from config_generator.etc.utils import bash_exec
+
+from shrub.v3.evg_build_variant import BuildVariant, DisplayTask
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_task import EvgTask, EvgTaskRef
+
+from itertools import product
+
+
+TAG = 'add-subdirectory'
+
+
+# pylint: disable=line-too-long
+# fmt: off
+MATRIX = [
+    ('debian10',     ['5.0',         ], ['single']),
+    ('debian11',     ['5.0',         ], ['single']),
+    ('debian12',     [       'latest'], ['single']),
+    ('rhel90',       [       'latest'], ['single']),
+    ('rhel90-arm64', [       'latest'], ['single']),
+    ('ubuntu1804',   ['5.0',         ], ['single']),
+    ('ubuntu2004',   ['5.0', 'latest'], ['single']),
+]
+# fmt: on
+# pylint: enable=line-too-long
+
+
+class BuildExamplesWithAddSubdirectory(Function):
+    name = 'build-examples-with-add-subdirectory'
+    commands = bash_exec(
+        command_type=EvgCommandType.TEST,
+        working_dir='mongo-cxx-driver',
+        script='.evergreen/scripts/build-examples-with-add-subdirectory.sh',
+    )
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return BuildExamplesWithAddSubdirectory.defn()
+
+
+def tasks():
+    res = []
+
+    build_type = 'Release'
+
+    for distro_name, mongodb_versions, topologies in MATRIX:
+        distro = find_large_distro(distro_name)
+
+        for mongodb_version, topology in product(mongodb_versions, topologies):
+            name = f'{TAG}-{make_distro_str(distro_name, None, None)}-{build_type.lower()}-{mongodb_version}-{topology}'
+            tags = [TAG, distro_name, build_type.lower(), mongodb_version, topology]
+
+            res.append(
+                EvgTask(
+                    name=name,
+                    tags=tags,
+                    run_on=distro.name,
+                    commands=[
+                        Setup.call(),
+                        StartMongod.call(mongodb_version=mongodb_version, topology=topology),
+                        BuildExamplesWithAddSubdirectory.call(),
+                    ],
+                )
+            )
+
+    return res
+
+
+def variants():
+    return [
+        BuildVariant(
+            name=f'{TAG}-examples',
+            display_name=f'{TAG}-examples',
+            tasks=[EvgTaskRef(name=f'.{TAG}')],
+            display_tasks=[
+                DisplayTask(
+                    name=f'{TAG}-examples',
+                    execution_tasks=[f'.{TAG}'],
+                )
+            ],
+        ),
+    ]