CDRIVER-3620 Add new config_generator

.evergreen/config.yml

@@ -1,2 +1,51 @@
+# Default timeout for all tasks is 1 hour.
+# May be overridden by individual tasks.
+exec_timeout_secs: 3600
+
+# Default command type is 'system' to indicate failures unrelated to tests or
+# their setup. Most commands should define their type as 'setup' or 'test'.
+#   - setup:  before test (lavendar).
+#   - test:   during test (red).
+#   - system: after test (purple).
+command_type: system
+
+# Ensure Evergreen tests earlier commits when a task fails.
+stepback: true
+
+# Ensure failure in pre commands is also considered task failure.
+pre_error_fails_task: true
+
+# Too many post commands are not designed to handle errors.
+# TODO: set to true once failing post commands have been moved into
+# teardown_group commands of appropriate task groups.
+post_error_fails_task: false
+
+# Unlikely, but document loudly when task failure is due to early termination.
+early_termination:
+  - func: early-termination
+
+# TODO: move into timeout commands of appropriate task groups.
+timeout:
+  - func: backtrace
+
+# Commands run before all tasks (excluding those in task groups).
+# Use sparingly and ensure they are error-proof.
+pre:
+  - func: fetch-source
+
+# Commands run after all tasks (excluding those in task groups).
+# Use sparingly and ensure they are error-proof.
+# TODO: move into teardown_group commands of appropriate task groups.
+post:
+  - func: backtrace
+  - func: upload-mo-artifacts
+  - func: upload-test-results
+  - func: stop-mongo-orchestration
+  - func: stop-load-balancer
+
 include:
   - filename: .evergreen/generated_configs/legacy-config.yml
+  - filename: .evergreen/generated_configs/functions.yml
+  - filename: .evergreen/generated_configs/task_groups.yml
+  - filename: .evergreen/generated_configs/tasks.yml
+  - filename: .evergreen/generated_configs/variants.yml

.evergreen/config_generator/components/abi_compliance_check.py

@@ -0,0 +1,56 @@
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_command import s3_put
+from shrub.v3.evg_task import EvgTask
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class CheckABICompliance(Function):
+    name = 'abi-compliance-check'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            working_dir='mongoc',
+            script='.evergreen/scripts/abi-compliance-check.sh'
+        ),
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            working_dir='mongoc',
+            env={
+                'AWS_ACCESS_KEY_ID': '${aws_key}',
+                'AWS_SECRET_ACCESS_KEY': '${aws_secret}',
+            },
+            script='''\
+                aws s3 cp abi-compliance/compat_reports s3://mciuploads/${project}/${build_variant}/${revision}/${version_id}/${build_id}/abi-compliance/compat_reports --recursive --acl public-read --region us-east-1
+                [[ ! -f ./abi-compliance/abi-error.txt ]]
+            '''
+        ),
+        s3_put(
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='text/html',
+            display_name='ABI Report:',
+            local_files_include_filter='mongoc/abi-compliance/compat_reports/**/*.html',
+            permissions='public-read',
+            remote_file='${project}/${build_variant}/${revision}/${version_id}/${build_id}/abi-compliance/compat_report.html',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return CheckABICompliance.defn()
+
+
+def tasks():
+    return [
+        EvgTask(
+            name=CheckABICompliance.name,
+            commands=[CheckABICompliance.call()],
+        )
+    ]

.evergreen/config_generator/components/check_mongoc_public_headers.py

@@ -0,0 +1,38 @@
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_task import EvgTask
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class CheckMongocPublicHeaders(Function):
+    name = 'check-headers'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            working_dir='mongoc',
+            script='.evergreen/scripts/check-public-decls.sh',
+        ),
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            working_dir='mongoc',
+            script='.evergreen/scripts/check-preludes.py .',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return CheckMongocPublicHeaders.defn()
+
+
+def tasks():
+    return [
+        EvgTask(
+            name=CheckMongocPublicHeaders.name,
+            commands=[CheckMongocPublicHeaders.call()],
+        )
+    ]

.evergreen/config_generator/components/early_termination.py

@@ -0,0 +1,22 @@
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class EarlyTermination(Function):
+    name = 'early-termination'
+    commands = [
+        bash_exec(
+            script='''\
+                echo 'EVERGREEN HOST WAS UNEXPECTEDLY TERMINATED!!!' 1>&2
+                echo 'Restart this Evergreen task and try again!' 1>&2
+            '''
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return EarlyTermination.defn()

.evergreen/config_generator/components/funcs/backtrace.py

@@ -0,0 +1,20 @@
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class Backtrace(Function):
+    name = 'backtrace'
+    commands = [
+        bash_exec(
+            working_dir='mongoc',
+            script='.evergreen/scripts/debug-core-evergreen.sh',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return Backtrace.defn()

.evergreen/config_generator/components/funcs/bootstrap_mongo_orchestration.py

@@ -0,0 +1,30 @@
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_command import expansions_update
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class BootstrapMongoOrchestration(Function):
+    name = 'bootstrap-mongo-orchestration'
+    command_type = EvgCommandType.SETUP
+    commands = [
+        bash_exec(
+            command_type=command_type,
+            working_dir='mongoc',
+            script='.evergreen/scripts/integration-tests.sh',
+            add_expansions_to_env=True,
+        ),
+        expansions_update(
+            command_type=command_type,
+            file='mongoc/mo-expansion.yml'
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return BootstrapMongoOrchestration.defn()

.evergreen/config_generator/components/funcs/fetch_build.py

@@ -0,0 +1,41 @@
+from shrub.v3.evg_command import archive_targz_extract
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_command import s3_get
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class FetchBuild(Function):
+    name = 'fetch-build'
+    command_type = EvgCommandType.SETUP
+    commands = [
+        bash_exec(command_type=command_type, script='rm -rf mongoc'),
+        s3_get(
+            command_type=command_type,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            local_file='build.tar.gz',
+            remote_file='${project}/${build_variant}/${revision}/${BUILD_NAME}/${build_id}.tar.gz',
+        ),
+        archive_targz_extract(path='build.tar.gz', destination='mongoc'),
+        # Scripts may not be executable on Windows.
+        bash_exec(
+            command_type=command_type,
+            working_dir='mongoc',
+            script='''\
+                for file in $(find .evergreen/scripts -type f); do
+                    chmod +rx "$file" || exit
+                done
+            '''
+        ),
+    ]
+
+    @classmethod
+    def call(cls, build_name, **kwargs):
+        return cls.default_call(vars={'BUILD_NAME': build_name}, **kwargs)
+
+
+def functions():
+    return FetchBuild.defn()

.evergreen/config_generator/components/funcs/fetch_det.py

@@ -0,0 +1,26 @@
+from shrub.v3.evg_command import EvgCommandType
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class FetchDET(Function):
+    name = 'fetch-det'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            script='''\
+                if [[ ! -d drivers-evergreen-tools ]]; then
+                    git clone --depth=1 [email protected]:mongodb-labs/drivers-evergreen-tools.git
+                fi
+            ''',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return FetchDET.defn()

.evergreen/config_generator/components/funcs/fetch_source.py

@@ -0,0 +1,61 @@
+from shrub.v3.evg_command import EvgCommandType
+from shrub.v3.evg_command import expansions_update
+from shrub.v3.evg_command import git_get_project
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class FetchSource(Function):
+    name = 'fetch-source'
+    command_type = EvgCommandType.SETUP
+    commands = [
+        git_get_project(command_type=command_type, directory='mongoc'),
+        bash_exec(
+            command_type=command_type,
+            working_dir='mongoc',
+            script='''\
+                set -o errexit
+                if [ -n "${github_pr_number}" -o "${is_patch}" = "true" ]; then
+                    # This is a GitHub PR or patch build, probably branched from master
+                    if command -v python3 2>/dev/null; then
+                        # Prefer python3 if it is available
+                        echo $(python3 ./build/calc_release_version.py --next-minor) > VERSION_CURRENT
+                    else
+                        echo $(python ./build/calc_release_version.py --next-minor) > VERSION_CURRENT
+                    fi
+                    VERSION=$VERSION_CURRENT-${version_id}
+                else
+                    VERSION=latest
+                fi
+                echo "CURRENT_VERSION: $VERSION" > expansion.yml
+            '''
+        ),
+        expansions_update(command_type=command_type,
+                          file='mongoc/expansion.yml'),
+        bash_exec(
+            command_type=command_type,
+            script='''\
+                rm -f *.tar.gz
+                curl --retry 5 --output mongoc.tar.gz -sS --max-time 120 https://s3.amazonaws.com/mciuploads/${project}/${branch_name}/mongo-c-driver-${CURRENT_VERSION}.tar.gz
+            '''
+        ),
+        # Scripts may not be executable on Windows.
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            working_dir='mongoc',
+            script='''\
+                for file in $(find .evergreen/scripts -type f); do
+                    chmod +rx "$file" || exit
+                done
+            '''
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return FetchSource.defn()

.evergreen/config_generator/components/funcs/prepare_kerberos.py

@@ -0,0 +1,36 @@
+from shrub.v3.evg_command import EvgCommandType
+
+from config_generator.etc.function import Function
+from config_generator.etc.utils import bash_exec
+
+
+class PrepareKerberos(Function):
+    name = 'prepare-kerberos'
+    commands = [
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            working_dir='mongoc',
+            silent=True,
+            script='''\
+            if test "${keytab|}" && [[ -f /etc/krb5.conf ]]; then
+                echo "${keytab}" > /tmp/drivers.keytab.base64
+                base64 --decode /tmp/drivers.keytab.base64 > /tmp/drivers.keytab
+                if touch /etc/krb5.conf 2>/dev/null; then
+                    cat .evergreen/etc/kerberos.realm | tee -a /etc/krb5.conf
+                elif command sudo true 2>/dev/null; then
+                    cat .evergreen/etc/kerberos.realm | sudo tee -a /etc/krb5.conf
+                else
+                    echo "Cannot append kerberos.realm to /etc/krb5.conf; skipping." 1>&2
+                fi
+            fi
+            '''
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return PrepareKerberos.defn()