[CDRIVER-4644] [CDRIVER-4602] Additional Earthly platforms, functionality, and cleanup (#1537)

* Remove ArchLinux matrix definitions that used ArchLinux distros
* Refactor of Earthly task generation, and parametrize on the compiler
* Missing alpine3.19 env
* Additional util scripts for container building
* Significant Earthly refactoring:
- Reusable build utils in tools/build.earth
- New ways of installing individual components
- Cleanup of earthly.py config generation
- New environment targets for Alpine 3.16, 3.17, and Ubuntu 20.04
* Drop archlinux distros from distros.py
* Aside: Remove the early-termination config
* Bump Earthly version
* Add Ubuntu 18.04 targets

Author: vector-of-bool

.evergreen/config.yml

@@ -20,10 +20,6 @@ pre_error_fails_task: true
 # teardown_group commands of appropriate task groups.
 post_error_fails_task: false
 
-# Unlikely, but document loudly when task failure is due to early termination.
-early_termination:
-  - func: early-termination
-
 # TODO: move into timeout commands of appropriate task groups.
 timeout:
   - func: backtrace

.evergreen/config_generator/components/c_std_compile.py

@@ -16,7 +16,6 @@
 # pylint: disable=line-too-long
 # fmt: off
 MATRIX = [
-    ('archlinux',         'clang',     None,   [11,   ]),
     ('debian92',          'clang',     None,   [11,   ]),
     ('ubuntu1604',        'clang',     'i686', [11,   ]),
     ('ubuntu1604',        'clang',     None,   [11,   ]),

.evergreen/config_generator/components/early_termination.py

@@ -1,38 +1,118 @@
 from __future__ import annotations
 
 import functools
-import itertools
-from typing import Any, Iterable, Literal, TypeVar, get_args, NamedTuple, get_type_hints
+import re
+from typing import Iterable, Literal, Mapping, NamedTuple, TypeVar
+
 from shrub.v3.evg_build_variant import BuildVariant
+from shrub.v3.evg_command import EvgCommandType, subprocess_exec
 from shrub.v3.evg_task import EvgTaskRef
-from ..etc.utils import Task
-from shrub.v3.evg_command import subprocess_exec, EvgCommandType
+
+from ..etc.utils import Task, all_possible
 
 T = TypeVar("T")
 
 _ENV_PARAM_NAME = "MONGOC_EARTHLY_ENV"
-"The name of the EVG expansion parameter used to key the Earthly build env"
-
-EnvKey = Literal["u22", "alpine3.18", "archlinux"]
-"Identifiers for environments. These correspond to special '*-env' targets in the Earthfile."
-
-_ENV_NAMES: dict[EnvKey, str] = {
-    "u22": "Ubuntu 22.04",
-    "alpine3.18": "Alpine 3.18",
-    "archlinux": "Arch Linux",
-}
-"A mapping from environment keys to 'pretty' environment names"
+_CC_PARAM_NAME = "MONGOC_EARTHLY_C_COMPILER"
+"The name of the EVG expansion for the Earthly c_compiler argument"
+
+
+EnvKey = Literal[
+    "u18",
+    "u20",
+    "u22",
+    "alpine3.16",
+    "alpine3.17",
+    "alpine3.18",
+    "alpine3.19",
+    "archlinux",
+]
+"Identifiers for environments. These correspond to special 'env.*' targets in the Earthfile."
+CompilerName = Literal["gcc", "clang"]
+"The name of the compiler program that is used for the build. Passed via --c_compiler to Earthly."
 
 # Other options: SSPI (Windows only), AUTO (not reliably test-able without more environments)
 SASLOption = Literal["Cyrus", "off"]
 "Valid options for the SASL configuration parameter"
 TLSOption = Literal["LibreSSL", "OpenSSL", "off"]
 "Options for the TLS backend configuration parameter (AKA 'ENABLE_SSL')"
-CxxVersion = Literal["master", "r3.8.0"]
+CxxVersion = Literal["r3.8.0", "r3.9.0"]
 "C++ driver refs that are under CI test"
 
-# A Unicode non-breaking space character
-_BULLET = "\N{Bullet}"
+# A separator character, since we cannot use whitespace
+_SEPARATOR = "\N{no-break space}\N{bullet}\N{no-break space}"
+
+
+def os_split(env: EnvKey) -> tuple[str, None | str]:
+    """Convert the environment key into a pretty name+version pair"""
+    match env:
+        # match 'alpine3.18' 'alpine53.123' etc.
+        case alp if mat := re.match(r"alpine(\d+\.\d+)", alp):
+            return ("Alpine", mat[1])
+        case "archlinux":
+            return "ArchLinux", None
+        # Match 'u22', 'u20', 'u71' etc.
+        case ubu if mat := re.match(r"u(\d\d)", ubu):
+            return "Ubuntu", f"{mat[1]}.04"
+        case _:
+            raise ValueError(
+                f"Failed to split OS env key {env=} into a name+version pair (unrecognized)"
+            )
+
+
+class EarthlyVariant(NamedTuple):
+    """
+    Define a "variant" that runs under a set of Earthly parameters. These are
+    turned into real EVG variants later on. The Earthly arguments are passed via
+    expansion parameters.
+    """
+
+    env: EnvKey
+    c_compiler: CompilerName
+
+    @property
+    def display_name(self) -> str:
+        """The pretty name for this variant"""
+        base: str
+        match os_split(self.env):
+            case name, None:
+                base = name
+            case name, version:
+                base = f"{name} {version}"
+        toolchain: str
+        match self.c_compiler:
+            case "clang":
+                toolchain = "LLVM/Clang"
+            case "gcc":
+                toolchain = "GCC"
+        return f"{base} ({toolchain})"
+
+    @property
+    def task_selector_tag(self) -> str:
+        """
+        The task tag that is used to select the tasks that want to run on this
+        variant.
+        """
+        return f"{self.env}-{self.c_compiler}"
+
+    @property
+    def expansions(self) -> Mapping[str, str]:
+        """
+        Expansion values that are defined for the build variant that is generated
+        from this object.
+        """
+        return {
+            _CC_PARAM_NAME: self.c_compiler,
+            _ENV_PARAM_NAME: self.env,
+        }
+
+    def as_evg_variant(self) -> BuildVariant:
+        return BuildVariant(
+            name=f"{self.task_selector_tag}",
+            tasks=[EvgTaskRef(name=f".{self.task_selector_tag}")],
+            display_name=self.display_name,
+            expansions=dict(self.expansions),
+        )
 
 
 class Configuration(NamedTuple):
@@ -42,63 +122,44 @@ class Configuration(NamedTuple):
 
     Adding/removing fields will add/remove dimensions on the task matrix.
 
-    The 'env' parameter is not encoded here, but is managed separately.
-
-    Keep this in sync with the 'PartialConfiguration' class defined below!
+    Some Earthly parameters are not encoded here, but are rather part of the variant (EarthlyVariant).
     """
 
     sasl: SASLOption
     tls: TLSOption
     test_mongocxx_ref: CxxVersion
 
-    @classmethod
-    def all(cls) -> Iterable[Configuration]:
-        """
-        Generate all configurations for all options of our parameters.
-        """
-        # Iter each configuration parameter:
-        fields: Iterable[tuple[str, type]] = get_type_hints(Configuration).items()
-        # Generate lists of pairs of parameter names their options:
-        all_pairs: Iterable[Iterable[tuple[str, str]]] = (
-            # Generate a (key, opt) pair for each option in parameter 'key'
-            [(key, opt) for opt in get_args(typ)]
-            # Over each parameter and type thereof:
-            for key, typ in fields
-        )
-        # Now generate the cross product of all alternative for all options:
-        matrix: Iterable[dict[str, Any]] = map(dict, itertools.product(*all_pairs))
-        for items in matrix:
-            # Convert each item to a Configuration:
-            yield Configuration(**items)
-
     @property
     def suffix(self) -> str:
-        return f"{_BULLET}".join(f"{k}={v}" for k, v in self._asdict().items())
+        return f"{_SEPARATOR}".join(f"{k}={v}" for k, v in self._asdict().items())
 
 
-def task_filter(env: EnvKey, conf: Configuration) -> bool:
+def task_filter(env: EarthlyVariant, conf: Configuration) -> bool:
     """
     Control which tasks are actually defined by matching on the platform and
     configuration values.
     """
     match env, conf:
         # We only need one task with "sasl=off"
-        case "u22", ("off", "OpenSSL", "master"):
+        case ["u22", "gcc"], ("off", "OpenSSL", "r3.8.0"):
             return True
+        # The Ubuntu 18.04 GCC has a bug that fails to build the 3.8.0 C++ driver
+        case ["u18", "gcc"], [_, _, "r3.8.0"]:
+            return False
         # Other sasl=off tasks we'll just ignore:
         case _, ("off", _tls, _cxx):
             return False
         # Ubuntu does not ship with a LibreSSL package:
-        case e, (_sasl, "LibreSSL", _cxx) if _ENV_NAMES[e].startswith("Ubuntu"):
+        case e, (_sasl, "LibreSSL", _cxx) if e.display_name.startswith("Ubuntu"):
             return False
         # Anything else: Allow it to run:
         case _:
             return True
 
 
-def envs_for(config: Configuration) -> Iterable[EnvKey]:
-    """Get all environment keys that are not excluded for the given configuration"""
-    all_envs: tuple[EnvKey, ...] = get_args(EnvKey)
+def variants_for(config: Configuration) -> Iterable[EarthlyVariant]:
+    """Get all Earthly variants that are not excluded for the given build configuration"""
+    all_envs = all_possible(EarthlyVariant)
     allow_env_for_config = functools.partial(task_filter, conf=config)
     return filter(allow_env_for_config, all_envs)
 
@@ -109,16 +170,26 @@ def earthly_task(
     targets: Iterable[str],
     config: Configuration,
 ) -> Task | None:
-    # Attach "earthly-xyz" tags to the task to allow build variants to select
+    """
+    Create an EVG task which executes earthly using the given parameters. If this
+    function returns `None`, then the task configuration is excluded from executing
+    and no task should be defined.
+    """
+    # Attach tags to the task to allow build variants to select
     # these tasks by the environment of that variant.
-    env_tags = sorted(f"earthly-{e}" for e in sorted(envs_for(config)))
+    env_tags = sorted(e.task_selector_tag for e in sorted(variants_for(config)))
     if not env_tags:
         # All environments have been excluded for this configuration. This means
         # the task itself should not be run:
         return
     # Generate the build-arg arguments based on the configuration options. The
     # NamedTuple field names must match with the ARG keys in the Earthfile!
     earthly_args = [f"--{key}={val}" for key, val in config._asdict().items()]
+    # Add arguments that come from parameter expansions defined in the build variant
+    earthly_args += [
+        f"--env=${{{_ENV_PARAM_NAME}}}",
+        f"--c_compiler=${{{_CC_PARAM_NAME}}}",
+    ]
     return Task(
         name=name,
         commands=[
@@ -131,7 +202,6 @@ def earthly_task(
                 args=[
                     "tools/earthly.sh",
                     "+env-warmup",
-                    f"--env=${{{_ENV_PARAM_NAME}}}",
                     *earthly_args,
                 ],
                 working_dir="mongoc",
@@ -144,7 +214,6 @@ def earthly_task(
                     "tools/earthly.sh",
                     "+run",
                     f"--targets={' '.join(targets)}",
-                    f"--env=${{{_ENV_PARAM_NAME}}}",
                     *earthly_args,
                 ],
                 working_dir="mongoc",
@@ -170,7 +239,7 @@ def earthly_task(
 
 
 def tasks() -> Iterable[Task]:
-    for conf in Configuration.all():
+    for conf in all_possible(Configuration):
         task = earthly_task(
             name=f"check:{conf.suffix}",
             targets=("test-example", "test-cxx-driver"),
@@ -181,15 +250,4 @@ def tasks() -> Iterable[Task]:
 
 
 def variants() -> list[BuildVariant]:
-    envs: tuple[EnvKey, ...] = get_args(EnvKey)
-    return [
-        BuildVariant(
-            name=f"earthly-{env}",
-            tasks=[EvgTaskRef(name=f".earthly-{env}")],
-            display_name=_ENV_NAMES[env],
-            expansions={
-                _ENV_PARAM_NAME: env,
-            },
-        )
-        for env in envs
-    ]
+    return [ev.as_evg_variant() for ev in all_possible(EarthlyVariant)]

.evergreen/config_generator/components/earthly.py

@@ -15,8 +15,6 @@
 # pylint: disable=line-too-long
 # fmt: off
 COMPILE_MATRIX = [
-    ('archlinux',         'clang',      None, ['cyrus']),
-    ('archlinux',         'gcc',        None, ['cyrus']),
     ('debian10',          'gcc',        None, ['cyrus']),
     ('debian11',          'gcc',        None, ['cyrus']),
     ('debian92',          'clang',      None, ['cyrus']),

.evergreen/config_generator/components/sasl/openssl.py

@@ -42,13 +42,6 @@ def validate_os_ver(cls, value):
 # See: https://evergreen.mongodb.com/distros
 # pylint: disable=line-too-long
 #fmt: off
-ARCHLINUX_DISTROS = [
-    Distro(name='archlinux-large', os='archlinux', os_type='linux', size='large'),
-    Distro(name='archlinux-small', os='archlinux', os_type='linux', size='small'),
-    Distro(name='archlinux-new-large', os='archlinux', os_type='linux', size='large'),
-    Distro(name='archlinux-new-small', os='archlinux', os_type='linux', size='small'),
-]
-
 DEBIAN_DISTROS = [
     Distro(name='debian10-large', os='debian', os_type='linux', os_ver='10', size='large'),
     Distro(name='debian10-small', os='debian', os_type='linux', os_ver='10', size='small'),
@@ -155,7 +148,6 @@ def validate_os_ver(cls, value):
 
 # Ensure no-arch distros are ordered before arch-specific distros.
 ALL_DISTROS = [] + \
-    ARCHLINUX_DISTROS + \
     DEBIAN_DISTROS + \
     MACOS_DISTROS + \
     MACOS_ARM64_DISTROS + \