CDRIVER-5904 update scripts and release instructions for SilkBomb 2.0 (#1882)

* Remove create-silk-asset-group and related references
* Remove Earthly sbom-download
* Update Earthly sbom-generate for SilkBomb 2.0
* Add Earthly sbom-validate and sbom-generate-new-serial-number
* Add SBOM task to EVG config

Author: eramongodb

.evergreen/config_generator/components/misc.py

@@ -0,0 +1,119 @@
+from config_generator.etc.distros import find_small_distro
+from config_generator.etc.function import Function, merge_defns
+from config_generator.etc.utils import bash_exec
+
+from shrub.v3.evg_build_variant import BuildVariant
+from shrub.v3.evg_command import BuiltInCommand, EvgCommandType, expansions_update, s3_put
+from shrub.v3.evg_task import EvgTask, EvgTaskRef
+
+from pydantic import ConfigDict
+from typing import Optional
+
+
+TAG = 'sbom'
+
+
+class CustomCommand(BuiltInCommand):
+    command: str
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+def ec2_assume_role(
+    role_arn: Optional[str] = None,
+    policy: Optional[str] = None,
+    duration_seconds: Optional[int] = None,
+    command_type: Optional[EvgCommandType] = None,
+) -> CustomCommand:
+    return CustomCommand(
+        command="ec2.assume_role",
+        params={
+            "role_arn": role_arn,
+            "policy": policy,
+            "duration_seconds": duration_seconds,
+        },
+        type=command_type,
+    )
+
+
+class SBOM(Function):
+    name = 'sbom'
+    commands = [
+        ec2_assume_role(
+            command_type=EvgCommandType.SETUP,
+            role_arn='${kondukto_role_arn}',
+        ),
+        bash_exec(
+            command_type=EvgCommandType.SETUP,
+            include_expansions_in_env=[
+                'AWS_ACCESS_KEY_ID',
+                'AWS_SECRET_ACCESS_KEY',
+                'AWS_SESSION_TOKEN',
+            ],
+            script='''\
+                set -o errexit
+                set -o pipefail
+                kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
+                printf "KONDUKTO_TOKEN: %s\\n" "$kondukto_token" >|expansions.kondukto.yml
+            ''',
+        ),
+        expansions_update(
+            command_type=EvgCommandType.SETUP,
+            file='expansions.kondukto.yml',
+        ),
+        bash_exec(
+            command_type=EvgCommandType.TEST,
+            working_dir='mongoc',
+            include_expansions_in_env=[
+                'artifactory_password',
+                'artifactory_username',
+                'branch_name',
+                'KONDUKTO_TOKEN',
+            ],
+            script='.evergreen/scripts/sbom.sh',
+        ),
+        s3_put(
+            command_type=EvgCommandType.TEST,
+            aws_key='${aws_key}',
+            aws_secret='${aws_secret}',
+            bucket='mciuploads',
+            content_type='application/json',
+            display_name='Augmented SBOM',
+            local_file='mongoc/augmented-sbom.json',
+            permissions='public-read',
+            remote_file='${project}/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented-sbom.json',
+        ),
+    ]
+
+    @classmethod
+    def call(cls, **kwargs):
+        return cls.default_call(**kwargs)
+
+
+def functions():
+    return merge_defns(
+        SBOM.defn(),
+    )
+
+
+def tasks():
+    distro_name = 'rhel80'
+    distro = find_small_distro(distro_name)
+
+    yield EvgTask(
+        name='sbom',
+        tags=[TAG, distro_name],
+        run_on=distro.name,
+        commands=[
+            SBOM.call(),
+        ],
+    )
+
+
+def variants():
+    return [
+        BuildVariant(
+            name=TAG,
+            display_name='SBOM',
+            tasks=[EvgTaskRef(name=f'.{TAG}')],
+        ),
+    ]

.evergreen/config_generator/components/sbom.py

@@ -499,6 +499,54 @@ functions:
         args:
           - -c
           - .evergreen/scripts/compile.sh
+  sbom:
+    - command: ec2.assume_role
+      type: setup
+      params:
+        role_arn: ${kondukto_role_arn}
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+        args:
+          - -c
+          - |
+            set -o errexit
+            set -o pipefail
+            kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
+            printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml
+    - command: expansions.update
+      type: setup
+      params:
+        file: expansions.kondukto.yml
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: mongoc
+        include_expansions_in_env:
+          - artifactory_password
+          - artifactory_username
+          - branch_name
+          - KONDUKTO_TOKEN
+        args:
+          - -c
+          - .evergreen/scripts/sbom.sh
+    - command: s3.put
+      type: test
+      params:
+        display_name: Augmented SBOM
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        bucket: mciuploads
+        content_type: application/json
+        local_file: mongoc/augmented-sbom.json
+        permissions: public-read
+        remote_file: ${project}/${build_variant}/${revision}/${version_id}/${build_id}/sbom/augmented-sbom.json
   scan-build:
     - command: subprocess.exec
       type: test

.evergreen/generated_configs/functions.yml

@@ -1410,27 +1410,6 @@ tasks:
     tags: [clang-format]
     commands:
       - func: clang-format
-  - name: create-silk-asset-group
-    run_on:
-      - ubuntu2204-large
-      - debian10-large
-      - debian11-large
-      - amazon2
-    tags: [misc, pr-merge-gate]
-    commands:
-      - command: subprocess.exec
-        type: setup
-        params:
-          binary: ./tools/earthly.sh
-          working_dir: mongoc
-          env:
-            SILK_CLIENT_ID: ${silk_client_id}
-            SILK_CLIENT_SECRET: ${silk_client_secret}
-          args:
-            - --secret=SILK_CLIENT_ID
-            - --secret=SILK_CLIENT_SECRET
-            - +create-silk-asset-group
-            - --branch=${branch_name}
   - name: cse-sasl-cyrus-darwinssl-macos-11-arm64-clang-compile
     run_on: macos-11-arm64
     tags: [cse-matrix-darwinssl, compile, macos-11-arm64, clang, cse, sasl-cyrus]
@@ -4934,6 +4913,11 @@ tasks:
       - func: bootstrap-mongo-orchestration
       - func: run-simple-http-server
       - func: run-tests
+  - name: sbom
+    run_on: rhel80-small
+    tags: [sbom, rhel80]
+    commands:
+      - func: sbom
   - name: scan-build-macos-14-arm64-clang
     run_on: macos-14-arm64
     tags: [scan-build-matrix, macos-14-arm64, clang]

.evergreen/generated_configs/tasks.yml

@@ -174,10 +174,6 @@ buildvariants:
     display_name: loadbalanced
     tasks:
       - name: .loadbalanced
-  - name: misc
-    display_name: Miscellaneous
-    tasks:
-      - name: .misc
   - name: mock-server-test
     display_name: Mock Server Test
     expansions:
@@ -282,6 +278,10 @@ buildvariants:
     expansions: {}
     tasks:
       - name: .sasl-matrix-winssl
+  - name: sbom
+    display_name: SBOM
+    tasks:
+      - name: .sbom
   - name: scan-build-matrix
     display_name: scan-build-matrix
     tasks:

.evergreen/generated_configs/variants.yml

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+: "${artifactory_username:?}"
+: "${artifactory_password:?}"
+: "${branch_name:?}"
+: "${KONDUKTO_TOKEN:?}"
+
+command -v podman >/dev/null || {
+  echo "missing required program podman" 1>&2
+  exit 1
+}
+
+podman login --password-stdin --username "${artifactory_username:?}" artifactory.corp.mongodb.com <<<"${artifactory_password:?}"
+
+silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0"
+
+# Ensure latest version of SilkBomb is being used.
+podman pull "${silkbomb:?}"
+
+silkbomb_augment_flags=(
+  --repo mongodb/mongo-c-driver
+  --branch "${branch_name:?}"
+  --sbom-in /pwd/etc/cyclonedx.sbom.json
+  --sbom-out /pwd/augmented-sbom.json
+
+  # Any notable updates to the Augmented SBOM version should be done manually after careful inspection.
+  # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`.
+  --no-update-sbom-version
+)
+
+# First validate the SBOM Lite.
+podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \
+  validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira
+
+# Then download the Augmented SBOM. Allow the timestamp to be updated.
+podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" \
+  augment "${silkbomb_augment_flags[@]:?}"
+
+[[ -f ./augmented-sbom.json ]] || {
+  echo "failed to download Augmented SBOM" 1>&2
+  exit 1
+}