Add Trivy image scanning

.github/workflows/stackhpc-container-image-build.yml

@@ -38,6 +38,12 @@ on:
         type: boolean
         required: false
         default: true
+      push-dirty:
+        description: Push scanned images that have vulnerabilities?
+        type: boolean
+        required: false
+        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
+        default: true
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -109,7 +115,15 @@ jobs:
       - name: Install package dependencies
         run: |
           sudo apt update
-          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv
+          sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
+
+      - name: Install gh
+        run: |
+          sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
+          sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh -y
 
       - name: Checkout
         uses: actions/checkout@v4
@@ -127,6 +141,10 @@ jobs:
         run: |
           docker ps
 
+      - name: Install Trivy
+        run: |
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
+
       - name: Install Kayobe
         run: |
           mkdir -p venvs &&
@@ -162,65 +180,124 @@ jobs:
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
-      - name: Build and push kolla overcloud images
+      - name: Create build logs output directory
+        run: mkdir image-build-logs 
+
+      - name: Build kolla overcloud images
+        id: build_overcloud_images
+        continue-on-error: true
         run: |
-          args="${{ github.event.inputs.regexes }}"
+          args="${{ inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          if ${{ inputs.push }} == 'true'; then
-            args="$args --push"
-          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe overcloud container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: github.event.inputs.overcloud == 'true'
+        if: inputs.overcloud
+
+      - name: Copy overcloud container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
+        if: inputs.overcloud
 
-      - name: Build and push kolla seed images
+      - name: Build kolla seed images
+        id: build_seed_images
+        continue-on-error: true
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
-          if ${{ inputs.push }} == 'true'; then
-            args="$args --push"
-          fi
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: github.event.inputs.seed == 'true'
+        if: inputs.seed
+
+      - name: Copy seed container image build logs to output directory
+        run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
+        if: inputs.seed
 
       - name: Get built container images
-        run: |
-          docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
 
-      - name: Upload container images artifact
+      - name: Scan built container images
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
+
+      - name: Move image scan logs to output artifact
+        run: mv image-scan-output image-build-logs/image-scan-output
+
+      - name: Fail if no images have passed scanning
+        run: if [ $(wc -l < image-build-logs/image-scan-output/clean-images.txt) -le 0 ]; then exit 1; fi
+        if: ${{ !inputs.push-dirty }}
+
+      - name: Copy clean images to push-attempt-images list
+        run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
+        if: inputs.push
+
+      - name: Append dirty images to push list
+        run: |
+          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push && inputs.push-dirty }}
+
+      - name: Push images
+        run: |
+          touch image-build-logs/push-failed-images.txt
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
+
+          while read -r image; do
+            # Retries!
+            for i in {1..5}; do 
+              if docker push $image; then
+                echo "Pushed $image"
+                break
+              elif $i == 5; then
+                echo "Failed to push $image"
+                echo $image >> image-build-logs/push-failed-images.txt
+              else
+                echo "Failed on retry $i"
+                sleep 5
+              fi;
+            done
+          done < image-build-logs/push-attempt-images.txt
+        shell: bash
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+        if: inputs.push
+
+      - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }} container images
-          path: ${{ matrix.distro }}-container-images
+          name: ${{ matrix.distro }}-logs
+          path: image-build-logs
           retention-days: 7
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed to build
+        run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
+        if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
+
+      - name: Fail when images failed to push
+        run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
+        if: ${{ !cancelled() }}
+
+      - name: Fail when images failed scanning
+        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+        if: ${{ !inputs.push-dirty && !cancelled() }}
 
-  sync-container-repositories:
-    name: Trigger container image repository sync
-    needs:
-      - container-image-build
-    if: github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push
-    runs-on: ubuntu-latest
-    permissions: {}
-    steps:
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.
       - name: Trigger container image repository sync
         run: |
           filter='${{ inputs.regexes }}'
-          if [[ -n $filter ]] && [[ ${{ github.event.inputs.seed }} == 'true' ]]; then
+          if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
             filter="$filter bifrost"
           fi
           gh workflow run \
@@ -231,7 +308,9 @@ jobs:
           -f sync-new-images=false
         env:
           GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}
 
       - name: Display link to container image repository sync workflows
         run: |
           echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
+        if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}

etc/kayobe/ansible/docker-registry-login.yml

@@ -0,0 +1,11 @@
+---
+- name: Login to docker registry
+  gather_facts: false
+  hosts: container-image-builders
+  tasks:
+    - name: Login to docker registry
+      docker_login:
+        registry_url: "{{ kolla_docker_registry or omit }}"
+        username: "{{ kolla_docker_registry_username }}"
+        password: "{{ kolla_docker_registry_password }}"
+        reauthorize: yes

releasenotes/notes/container-image-scanning-e5adf2c6b540b502.yaml

@@ -0,0 +1,6 @@
+---
+security:
+  - |
+    Kolla container images created using the
+    ``stackhpc-container-image-build.yml`` workflow are now automatically
+    scanned for vulnerablilities.

tools/scan-images.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Check correct usage
+if [[ ! $2 ]]; then
+  echo "Usage: scan-images.sh <os-distribution> <image-tag>"
+  exit 2
+fi
+
+set -u
+
+# Check that trivy is installed
+if ! trivy --version; then
+  echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1'
+fi
+
+# Clear any previous outputs
+rm -rf image-scan-output
+
+# Make a fresh output directory
+mkdir -p image-scan-output
+
+# Get built container images
+docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/$1-*:$2" > $1-scanned-container-images.txt
+
+# Make a file of imagename:tag
+images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:)
+
+# Ensure output files exist
+touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt
+
+# If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
+# If there are vulnerabilities detected, add it to dirty-images.txt and
+# generate a csv summary
+for image in $images; do
+  filename=$(basename $image | sed 's/:/\./g')
+  if $(trivy image \
+          --quiet \
+          --exit-code 1 \
+          --scanners vuln \
+          --format json \
+          --severity HIGH,CRITICAL \
+          --output image-scan-output/${filename}.json \
+          --ignore-unfixed \
+          $image); then
+    # Clean up the output file for any images with no vulnerabilities
+    rm -f image-scan-output/${filename}.json
+
+    # Add the image to the clean list
+    echo "${image}" >> image-scan-output/clean-images.txt
+  else
+    # Add the image to the dirty list
+    echo "${image}" >> image-scan-output/dirty-images.txt
+
+    # Write a header for the summary CSV
+    echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv
+
+    # Write the summary CSV data
+    jq -r '.Results[] 
+            | select(.Vulnerabilities) 
+            | .Vulnerabilities
+            # Ignore packages with "kernel" in the PkgName
+            | map(select(.PkgName | test("kernel") | not ))
+            | group_by(.VulnerabilityID)
+            | map(
+                  [
+                    (map(.PkgName) | unique | join(";")),
+                    (map(.PkgPath | select( . != null )) | join(";")),
+                    .[0].PkgID,
+                    .[0].VulnerabilityID,
+                    .[0].FixedVersion,
+                    .[0].PrimaryURL,
+                    .[0].Severity
+                    ]
+                ) 
+            | .[] 
+            | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
+  fi
+done