CI: Avoid leaking credentials in container image builds

.github/workflows/stackhpc-all-in-one.yml

@@ -86,9 +86,9 @@ jobs:
         with:
            submodules: true
 
-      - name: Make sure dockerd is running and test Docker.
+      - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Output image tag
         id: image_tag

.github/workflows/stackhpc-container-image-build.yml

@@ -125,7 +125,7 @@ jobs:
 
       - name: Make sure dockerd is running and test Docker
         run: |
-          docker run --rm hello-world
+          docker ps
 
       - name: Install Kayobe
         run: |
@@ -136,10 +136,11 @@ jobs:
           pip install -U pip &&
           pip install ../src/kayobe
 
-      # Required for Docker registry login. Normally installed during host configure.
+      # Required for Pulp auth proxy deployment and Docker registry login.
+      # Normally installed during host configure.
       - name: Install Docker Python SDK
         run: |
-          pip install --user docker
+          sudo pip install docker
 
       - name: Configure localhost as a seed
         run: |
@@ -150,11 +151,23 @@ jobs:
           localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
           EOF
 
+      # See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
+      # NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
+      # runner and dind containers.
+      - name: Deploy an authenticating package repository mirror proxy
+        run: |
+          source venvs/kayobe/bin/activate &&
+          source src/kayobe-config/kayobe-env --environment ci-builder &&
+          kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
+        env:
+          KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
+
       - name: Build and push kolla overcloud images
         run: |
           args="${{ github.event.inputs.regexes }}"
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi
@@ -169,6 +182,7 @@ jobs:
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
           args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
+          args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           if ${{ inputs.push }} == 'true'; then
             args="$args --push"
           fi

.gitignore

@@ -58,3 +58,7 @@ etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring
 
 # Tempest logs
 tempest-artifacts
+
+# Ansible Galaxy roles & collections
+etc/kayobe/ansible/roles/*\.*/
+etc/kayobe/ansible/collections/

doc/source/contributor/environments/ci-builder.rst

@@ -95,6 +95,34 @@ Next, configure the host OS & services.
 
    kayobe seed host configure
 
+.. _authenticating-pulp-proxy:
+
+Authenticating Pulp proxy
+-------------------------
+
+If you are building against authenticated package repositories such as those in
+`Ark <https://ark.stackhpc.com>`_, you will need to provide secure access to
+the repositories without leaking credentials into the built images or their
+metadata.  This is typically not the case for a client-local Pulp, which
+provides unauthenticated read-only access to the repositories on a trusted
+network.
+
+Docker provides `build
+secrets <https://docs.docker.com/build/building/secrets/>`_, but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+StackHPC Kayobe Configuration provides support for deploying an authenticating
+Pulp proxy that injects an HTTP basic auth header into requests that it
+proxies. Because this proxy bypasses Pulp's authentication, it must not be
+exposed to any untrusted environment.
+
+To deploy the proxy:
+
+.. parsed-literal::
+
+   kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml
+
 Building images
 ===============
 
@@ -105,6 +133,9 @@ At this point you are ready to build and push some container images.
    kayobe seed container image build --push
    kayobe overcloud container image build --push
 
+If using an :ref:`authenticating Pulp proxy <authenticating-pulp-proxy>`,
+append ``-e stackhpc_repo_mirror_auth_proxy_enabled=true`` to these commands.
+
 The container images are tagged as |current_release|-<datetime>.
 
 To use the new images, edit

etc/kayobe/ansible/pulp-auth-proxy.yml

@@ -0,0 +1,14 @@
+---
+# See roles/pulp_auth_proxy/README.md for details.
+
+- name: Deploy Pulp auth proxy
+  hosts: container-image-builders
+  gather_facts: false
+  tasks:
+    - import_role:
+        name: pulp_auth_proxy
+      vars:
+        pulp_auth_proxy_url: "{{ stackhpc_repo_mirror_url }}"
+        pulp_auth_proxy_username: "{{ stackhpc_repo_mirror_username }}"
+        pulp_auth_proxy_password: "{{ stackhpc_repo_mirror_password }}"
+        pulp_auth_proxy_conf_path: "{{ base_path }}/containers/pulp_proxy"

etc/kayobe/ansible/roles/pulp_auth_proxy/README.md

@@ -0,0 +1,26 @@
+# Pulp Auth Proxy
+
+There is currently no practical, secure way to provide credentials for
+accessing Ark's authenticated package repositories from within a Kolla build.
+Docker provides [build
+secrets](https://docs.docker.com/build/building/secrets/), but these must be
+explicitly requested for each RUN statement, making them challenging to use in
+Kolla.
+
+This role deploys an Nginx container that runs as a reverse proxy, injecting an
+HTTP basic authentication header into requests.
+
+Because this proxy bypasses Pulp's authentication, it must not be exposed to
+any untrusted environment.
+
+## Role variables
+
+* `pulp_auth_proxy_pulp_url`: URL of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_username`: Username of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_password`: Password of the Pulp server to proxy requests to.
+* `pulp_auth_proxy_conf_path`: Path to a directory in which to write Nginx
+  configuration.
+* `pulp_auth_proxy_listen_ip`: IP address on the Docker host on which to
+  listen. Default is `127.0.0.1`.
+* `pulp_auth_proxy_listen_port`: Port on the Docker host on which to listen.
+  Default is 80.

etc/kayobe/ansible/roles/pulp_auth_proxy/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+pulp_auth_proxy_url:
+pulp_auth_proxy_username:
+pulp_auth_proxy_password:
+pulp_auth_proxy_conf_path:
+pulp_auth_proxy_listen_ip: 127.0.0.1
+pulp_auth_proxy_listen_port: 80

etc/kayobe/ansible/roles/pulp_auth_proxy/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+- name: "Ensure {{ pulp_auth_proxy_conf_path }} exists"
+  ansible.builtin.file:
+    path: "{{ pulp_auth_proxy_conf_path }}"
+    state: directory
+    mode: 0700
+  become: true
+
+- name: Ensure pulp_proxy.conf is templated
+  ansible.builtin.template:
+    src: pulp_proxy.conf.j2
+    dest: "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf"
+    mode: 0600
+  become: true
+  register: pulp_proxy_conf
+
+- name: Ensure pulp_proxy container is running
+  community.docker.docker_container:
+    name: pulp_proxy
+    image: nginx:stable-alpine
+    ports:
+      - "{{ pulp_auth_proxy_listen_ip }}:{{ pulp_auth_proxy_listen_port }}:80"
+    restart_policy: "no"
+    restart: "{{ pulp_proxy_conf is changed }}"
+    volumes:
+      - "{{ pulp_auth_proxy_conf_path }}/pulp_proxy.conf:/etc/nginx/conf.d/default.conf:ro"

etc/kayobe/ansible/roles/pulp_auth_proxy/templates/pulp_proxy.conf.j2

@@ -0,0 +1,17 @@
+server {
+    listen {{ pulp_auth_proxy_listen_port }};
+    server_name pulp_proxy;
+    location / {
+        proxy_pass {{ pulp_auth_proxy_url }};
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host {{ pulp_auth_proxy_url | urlsplit('hostname') }};
+        # The important part: add basic auth header
+        proxy_set_header Authorization "Basic {{ (pulp_auth_proxy_username ~ ':' ~ pulp_auth_proxy_password) | b64encode }}";
+        proxy_pass_header Authorization;
+        # See https://stackoverflow.com/questions/25329941/nginx-caching-proxy-fails-with-ssl23-get-server-hellosslv3-alert-handshake-fail/25330027#25330027
+        proxy_ssl_server_name on;
+        proxy_ssl_protocols TLSv1.2;
+    }
+}

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -43,7 +43,7 @@ resolv_is_managed: false
 # Host and port of a package repository mirror.
 # Build against the development Pulp service repositories.
 # Use Ark's package repositories to install packages.
-stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}"
+stackhpc_repo_mirror_url: "{{ stackhpc_repo_mirror_auth_proxy_url if stackhpc_repo_mirror_auth_proxy_enabled | bool else stackhpc_release_pulp_url }}"
 stackhpc_repo_mirror_username: "{{ stackhpc_docker_registry_username }}"
 stackhpc_repo_mirror_password: "{{ stackhpc_docker_registry_password }}"
 

etc/kayobe/kolla.yml

@@ -287,16 +287,21 @@ base_centos_repo_overrides_post_yum_rocky_list: "{{ stackhpc_rocky_9_repos + sta
 base_centos_repo_overrides_post_yum_list: "{{ base_centos_repo_overrides_post_yum_centos_list if kolla_base_distro == 'centos' else base_centos_repo_overrides_post_yum_rocky_list }}"
 stackhpc_yum_repos: "{{ stackhpc_centos_stream_repos if kolla_base_distro == 'centos' else stackhpc_rocky_9_repos }}"
 
+# Apt sources.list entry prefix.
+# If using an authenticating Pulp proxy we need to trust the repository because
+# the certificate provided by the upstream repo will not match the proxy's IP.
+stackhpc_ubuntu_repo_prefix: "deb {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}[trusted=yes] {% endif %}"
+
 # List of base repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_base_repos:
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
-  - "deb {{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-updates main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_url }} focal-backports main universe"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_focal_security_url }} focal-security main universe"
 
 # List of UCA repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_uca_repos:
-  - "deb {{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main"
+  - "{{ stackhpc_ubuntu_repo_prefix }}{{ stackhpc_repo_ubuntu_cloud_archive_url }} focal-updates/{{ openstack_release }} main"
 
 # List of repositories for Ubuntu Focal.
 stackhpc_ubuntu_focal_repos: "{{ stackhpc_ubuntu_focal_base_repos + stackhpc_ubuntu_focal_uca_repos }}"
@@ -317,26 +322,16 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
-    {% if stackhpc_repo_mirror_username is truthy %}
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
-    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
-        rm -f /etc/apt/auth.conf && \
-    {% if stackhpc_repo_mirror_url | urlsplit('scheme') == 'https' %}
-    {# We lack the ca-certificates package at this stage, so don't verify the CA #}
+    {% if stackhpc_repo_mirror_auth_proxy_enabled | bool %}
+    {# We lack the ca-certificates package at this stage, so don't verify the CA initially #}
         echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/90no-verify-peer && \
     {% endif %}
-    {% if stackhpc_repo_mirror_username is truthy %}
-        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
-        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
-        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
-    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_base_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}
@@ -354,10 +349,6 @@ kolla_build_blocks:
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
-    {% if stackhpc_repo_mirror_username is truthy %}
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nusername={{ stackhpc_repo_mirror_username }}|' \
-               -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\npassword={{ stackhpc_repo_mirror_password }}|' \
-    {% endif %}
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
     {% endif %}
@@ -368,13 +359,7 @@ kolla_build_blocks:
     {% endif %}
     RUN \
         rm /etc/apt/sources.list && \
-        rm -f /etc/apt/auth.conf && \
         rm -f /etc/apt/apt.conf.d/90no-verify-peer && \
-    {% if stackhpc_repo_mirror_username is truthy %}
-        echo 'machine {{ stackhpc_repo_mirror_url }}' >> /etc/apt/auth.conf && \
-        echo 'login {{ stackhpc_repo_mirror_username }}' >> /etc/apt/auth.conf && \
-        echo 'password {{ stackhpc_repo_mirror_password }}' >> /etc/apt/auth.conf && \
-    {% endif %}
     {% for repo in stackhpc_ubuntu_focal_repos %}
         echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
     {% endif %}

etc/kayobe/stackhpc.yml

@@ -8,6 +8,17 @@ stackhpc_repo_mirror_username:
 # Password of a package repository mirror.
 stackhpc_repo_mirror_password:
 
+# Whether to use an authenticating reverse proxy to access the package
+# repository mirror.  This may be used when building container images, to avoid
+# injecting package repository mirror credentials into the built images.  See
+# ansible/roles/pulp_auth_proxy/README.md for details.
+stackhpc_repo_mirror_auth_proxy_enabled: false
+
+# URL of an authenticating reverse proxy used to access the package repository
+# mirror. Used during container image builds when
+# stackhpc_repo_mirror_auth_proxy_enabled is true.
+stackhpc_repo_mirror_auth_proxy_url: "http://localhost"
+
 # Distribution name. Either 'development' or 'production'.
 stackhpc_repo_distribution: "development"
 

releasenotes/notes/pulp-auth-proxy-24f0b31a4498441b.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Adds a custom playbook (``pulp-auth-proxy.yml``) for deploying an
+    authenticating proxy for Pulp. This can be used when building container
+    images to avoid leaking credentials for package repositories into the built
+    images or their metadata.