Merge pull request #490 from stackhpc/feat/pulp-builds

Support site Pulp server for image builds

Author: wtripp180901

.github/workflows/fatimage.yml

@@ -33,6 +33,7 @@ jobs:
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/nightlybuild.yml

@@ -35,6 +35,7 @@ jobs:
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud || vars.CI_CLOUD }}
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
+      LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
       - uses: actions/checkout@v2

README.md

@@ -32,7 +32,7 @@ It requires an OpenStack cloud, and an Ansible "deploy host" with access to that
 Before starting ensure that:
 - You have root access on the deploy host.
 - You can create instances using a Rocky 9 GenericCloud image (or an image based on that).
-    - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI. However the appliance will install the necessary packages if a GenericCloud image is used.
+    - **NB**: In general it is recommended to use the [latest released image](https://github.com/stackhpc/ansible-slurm-appliance/releases) which already contains the required packages. This is built and tested in StackHPC's CI.
 - You have a SSH keypair defined in OpenStack, with the private key available on the deploy host.
 - Created instances have access to internet (note proxies can be setup through the appliance if necessary).
 - Created instances have accurate/synchronised time (for VM instances this is usually provided by the hypervisor; if not or for bare metal instances it may be necessary to configure a time service via the appliance).

ansible/.gitignore

@@ -66,5 +66,7 @@ roles/*
 !roles/lustre/**
 !roles/dnf_repos/
 !roles/dnf_repos/**
+!roles/pulp_site/
+!roles/pulp_site/**
 !roles/doca/
 !roles/doca/**

ansible/adhoc/deploy-pulp.yml

@@ -0,0 +1,26 @@
+# Usage: ansible-playbook ansible/adhoc/deploy-pulp.yml -e "pulp_server=<pulp server hostname>"
+
+- name: Add temporary pulp server host
+  hosts: localhost
+  tasks:
+  - ansible.builtin.add_host:
+      name: "{{ pulp_server }}"
+      group: "_pulp_host"
+
+- name: Install pulp on server and add to config
+  become: yes
+  hosts: _pulp_host
+  tasks:
+  - name: Install pulp
+    ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: install.yml
+      public: true
+
+  - name: Print Pulp endpoint
+    become: no
+    debug:
+      msg: | 
+        Server configured, override 'appliances_pulp_url' with
+          appliances_pulp_url: "http://{{ pulp_server }}:{{ pulp_site_port }}"
+        in your environments

ansible/adhoc/sync-pulp.yml

@@ -0,0 +1,10 @@
+- hosts: localhost
+  tasks:
+    - ansible.builtin.include_role:
+        name: pulp_site
+        tasks_from: sync.yml
+      vars:
+        pulp_site_target_arch: "x86_64"
+        pulp_site_target_distribution: "rocky"
+        pulp_site_target_distribution_version: "9.4"
+        pulp_site_target_distribution_version_major: "9"

ansible/bootstrap.yml

@@ -110,6 +110,20 @@
         policy: "{{ selinux_policy }}"
       register: sestatus
 
+- hosts: dnf_repos
+  become: yes
+  tasks:
+  - name: Check that creds won't be leaked to users
+    ansible.builtin.assert:
+      that: dnf_repos_password is undefined
+      fail_msg: Passwords should not be templated into repofiles during configure, unset 'dnf_repos_password'
+    when: appliances_mode == 'configure'
+  - name: Replace system repos with pulp repos 
+    ansible.builtin.include_role:
+      name: dnf_repos
+      tasks_from: set_repos.yml
+    when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided
+
 # --- tasks after here require access to package repos ---
 - hosts: squid
   tags: squid

ansible/disable-repos.yml

@@ -0,0 +1,8 @@
+- hosts: dnf_repos
+  become: yes
+  tasks:
+    - name: Disable pulp repos
+      ansible.builtin.include_role:
+        name: dnf_repos
+        tasks_from: disable_repos.yml
+      when: ansible_distribution_major_version == "9" #TODO update role once RL8 config decided

ansible/fatimage.yml

@@ -17,6 +17,16 @@
   import_playbook: "{{ hook_path if hook_path | exists else 'noop.yml' }}"
   when: hook_path | exists
 
+- name: Sync pulp repos with upstream
+  hosts: pulp
+  tasks:
+  - ansible.builtin.include_role:
+      name: pulp_site
+      tasks_from: sync.yml
+      apply:
+        delegate_to: localhost
+    when: appliances_mode != 'configure'
+
 - import_playbook: bootstrap.yml
 
 - name: Run post-bootstrap.yml hook
@@ -210,6 +220,8 @@
       import_role:
         name: doca
 
+- import_playbook: disable-repos.yml
+
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"

ansible/roles/dnf_repos/defaults/main.yml

@@ -1,25 +1,23 @@
-dnf_repos_rocky_ark_prefix: https://ark.stackhpc.com/pulp/content/{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}
-dnf_repos_rocky_ark_suffix: "{{ ansible_architecture }}/os/{{ dnf_repos_rocky_ark_timestamp }}/"
-# most stable from https://github.com/stackhpc/stackhpc-kayobe-config/blob/stackhpc/2024.1/etc/kayobe/pulp-repo-versions.yml
-# note that some timestamps can't be used because not all repos have snapshots for them
-dnf_repos_rocky_ark_timestamp: 20240816T002610
-dnf_repos_username: slurm-app-ci
-dnf_repos_password: "{{ lookup('ansible.builtin.env', 'ARK_PASSWORD') }}"
+dnf_repos_pulp_content_url: "{{ appliances_pulp_url }}/pulp/content"
+dnf_repos_rocky_prefix: "{{ ansible_distribution | lower }}/{{ ansible_distribution_version }}"
+dnf_repos_epel_prefix: "epel/{{ ansible_distribution_major_version }}"
+dnf_repos_username: "{{ omit }}"
+dnf_repos_password: "{{ omit }}"
 
 # epel installed separately
 dnf_repos_repolist:
 - file: rocky
   name: baseos
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/BaseOS/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/BaseOS/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.baseos[ansible_distribution_version] }}"
 - file: rocky
   name: appstream
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/AppStream/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/AppStream/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.appstream[ansible_distribution_version] }}"
 - file: rocky
   name: crb
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/CRB/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/CRB/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.crb[ansible_distribution_version] }}"
 - file: rocky-extras
   name: extras
-  base_url: "{{ dnf_repos_rocky_ark_prefix }}/extras/{{ dnf_repos_rocky_ark_suffix }}"
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ dnf_repos_rocky_prefix }}/extras/{{ ansible_architecture }}/os/{{ appliances_repo_timestamps.extras[ansible_distribution_version] }}"
 
-dnf_repos_epel_timestamp: 20240902T080424
-dnf_repos_epel_baseurl: "https://ark.stackhpc.com/pulp/content/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ dnf_repos_epel_timestamp }}"
+dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/epel/{{ ansible_distribution_major_version }}/Everything/{{ ansible_architecture }}/{{ appliances_repo_timestamps.epel[ansible_distribution_major_version] }}"
+dnf_repos_epel_description: "epel"

ansible/roles/dnf_repos/tasks/disable_repos.yml

@@ -1,5 +1,5 @@
 ---
-- name: Disable Pulp repos and remove creds
+- name: Disable Pulp repos
   ansible.builtin.yum_repository:
     file: "{{ item.file }}"
     name: "{{ item.name }}"
@@ -8,11 +8,11 @@
     enabled: false
   loop: "{{ dnf_repos_repolist }}"
 
-- name: Disable EPEL repo and remove creds
+- name: Disable EPEL repo
   ansible.builtin.yum_repository:
     name: epel
     file: epel
-    description: epel
+    description: "{{ dnf_repos_epel_description }}"
     baseurl: "{{ dnf_repos_epel_baseurl }}"
     gpgcheck: false
     enabled: false

ansible/roles/dnf_repos/tasks/set_repos.yml

@@ -19,8 +19,8 @@
   ansible.builtin.yum_repository:
     name: epel
     file: epel
-    description: epel
+    description: "{{ dnf_repos_epel_description }}"
     gpgcheck: false
+    baseurl: "{{ dnf_repos_epel_baseurl }}"
     username: "{{ dnf_repos_username }}"
     password: "{{ dnf_repos_password }}"
-    baseurl: "{{ dnf_repos_epel_baseurl }}"

ansible/roles/passwords/defaults/main.yml

@@ -9,6 +9,7 @@ slurm_appliance_secrets:
   vault_freeipa_ds_password: "{{ vault_freeipa_ds_password | default(lookup('password', '/dev/null')) }}"
   vault_freeipa_admin_password: "{{ vault_freeipa_admin_password | default(lookup('password', '/dev/null')) }}"
   vault_k3s_token: "{{ vault_k3s_token | default(lookup('ansible.builtin.password', '/dev/null', length=64)) }}"
+  vault_pulp_admin_password: "{{ vault_pulp_admin_password | default(lookup('password', '/dev/null', chars=['ascii_letters', 'digits'])) }}"
 
 secrets_openhpc_mungekey_default:
   content: "{{ lookup('pipe', 'dd if=/dev/urandom bs=1 count=1024 2>/dev/null | base64') }}"

ansible/roles/pulp_site/.gitignore

@@ -0,0 +1 @@
+filter_plugins/__pycache__

ansible/roles/pulp_site/defaults/main.yml

@@ -0,0 +1,39 @@
+pulp_site_url: "{{ appliances_pulp_url }}"
+pulp_site_port: 8080
+pulp_site_username: admin # shouldn't be changed
+pulp_site_password: "{{ vault_pulp_admin_password }}"
+pulp_site_upstream_content_url: https://ark.stackhpc.com/pulp/content
+_pulp_site_rocky_prefix: "{{ pulp_site_target_distribution }}/{{ pulp_site_target_distribution_version }}"
+pulp_site_default_upstream_suffix: "{{ pulp_site_target_arch }}/os"
+pulp_site_validate_certs: false
+pulp_site_install_dir: '/home/rocky/pulp'
+pulp_site_selinux_suffix: "{{ ':Z' if ansible_selinux.status == 'enabled' else '' }}"
+pulp_site_target_facts: "{{ hostvars[groups['builder'][0]]['ansible_facts'] }}"
+pulp_site_target_arch: "{{ pulp_site_target_facts['architecture'] }}"
+pulp_site_target_distribution: "{{ pulp_site_target_facts['distribution'] | lower }}"
+pulp_site_target_distribution_version: "{{ pulp_site_target_facts['distribution_version'] }}"
+pulp_site_target_distribution_version_major: "{{ pulp_site_target_facts['distribution_major_version'] }}"
+
+pulp_site_rpm_info:
+- name: "baseos-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.baseos[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/BaseOS/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.baseos[pulp_site_target_distribution_version] }}"
+- name: "appstream-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.appstream[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/AppStream/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.appstream[pulp_site_target_distribution_version] }}"
+- name: "crb-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.crb[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/{{ 'PowerTools' if pulp_site_target_distribution_version_major == '8' else 'CRB' }}/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.crb[pulp_site_target_distribution_version] }}"
+- name: "extras-{{ pulp_site_target_distribution_version }}-{{ appliances_repo_timestamps.extras[pulp_site_target_distribution_version] }}"
+  subpath: "{{ _pulp_site_rocky_prefix }}/extras/{{ pulp_site_default_upstream_suffix }}/{{ appliances_repo_timestamps.extras[pulp_site_target_distribution_version] }}"
+- name: "epel-{{ pulp_site_target_distribution_version_major }}-{{ appliances_repo_timestamps.epel[pulp_site_target_distribution_version_major] }}"
+  subpath: "epel/{{ pulp_site_target_distribution_version_major }}/Everything/{{ pulp_site_target_arch }}/{{ appliances_repo_timestamps.epel[pulp_site_target_distribution_version_major] }}"
+
+pulp_site_rpm_repo_defaults:
+  remote_username: "{{ pulp_site_upstream_username }}"
+  remote_password: "{{ pulp_site_upstream_password }}"
+  policy: on_demand
+  state: present
+
+_pulp_site_rpm_info_all: "{{ pulp_site_rpm_info | map('combine', pulp_site_rpm_repo_defaults) }}"
+
+pulp_site_rpm_repos: "{{ _pulp_site_rpm_info_all | to_rpm_repos(pulp_site_upstream_content_url) }}"
+pulp_site_rpm_publications: "{{ _pulp_site_rpm_info_all | to_rpm_pubs }}"
+pulp_site_rpm_distributions: "{{ _pulp_site_rpm_info_all | to_rpm_distros }}"

ansible/roles/pulp_site/filter_plugins/pulp-list-filters.py

@@ -0,0 +1,31 @@
+class FilterModule(object):
+    def filters(self):
+        return {
+            'to_rpm_repos': self.to_rpm_repos,
+            'to_rpm_pubs': self.to_rpm_pubs,
+            'to_rpm_distros': self.to_rpm_distros
+        }
+
+    def to_rpm_repos(self, list, pulp_url):
+        repo_list = map(lambda x: {
+            'name': x['name'],
+            'url': pulp_url+'/'+x['subpath'],
+            'remote_username': x['remote_username'],
+            'remote_password': x['remote_password'],
+            'policy': x['policy'],
+            'state': x['state'] }, list)
+        return repo_list
+    
+    def to_rpm_pubs(self, list):
+        pub_list = map(lambda x: {
+            'repository': x['name'],
+            'state': x['state'] }, list)
+        return pub_list
+    
+    def to_rpm_distros(self, list):
+        distro_list = map(lambda x: {
+            'name': x['name'],
+            'repository': x['name'],
+            'base_path': x['subpath'],
+            'state': x['state'] }, list)
+        return distro_list

ansible/roles/pulp_site/tasks/install.yml

@@ -0,0 +1,43 @@
+---
+
+- name: Install packages
+  dnf:
+    name:
+    - podman
+
+- name: Create install directories
+  ansible.builtin.file:
+    state: directory
+    path: "{{ pulp_site_install_dir }}/{{ item }}"
+  loop:
+  - settings/certs
+  - pulp_storage
+  - pgsql
+  - containers
+
+- name: Template settings file
+  ansible.builtin.template:
+    src: settings.py.j2
+    dest: "{{ pulp_site_install_dir }}/settings/settings.py"
+
+- name: Install pulp podman container
+  containers.podman.podman_container:
+    name: pulp
+    publish:
+      - "{{ pulp_site_port }}:80"
+    volume:
+    - "{{ pulp_site_install_dir }}/settings:/etc/pulp{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/pulp_storage:/var/lib/pulp{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/pgsql:/var/lib/pgsql{{ pulp_site_selinux_suffix }}"
+    - "{{ pulp_site_install_dir }}/containers:/var/lib/containers{{ pulp_site_selinux_suffix }}"
+    device: /dev/fuse
+    image: docker.io/pulp/pulp:3.68.1
+
+- name: Reset admin password once container has initialised
+  no_log: true
+  ansible.builtin.shell:
+    cmd: "podman exec pulp bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_site_password }}'"
+  register: _admin_reset_output
+  until: 0 == _admin_reset_output.rc
+  retries: 6
+  delay: 30