Fix container image promotion (hopefully)

This change modifies the way in which we promote container image tags.
Previously, the stackhpc-dev container repository was configured as a
remote, and the stackhpc repository synced with it, using include_tags
to limit to a specific tag.

Now, we use the "Add content" [1] API to add container tags from
stackhpc-dev to stackhpc. Currently we're still promoting a single tag
across all repositories, so a non-existent tag is not an error. This
will change in future.

[1] https://docs.pulpproject.org/pulp_container/restapi.html#tag/Repositories:-Container/operation/repositories_container_container_add

Author: markgoddard

ansible/dev-pulp-container-promote.yml

@@ -33,3 +33,24 @@
         pulp_username: "{{ dev_pulp_username }}"
         pulp_password: "{{ dev_pulp_password }}"
         pulp_distribution_container: "{{ dev_pulp_distribution_container_release }}"
+
+    # Copy tags from stackhpc-dev to stackhpc repositories.
+    - import_role:
+        name: pulp-container-content
+      vars:
+        pulp_url: "{{ dev_pulp_url }}"
+        pulp_username: "{{ dev_pulp_username }}"
+        pulp_password: "{{ dev_pulp_password }}"
+        pulp_container_content: >-
+          {%- set contents = [] -%}
+          {%- for base_distro in kolla_base_distros -%}
+          {%- for image in kolla_container_images_filtered -%}
+          {%- if image not in kolla_unbuildable_images[base_distro]-%}
+          {%- set src_image_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%}
+          {%- set dest_image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%}
+          {%- set content = {"src_repo": src_image_repo, "src_is_push": true, "dest_repo": dest_image_repo, "tags": [dev_pulp_repository_container_promotion_tag]} -%}
+          {%- set _ = contents.append(content) -%}
+          {%- endif -%}
+          {%- endfor -%}
+          {%- endfor -%}
+          {{ contents }}

ansible/inventory/group_vars/all/dev-pulp-containers

@@ -8,12 +8,6 @@ dev_release_pulp_registry_url: "{{ dev_pulp_url }}"
 
 # Common parameters for release image repositories.
 dev_pulp_repository_container_repo_release_common:
-  url: "{{ dev_release_pulp_registry_url }}"
-  include_tags:
-    - "{{ dev_pulp_repository_container_promotion_tag.strip() | mandatory }}"
-  policy: on_demand
-  remote_username: "{{ dev_pulp_username }}"
-  remote_password: "{{ dev_pulp_password }}"
   state: present
 
 # List of release container image repositories.
@@ -23,8 +17,7 @@ dev_pulp_repository_container_repos_release: >-
   {%- for image in kolla_container_images_filtered -%}
   {%- if image not in kolla_unbuildable_images[base_distro]-%}
   {%- set image_repo = "stackhpc/" ~ base_distro ~ "-source-" ~ image -%}
-  {%- set upstream_repo = "stackhpc-dev/" ~ base_distro ~ "-source-" ~ image -%}
-  {%- set repo = {"name": image_repo, "upstream_name": upstream_repo} -%}
+  {%- set repo = {"name": image_repo} -%}
   {%- set _ = repos.append(dev_pulp_repository_container_repo_release_common | combine(repo)) -%}
   {%- endif -%}
   {%- endfor -%}

ansible/roles/pulp-container-content/README.md

@@ -0,0 +1,49 @@
+pulp_container_content
+======================
+
+This role adds and removes content in Pulp container repositories.
+
+Currently only supports tags.
+
+Role variables
+--------------
+
+* `pulp_url`: URL of Pulp server. Default is `https://localhost:8080`
+* `pulp_username`: Username used to access Pulp server. Default is `admin`
+* `pulp_password`: Password used to access Pulp server. Default is unset
+* `pulp_container_content`: List of content to add or remove. Each item is a dict with the following keys:
+
+  * `src_repo`: Name of the repository to copy from when `state` is `present`,
+    or the repository to remove from when `state` is `absent`.
+  * `src_is_push`: Whether `src_repo` is a push repository. Default is `false`.
+  * `dest_repo`: Name of the repository to copy to when `state is `present`.
+  * `tags`: List of names of tags to add or remove.
+  * `state`: Whether to add (`present`) or remove (`absent`) content.
+
+Example playbook
+----------------
+
+```yaml
+---
+- name: Add or remove container content
+  any_errors_fatal: True
+  gather_facts: True
+  hosts: all
+  roles:
+    - role: pulp-container-content
+      pulp_username: admin
+      pulp_password: "{{ secrets_pulp_admin_password }}"
+      pulp_container_content:
+        # Copy tag1 and tag2 from repo1 to repo2
+        - src_repo: repo1
+          src_is_push: true
+          dest_repo: repo2
+          tags:
+            - tag1
+            - tag2
+        # Remove tag3 from repo3
+        - src_repo: repo3
+          tags:
+            - tag3
+          state: absent
+```

ansible/roles/pulp-container-content/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+pulp_url: https://localhost:8080
+pulp_username: admin
+pulp_password:
+
+pulp_container_content: []

ansible/roles/pulp-container-content/tasks/content.yml

@@ -0,0 +1,73 @@
+---
+- name: Query source repository
+  uri:
+    url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container{% if item.src_is_push | default(false) | bool %}-push{% endif %}/?name={{ item.src_repo | urlencode | regex_replace('/','%2F') }}"
+    user: "{{ pulp_username }}"
+    password: "{{ pulp_password }}"
+    method: GET
+    status_code: 200
+    force_basic_auth: true
+  register: src_repo
+  when: item.state | default != 'absent'
+
+- name: Assert that source repository exists
+  assert:
+    that:
+      src_repo.json.results | length > 0
+  when: item.state | default != 'absent'
+
+- name: Query destination repository
+  uri:
+    url: "{{ pulp_url }}/pulp/api/v3/repositories/container/container/?name={{ item.dest_repo | urlencode | regex_replace('/','%2F') }}"
+    user: "{{ pulp_username }}"
+    password: "{{ pulp_password }}"
+    method: GET
+    status_code: 200
+    force_basic_auth: true
+  register: dest_repo
+
+- name: Assert that destination repository exists
+  assert:
+    that:
+      dest_repo.json.results | length > 0
+
+- name: Query tags
+  uri:
+    url: "{{ pulp_url }}/pulp/api/v3/content/container/tags/?name__in={{ item.tags | join(',') | urlencode | regex_replace('/','%2F') }}&repository_version={{ repo.json.results[0].latest_version_href | urlencode | regex_replace('/','%2F')  }}"
+    user: "{{ pulp_username }}"
+    password: "{{ pulp_password }}"
+    method: GET
+    status_code: 200
+    force_basic_auth: true
+  register: tags
+  vars:
+    repo: "{{ src_repo if item.state | default != 'absent' else dest_repo }}"
+
+# NOTE: Currently we don't verify that all tags exist. This can be fixed when we specify a different tag for each repo.
+
+- name: Add or remove content units
+  uri:
+    url: "{{ pulp_url }}{{ dest_repo.json.results[0].pulp_href }}{% if item.state | default != 'absent' %}add{% else %}remove{% endif %}/"
+    user: "{{ pulp_username }}"
+    password: "{{ pulp_password }}"
+    method: POST
+    status_code: 202
+    force_basic_auth: true
+    body:
+      content_units: "{{ tags.json.results | map(attribute='pulp_href') | list }}"
+    body_format: json
+  register: task
+
+- name: Wait for task to complete
+  uri:
+    url: "{{ pulp_url }}{{ task.json.task }}"
+    user: "{{ pulp_username }}"
+    password: "{{ pulp_password }}"
+    method: GET
+    status_code: 200
+    force_basic_auth: true
+  register: result
+  until: result.json.state not in ['waiting', 'running']
+  retries: 30
+  delay: 2
+  failed_when: result.json.state != 'completed'

ansible/roles/pulp-container-content/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Include content.yml
+  include_tasks: content.yml
+  loop: "{{ pulp_container_content }}"