Support ansible-init for remote collections (#411)

* Add ansible-init role to requirements.yml

* Add ansible-init to groups and plays

* Configure cluster_infra ansible-init metadata

* Only run site.yml once ansible-init has completed

* Wait for ansible init to finish before running bootstrap

* revert to using cluster_infra metadata defaults

* update image

* revert sausage bastion changes

* set ansible_init_wait as common var

* use run_number as a shorter ID for CI

* install ood apps in fatimage

* add ood jupyter install to fatimage

* bump image

* jupyter_compute ood into fatimage

* bump fatimage for jupyter_compute ood

* Update stackhpc.yml

* duplicate tuned inventory group name

* Fix invalid group name for slurmci

* Update stackhpc.yml

undo groupname changes

* slurmci group name warning

* rm ood changes

* bump fatimage

* change azimuth collection in bootstrap

* update azimuth image utils version

* update requirements

* Update bastion.yml

* Use azimuth image utils collection for ansible-init

* bump fatimage

---------

Co-authored-by: bertie <[email protected]>
Co-authored-by: bertiethorpe <[email protected]>

Author: sjpb

ansible/bootstrap.yml

@@ -1,5 +1,20 @@
 ---
 
+- hosts: cluster
+  gather_facts: false
+  become: yes
+  tasks:
+    - name: Check if ansible-init is installed
+      stat:
+        path: /etc/systemd/system/ansible-init.service
+      register: _stat_ansible_init_unitfile
+    
+    - name: Wait for ansible-init to finish
+      wait_for:
+        path: /var/lib/ansible-init.done
+        timeout: "{{ ansible_init_wait }}" # seconds
+      when: _stat_ansible_init_unitfile.stat.exists
+
 - hosts: localhost
   gather_facts: false
   become: false
@@ -235,3 +250,11 @@
   tasks:
     - include_role:
         name: ofed
+
+- hosts: ansible_init
+  gather_facts: yes
+  become: yes
+  tags: linux_ansible_init
+  tasks:
+    - include_role:
+        name: azimuth_cloud.image_utils.linux_ansible_init

ansible/fatimage.yml

@@ -68,11 +68,13 @@
         tasks_from: install-package.yml
         vars_from: "Rocky/{{ ansible_distribution_major_version }}.yml"
     # # FUTURE: install-apps.yml - this is git clones
+
     - name: Open Ondemand server (apps)
       include_role:
         name: osc.ood
         tasks_from: install-apps.yml
         vars_from: "Rocky/{{ ansible_distribution_major_version }}.yml"
+
     - name: Open Ondemand remote desktop
       import_role:
         name: openondemand

ansible/roles/cluster_infra/defaults/main.yml

@@ -0,0 +1,2 @@
+ansible_init_collections: []
+ansible_init_playbooks: []

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -370,6 +370,23 @@ resource "openstack_compute_instance_v2" "login" {
       - "${openstack_compute_keypair_v2.cluster_keypair.public_key}"
     {%- endif %}
   EOF
+
+  metadata = {
+    {% for playbook in ansible_init_playbooks %}
+      ansible_init_pb_{{ loop.index0 }}_name = "{{ playbook.name }}"
+      {% if playbook.stage is defined %}
+        ansible_init_pb_{{ loop.index0 }}_stage = "{{ playbook.stage }}"
+      {% endif %}
+    {% endfor %}
+    {% for collection in ansible_init_collections %}
+      ansible_init_coll_{{ loop.index0 }}_name = "{{ collection.name }}"
+      ansible_init_coll_{{ loop.index0 }}_type = "{{ collection.type }}"
+      ansible_init_coll_{{ loop.index0 }}_version = "{{ collection.version }}"
+      {% if collection.source is defined %}
+        ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
+      {% endif %}
+    {% endfor %} 
+  }
 }
 
 resource "openstack_compute_instance_v2" "control" {
@@ -446,6 +463,23 @@ resource "openstack_compute_instance_v2" "control" {
         - [LABEL=home, /exports/home, auto]
         {% endif %}
   EOF
+
+  metadata = {
+    {% for playbook in ansible_init_playbooks %}
+      ansible_init_pb_{{ loop.index0 }}_name = "{{ playbook.name }}"
+      {% if playbook.stage is defined %}
+        ansible_init_pb_{{ loop.index0 }}_stage = "{{ playbook.stage }}"
+      {% endif %}
+    {% endfor %}
+    {% for collection in ansible_init_collections %}
+      ansible_init_coll_{{ loop.index0 }}_name = "{{ collection.name }}"
+      ansible_init_coll_{{ loop.index0 }}_type = "{{ collection.type }}"
+      ansible_init_coll_{{ loop.index0 }}_version = "{{ collection.version }}"
+      {% if collection.source is defined %}
+        ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
+      {% endif %}
+    {% endfor %} 
+  }
 }
 
 {% for partition in openhpc_slurm_partitions %}
@@ -498,6 +532,23 @@ resource "openstack_compute_instance_v2" "{{ partition.name }}" {
       - "${openstack_compute_keypair_v2.cluster_keypair.public_key}"
     {%- endif %}
   EOF
+
+  metadata = {
+    {% for playbook in ansible_init_playbooks %}
+      ansible_init_pb_{{ loop.index0 }}_name = "{{ playbook.name }}"
+      {% if playbook.stage is defined %}
+        ansible_init_pb_{{ loop.index0 }}_stage = "{{ playbook.stage }}"
+      {% endif %}
+    {% endfor %}
+    {% for collection in ansible_init_collections %}
+      ansible_init_coll_{{ loop.index0 }}_name = "{{ collection.name }}"
+      ansible_init_coll_{{ loop.index0 }}_type = "{{ collection.type }}"
+      ansible_init_coll_{{ loop.index0 }}_version = "{{ collection.version }}"
+      {% if collection.source is defined %}
+        ansible_init_coll_{{ loop.index0 }}_source = "{{ collection.source }}"
+      {% endif %}
+    {% endfor %} 
+  }
 }
 
 {% endfor %}

environments/.stackhpc/terraform/main.tf

@@ -29,9 +29,9 @@ variable "cluster_image" {
     description = "single image for all cluster nodes, keyed by os_version - a convenience for CI"
     type = map(string)
     default = {
-        # https://github.com/stackhpc/ansible-slurm-appliance/pull/414
-        RL8: "openhpc-RL8-240723-0907-b560bf4c"
-        RL9: "openhpc-ofed-RL9-240723-0907-b560bf4c"
+        # https://github.com/stackhpc/ansible-slurm-appliance/pull/411
+        RL8: "openhpc-RL8-240725-1710-325c7b47"
+        RL9: "openhpc-ofed-RL9-240725-1710-325c7b47"
     }
 }
 

environments/common/inventory/group_vars/all/ansible_init.yml

@@ -0,0 +1 @@
+ansible_init_wait: 1200 # seconds

environments/common/inventory/groups

@@ -132,3 +132,6 @@ freeipa_client
 
 [tuned]
 # Hosts to run TuneD configuration
+
+[ansible_init]
+# Hosts to run linux-anisble-init

environments/common/layouts/everything

@@ -78,3 +78,7 @@ openhpc
 
 [tuned:children]
 # Hosts to run TuneD configuration
+
+[ansible_init:children]
+# Hosts to run ansible-init
+cluster

requirements.yml

@@ -46,4 +46,7 @@ collections:
   - name: https://github.com/stackhpc/ansible-collection-terraform
     type: git
     version: 0.2.0
+  - name: https://github.com/azimuth-cloud/ansible-collection-image-utils
+    type: git
+    version: main # update on release
 ...