Disable updates and test OFED build in arcus CI

.github/workflows/stackhpc.yml

@@ -7,15 +7,32 @@ on:
   pull_request:
 jobs:
   openstack:
-    name: openstack-ci-${{ matrix.cloud }}
+    name: openstack-ci-${{ matrix.name }}
     strategy:
       matrix:
-        cloud:
-          - "smslabs" # SMS-Labs OpenStack in stackhpc-ci project
-          - "arcus"   # Arcus OpenStack in rcp-cloud-portal-demo project, with RoCE
+        include:
+          # SMS-Labs OpenStack in stackhpc-ci project
+          - cloud: smslabs
+            base_image_name: openhpc-220510-1910.qcow2
+            name: smslabs
+            cluster_name_suffix: ''
+          # Arcus OpenStack in rcp-cloud-portal-demo project, with RoCE
+          - cloud: arcus
+            base_image_name: openhpc-220510-1910.qcow2
+            name: arcus
+            cluster_name_suffix: ''
+          # As above using OFED
+          - cloud: arcus
+            base_image_name: openhpc-220510-1911-ofed.qcow2
+            name: arcus-ofed
+            cluster_name_suffix: ofed
       fail-fast: false # as want clouds to continue independently
-    concurrency: ${{ matrix.cloud }}
+    concurrency: ${{ matrix.name }}
     runs-on: ubuntu-20.04
+    env:
+      TF_VAR_cluster_name: ci${{ github.run_id }}${{ matrix.cluster_name_suffix }}
+      OS_CLOUD: openstack
+      ANSIBLE_FORCE_COLOR: True
     steps:
       - uses: actions/checkout@v2
 
@@ -60,8 +77,7 @@ jobs:
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
         env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
+          TF_VAR_base_image_name: ${{ matrix.base_image_name }}
 
       - name: Get server provisioning failure messages
         id: provision_failure
@@ -70,9 +86,6 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           echo "::set-output name=messages::$(../../skeleton/\{\{cookiecutter.environment\}\}/terraform/getfaults.py  $PWD)"
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: always() && steps.provision.outcome == 'failure'
 
       - name: Delete infrastructure if failed due to lack of hosts
@@ -81,9 +94,6 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
-        env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
 
       - name: Directly configure cluster and build compute, login and control images
@@ -96,9 +106,8 @@ jobs:
           echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           ansible-playbook -vv ansible/site.yml
         env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
           TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          PKR_VAR_source_image_name: ${{ matrix.base_image_name }}
 
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
@@ -138,18 +147,12 @@ jobs:
           . environments/${{ matrix.cloud }}/activate
           ansible all -m wait_for_connection
           ansible-playbook -vv ansible/ci/test_reimage.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True
-
+
       - name: Run MPI-based tests
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/adhoc/hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-          OS_CLOUD: openstack
 
       - name: Delete infrastructure
         run: |
@@ -158,15 +161,11 @@ jobs:
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
         env:
-          OS_CLOUD: openstack
-          TF_VAR_cluster_name: ci${{ github.run_id }}
+          TF_VAR_base_image_name: ${{ matrix.base_image_name }}
         if: ${{ success() || cancelled() }}
 
       - name: Delete images
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
           ansible-playbook -vv ansible/ci/delete_images.yml
-        env:
-          OS_CLOUD: openstack
-          ANSIBLE_FORCE_COLOR: True

ansible/bootstrap.yml

@@ -52,41 +52,6 @@
     - import_role:
         name: fail2ban
 
-- hosts: update
-  gather_facts: false
-  become: yes
-  tags:
-    - update
-  tasks:
-    - block:
-      - name: Update selected packages
-        yum:
-          name: "{{ update_name }}"
-          state: "{{ update_state }}"
-          exclude: "{{ update_exclude }}"
-          disablerepo: "{{ update_disablerepo }}"
-        async: "{{ 30 * 60 }}" # wait for up to 30 minutes
-        poll: 15 # check every 15 seconds
-        register: updates
-      - debug:
-          var: updates
-      - name: Ensure update log directory on localhost exists
-        file:
-          path: "{{ update_log_path | dirname }}"
-          state: directory
-        become: false
-        delegate_to: localhost
-        run_once: true
-      - name: Log updated packages
-        copy:
-          content: "{{ updates.results | join('\n') }}"
-          dest: "{{ update_log_path }}"
-        delegate_to: localhost
-        become: no
-      - debug:
-          msg: "{{ updates.results | length }} changes to packages - see {{ update_log_path }} for details"
-      when: "update_enable | default('false') | bool"
-
 - hosts:
     - selinux
     - update
@@ -110,4 +75,4 @@
         sleep: 15
     - name: update facts
       setup:
-      when: (sestatus.changed | default(false)) or (sestatus.reboot_required | default(false))
+      when: (sestatus.changed | default(false)) or (sestatus.reboot_required | default(false))

environments/arcus/README.md

@@ -0,0 +1,15 @@
+Provides a CI environment on Arcus in the `rcp-cloud-portal-demo` project. Uses "direct"-mode VNICs.
+
+This can be deployed manually on Arcus but will require the following to be defined:
+- Terraform variables `cluster_name` and `base_image_name`. E.g.:
+
+    ```
+    environments/smslabs/terraform/terraform.tfvars:
+    cluster_name = "ofed"
+    base_image_name = "openhpc-220510-1911-ofed.qcow2"
+    ```
+- Ansible variable `vault_testuser_password` defining the password for `testuser` (used for accessing Open Ondemand), e.g.:
+    ```
+    environments/smslabs/inventory/group_vars/all/dev.yml:
+    vault_testuser_password: somesecretstring
+    ```

environments/arcus/builder.pkrvars.hcl

@@ -1,6 +1,5 @@
 flavor = "vm.alaska.cpu.general.small"
 networks = ["a262aabd-e6bf-4440-a155-13dbc1b5db0e"] # WCDC-iLab-60
-source_image_name = "openhpc-220413-1545.qcow2"
 ssh_keypair_name = "slurm-app-ci"
 security_groups = ["default", "SSH"]
 ssh_bastion_host = "128.232.222.183"

environments/arcus/terraform/main.tf

@@ -8,6 +8,11 @@ variable "cluster_name" {
     description = "Name for cluster, used as prefix for resources - set by environment var in CI"
 }
 
+variable "base_image_name" {
+    type = string
+    description = "Name of base image for all nodes - set by environment var in CI"
+}
+
 module "cluster" {
     source = "../../skeleton/{{cookiecutter.environment}}/terraform/"
 
@@ -18,18 +23,18 @@ module "cluster" {
     key_pair = "slurm-app-ci"
     control_node = {
         flavor: "vm.alaska.cpu.general.small"
-        image: "openhpc-220504-0904.qcow2"
+        image: var.base_image_name
     }
     login_nodes = {
         login-0: {
             flavor: "vm.alaska.cpu.general.small"
-            image: "openhpc-220504-0904.qcow2"
+            image: var.base_image_name
         }
     }
     compute_types = {
         small: {
             flavor: "vm.alaska.cpu.general.small"
-            image: "openhpc-220504-0904.qcow2"
+            image: var.base_image_name
         }
     }
     compute_nodes = {

environments/common/inventory/group_vars/builder/defaults.yml

@@ -5,7 +5,6 @@
 openhpc_slurm_service_started: false
 nfs_client_mnt_state: present
 openhpc_rebuild_reconfigure: false
-update_enable: true
 block_devices_partition_state: skip
 block_devices_filesystem_state: skip
 block_devices_mount_state: present

environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl

@@ -3,16 +3,16 @@ ansible_user=rocky
 openhpc_cluster_name=${cluster_name}
 
 [control]
-${control.name} ansible_host=${[for n in control.network: n.fixed_ip_v4 if n.access_network][0]} server_networks='${jsonencode({for net in control.network: net.name => [ net.fixed_ip_v4 ] })}'
+${control.name} ansible_host=${[for n in control.network: n.fixed_ip_v4 if n.access_network][0]} instance_id=${control.id} server_networks='${jsonencode({for net in control.network: net.name => [ net.fixed_ip_v4 ] })}'
 
 [login]
 %{ for login in logins ~}
-${login.name} ansible_host=${[for n in login.network: n.fixed_ip_v4 if n.access_network][0]} server_networks='${jsonencode({for net in login.network: net.name => [ net.fixed_ip_v4 ] })}'
+${login.name} ansible_host=${[for n in login.network: n.fixed_ip_v4 if n.access_network][0]} instance_id=${login.id} server_networks='${jsonencode({for net in login.network: net.name => [ net.fixed_ip_v4 ] })}'
 %{ endfor ~}
 
 [compute]
 %{ for compute in computes ~}
-${compute.name} ansible_host=${[for n in compute.network: n.fixed_ip_v4 if n.access_network][0]} server_networks='${jsonencode({for net in compute.network: net.name => [ net.fixed_ip_v4 ] })}'
+${compute.name} ansible_host=${[for n in compute.network: n.fixed_ip_v4 if n.access_network][0]} instance_id=${compute.id} server_networks='${jsonencode({for net in compute.network: net.name => [ net.fixed_ip_v4 ] })}'
 %{ endfor ~}
 
 # Define groups for slurm parititions:

environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf

@@ -36,7 +36,6 @@ resource "openstack_networking_port_v2" "nonlogin" {
   }
 }
 
-
 resource "openstack_compute_instance_v2" "control" {
 
   name = "${var.cluster_name}-control"

environments/smslabs/README.md

@@ -0,0 +1,15 @@
+Provides a CI environment on SMS-labs in the `stackhpc-ci` project. Uses "normal"-mode VNICs.
+
+This can be deployed manually on SMS-labs but will require the following to be defined:
+- Terraform variables `cluster_name` and `base_image_name`. E.g.:
+
+    ```
+    environments/smslabs/terraform/terraform.tfvars:
+    cluster_name = "ofed"
+    base_image_name = "openhpc-220510-1911-ofed.qcow2"
+    ```
+- Ansible variable `vault_testuser_password` defining the password for `testuser` (used for accessing Open Ondemand), e.g.:
+    ```
+    environments/smslabs/inventory/group_vars/all/dev.yml:
+    vault_testuser_password: somesecretstring
+    ```

environments/smslabs/builder.pkrvars.hcl

@@ -1,6 +1,5 @@
-flavor = "general.v1.tiny"
+flavor = "general.v1.small"
 networks = ["c245901d-6b84-4dc4-b02b-eec0fb6122b2"] # stackhpc-ci-geneve
-source_image_name = "openhpc-220413-1545.qcow2"
 ssh_keypair_name = "slurm-app-ci"
 security_groups = ["default", "SSH"]
 ssh_bastion_host = "185.45.78.150"

environments/smslabs/terraform/main.tf

@@ -8,6 +8,11 @@ variable "cluster_name" {
     description = "Name for cluster, used as prefix for resources - set by environment var in CI"
 }
 
+variable "base_image_name" {
+    type = string
+    description = "Name of base image for all nodes - set by environment var in CI"
+}
+
 module "cluster" {
     source = "../../skeleton/{{cookiecutter.environment}}/terraform/"
 
@@ -20,19 +25,19 @@ module "cluster" {
     ]
     key_pair = "slurm-app-ci"
     control_node = {
-        flavor: "general.v1.tiny"
-        image: "openhpc-220504-0904.qcow2"
+        flavor: "general.v1.small"
+        image: var.base_image_name
     }
     login_nodes = {
         login-0: {
-          flavor: "general.v1.tiny"
-          image: "openhpc-220504-0904.qcow2"
+          flavor: "general.v1.small"
+          image: var.base_image_name
         }
     }
     compute_types = {
         small: {
-            flavor: "general.v1.tiny"
-            image: "openhpc-220504-0904.qcow2"
+            flavor: "general.v1.small"
+            image: var.base_image_name
         }
     }
     compute_nodes = {