Fix broken DNS in arcus CI (#202)

* use image without resolv.conf in arcus

* add ssh reset after user changes for robustness

* move ci image build to separate from direct deployment

* use image without resolv.conf in arcus packer build too

* update arcus image to use openhpc-220808-1510.qcow2 built in slurm_image_builder:main

Author: sjpb

.github/workflows/stackhpc.yml

@@ -87,7 +87,7 @@ jobs:
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
 
-      - name: Directly configure cluster and build compute, login and control images
+      - name: Directly configure cluster and build compute
         # see pre-hook for the image build
         run: |
           . venv/bin/activate
@@ -131,9 +131,20 @@ jobs:
           (echo $statuscode | grep "200 OK") || (echo $statuscode  && exit 1)
         env:
           TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-          
-      - name: Test reimage of login and compute nodes
-        # TODO: test control node reimage
+
+      - name: Build packer images
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
+          cd packer/
+          PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True
+          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
+      - name: Test reimage of nodes
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate

ansible/bootstrap.yml

@@ -23,6 +23,8 @@
       # Need to change working directory otherwise we try to switch back to non-existent directory.
       become_flags: '-i'
       become: true
+    - name: Reset ssh connection to allow user changes to affect ansible_user
+      meta: reset_connection
 
 - hosts: selinux
   gather_facts: false

environments/arcus/builder.pkrvars.hcl

@@ -1,6 +1,6 @@
 flavor = "vm.alaska.cpu.general.small"
 networks = ["a262aabd-e6bf-4440-a155-13dbc1b5db0e"] # WCDC-iLab-60
-source_image_name = "openhpc-220526-1354.qcow2"
+source_image_name = "openhpc-220808-1510.qcow2"
 ssh_keypair_name = "slurm-app-ci"
 security_groups = ["default", "SSH"]
 ssh_bastion_host = "128.232.222.183"

environments/arcus/hooks/pre.yml

@@ -1,22 +1,3 @@
-- hosts: localhost
-  become: false
-  tags: build
-  tasks:
-    - name: Ensure secrets generated
-      include_role:
-        name: passwords
-    
-    - name: Build packer images
-      shell:
-        cmd: |
-          cd packer
-          PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-        chdir: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}"
-      when: "'builder' not in group_names" # avoid recursion!
-      register: packer_run
-      async: 2700 # 45 minutes
-      poll: 0
-
 - hosts: all
   become: true
   tags: etc_hosts

environments/arcus/terraform/main.tf

@@ -18,18 +18,18 @@ module "cluster" {
     key_pair = "slurm-app-ci"
     control_node = {
         flavor: "vm.alaska.cpu.general.small"
-        image: "openhpc-220526-1354.qcow2"
+        image: "openhpc-220808-1510.qcow2"
     }
     login_nodes = {
         login-0: {
             flavor: "vm.alaska.cpu.general.small"
-            image: "openhpc-220526-1354.qcow2"
+            image: "openhpc-220808-1510.qcow2"
         }
     }
     compute_types = {
         small: {
             flavor: "vm.alaska.cpu.general.small"
-            image: "openhpc-220526-1354.qcow2"
+            image: "openhpc-220808-1510.qcow2"
         }
     }
     compute_nodes = {

environments/smslabs/hooks/pre.yml

@@ -1,22 +1,3 @@
-- hosts: localhost
-  become: false
-  tags: build
-  tasks:
-    - name: Ensure secrets generated
-      include_role:
-        name: passwords
-    
-    - name: Build packer images
-      shell:
-        cmd: |
-          cd packer
-          PACKER_LOG=1 packer build -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
-        chdir: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}"
-      when: "'builder' not in group_names" # avoid recursion!
-      register: packer_run
-      async: 2700 # 45 minutes
-      poll: 0
-
 - hosts: all
   become: true
   tags: etc_hosts