Merge branch 'main' into fix/skelton-ssh-persist

Author: sjpb

.github/workflows/fatimage.yml

@@ -4,26 +4,26 @@ on:
   workflow_dispatch:
 jobs:
   openstack:
-    name: openstack-build-arcus
+    name: openstack-imagebuild
     concurrency: ${{ github.ref }} # to branch/PR
     runs-on: ubuntu-20.04
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
+      CI_CLOUD: ${{ vars.CI_CLOUD }}
     steps:
       - uses: actions/checkout@v2
 
       - name: Setup ssh
         run: |
           set -x
           mkdir ~/.ssh
-          echo "${arcus_SSH_KEY}" > ~/.ssh/id_rsa
+          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
-        env:
-          arcus_SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
+        shell: bash
 
       - name: Add bastion's ssh key to known_hosts
-        run: cat environments/.stackhpc/bastion_fingerprint >> ~/.ssh/known_hosts
+        run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
 
       - name: Install ansible etc
@@ -32,11 +32,9 @@ jobs:
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
-          echo "${arcus_CLOUDS_YAML}" > ~/.config/openstack/clouds.yaml
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
-        env:
-          arcus_CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
-            
+
       - name: Setup environment
         run: |
           . venv/bin/activate
@@ -49,7 +47,7 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=ask -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+          PACKER_LOG=1 packer build -only openstack.openhpc -on-error=ask -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
 
       - name: Get created image name from manifest
         id: manifest

.github/workflows/stackhpc.yml

@@ -40,6 +40,8 @@ jobs:
 
       - name: Install terraform
         uses: hashicorp/setup-terraform@v1
+        with:
+          terraform: v1.5.5
 
       - name: Initialise terraform
         run: terraform init

ansible/fatimage.yml

@@ -59,6 +59,11 @@
         tasks_from: vnc_compute.yml
 
     # - import_playbook: monitoring.yml:
+    - import_role:
+        name: opensearch
+        tasks_from: install.yml
+      become: true
+
     #   opensearch - containerised, nothing to do
     # slurm_stats - nothing to do
     # filebeat - containerised - nothing to do

ansible/monitoring.yml

@@ -7,6 +7,11 @@
   tasks:
     - import_role:
         name: opensearch
+        tasks_from: install.yml
+      become: true
+    - import_role:
+        name: opensearch
+        tasks_from: runtime.yml
       become: true
 
 - name: Setup slurm stats

ansible/roles/opensearch/defaults/main.yml

@@ -3,7 +3,7 @@
 #opensearch_internal_users_path:
 
 opensearch_podman_user: "{{ ansible_user }}"
-opensearch_version: '2.4.0' # https://hub.docker.com/r/opensearchproject/opensearch/tags
+opensearch_version: '2.9.0' # https://hub.docker.com/r/opensearchproject/opensearch/tags
 opensearch_config_path: /usr/share/opensearch/config
 opensearch_data_path: /usr/share/opensearch/data
 opensearch_state: started # will be restarted if required

ansible/roles/opensearch/handlers/main.yml

@@ -5,5 +5,4 @@
     name: opensearch.service
     state: "{{ 'restarted' if 'started' in opensearch_state else opensearch_state }}"
     enabled: "{{ opensearch_systemd_service_enabled }}"
-    daemon_reload: "{{ 'started' in opensearch_state }}"
   become: true

ansible/roles/opensearch/tasks/archive_data.yml

@@ -0,0 +1,17 @@
+# Remove data which was NOT indexed by Slurm Job ID
+# It will be re-ingested by filebeat from the slurmdbd, with that index
+
+- name: Ensure opensearch stopped
+  systemd:
+    name: opensearch
+    state: stopped
+  register: _opensearch_stop
+  until: "_opensearch_stop.status.ActiveState in ['inactive', 'failed']"
+  retries: 15
+  delay: 5
+
+- name: Archive existing data
+  community.general.archive:
+    path: "{{ opensearch_data_path }}"
+    dest: "{{ opensearch_data_path | dirname }}/data-{{ lookup('pipe', 'date --iso-8601=minutes') }}.tar.gz"
+    remove: true

ansible/roles/opensearch/tasks/install.yml

@@ -0,0 +1,19 @@
+# safe to use during build
+
+- name: Increase maximum number of virtual memory maps
+  # see https://opensearch.org/docs/2.0/opensearch/install/important-settings/
+  ansible.posix.sysctl:
+    name: vm.max_map_count
+    value: '262144'
+    state: present
+    reload: yes
+
+- name: Create systemd unit file
+  template:
+    dest: /etc/systemd/system/opensearch.service
+    src: opensearch.service.j2
+  register: _opensearch_unit
+
+- name: Reload opensearch unit file
+  command: systemctl daemon-reload
+  when: _opensearch_unit.changed

ansible/roles/opensearch/tasks/main.yml renamed to ansible/roles/opensearch/tasks/runtime.yml

@@ -15,14 +15,16 @@
     path: /etc/systemd/system/opendistro.service
     state: absent
 
-- name: Increase maximum number of virtual memory maps
-  # see https://opensearch.org/docs/2.0/opensearch/install/important-settings/
-  ansible.posix.sysctl:
-    name: vm.max_map_count
-    value: '262144'
-    state: present
-    reload: yes
-  become: true
+- name: Enumerate files in data directory
+  find:
+    path: "{{ opensearch_data_path }}"
+  register: _find_opensearch_data
+
+- name: Archive incorrectly indexed data
+  import_tasks: archive_data.yml
+  when:
+    - _find_opensearch_data.files | length > 0
+    - "'slurm_jobid_index' not in _find_opensearch_data.files | map(attribute='path') | map('basename')"
 
 - name: Ensure required opensearch host directories exist
   file:
@@ -35,11 +37,18 @@
   loop:
     - "{{ opensearch_config_path }}"
     - "{{ opensearch_data_path }}"
-  when: "'started' in opensearch_state" # don't run during image build
+
+- name: Set indexed data flag
+  copy:
+    dest: "{{ opensearch_data_path }}/slurm_jobid_index"
+    content: |
+      This is a flag file to indicate that filebeat is pushing data
+      indexed by Slurm JobID to prevent duplicate OpenSearch records
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
 
 - name: Create certs
   import_tasks: certs.yml
-  when: "'started' in opensearch_state" # don't run during image build
 
 - name: Template general configuration
   ansible.builtin.template:
@@ -52,7 +61,6 @@
     mode: 0660
   notify: Restart opensearch service
   become: true
-  when: "'started' in opensearch_state" # don't run during image build
 
 - name: Template internal user configuration
   template:
@@ -65,14 +73,11 @@
       mode: 0660
   notify: Restart opensearch service
   become: true
-  when: "'started' in opensearch_state" # don't run during image build
 
-- name: Create systemd unit file
-  template:
-    dest: /etc/systemd/system/opensearch.service
-    src: opensearch.service.j2
-  become: true
-  notify: Restart opensearch service
+- name: Pull container
+  containers.podman.podman_image:
+    name: "opensearchproject/opensearch:{{ opensearch_version }}"
+  become_user: "{{ opensearch_podman_user }}"
 
 - name: Flush handlers
   meta: flush_handlers

environments/.stackhpc/builder.pkrvars.hcl renamed to environments/.stackhpc/ARCUS.pkrvars.hcl

@@ -1,6 +1,6 @@
 flavor = "vm.ska.cpu.general.small"
 networks = ["a262aabd-e6bf-4440-a155-13dbc1b5db0e"] # WCDC-iLab-60
-source_image_name = "openhpc-230503-0944-bf8c3f63.qcow2" # https://github.com/stackhpc/ansible-slurm-appliance/pull/252
+source_image_name = "openhpc-230804-1754-80b8d714" # https://github.com/stackhpc/ansible-slurm-appliance/pull/298
 fatimage_source_image_name = "Rocky-8-GenericCloud-8.6.20220702.0.x86_64.qcow2"
 ssh_keypair_name = "slurm-app-ci"
 ssh_private_key_file = "~/.ssh/id_rsa"

environments/.stackhpc/SMS.pkrvars.hcl

@@ -0,0 +1,9 @@
+flavor = "general.v1.tiny"
+networks = ["26023e3d-bc8e-459c-8def-dbd47ab01756"] # stackhpc-ipv4-geneve
+source_image_name = "openhpc-230503-0944-bf8c3f63" # https://github.com/stackhpc/ansible-slurm-appliance/pull/252
+fatimage_source_image_name = "Rocky-8-GenericCloud-8.6.20220702.0.x86_64.qcow2"
+ssh_keypair_name = "slurm-app-ci"
+ssh_private_key_file = "~/.ssh/id_rsa"
+security_groups = ["default", "SSH"]
+ssh_bastion_host = "185.45.78.150"
+ssh_bastion_username = "steveb"

environments/.stackhpc/inventory/group_vars/all/rebuild.yml

@@ -1,8 +1,7 @@
-cluster_net = "WCDC-iLab-60"
-cluster_subnet = "WCDC-iLab-60"
-vnic_type = "direct"
+cluster_net = "portal-internal"
+cluster_subnet = "portal-internal"
+vnic_type = "normal"
 control_node_flavor = "vm.ska.cpu.general.quarter"
 other_node_flavor = "vm.ska.cpu.general.small"
-volume_backed_instances = false
 state_volume_device_path = "/dev/sdb"
 home_volume_device_path = "/dev/sdc"

environments/.stackhpc/terraform/ARCUS.tfvars

@@ -3,6 +3,5 @@ cluster_subnet = "stackhpc-ipv4-geneve-subnet"
 vnic_type = "normal"
 control_node_flavor = "general.v1.medium"
 other_node_flavor = "general.v1.tiny"
-volume_backed_instances = true
 state_volume_device_path = "/dev/vdb"
 home_volume_device_path = "/dev/vdc"

environments/.stackhpc/terraform/SMS.tfvars

@@ -13,7 +13,7 @@ variable "cluster_name" {
 variable "cluster_image" {
     description = "single image for all cluster nodes - a convenience for CI"
     type = string
-    default = "openhpc-230503-0944-bf8c3f63" # https://github.com/stackhpc/ansible-slurm-appliance/pull/252
+    default = "openhpc-230811-1548-a49164d1" # https://github.com/stackhpc/ansible-slurm-appliance/pull/301
     # default = "Rocky-8-GenericCloud-Base-8.7-20221130.0.x86_64.qcow2"
     # default = "Rocky-8-GenericCloud-8.6.20220702.0.x86_64.qcow2"
 }
@@ -28,7 +28,9 @@ variable "control_node_flavor" {}
 
 variable "other_node_flavor" {}
 
-variable "volume_backed_instances" {}
+variable "volume_backed_instances" {
+    default = false
+}
 
 variable "state_volume_device_path" {}
 

environments/.stackhpc/terraform/main.tf

@@ -22,6 +22,11 @@ filebeat.inputs:
     fields_under_root: true
 
 processors:
+  # Want to use the Slurm JobID as the ElasticSearch id to avoid duplicated records
+  # Don't use filebeat.inputs:json.document_id as this removes the JobID from the record
+  - fingerprint:
+      fields: ["json.JobID"]
+      target_field: "@metadata._id"
   - timestamp:
       field: json.End
       layouts: