Move CI to arcus (#4)

* add ofed with ci on arcus

* create port in CI

* install all OFED packages by default

* don't remove ofed build deps (as removing ofed packages too)

* do both OFED and non-ofed builds in CI

* use github actions matrix w/ single packer build for ofed/non-ofed builds

* add ofed flag to CI job name

* Revert "add ofed flag to CI job name"

This reverts commit 1a4fc93.

* try to fix github matrix

* delete TF-provisioned infra on success or cancellation

* Reduce to 20GB root disk size

* upload image to S3 prerelease bucket

* fix conditional templating warning

* ensure kernel upgrades cause reboot

* change to Rocky8.6 image

* convert from ofed branch

* temporarily disable CI on PR

* reduce diff to main

* reenable CI on PR

* remove terraform

* remove ofed suffix

* fix network

* update docs

* use RL8.5 instead of broken (no sudo/hostname) 8.6 image

* fix command not found in CI

* retry getting image name

* remove quotes from jq output

* fix download step

* add s3cmd install

Author: sjpb

.github/arcus_bastion_fingerprint

@@ -0,0 +1,3 @@
+|1|BwhEZQPqvZcdf9Phmh2mTPmIivU=|bHi1Nf8dYI8z1C+qsqQFPAty1xA= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChxwhZggdwj55gNzfDBzah0G8IeTPQjgMZrpboxp2BO4J+o1iZSwDj+2fqyhBGTE43vCJR13uEygz49XIy+t17qBNwHz4fVVR7jdMNymtbZoOsq9oAoBdGEICHrMzQsYZmT9+Wt74ZP2PKOOn+a+f2vg7YdeSy1UhT08iJlbXwCx56fCQnMJMOnZM9MXVLd4NUFN1TeOCIBQHwRiMJyJ7S7CdUKpyUqHOG85peKiPJ07C0RZ/W5HkYKqltwtvPGQd262p5eLC9j3nhOYSG2meRV8yTxYz3lDIPDx0+189CZ5NaxFSPCgqSYA24zavhPVLQqoct7nd7fcEw9JiTs+abZC6GckCONSHDLM+iRtWC/i5u21ZZDLxM9SIqPI96cYFszGeqyZoXxS5qPaIDHbQNAEqJp9ygNXgh9vuBo7E+aWYbFDTG0RuvW02fbmFfZw2/yXIr37+cQX+GPOnkfIRuHE3Hx5eN8C04v+BMrAfK2minawhG3A2ONJs9LI6QoeE=
+|1|whGSPLhKW4xt/7PWOZ1treg3PtA=|F5gwV8j0JYWDzjb6DvHHaqO+sxs= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpCG881Gt3dr+nuVIC2uGEQkeVwG6WDdS1WcCoxXC7AG+Oi5bfdqtf4IfeLpWmeuEaAaSFH48ODFr76ViygSjU=
+|1|0V6eQ1FKO5NMKaHZeNFbw62mrJs=|H1vuGTbbtZD2MEgZxQf1PXPk+yU= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEnOtYByM3s2qvRT8SS1sn5z5sbwjzb1alm0B3emPcHJ

.github/smslabs_bastion_fingerprint

@@ -0,0 +1,86 @@
+name: Build images on arcus:rcp-cloud-portal-demo
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+concurrency: rcp-cloud-portal-demo # openstack project
+jobs:
+  arcus:
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        working-directory: ./collections/ansible_collections/stackhpc/slurm_image_builder
+    outputs:
+      image_name:  ${{ steps.manifest.outputs.IMAGE_NAME }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: collections/ansible_collections/stackhpc/slurm_image_builder
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir ~/.ssh
+          echo "$SSH_KEY" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+        env:
+          SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
+
+      - name: Add bastion's ssh key to known_hosts
+        run: cat .github/arcus_bastion_fingerprint >> ~/.ssh/known_hosts
+        shell: bash
+      
+      - name: Install ansible etc
+        run: ./setup.sh
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "$CLOUDS_YAML" > ~/.config/openstack/clouds.yaml
+        shell: bash
+        env:
+          CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
+
+      - name: Run image build
+        id: image_build
+        run: |
+          . venv/bin/activate
+          PACKER_LOG=1 packer build --on-error=ask -var-file=arcus.builder.pkrvars.hcl openstack.pkr.hcl
+        env:
+          OS_CLOUD: openstack
+
+      - name: Delete infrastructure
+        run: terraform destroy -auto-approve
+        env:
+          OS_CLOUD: openstack
+        if: ${{ success() || cancelled() }}
+
+      - name: Get created image name from manifest
+        id: manifest
+        run: |
+          . venv/bin/activate
+          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer-manifest.json)
+          while ! openstack image show -f value -c name $IMAGE_ID; do
+            sleep 30
+          done
+          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo "::set-output name=IMAGE_ID::$IMAGE_ID"
+          echo "::set-output name=IMAGE_NAME::$IMAGE_NAME"
+        env:
+          OS_CLOUD: openstack
+
+      - name: Download image to runner
+        run: |
+          . venv/bin/activate
+          openstack image save --file ${{ steps.manifest.outputs.IMAGE_NAME }} ${{ steps.manifest.outputs.IMAGE_ID }}
+        env:
+          OS_CLOUD: openstack
+
+      - name: Upload image to S3 prerelease bucket
+        run: |
+          echo "$S3_CFG" >  ~/.s3cfg
+          sudo apt-get install s3cmd
+          s3cmd put ${{ steps.manifest.outputs.IMAGE_NAME }} s3://openhpc-images-prerelease
+        env:
+          S3_CFG: ${{ secrets.ARCUS_S3_CFG }}

.github/workflows/arcus.yml

@@ -10,14 +10,18 @@ Images contain the git description of this repo state used to build them in `/va
 
 # Creating Images
 
-TODO: Describe how to run in CI.
+For CI, simply trigger the workflow. Built images will be uploaded to Arcus `s3://openhpc-images-prerelease`.
 
-Current manual steps, assuming a Rocky Linux 8.5 host on [sms-lab](https://api.sms-lab.cloud/):
+Current manual steps, assuming a Rocky Linux 8.5 host on Arcus:
+
+1. Create an appropriate collections path, e.g:
+
+    mkdir -p collections/ansible_collections/stackhpc/slurm_image_builder
 
 1. Clone the repo
 1. Install environment: `./setup.sh`
 1. Activate venv if necessary: `. venv/bin/activate`
-1. Build image: `PACKER_LOG=1 packer build --on-error=ask -var-file=smslabs.builder.pkrvars.hcl openstack.pkr.hcl`
+1. Build image: `PACKER_LOG=1 packer build --on-error=ask -var-file=arcus.builder.pkrvars.hcl openstack.pkr.hcl`
 
 # Usage of Images
 

.github/workflows/smslabs.yml

@@ -0,0 +1 @@
+port_id = "b04da514-c51f-4f33-885c-a57b352aa1b9"

README.md

@@ -2,68 +2,34 @@
 #  $ PACKER_LOG=1 packer build --on-error=ask -var-file=<something>.pkrvars.hcl openstack.pkr.hcl
 
 # "timestamp" template function replacement:s
-locals { timestamp = formatdate("YYMMDD-hhmm", timestamp())}
-
-variable "networks" {
-  type = list(string)
-}
+locals {timestamp = formatdate("YYMMDD-hhmm", timestamp())}
 
 variable "source_image_name" {
   type = string
-}
-
-variable "flavor" {
-  type = string
-}
-
-variable "ssh_username" {
-  type = string
-  default = "rocky"
-}
-
-variable "ssh_private_key_file" {
-  type = string
-  default = "~/.ssh/id_rsa"
-}
-
-variable "ssh_keypair_name" {
-  type = string
-}
-
-variable "security_groups" {
-  type = list(string)
-}
-
-variable "image_visibility" {
-  type = string
-  default = "Private"
+  default = "Rocky-8-GenericCloud-8.5-20211114.2.x86_64"
 }
 
 variable "ssh_bastion_host" {
   type = string
+  default = "128.232.222.183"
 }
 
 variable "ssh_bastion_username" {
   type = string
-}
-
-variable "ssh_bastion_private_key_file" {
-  type = string
-  default = "~/.ssh/id_rsa"
+  default = "slurm-app-ci"
 }
 
 source "openstack" "openhpc" {
-  flavor = "${var.flavor}"
-  networks = "${var.networks}"
+  flavor = "vm.alaska.cpu.general.tiny"
+  networks = ["4b6b2722-ee5b-40ec-8e52-a6610e14cc51"] # portal-internal
   source_image_name = "${var.source_image_name}" # NB: must already exist in OpenStack
-  ssh_username = "${var.ssh_username}"
+  ssh_username = "rocky"
   ssh_timeout = "20m"
-  ssh_private_key_file = "${var.ssh_private_key_file}" # TODO: doc same requirements as for qemu build?
-  ssh_keypair_name = "${var.ssh_keypair_name}" # TODO: doc this
+  ssh_private_key_file = "~/.ssh/id_rsa"
+  ssh_keypair_name = "slurm-app-ci"
   ssh_bastion_host = "${var.ssh_bastion_host}"
   ssh_bastion_username = "${var.ssh_bastion_username}"
-  ssh_bastion_private_key_file = "${var.ssh_bastion_private_key_file}"
-  security_groups = "${var.security_groups}"
+  ssh_bastion_private_key_file = "~/.ssh/id_rsa"
   image_name = "${source.name}-${local.timestamp}.qcow2"
 }
 
@@ -75,8 +41,8 @@ build {
     playbook_file = "playbooks/build.yml" # can't use ansible FQCN here
     use_proxy = false # see https://www.packer.io/docs/provisioners/ansible#troubleshooting
     extra_arguments = ["-v"]
-    # ansible_ssh_common_args: '-o ProxyCommand="ssh {{ bastion_user }}@{{ bastion_ip }} -W %h:%p"'
     ansible_ssh_extra_args = ["-o ProxyCommand='ssh ${var.ssh_bastion_username }@${ var.ssh_bastion_host} -W %h:%p'"]
+    # keep_inventory_file = true
   }
 
   post-processor "manifest" {

arcus.builder.pkrvars.hcl

@@ -1,41 +1,5 @@
 ---
 
-- name: Install rpm keys
-  ansible.builtin.rpm_key:
-    key: "{{ item }}"
-    state: present
-  loop: "{{ rpm_keys }}"
-
-- name: Add dnf repos
-  ansible.builtin.get_url:
-    url: "{{ item }}"
-    dest: "/etc/yum.repos.d/{{ item.split('/')[-1] }}"
-  loop: "{{ dnf_add_repos }}"
-
-- name: Enable dnf repos
-  # NB: Doesn't use `dnf config-manager --set-enabled ...` as can't make that idempotent
-  lineinfile:
-    path: "{{ item }}"
-    create: false # raises error if not already installed
-    regexp: enabled=
-    line: enabled=1
-  loop: "{{ dnf_enabled_repos }}"    
-
-- name: Upgrade dnf packages
-  dnf:
-    name: '*'
-    state: latest
-    exclude: "{{ dnf_update_exclude }}"
-
-- name: Reboot if required due to package upgrades
-  reboot:
-    post_reboot_delay: 30
-  when: lookup('fileglob', '/var/run/reboot-required') | length > 0
-
-- name: Wait for hosts to be reachable
-  wait_for_connection:
-    sleep: 15
-
 - name: Install dnf release packages
   dnf: "{{ item }}"
   loop: "{{ dnf_release_packages }}"
@@ -45,6 +9,6 @@
     name: "{{ dnf_latest_packages }}"
     state: latest
 
-- name: Install dnf packages:2
+- name: Install dnf packages at specific versions
   dnf:
     name: "{{ dnf_specific_packages }}"

openstack.pkr.hcl

@@ -0,0 +1,20 @@
+- name: Install rpm keys
+  ansible.builtin.rpm_key:
+    key: "{{ item }}"
+    state: present
+  loop: "{{ rpm_keys }}"
+
+- name: Add dnf repos
+  ansible.builtin.get_url:
+    url: "{{ item }}"
+    dest: "/etc/yum.repos.d/{{ item.split('/')[-1] }}"
+  loop: "{{ dnf_add_repos }}"
+
+- name: Enable dnf repos
+  # NB: Doesn't use `dnf config-manager --set-enabled ...` as can't make that idempotent
+  lineinfile:
+    path: "{{ item }}"
+    create: false # raises error if not already installed
+    regexp: enabled=
+    line: enabled=1
+  loop: "{{ dnf_enabled_repos }}"