Skip to content

Commit 9ea95e6

Browse files
markgoddardAlex-Welsh
markgoddard
authored and
Alex-Welsh
committed
Fix host image builds on Arc runners
Arc runners are kubernetes-orchestrated github runners. Host image builds do not work on these runners, so this commit adapts the host image build workflow to spin up a worker VM which executes the build.
1 parent 8c55e94 commit 9ea95e6

File tree

13 files changed

+476
-186
lines changed

13 files changed

+476
-186
lines changed

‎.github/workflows/overcloud-host-image-build.yml

Lines changed: 239 additions & 163 deletions
Large diffs are not rendered by default.

‎.github/workflows/stackhpc-ci-cleanup.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,23 @@ jobs:
5555
OS_CLOUD: openstack
5656
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
5757
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
58+
59+
- name: Clean up host image builder instances over 5 hours old
60+
run: |
61+
result=0
62+
changes_before=$(date -Imin -d -5hours)
63+
for status in ACTIVE BUILD ERROR SHUTOFF; do
64+
for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do
65+
echo "Cleaning up $status instance $instance"
66+
openstack server show $instance
67+
if ! openstack server delete $instance; then
68+
echo "Failed to delete $status instance $instance"
69+
result=1
70+
fi
71+
done
72+
done
73+
exit $result
74+
env:
75+
OS_CLOUD: openstack
76+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
77+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

‎etc/kayobe/ansible/openstack-host-image-upload.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
---
2+
# This playbook is desinged to be used by the overcloud-host-image-build.yml
3+
# GitHub workflow to upload newly-built images to a development cloud for
4+
# testing and use in CI.
5+
- name: Upload an OS image to Glance
6+
hosts: seed
7+
vars:
8+
local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2"
9+
image_name: "overcloud-{{ os_distribution }}-{{ os_release }}"
10+
tasks:
11+
- name: Write out clouds.yml
12+
copy:
13+
content: "{{ lookup('ansible.builtin.env', 'CLOUDS_YAML') }}"
14+
dest: clouds.yaml
15+
mode: 0664
16+
17+
- name: Write out secure.yml
18+
no_log: true
19+
vars:
20+
- os_secrets:
21+
clouds:
22+
openstack:
23+
auth:
24+
application_credential_id: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}"
25+
application_credential_secret: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}"
26+
copy:
27+
content: "{{ os_secrets | to_nice_yaml }}"
28+
dest: secure.yaml
29+
mode: 0664
30+
31+
- name: Ensure dependencies are installed
32+
pip:
33+
name: openstacksdk
34+
35+
- name: Upload an image to Glance
36+
openstack.cloud.image:
37+
cloud: openstack
38+
name: "{{ image_name }}"
39+
container_format: bare
40+
disk_format: qcow2
41+
state: present
42+
filename: "{{ local_image_path }}"

‎etc/kayobe/ansible/pulp-host-image-upload.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
---
22
- name: Upload and create a distribution for an image
3-
hosts: localhost
3+
hosts: seed
44
vars:
55
remote_pulp_url: "{{ stackhpc_release_pulp_url }}"
66
remote_pulp_username: "{{ stackhpc_image_repository_username }}"
77
remote_pulp_password: "{{ stackhpc_image_repository_password }}"
88
repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}"
9-
base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
9+
pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}"
1010
tasks:
1111
- name: Print image tag
1212
debug:
@@ -74,7 +74,7 @@
7474
username: "{{ remote_pulp_username }}"
7575
password: "{{ remote_pulp_password }}"
7676
name: "{{ repository_name }}_latest"
77-
base_path: "{{ base_path }}/latest"
77+
base_path: "{{ pulp_base_path }}/latest"
7878
publication: "{{ publication_details.publication.pulp_href }}"
7979
content_guard: development
8080
state: present
@@ -86,7 +86,7 @@
8686
username: "{{ remote_pulp_username }}"
8787
password: "{{ remote_pulp_password }}"
8888
name: "{{ repository_name }}_{{ host_image_tag }}"
89-
base_path: "{{ base_path }}/{{ host_image_tag }}"
89+
base_path: "{{ pulp_base_path }}/{{ host_image_tag }}"
9090
publication: "{{ publication_details.publication.pulp_href }}"
9191
content_guard: development
9292
state: present
@@ -95,26 +95,26 @@
9595
- name: Update new images file with versioned path
9696
lineinfile:
9797
path: /tmp/updated_images.txt
98-
line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
98+
line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
9999
{{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
100100
create: true
101101

102102
- name: Update new images file with latest path
103103
lineinfile:
104104
path: /tmp/updated_images.txt
105-
line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
105+
line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
106106
latest/{{ found_files.files[0].path | basename }}"
107107
when: latest_distribution_details.changed
108108

109109
- name: Print versioned path
110110
debug:
111-
msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
111+
msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
112112
{{ host_image_tag }}/{{ found_files.files[0].path | basename }}"
113113
when: latest_distribution_details.changed
114114

115115
- name: Print latest path
116116
debug:
117-
msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\
117+
msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\
118118
latest/{{ found_files.files[0].path | basename }}"
119119
when: latest_distribution_details.changed
120120

‎etc/kayobe/environments/ci-builder/inventory/group_vars/seed/network-interfaces

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,5 @@
22
###############################################################################
33
# Network interface definitions for the controller group.
44

5-
# Controller interface on all-in-one network.
6-
aio_interface: breth1
7-
# Use dummy1 if it exists, otherwise the bridge will have no ports.
8-
aio_bridge_ports: "{{ ['dummy1'] if 'dummy1' in hostvars[inventory_hostname].ansible_facts else [] }}"
9-
10-
###############################################################################
11-
# Dummy variable to allow Ansible to accept this file.
12-
workaround_ansible_issue_8743: yes
13-
5+
# Seed interface on all-in-one network.
6+
aio_interface: eth0

‎etc/kayobe/overcloud-dib.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra:
7171
overcloud_dib_git_elements_extra:
7272
- repo: "https://github.com/stackhpc/stackhpc-image-elements"
7373
local: "{{ source_checkout_path }}/stackhpc-image-elements"
74-
version: "v1.6.0"
74+
version: "v1.6.1"
7575
elements_path: "elements"
7676

7777
# List of git repositories containing Diskimage Builder (DIB) elements. See

‎etc/kayobe/pulp-host-image-versions.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
---
22
# Overcloud host image versioning tags
33
# These images must be in SMS, since they are used by our AIO CI runners
4-
stackhpc_centos_8_stream_overcloud_host_image_version: "yoga-20230525T095243"
5-
stackhpc_rocky_8_overcloud_host_image_version: "yoga-20230629T135322"
6-
stackhpc_rocky_9_overcloud_host_image_version: "yoga-20240124T094316"
7-
stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20230609T120720"
8-
stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20231012T121552"
4+
stackhpc_centos_8_stream_overcloud_host_image_version: "yoga-20240326T091511"
5+
stackhpc_rocky_8_overcloud_host_image_version: "yoga-20240326T091511"
6+
stackhpc_rocky_9_overcloud_host_image_version: "yoga-20240326T091511"
7+
stackhpc_ubuntu_focal_overcloud_host_image_version: "yoga-20240326T091511"
8+
stackhpc_ubuntu_jammy_overcloud_host_image_version: "yoga-20240326T091511"

‎terraform/host-image-builder/README.rst

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
============================
2+
Terraform Host Image Builder
3+
============================
4+
5+
This Terraform configuration deploys a single VM on an OpenStack cloud, to
6+
build overcloud host images.
7+
8+
This configuration is used in the GitHub Actions overcloud-host-image-build.yml
9+
workflow.

‎terraform/host-image-builder/outputs.tf

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
output "access_ip_v4" {
2+
value = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4
3+
}
4+
5+
output "access_interface" {
6+
value = var.host_image_builder_interface
7+
}

‎terraform/host-image-builder/provider.tf

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#provider "openstack" {
2+
# use environment variables
3+
#}
4+
5+
terraform {
6+
required_version = ">= 0.14"
7+
backend "local" {
8+
}
9+
required_providers {
10+
openstack = {
11+
source = "terraform-provider-openstack/openstack"
12+
}
13+
}
14+
}

‎terraform/host-image-builder/templates/userdata.cfg.tpl

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#cloud-config
2+
# Don't automatically mount ephemeral disk
3+
mounts:
4+
- [/dev/vdb, null]
5+
# WORKAROUND: internal DNS missing from SMS lab. (currently unused)
6+
runcmd:
7+
- 'echo "10.0.0.34 pelican pelican.service.compute.sms-lab.cloud" >> /etc/hosts'
8+
- 'echo "10.205.3.187 pulp-server pulp-server.internal.sms-cloud" >> /etc/hosts'
9+
# Configure SSH keys here, to avoid creating an ephemeral keypair.
10+
# This means only the instance needs to be cleaned up if the destroy fails.
11+
ssh_authorized_keys:
12+
- ${ssh_public_key}
13+
14+
write_files:
15+
# WORKAROUND: https://bugs.launchpad.net/kolla-ansible/+bug/1995409
16+
- content: |
17+
#!/bin/bash
18+
docker exec openvswitch_vswitchd ovs-vsctl "$@"
19+
owner: root:root
20+
path: /usr/bin/ovs-vsctl
21+
permissions: '0755'

‎terraform/host-image-builder/terraform.tfvars

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
ssh_public_key = "id_rsa.pub"
2+
ssh_username = "rocky"
3+
host_image_builder_name = "skc-ci-host-image-builder"
4+
# Must be a Rocky Linux 9 host to successfully build all images
5+
# This MUST NOT be an LVM image. It can cause confusing conficts with the built image.
6+
host_image_builder_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2"
7+
host_image_builder_flavor = "en1.medium"
8+
host_image_builder_network = "stackhpc-ci"
9+
host_image_builder_subnet = "stackhpc-ci"
10+
host_image_builder_interface = "eth0"

‎terraform/host-image-builder/vm.tf

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
variable "ssh_public_key" {
2+
type = string
3+
}
4+
5+
variable "ssh_username" {
6+
type = string
7+
}
8+
9+
variable "host_image_builder_name" {
10+
type = string
11+
default = "kayobe-host-image-builder"
12+
}
13+
14+
variable "host_image_builder_image" {
15+
type = string
16+
default = "Rocky-9"
17+
}
18+
19+
variable "host_image_builder_interface" {
20+
type = string
21+
default = "eth0"
22+
}
23+
24+
variable "host_image_builder_flavor" {
25+
type = string
26+
}
27+
28+
variable "host_image_builder_network" {
29+
type = string
30+
}
31+
32+
variable "host_image_builder_subnet" {
33+
type = string
34+
}
35+
36+
variable "host_image_builder_volume_size" {
37+
type = number
38+
default = 150
39+
}
40+
41+
variable "host_image_builder_tags" {
42+
type = list(string)
43+
default = []
44+
}
45+
46+
locals {
47+
image_is_uuid = length(regexall("^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", var.host_image_builder_image)) > 0
48+
}
49+
50+
data "openstack_images_image_v2" "image" {
51+
name = var.host_image_builder_image
52+
most_recent = true
53+
count = local.image_is_uuid ? 0 : 1
54+
}
55+
56+
data "openstack_networking_subnet_v2" "network" {
57+
name = var.host_image_builder_subnet
58+
}
59+
60+
resource "openstack_compute_instance_v2" "kayobe-host-image-builder" {
61+
name = var.host_image_builder_name
62+
flavor_name = "en1.medium"
63+
config_drive = true
64+
user_data = templatefile("templates/userdata.cfg.tpl", {ssh_public_key = file(var.ssh_public_key)})
65+
network {
66+
name = var.host_image_builder_network
67+
}
68+
69+
block_device {
70+
uuid = local.image_is_uuid ? var.host_image_builder_image: data.openstack_images_image_v2.image[0].id
71+
source_type = "image"
72+
volume_size = var.host_image_builder_volume_size
73+
boot_index = 0
74+
destination_type = "volume"
75+
delete_on_termination = true
76+
}
77+
78+
tags = var.host_image_builder_tags
79+
}
80+
81+
# Wait for the instance to be accessible via SSH before progressing.
82+
resource "null_resource" "kayobe-host-image-builder" {
83+
provisioner "remote-exec" {
84+
connection {
85+
host = openstack_compute_instance_v2.kayobe-host-image-builder.access_ip_v4
86+
user = var.ssh_username
87+
private_key = file("id_rsa")
88+
# Terraform will run the start script from /tmp by default. For the
89+
# current images, /tmp is noexec, so the path must be changed
90+
script_path = "/home/${var.ssh_username}/start.sh"
91+
}
92+
93+
inline = [
94+
"#!/bin/sh",
95+
"echo 'connected!'"
96+
]
97+
}
98+
}

0 commit comments

Comments
 (0)