Skip to content

Commit b0979ab

Browse files
technowhizztechnowhizz
authored
Merge branch 'stackhpc/2023.1' into logs_in_grafana
2 parents cd2c2f4 + f00221e commit b0979ab

File tree

65 files changed

+1064
-284
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+1064
-284
lines changed

‎.github/workflows/overcloud-host-image-build.yml

Lines changed: 187 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -23,22 +23,39 @@ on:
2323

2424
env:
2525
ANSIBLE_FORCE_COLOR: True
26+
KAYOBE_ENVIRONMENT: ci-builder
27+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
2628
jobs:
2729
overcloud-host-image-build:
2830
name: Build overcloud host images
2931
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
30-
runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9]
32+
runs-on: arc-skc-host-image-builder-runner
3133
permissions: {}
3234
steps:
33-
- uses: actions/checkout@v4
35+
- name: Install Package
36+
uses: ConorMacBride/install-package@main
37+
with:
38+
apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
39+
40+
- name: Start the SSH service
41+
run: |
42+
sudo /etc/init.d/ssh start
43+
44+
- name: Checkout
45+
uses: actions/checkout@v4
3446
with:
3547
path: src/kayobe-config
3648

49+
- name: Output image tag of the builder
50+
id: builder_image_tag
51+
run: |
52+
echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
53+
3754
- name: Determine OpenStack release
3855
id: openstack_release
3956
run: |
4057
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
41-
echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
58+
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
4259
4360
# Generate a tag to apply to all built overcloud host images.
4461
- name: Generate overcloud host image tag
@@ -50,36 +67,13 @@ jobs:
5067
run: |
5168
echo "${{ steps.host_image_tag.outputs.host_image_tag }}"
5269
53-
- name: Clean any previous build artifact
54-
run: |
55-
rm -f /tmp/updated_images.txt
56-
5770
- name: Clone StackHPC Kayobe repository
5871
uses: actions/checkout@v4
5972
with:
6073
repository: stackhpc/kayobe
6174
ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }}
6275
path: src/kayobe
6376

64-
- name: Install dependencies
65-
run: |
66-
sudo dnf -y install zstd debootstrap
67-
68-
- name: Setup networking
69-
run: |
70-
if ! ip l show breth1 >/dev/null 2>&1; then
71-
sudo ip l add breth1 type bridge
72-
fi
73-
sudo ip l set breth1 up
74-
if ! ip a show breth1 | grep 192.168.33.3/24; then
75-
sudo ip a add 192.168.33.3/24 dev breth1
76-
fi
77-
if ! ip l show dummy1 >/dev/null 2>&1; then
78-
sudo ip l add dummy1 type dummy
79-
fi
80-
sudo ip l set dummy1 up
81-
sudo ip l set dummy1 master breth1
82-
8377
- name: Install Kayobe
8478
run: |
8579
mkdir -p venvs &&
@@ -89,36 +83,132 @@ jobs:
8983
pip install -U pip &&
9084
pip install ../src/kayobe
9185
86+
- name: Install terraform
87+
uses: hashicorp/setup-terraform@v2
88+
89+
- name: Initialise terraform
90+
run: terraform init
91+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
92+
93+
- name: Generate SSH keypair
94+
run: ssh-keygen -f id_rsa -N ''
95+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
96+
97+
- name: Generate clouds.yaml
98+
run: |
99+
cat << EOF > clouds.yaml
100+
${{ secrets.CLOUDS_YAML }}
101+
EOF
102+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
103+
104+
- name: Generate terraform.tfvars
105+
run: |
106+
cat << EOF > terraform.tfvars
107+
ssh_public_key = "id_rsa.pub"
108+
ssh_username = "rocky"
109+
aio_vm_name = "skc-host-image-builder"
110+
# Must be a Rocky Linux 9 host to successfully build all images
111+
# This MUST NOT be an LVM image. It can cause confusing conficts with the built image.
112+
aio_vm_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2"
113+
aio_vm_flavor = "en1.medium"
114+
aio_vm_network = "stackhpc-ci"
115+
aio_vm_subnet = "stackhpc-ci"
116+
aio_vm_interface = "eth0"
117+
EOF
118+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
119+
120+
- name: Terraform Plan
121+
run: terraform plan
122+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
123+
env:
124+
OS_CLOUD: "openstack"
125+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
126+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
127+
128+
- name: Terraform Apply
129+
run: |
130+
for attempt in $(seq 5); do
131+
if terraform apply -auto-approve; then
132+
echo "Created infrastructure on attempt $attempt"
133+
exit 0
134+
fi
135+
echo "Failed to create infrastructure on attempt $attempt"
136+
sleep 10
137+
terraform destroy -auto-approve
138+
sleep 60
139+
done
140+
echo "Failed to create infrastructure after $attempt attempts"
141+
exit 1
142+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
143+
env:
144+
OS_CLOUD: "openstack"
145+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
146+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
147+
148+
- name: Get Terraform outputs
149+
id: tf_outputs
150+
run: |
151+
terraform output -json
152+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
153+
154+
- name: Write Terraform outputs
155+
run: |
156+
cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml
157+
${{ steps.tf_outputs.outputs.stdout }}
158+
EOF
159+
160+
- name: Write Terraform network config
161+
run: |
162+
cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml
163+
---
164+
aio_ips:
165+
builder: "{{ access_ip_v4.value }}"
166+
EOF
167+
168+
- name: Write Terraform network interface config
169+
run: |
170+
mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed
171+
rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
172+
cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces
173+
admin_interface: "{{ access_interface.value }}"
174+
aio_interface: "{{ access_interface.value }}"
175+
EOF
176+
177+
- name: Manage SSH keys
178+
run: |
179+
mkdir -p ~/.ssh
180+
touch ~/.ssh/authorized_keys
181+
cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys
182+
cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/
183+
92184
- name: Bootstrap the control host
93185
run: |
94186
source venvs/kayobe/bin/activate &&
95187
source src/kayobe-config/kayobe-env --environment ci-builder &&
96188
kayobe control host bootstrap
97189
98-
- name: Configure the seed host
190+
- name: Configure the seed host (Builder VM)
191+
run: |
192+
source venvs/kayobe/bin/activate &&
193+
source src/kayobe-config/kayobe-env --environment ci-builder &&
194+
kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network
195+
196+
- name: Install dependencies
99197
run: |
100198
source venvs/kayobe/bin/activate &&
101199
source src/kayobe-config/kayobe-env --environment ci-builder &&
102-
kayobe seed host configure
200+
kayobe seed host command run \
201+
--command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output
103202
env:
104203
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
105204

106205
- name: Create bifrost_httpboot Docker volume
107-
run: |
108-
if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then
109-
sudo docker volume create bifrost_httpboot
110-
fi
111-
112-
- name: Generate clouds.yaml
113-
run: |
114-
cat << EOF > clouds.yaml
115-
${{ secrets.CLOUDS_YAML }}
116-
EOF
117-
118-
- name: Install OpenStack client
119206
run: |
120207
source venvs/kayobe/bin/activate &&
121-
pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }}
208+
source src/kayobe-config/kayobe-env --environment ci-builder &&
209+
kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output
210+
env:
211+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
122212

123213
- name: Build a Rocky Linux 9 overcloud host image
124214
id: build_rocky_9
@@ -134,6 +224,16 @@ jobs:
134224
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
135225
if: inputs.rocky9
136226

227+
- name: Show last error logs
228+
continue-on-error: true
229+
run: |
230+
source venvs/kayobe/bin/activate &&
231+
source src/kayobe-config/kayobe-env --environment ci-builder &&
232+
kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output
233+
env:
234+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
235+
if: steps.build_rocky_9.outcome == 'failure'
236+
137237
- name: Upload Rocky Linux 9 overcloud host image to Ark
138238
run: |
139239
source venvs/kayobe/bin/activate &&
@@ -147,19 +247,17 @@ jobs:
147247
env:
148248
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
149249
if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
150-
151-
- name: Upload Rocky Linux 9 overcloud host image to SMS
250+
251+
- name: Upload Rocky Linux 9 overcloud host image to Dev Cloud
152252
run: |
153253
source venvs/kayobe/bin/activate &&
154-
openstack image create \
155-
overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \
156-
--container-format bare \
157-
--disk-format qcow2 \
158-
--file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \
159-
--private \
160-
--os-cloud sms-lab-release \
161-
--progress
254+
source src/kayobe-config/kayobe-env --environment ci-builder &&
255+
kayobe playbook run \
256+
src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
257+
-e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \
258+
-e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }}
162259
env:
260+
CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
163261
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
164262
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
165263
if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success'
@@ -178,6 +276,16 @@ jobs:
178276
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
179277
if: inputs.ubuntu-jammy
180278

279+
- name: Show last error logs
280+
continue-on-error: true
281+
run: |
282+
source venvs/kayobe/bin/activate &&
283+
source src/kayobe-config/kayobe-env --environment ci-builder &&
284+
kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output
285+
env:
286+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
287+
if: steps.build_ubuntu_jammy.outcome == 'failure'
288+
181289
- name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark
182290
run: |
183291
source venvs/kayobe/bin/activate &&
@@ -192,50 +300,27 @@ jobs:
192300
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
193301
if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
194302

195-
- name: Upload Ubuntu Jammy 22.04 overcloud host image to SMS
303+
- name: Upload Ubuntu Jammy overcloud host image to Dev Cloud
196304
run: |
197305
source venvs/kayobe/bin/activate &&
198-
openstack image create \
199-
overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \
200-
--container-format bare \
201-
--disk-format qcow2 \
202-
--file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \
203-
--private \
204-
--os-cloud sms-lab-release \
205-
--progress
306+
source src/kayobe-config/kayobe-env --environment ci-builder &&
307+
kayobe playbook run \
308+
src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \
309+
-e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \
310+
-e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }}
206311
env:
312+
CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }}
207313
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
208314
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
209315
if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success'
210316

211-
- name: Upload updated images artifact
212-
uses: actions/upload-artifact@v4
213-
with:
214-
name: Updated images list
215-
path: /tmp/updated_images.txt
216-
retention-days: 7
217-
if: steps.build_rocky_9.outcome == 'success' ||
218-
steps.build_ubuntu_jammy.outcome == 'success'
219-
220-
- name: Upload Rocky 9 build logs if build failed
221-
uses: actions/upload-artifact@v4
222-
with:
223-
name: Rocky 9 build logs
224-
path: |
225-
/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout
226-
/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr
227-
retention-days: 7
228-
if: steps.build_rocky_9.outcome == 'failure'
229-
230-
- name: Upload Ubuntu Jammy 22.04 build logs if build failed
231-
uses: actions/upload-artifact@v4
232-
with:
233-
name: Ubuntu Jammy 22.04 build logs
234-
path: |
235-
/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout
236-
/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr
237-
retention-days: 7
238-
if: steps.build_ubuntu_jammy.outcome == 'failure'
317+
- name: Copy logs back
318+
continue-on-error: true
319+
run: |
320+
mkdir logs
321+
scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/
322+
scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true
323+
if: always()
239324

240325
- name: Fail if any overcloud host image builds failed
241326
run: |
@@ -244,7 +329,18 @@ jobs:
244329
if: steps.build_rocky_9.outcome == 'failure' ||
245330
steps.build_ubuntu_jammy.outcome == 'failure'
246331

247-
- name: Clean up build artifacts
248-
run: |
249-
sudo rm -rf /opt/kayobe/images/
332+
- name: Upload logs artifact
333+
uses: actions/upload-artifact@v4
334+
with:
335+
name: Build logs
336+
path: ./logs
337+
if: always()
338+
339+
- name: Destroy
340+
run: terraform destroy -auto-approve
341+
working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio
342+
env:
343+
OS_CLOUD: openstack
344+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
345+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
250346
if: always()

‎.github/workflows/overcloud-host-image-promote.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
id: openstack_release
3232
run: |
3333
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
34-
echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
34+
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
3535
working-directory: src/kayobe-config
3636

3737
- name: Clone StackHPC Kayobe repository

‎.github/workflows/overcloud-host-image-upload.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
id: openstack_release
4848
run: |
4949
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview)
50-
echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT
50+
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
5151
5252
- name: Clone StackHPC Kayobe repository
5353
uses: actions/checkout@v4

0 commit comments

Comments
 (0)