Skip to content

Commit d0c8781

Browse files
wtripp180901wtripp180901
committed
merge conflicts
2 parents d712f69 + 04db97d commit d0c8781

22 files changed

+559
-149
lines changed

‎.github/workflows/nightly-cleanup.yml

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: Cleanup CI clusters
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
ci_cloud:
6+
description: 'Select the CI_CLOUD'
7+
required: true
8+
type: choice
9+
options:
10+
- LEAFCLOUD
11+
- SMS
12+
- ARCUS
13+
schedule:
14+
- cron: '0 20 * * *' # Run at 8PM - image sync runs at midnight
15+
16+
jobs:
17+
ci_cleanup:
18+
name: ci-cleanup
19+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
cloud:
24+
- LEAFCLOUD
25+
- SMS
26+
- ARCUS
27+
runs-on: ubuntu-22.04
28+
env:
29+
OS_CLOUD: openstack
30+
CI_CLOUD: ${{ matrix.cloud }}
31+
steps:
32+
- uses: actions/checkout@v2
33+
34+
- name: Record which cloud CI is running on
35+
run: |
36+
echo CI_CLOUD: ${{ env.CI_CLOUD }}
37+
38+
- name: Setup environment
39+
run: |
40+
python3 -m venv venv
41+
. venv/bin/activate
42+
pip install -U pip
43+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
44+
shell: bash
45+
46+
- name: Write clouds.yaml
47+
run: |
48+
mkdir -p ~/.config/openstack/
49+
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
50+
shell: bash
51+
52+
- name: Find CI clusters
53+
run: |
54+
. venv/bin/activate
55+
CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+' | sort | uniq)
56+
echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
57+
shell: bash
58+
59+
- name: Delete clusters if control node not tagged with keep
60+
run: |
61+
. venv/bin/activate
62+
for cluster_prefix in ${CI_CLUSTERS}
63+
do
64+
TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
65+
if [[ $TAGS =~ "keep" ]]; then
66+
echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
67+
else
68+
yes | ./dev/delete-cluster.py ${cluster_prefix}
69+
fi
70+
done
71+
shell: bash

‎.github/workflows/nightlybuild.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ jobs:
8989
-on-error=${{ vars.PACKER_ON_ERROR }} \
9090
-only=${{ matrix.build }} \
9191
-var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
92-
-var "source_image_name=${{ env.SOURCE_IMAGE }}"
92+
-var "source_image_name=${{ env.SOURCE_IMAGE }}" \
9393
openstack.pkr.hcl
9494
9595
env:

‎.github/workflows/s3-image-sync.yml

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
name: Upload CI-tested images to Arcus S3 and sync clouds
2+
on:
3+
workflow_dispatch:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
9+
env:
10+
S3_BUCKET: openhpc-images-prerelease
11+
IMAGE_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
12+
13+
jobs:
14+
s3_cleanup:
15+
runs-on: ubuntu-22.04
16+
concurrency: ${{ github.workflow }}-${{ github.ref }}
17+
strategy:
18+
fail-fast: false
19+
steps:
20+
- uses: actions/checkout@v2
21+
22+
- name: Write s3cmd configuration
23+
run: |
24+
echo "${{ secrets['ARCUS_S3_CFG'] }}" > ~/.s3cfg
25+
shell: bash
26+
27+
- name: Install s3cmd
28+
run: |
29+
sudo apt-get --yes install s3cmd
30+
31+
- name: Cleanup S3 bucket
32+
run: |
33+
s3cmd rm s3://${{ env.S3_BUCKET }} --recursive --force
34+
35+
image_upload:
36+
runs-on: ubuntu-22.04
37+
needs: s3_cleanup
38+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }}
39+
strategy:
40+
fail-fast: false
41+
matrix:
42+
build:
43+
- RL8
44+
- RL9
45+
- RL9-cuda
46+
env:
47+
ANSIBLE_FORCE_COLOR: True
48+
OS_CLOUD: openstack
49+
CI_CLOUD: ${{ vars.CI_CLOUD }}
50+
outputs:
51+
ci_cloud: ${{ steps.ci.outputs.CI_CLOUD }}
52+
steps:
53+
- uses: actions/checkout@v2
54+
55+
- name: Record which cloud CI is running on
56+
id: ci
57+
run: |
58+
echo "CI_CLOUD=${{ env.CI_CLOUD }}" >> "$GITHUB_OUTPUT"
59+
60+
- name: Setup environment
61+
run: |
62+
python3 -m venv venv
63+
. venv/bin/activate
64+
pip install -U pip
65+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
66+
shell: bash
67+
68+
- name: Write clouds.yaml
69+
run: |
70+
mkdir -p ~/.config/openstack/
71+
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
72+
shell: bash
73+
74+
- name: Write s3cmd configuration
75+
run: |
76+
echo "${{ secrets['ARCUS_S3_CFG'] }}" > ~/.s3cfg
77+
shell: bash
78+
79+
- name: Install s3cmd
80+
run: |
81+
sudo apt-get --yes install s3cmd
82+
83+
- name: Retrieve image name
84+
run: |
85+
TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
86+
echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
87+
shell: bash
88+
89+
- name: Download image to runner
90+
run: |
91+
. venv/bin/activate
92+
openstack image save --file ${{ env.TARGET_IMAGE }} ${{ env.TARGET_IMAGE }}
93+
shell: bash
94+
95+
- name: Upload Image to S3
96+
run: |
97+
echo "Uploading Image: ${{ env.TARGET_IMAGE }} to S3..."
98+
s3cmd --multipart-chunk-size-mb=150 put ${{ env.TARGET_IMAGE }} s3://${{ env.S3_BUCKET }}
99+
shell: bash
100+
101+
image_sync:
102+
needs: image_upload
103+
runs-on: ubuntu-22.04
104+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}-${{ matrix.build }}
105+
strategy:
106+
fail-fast: false
107+
matrix:
108+
cloud:
109+
- LEAFCLOUD
110+
- SMS
111+
- ARCUS
112+
build:
113+
- RL8
114+
- RL9
115+
- RL9-cuda
116+
exclude:
117+
- cloud: ${{ needs.image_upload.outputs.ci_cloud }}
118+
119+
env:
120+
ANSIBLE_FORCE_COLOR: True
121+
OS_CLOUD: openstack
122+
CI_CLOUD: ${{ matrix.cloud }}
123+
steps:
124+
- uses: actions/checkout@v2
125+
126+
- name: Record which cloud CI is running on
127+
run: |
128+
echo CI_CLOUD: ${{ env.CI_CLOUD }}
129+
130+
- name: Setup environment
131+
run: |
132+
python3 -m venv venv
133+
. venv/bin/activate
134+
pip install -U pip
135+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
136+
shell: bash
137+
138+
- name: Write clouds.yaml
139+
run: |
140+
mkdir -p ~/.config/openstack/
141+
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
142+
shell: bash
143+
144+
- name: Retrieve image name
145+
run: |
146+
TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
147+
echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
148+
149+
- name: Download latest image if missing
150+
run: |
151+
. venv/bin/activate
152+
bash .github/bin/get-s3-image.sh ${{ env.TARGET_IMAGE }} ${{ env.S3_BUCKET }}
153+
154+
- name: Cleanup OpenStack Image (on error or cancellation)
155+
if: cancelled() || failure()
156+
run: |
157+
. venv/bin/activate
158+
image_hanging=$(openstack image list --name ${{ env.TARGET_IMAGE }} -f value -c ID -c Status | grep -v ' active$' | awk '{print $1}')
159+
if [ -n "$image_hanging" ]; then
160+
echo "Cleaning up OpenStack image with ID: $image_hanging"
161+
openstack image delete $image_hanging
162+
else
163+
echo "No image ID found, skipping cleanup."
164+
fi
165+
shell: bash

‎.github/workflows/stackhpc.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ on:
1212
- '!docs/**'
1313
- '!README.md'
1414
- '!.gitignore'
15+
- '!.github/workflows/'
16+
- '.github/workflows/stackhpc'
1517
pull_request:
1618
paths:
1719
- '**'
@@ -20,6 +22,8 @@ on:
2022
- '!docs/**'
2123
- '!README.md'
2224
- '!.gitignore'
25+
- '!.github/workflows/'
26+
- '.github/workflows/stackhpc'
2327
jobs:
2428
openstack:
2529
name: openstack-ci

‎.github/workflows/upload-release-image.yml.sample

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
bash .github/bin/get-s3-image.sh ${{ inputs.image_name }} ${{ inputs.bucket_name }}
5454

5555
- name: Cleanup OpenStack Image (on error or cancellation)
56-
if: cancelled()
56+
if: cancelled() || failure()
5757
run: |
5858
. venv/bin/activate
5959
image_hanging=$(openstack image list --name ${{ inputs.image_name }} -f value -c ID -c Status | grep -v ' active$' | awk '{print $1}')

0 commit comments

Comments
 (0)