Skip to content

Commit 1dadeb2

Browse files
aelovikov-intelaelovikov-intel
committed
[CI][6/N] Another approach for experimental pre-commit
1 parent cee07d3 commit 1dadeb2

File tree

3 files changed

+351
-17
lines changed

3 files changed

+351
-17
lines changed

‎.github/workflows/sycl_exp_precommit.yml

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,36 @@ on:
99

1010
jobs:
1111
build:
12-
runs-on: [cuda]
13-
steps:
14-
- run: echo "Build successful"
12+
uses: ./.github/workflows/sycl_linux_build.yml
13+
with:
14+
build_ref: ${{ github.sha }}
15+
merge_ref: ''
16+
build_cache_root: "/__w/"
17+
build_artifact_suffix: "default"
18+
build_cache_suffix: "default"
19+
changes: '[]'
1520

16-
e2e:
21+
test:
1722
needs: [build]
18-
runs-on: aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}
19-
steps:
20-
- run: echo "E2E on AWS CUDA successful"
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
include:
27+
- name: ESIMD Emu
28+
runner: '["Linux", "x86-cpu"]'
29+
image: ghcr.io/intel/llvm/ubuntu2204_build:latest
30+
image_options: -u 1001
31+
target_devices: ext_intel_esimd_emulator:gpu
32+
uses: ./.github/workflows/sycl_linux_run_tests.yml
33+
with:
34+
name: ${{ matrix.name }}
35+
runner: ${{ matrix. runner }}
36+
image: ${{ matrix.image }}
37+
image_options: ${{ matrix.image_options }}
38+
target_devices: ${{ matrix.target_devices }}
39+
ref: ${{ github.sha }}
40+
merge_ref: ''
41+
42+
sycl_toolchain_artifact: sycl_linux_default
43+
sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }}
44+
sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }}

‎.github/workflows/sycl_linux_build.yml

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
name: Reusable SYCL Linux build workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
cc:
7+
type: string
8+
required: false
9+
default: "gcc"
10+
cxx:
11+
type: string
12+
required: false
13+
default: "g++"
14+
build_image:
15+
type: string
16+
required: false
17+
default: "ghcr.io/intel/llvm/ubuntu2204_build:latest"
18+
build_ref:
19+
type: string
20+
required: false
21+
build_cache_root:
22+
type: string
23+
required: true
24+
build_cache_suffix:
25+
type: string
26+
required: false
27+
default: "default"
28+
build_configure_extra_args:
29+
type: string
30+
required: false
31+
default: "--hip --cuda --enable-esimd-emulator"
32+
build_artifact_suffix:
33+
type: string
34+
required: true
35+
artifact_archive_name:
36+
type: string
37+
default: llvm_sycl.tar.zst
38+
changes:
39+
type: string
40+
description: 'Filter matches for the changed files in the PR'
41+
default: '[llvm, clang, sycl, llvm_spirv, xptifw, libclc, libdevice]'
42+
required: false
43+
merge_ref:
44+
description: |
45+
Commit-ish to merge post-checkout if non-empty. Must be reachable from
46+
the default_branch input paramter.
47+
type: string
48+
default: 'FETCH_HEAD'
49+
retention-days:
50+
description: 'Artifacts retention period'
51+
type: string
52+
default: 3
53+
54+
outputs:
55+
artifact_archive_name:
56+
value: ${{ jobs.build.outputs.artifact_archive_name }}
57+
artifact_decompress_command:
58+
value: ${{ jobs.build.outputs.artifact_decompress_command }}
59+
60+
workflow_dispatch:
61+
inputs:
62+
changes:
63+
description: 'Filter matches for the changed files in the PR'
64+
type: choice
65+
options:
66+
- "[]"
67+
- '[llvm, clang, sycl, llvm_spirv, xptifw, libclc, libdevice]'
68+
build_image:
69+
type: choice
70+
options:
71+
- "ghcr.io/intel/llvm/sycl_ubuntu2204_nightly:build"
72+
cc:
73+
type: choice
74+
options:
75+
- gcc
76+
cxx:
77+
type: choice
78+
options:
79+
- g++
80+
build_configure_extra_args:
81+
type: choice
82+
options:
83+
- "--hip --cuda --enable-esimd-emulator"
84+
# Cache properties need to match CC/CXX/CMake opts. Any additional choices
85+
# would need extra care.
86+
build_cache_root:
87+
type: choice
88+
options:
89+
- "/__w/"
90+
build_cache_suffix:
91+
type: choice
92+
options:
93+
- "default"
94+
95+
build_artifact_suffix:
96+
type: choice
97+
options:
98+
- "default"
99+
retention-days:
100+
type: choice
101+
options:
102+
- 3
103+
104+
jobs:
105+
build:
106+
name: Build + LIT
107+
runs-on: [Linux, build]
108+
container:
109+
image: ${{ inputs.build_image }}
110+
options: -u 1001:1001
111+
outputs:
112+
build_conclusion: ${{ steps.build.conclusion }}
113+
artifact_archive_name: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
114+
artifact_decompress_command: ${{ steps.artifact_info.outputs.DECOMPRESS }}
115+
env:
116+
CCACHE_DIR: ${{ inputs.build_cache_root }}/build_cache_${{ inputs.build_cache_suffix }}
117+
CCACHE_MAXSIZE: 8G
118+
steps:
119+
- name: Deduce artifact archive params
120+
# To reduce number of inputs parameters that is limited for manual triggers.
121+
id: artifact_info
122+
run: |
123+
NAME="${{inputs.artifact_archive_name}}"
124+
if [ -z "$NAME" ]; then
125+
NAME=llvm_sycl.tar.zst
126+
fi
127+
echo ARCHIVE_NAME="$NAME" >> $GITHUB_OUTPUT
128+
if [ "${NAME}" != "${NAME%.tar.gz}" ]; then
129+
echo COMPRESS="gzip" >> $GITHUB_OUTPUT
130+
echo DECOMPRESS="gunzip" >> $GITHUB_OUTPUT
131+
elif [ "${NAME}" != "${NAME%.tar.zst}" ]; then
132+
echo COMPRESS="zstd -9" >> $GITHUB_OUTPUT
133+
echo DECOMPRESS="zstd" >> $GITHUB_OUTPUT
134+
else
135+
echo "Unsupported extension"
136+
exit 1
137+
fi
138+
- uses: actions/checkout@v3
139+
with:
140+
sparse-checkout: |
141+
devops/actions
142+
# Cleanup will be run after all actions are completed.
143+
- name: Register cleanup after job is finished
144+
uses: ./devops/actions/cleanup
145+
- uses: ./devops/actions/cached_checkout
146+
with:
147+
path: src
148+
ref: ${{ inputs.build_ref || github.sha }}
149+
merge_ref: ${{ inputs.merge_ref }}
150+
cache_path: "/__w/repo_cache/"
151+
- name: Configure
152+
env:
153+
CC: ${{ inputs.cc }}
154+
CXX: ${{ inputs.cxx }}
155+
ARGS: ${{ inputs.build_configure_extra_args }}
156+
CUDA_LIB_PATH: "/usr/local/cuda/lib64/stubs"
157+
run: |
158+
mkdir -p $CCACHE_DIR
159+
mkdir -p $GITHUB_WORKSPACE/build
160+
cd $GITHUB_WORKSPACE/build
161+
python3 $GITHUB_WORKSPACE/src/buildbot/configure.py -w $GITHUB_WORKSPACE \
162+
-s $GITHUB_WORKSPACE/src -o $GITHUB_WORKSPACE/build -t Release \
163+
--ci-defaults $ARGS \
164+
--cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache \
165+
--cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
166+
--cmake-opt="-DLLVM_INSTALL_UTILS=ON" \
167+
--cmake-opt="-DSYCL_PI_TESTS=OFF"
168+
- name: Compile
169+
id: build
170+
run: cmake --build $GITHUB_WORKSPACE/build
171+
- name: check-llvm
172+
if: always() && !cancelled() && contains(inputs.changes, 'llvm')
173+
run: |
174+
cmake --build $GITHUB_WORKSPACE/build --target check-llvm
175+
- name: check-clang
176+
if: always() && !cancelled() && contains(inputs.changes, 'clang')
177+
run: |
178+
# Can we move this to Dockerfile? Hopefully, noop on Windows.
179+
export XDG_CACHE_HOME=$GITHUB_WORKSPACE/os_cache
180+
cmake --build $GITHUB_WORKSPACE/build --target check-clang
181+
- name: check-sycl
182+
if: always() && !cancelled() && contains(inputs.changes, 'sycl')
183+
run: |
184+
# TODO consider moving this to Dockerfile.
185+
export LD_LIBRARY_PATH=/usr/local/cuda/compat/:/usr/local/cuda/lib64:$LD_LIBRARY_PATH
186+
cmake --build $GITHUB_WORKSPACE/build --target check-sycl
187+
- name: check-llvm-spirv
188+
if: always() && !cancelled() && contains(inputs.changes, 'llvm_spirv')
189+
run: |
190+
cmake --build $GITHUB_WORKSPACE/build --target check-llvm-spirv
191+
- name: check-xptifw
192+
if: always() && !cancelled() && contains(inputs.changes, 'xptifw')
193+
run: |
194+
cmake --build $GITHUB_WORKSPACE/build --target check-xptifw
195+
- name: check-libclc
196+
if: always() && !cancelled() && contains(inputs.changes, 'libclc')
197+
run: |
198+
cmake --build $GITHUB_WORKSPACE/build --target check-libclc
199+
- name: check-libdevice
200+
if: always() && !cancelled() && contains(inputs.changes, 'libdevice')
201+
run: |
202+
cmake --build $GITHUB_WORKSPACE/build --target check-libdevice
203+
- name: Install
204+
if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }}
205+
# TODO replace utility installation with a single CMake target
206+
run: |
207+
cmake --build $GITHUB_WORKSPACE/build --target deploy-sycl-toolchain
208+
cmake --build $GITHUB_WORKSPACE/build --target utils/FileCheck/install
209+
cmake --build $GITHUB_WORKSPACE/build --target utils/count/install
210+
cmake --build $GITHUB_WORKSPACE/build --target utils/not/install
211+
cmake --build $GITHUB_WORKSPACE/build --target utils/lit/install
212+
cmake --build $GITHUB_WORKSPACE/build --target utils/llvm-lit/install
213+
cmake --build $GITHUB_WORKSPACE/build --target install-llvm-size
214+
cmake --build $GITHUB_WORKSPACE/build --target install-llvm-cov
215+
cmake --build $GITHUB_WORKSPACE/build --target install-llvm-profdata
216+
cmake --build $GITHUB_WORKSPACE/build --target install-compiler-rt
217+
- name: Additional Install for "--shared-libs" build
218+
if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' && contains(inputs.build_configure_extra_args, '--shared-libs') }}
219+
run: |
220+
cmake --build $GITHUB_WORKSPACE/build --target install-clang-libraries
221+
cmake --build $GITHUB_WORKSPACE/build --target install-llvm-libraries
222+
223+
- name: Install lint utilities
224+
# We install these into our nightly container that CI uses to run lint
225+
# checks.
226+
run: |
227+
cmake --build $GITHUB_WORKSPACE/build --target install-clang-format
228+
cmake --build $GITHUB_WORKSPACE/build --target install-clang-tidy
229+
230+
- name: Pack toolchain
231+
if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }}
232+
run: tar -I '${{ steps.artifact_info.outputs.COMPRESS }}' -cf ${{ steps.artifact_info.outputs.ARCHIVE_NAME }} -C $GITHUB_WORKSPACE/build/install .
233+
- name: Upload toolchain
234+
if: ${{ always() && !cancelled() && steps.build.conclusion == 'success' }}
235+
uses: actions/upload-artifact@v3
236+
with:
237+
name: sycl_linux_${{ inputs.build_artifact_suffix }}
238+
path: ${{ steps.artifact_info.outputs.ARCHIVE_NAME }}
239+
retention-days: ${{ inputs.retention-days }}

‎.github/workflows/sycl_precommit_aws.yml

Lines changed: 81 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,99 @@ on:
44
workflow_run:
55
workflows: [SYCL Experimental Pre-Commit]
66
types:
7-
- in_progress
87
- completed
98

109
jobs:
11-
aws:
10+
create-check:
11+
runs-on: [Linux, build]
12+
outputs:
13+
id: ${{ steps.create_check.outputs.result }}
14+
steps:
15+
- uses: actions/github-script@v6
16+
id: create_check
17+
with:
18+
script: |
19+
const sha = context.payload.workflow_run.head_sha
20+
const run_id = '${{ github.run_id }}'
21+
const this_run_url = 'https://github.com/' + context.repo.owner + '/' + context.repo.repo + '/actions/runs/' + run_id
22+
23+
const result = await github.request('POST /repos/{owner}/{repo}/check-runs', {
24+
owner: context.repo.owner,
25+
repo: context.repo.repo,
26+
name: 'e2e-aws-cuda',
27+
head_sha: sha,
28+
details_url: this_run_url,
29+
status: 'in_progress',
30+
output: {
31+
title: 'SYCL E2E on AWS CUDA',
32+
summary: 'URL: ' + this_run_url
33+
}
34+
})
35+
console.log(result)
36+
return result.data.id
37+
38+
aws-start:
39+
runs-on: ubuntu-20.04
40+
environment: aws
41+
steps:
42+
- uses: actions/checkout@v3
43+
with:
44+
sparse-checkout: devops/actions/aws-ec2
45+
- run: npm install ./devops/actions/aws-ec2
46+
- uses: ./devops/actions/aws-ec2
47+
with:
48+
mode: start
49+
runs-on-list: '[{"runs-on":"aws_cuda-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}","aws-ami":"ami-01cb0573cb039ab24","aws-type":["g5.2xlarge","g5.4xlarge"],"aws-disk":"/dev/sda1:64","aws-spot":"false"}]'
50+
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
51+
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
52+
AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
53+
54+
e2e-cuda:
55+
needs: [aws-start]
56+
uses: ./.github/workflows/sycl_linux_run_tests.yml
57+
with:
58+
name: CUDA E2E
59+
runner: 'aws_cuda-${{ github.run_id }}-${{ github.run_attempt }}'
60+
image: ghcr.io/intel/llvm/ubuntu2204_build:latest
61+
image_options: -u 1001 --gpus all
62+
target_devices: ext_oneapi_cuda:gpu
63+
ref: ${{ github.sha }}
64+
merge_ref: ${{ github.event.pull_request.base.sha }}
65+
env: '{"LIT_FILTER":"Basic/accessor"}'
66+
67+
sycl_toolchain_artifact: sycl_linux_default
68+
sycl_toolchain_archive: llvm_sycl.tar.zst
69+
sycl_toolchain_decompress_command: zstd
70+
71+
update-check:
72+
needs: [create-check, e2e-cuda]
73+
if: always()
74+
runs-on: [Linux, build]
75+
steps:
76+
- uses: actions/github-script@v6
77+
with:
78+
script: |
79+
const result = await github.request('PATCH /repos/{owner}/{repo}/check-runs/{check_run_id}', {
80+
owner: context.repo.owner,
81+
repo: context.repo.repo,
82+
check_run_id: '${{ needs.create-check.outputs.id }}',
83+
status: 'completed',
84+
conclusion: '${{ needs.e2e-cuda.conclusion }}'
85+
})
86+
console.log(result)
87+
88+
aws-stop:
89+
needs: [aws-start, e2e-cuda]
1290
runs-on: ubuntu-20.04
1391
environment: aws
1492
steps:
1593
- uses: actions/checkout@v3
1694
with:
1795
sparse-checkout: devops/actions/aws-ec2
1896
- run: npm install ./devops/actions/aws-ec2
19-
- id: mode
20-
run: |
21-
if [ "${{ github.event.action }}" == "completed" ]; then
22-
echo 'MODE=stop' >> $GITHUB_OUTPUT
23-
else
24-
echo 'MODE=start' >> $GITHUB_OUTPUT
25-
fi
2697
- uses: ./devops/actions/aws-ec2
2798
with:
28-
mode: ${{ steps.mode.outputs.MODE }}
99+
mode: stop
29100
runs-on-list: '[{"runs-on":"aws_cuda-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}","aws-ami":"ami-01cb0573cb039ab24","aws-type":["g5.2xlarge","g5.4xlarge"],"aws-disk":"/dev/sda1:64","aws-spot":"false"}]'
30101
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
31102
AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}

0 commit comments

Comments
 (0)