Skip to content

Commit 072cbd9

Browse files
sandeepd-nvsandeepd-nv
committed
Copied all files from legate-gh-ci into repo.
1 parent ad574fb commit 072cbd9

File tree

20 files changed

+1516
-0
lines changed

20 files changed

+1516
-0
lines changed

‎.github/actions/build/action.yml.j2

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
name: build
2+
3+
description: Build specified project
4+
5+
inputs:
6+
build-type:
7+
required: true
8+
type: string
9+
description: One of ci / release
10+
target-device:
11+
required: true
12+
type: string
13+
platform:
14+
required: true
15+
type: string
16+
use-container:
17+
required: true
18+
type: boolean
19+
docker-image:
20+
type: string
21+
required: true
22+
inter-repos-ro-access-token:
23+
type: string
24+
required: true
25+
upload-enabled:
26+
required: true
27+
type: boolean
28+
29+
runs:
30+
using: composite
31+
steps:
32+
33+
<% for package_id, package_info in packages.items() %>
34+
- name: Download <<package_info.repo>> (artifacts)
35+
uses: ./legate-gh-ci/.github/actions/download-artifacts
36+
with:
37+
artifact-repo: "<<package_info.repo>>"
38+
artifact-name: "<<package_info.artifact_name | replace_placeholder('repo', package_info.repo) | replace_placeholder('git_tag', package_info.git_tag) >>"
39+
target-device: "${{ inputs.target-device }}"
40+
git_sha: "<<package_info.git_tag>>"
41+
inter-repos-ro-access-token: ${{ inputs.inter-repos-ro-access-token }}
42+
platform: ${{ inputs.platform }}
43+
dest-dir: ${{ env.ARTIFACTS_DIR }}
44+
dependencies-workflow: <<package_info.artifact_workflow>>
45+
<% endfor %>
46+
47+
<% if packages %>
48+
49+
- name: Display structure of downloaded artifacts
50+
shell: bash --noprofile --norc -xeuo pipefail {0}
51+
run: |
52+
pwd
53+
ls -lahR ${{ env.ARTIFACTS_DIR }}
54+
<% endif %>
55+
56+
- if: github.repository_owner == 'nv-legate'
57+
name: Get AWS credentials for sccache bucket
58+
uses: aws-actions/configure-aws-credentials@v4
59+
with:
60+
aws-region: us-east-2
61+
role-duration-seconds: 28800 # 8 hours
62+
role-to-assume: arn:aws:iam::279114543810:role/gha-oidc-nv-legate
63+
64+
- if: ${{ inputs.use-container }}
65+
name: Build (in container)
66+
shell: bash --noprofile --norc -xeuo pipefail {0}
67+
run: |
68+
69+
docker run \
70+
-e AWS_REGION \
71+
-e AWS_SESSION_TOKEN \
72+
-e AWS_ACCESS_KEY_ID \
73+
-e AWS_SECRET_ACCESS_KEY \
74+
-e GITHUB_TOKEN \
75+
-e ARTIFACTS_DIR="$ARTIFACTS_DIR" \
76+
-e UPLOAD_ENABLED="$UPLOAD_ENABLED" \
77+
-e USE_CUDA="$USE_CUDA" \
78+
-e REPO_DIR="$REPO_DIR" \
79+
-e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \
80+
-e LEGATE_GH_CI_DIR="${{ env.LEGATE_GH_CI_DIR }}" \
81+
-e PYTHON_VERSION="$PYTHON_VERSION" \
82+
-v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \
83+
-v "${{ env.LEGATE_GH_CI_DIR }}:${{ env.LEGATE_GH_CI_DIR }}" \
84+
-v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \
85+
--rm "${{ inputs.docker-image }}" \
86+
/bin/bash -c "${{ env.LEGATE_GH_CI_DIR }}/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}"
87+
88+
- if: ${{ !inputs.use-container }}
89+
name: Build (without container)
90+
shell: bash --noprofile --norc -xeuo pipefail {0}
91+
run: |
92+
"${{ env.LEGATE_GH_CI_DIR }}/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}"
93+
94+
- name: Display structure of the artifacts folder (post build)
95+
shell: bash --noprofile --norc -xeuo pipefail {0}
96+
run: |
97+
sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }}
98+
ls -lahR ${{ env.ARTIFACTS_DIR }}
99+
100+
- name: Upload build artifacts
101+
uses: actions/upload-artifact@v4
102+
with:
103+
name: ${{ env.ARTIFACT_NAME }}
104+
path: ${{ env.ARTIFACTS_DIR }}

‎.github/actions/download-artifacts/action.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
name: download-artifacts
2+
3+
description: Download dependencies (artifacts)
4+
5+
inputs:
6+
artifact-repo:
7+
type: string
8+
require: true
9+
artifact-name:
10+
type: string
11+
require: true
12+
target-device:
13+
type: string
14+
required: true
15+
git_sha:
16+
type: string
17+
required: true
18+
inter-repos-ro-access-token:
19+
type: string
20+
required: true
21+
platform:
22+
type: string
23+
required: true
24+
dest-dir:
25+
type: string
26+
required: true
27+
dependencies-workflow:
28+
required: true
29+
type: string
30+
31+
runs:
32+
using: composite
33+
steps:
34+
35+
- id: cache
36+
name: Cache conda artifacts
37+
uses: actions/cache@v4
38+
with:
39+
key: "nv-legate/${{ inputs.artifact-repo }}@${{ inputs.platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}"
40+
path: ${{ inputs.dest-dir }}
41+
42+
- if: steps.cache.outputs.cache-hit != 'true'
43+
name: Download ${{ inputs.artifact-repo }} artifacts
44+
uses: dawidd6/action-download-artifact@v3
45+
with:
46+
github_token: ${{ inputs.inter-repos-ro-access-token }}
47+
path: ${{ inputs.dest-dir }}
48+
repo: nv-legate/${{ inputs.artifact-repo }}
49+
check_artifacts: true
50+
commit: ${{ inputs.git_sha }}
51+
workflow_conclusion: ""
52+
workflow: ${{ inputs.dependencies-workflow }}
53+
name: ${{ inputs.artifact-name }}
54+
skip_unpack: true
55+
if_no_artifact_found: fail
56+
allow_forks: false
57+
58+
- if: steps.cache.outputs.cache-hit != 'true'
59+
name: Unpack artifact
60+
shell: bash --noprofile --norc -xeuo pipefail {0}
61+
run: |
62+
cd ${{ inputs.dest-dir }}
63+
unzip *.zip

‎.github/actions/setup/action.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Common setup
2+
3+
inputs:
4+
client-repo:
5+
required: true
6+
type: string
7+
build-type:
8+
required: true
9+
type: string
10+
target-device:
11+
required: true
12+
type: string
13+
platform:
14+
required: true
15+
type: string
16+
legate-gh-ci-tag:
17+
required: true
18+
type: string
19+
build-mode:
20+
required: true
21+
type: string
22+
upload-enabled:
23+
required: true
24+
type: boolean
25+
python-version:
26+
required: false
27+
type: string
28+
29+
runs:
30+
using: composite
31+
steps:
32+
- name: Set LEGATE_GH_CI_DIR dir
33+
shell: bash --noprofile --norc -xeuo pipefail {0}
34+
run: |
35+
cd legate-gh-ci
36+
echo "LEGATE_GH_CI_DIR=$(pwd)" >> $GITHUB_ENV
37+
env
38+
39+
- name: Checkout ${{ inputs.client-repo }}
40+
uses: actions/checkout@v4
41+
with:
42+
fetch-depth: 0
43+
path: ${{ inputs.client-repo }}
44+
45+
- name: Set REPO_DIR and Dump environment
46+
shell: bash --noprofile --norc -xeuo pipefail {0}
47+
run: |
48+
cd ${{ inputs.client-repo }}
49+
echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV
50+
env
51+
52+
- name: Set environment variables
53+
shell: bash --noprofile --norc -xeuo pipefail {0}
54+
run: |
55+
56+
WITH_TESTS_STR=''
57+
if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then
58+
WITH_TESTS_STR='-with_tests'
59+
fi
60+
61+
TARGET_PLATFORM='linux-64'
62+
if [[ "${{ inputs.platform }}" == "linux-aarch64" ]]; then
63+
TARGET_PLATFORM='linux-aarch64'
64+
fi
65+
66+
BUILD_MODE="${{ inputs.build-mode }}"
67+
BUILD_MODE_STR=""
68+
[ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}"
69+
70+
if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then
71+
# We upload release versions in the default folder.
72+
PKG_DIR="${TARGET_PLATFORM}"
73+
else
74+
PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
75+
fi
76+
77+
echo "ARTIFACT_NAME=${{ inputs.platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
78+
echo "ARTIFACTS_DIR=$(realpath "$(pwd)/../artifacts")" >> $GITHUB_ENV
79+
echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV
80+
echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
81+
echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV
82+
echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
83+
echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
84+
echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
85+
echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV

‎.github/actions/test/action.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: test
2+
3+
description: Run tests in specified project
4+
5+
inputs:
6+
test-options:
7+
required: true
8+
type: string
9+
has-gpu:
10+
required: true
11+
type: boolean
12+
description: "The runner has GPU(s)."
13+
14+
runs:
15+
using: composite
16+
steps:
17+
- if: ${{ inputs.has-gpu == true }}
18+
name: Run nvidia-smi to make sure GPU is working
19+
shell: bash --noprofile --norc -xeuo pipefail {0}
20+
run: nvidia-smi
21+
22+
- name: Download build artifacts
23+
uses: actions/download-artifact@v4
24+
with:
25+
name: ${{ env.ARTIFACT_NAME }}
26+
path: ${{ env.ARTIFACTS_DIR }}
27+
28+
- name: Display structure of downloaded artifacts
29+
shell: bash --noprofile --norc -xeuo pipefail {0}
30+
run: |
31+
pwd
32+
ls -lahR $ARTIFACTS_DIR
33+
34+
- name: Run test / analysis
35+
shell: bash --noprofile --norc -xeuo pipefail {0}
36+
run: |
37+
"${{ env.LEGATE_GH_CI_DIR }}/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/test" ${{ inputs.test-options }}

‎.github/actions/upload/action.yml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: upload action
2+
3+
description: Run upload actions in specified project
4+
5+
inputs:
6+
urmToken:
7+
required: true
8+
type: string
9+
upload-action:
10+
required: true
11+
type: string
12+
pkgSubString:
13+
required: true
14+
type: string
15+
16+
17+
runs:
18+
using: composite
19+
steps:
20+
- name: Download build artifacts
21+
uses: actions/download-artifact@v4
22+
with:
23+
name: ${{ env.ARTIFACT_NAME }}
24+
path: ${{ env.ARTIFACTS_DIR }}
25+
26+
- name: Display structure of downloaded artifacts
27+
shell: bash --noprofile --norc -xeuo pipefail {0}
28+
run: |
29+
pwd
30+
ls -lahR $ARTIFACTS_DIR
31+
32+
- name: Install pre-requisites
33+
shell: bash --noprofile --norc -xeuo pipefail {0}
34+
run: |
35+
echo "Install pre-requisites"
36+
apt-get update
37+
apt-get install -y curl jq
38+
39+
- name: Call action, Upload Package
40+
if: inputs.upload-action == 'upload-package'
41+
shell: bash --noprofile --norc -xeuo pipefail {0}
42+
run: |
43+
echo "Call action, Upload Package"
44+
for f in $(find $ARTIFACTS_DIR/$OUTPUT_FOLDER/$TARGET_PLATFORM/. -name "${{ inputs.pkgSubString }}"'*.tar.bz2'); do
45+
fname=$(basename $f)
46+
packageFound=$(curl -usvc-legate-github:${{ inputs.urmToken }} -X POST -H "content-type: text/plain" "$ARTIFACT_SERVER/api/search/aql" -d 'items.find({"repo":"'"$ARTIFACT_REPOS"'","path": {"$match":"'"$PKG_DIR"'"}, "name":{"$eq":"'"$fname"'"}})' | jq -r .results[].name)
47+
if [ -z $packageFound ]; then
48+
echo "Uploading the package: $fname"
49+
curl -usvc-legate-github:${{ inputs.urmToken }} -T $f "$ARTIFACT_SERVER/$ARTIFACT_REPOS/$PKG_DIR/$fname;buildDate=$BUILD_DATE;automatedTestingPassed=0;sha=${{ github.sha }}"
50+
else
51+
echo "The package: $fname already exists on the server..skipping upload"
52+
fi
53+
done
54+
55+
- name: Call action, Update Test status
56+
if: inputs.upload-action == 'update-test-status'
57+
shell: bash --noprofile --norc -xeuo pipefail {0}
58+
run: |
59+
echo "Call action, Update Test status"
60+
for f in $(find $ARTIFACTS_DIR/$OUTPUT_FOLDER/$TARGET_PLATFORM/. -name "${{ inputs.pkgSubString }}"'*.tar.bz2'); do
61+
fname=$(basename $f)
62+
packageFound=$(curl -usvc-legate-github:${{ inputs.urmToken }} -X POST -H "content-type: text/plain" "$ARTIFACT_SERVER/api/search/aql" -d 'items.find({"repo":"'"$ARTIFACT_REPOS"'","path": {"$match":"'"$PKG_DIR"'"}, "name":{"$eq":"'"$fname"'"}})' | jq -r .results[].name)
63+
if [ -z $packageFound ]; then
64+
echo "Cannot update. Package:$fname not found on server"
65+
else
66+
testStatus=$(curl -usvc-legate-github:${{ inputs.urmToken }} -X GET -H "content-type: text/plain" "$ARTIFACT_SERVER/api/storage/$ARTIFACT_REPOS/$PKG_DIR/$fname?properties" | grep automatedTestingPassed | cut -d '"' -f 4)
67+
if [[ "$testStatus" == "0" ]]; then
68+
echo "Updating Test status of Package: $fname"
69+
curl -H "Content-Type: application/json" -X PATCH -usvc-legate-github:${{ inputs.urmToken }} -d '{"props":{"automatedTestingPassed": "1"}}' "$ARTIFACT_SERVER/api/metadata/$ARTIFACT_REPOS/$PKG_DIR/$fname"
70+
else
71+
echo "status already set..Skipping Update"
72+
fi
73+
fi
74+
done

0 commit comments

Comments
 (0)