Skip to content

Commit c8a325a

Browse files
tannewttannewt
authored
Merge pull request #7567 from MicroDev1/ci
CI Enhancements & Refactoring
2 parents 0be5397 + 50c52fc commit c8a325a

File tree

18 files changed

+838
-450
lines changed

18 files changed

+838
-450
lines changed

‎.github/actions/deps/external/action.yml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: Fetch external deps
2+
3+
inputs:
4+
platform:
5+
required: false
6+
default: none
7+
type: choice
8+
options:
9+
- arm
10+
- aarch
11+
- esp
12+
- riscv
13+
- none
14+
15+
runs:
16+
using: composite
17+
steps:
18+
# aarch
19+
- name: Get aarch toolchain
20+
if: inputs.platform == 'aarch'
21+
run: |
22+
wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
23+
sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
24+
sudo apt-get install -y mtools
25+
shell: bash
26+
- name: Install mkfs.fat
27+
if: inputs.platform == 'aarch'
28+
run: |
29+
wget https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz
30+
tar -xaf dosfstools-4.2.tar.gz
31+
cd dosfstools-4.2
32+
./configure
33+
make -j 2
34+
cd src
35+
echo >> $GITHUB_PATH $(pwd)
36+
shell: bash
37+
38+
# arm
39+
- name: Get arm toolchain
40+
if: inputs.platform == 'aarch' || inputs.platform == 'arm'
41+
uses: carlosperate/arm-none-eabi-gcc-action@v1
42+
with:
43+
release: '10-2020-q4'
44+
45+
# esp
46+
- name: Get esp toolchain
47+
if: inputs.platform == 'esp'
48+
run: sudo apt-get install -y ninja-build
49+
shell: bash
50+
- name: Install IDF tools
51+
if: inputs.platform == 'esp'
52+
run: |
53+
echo "Installing ESP-IDF tools"
54+
$IDF_PATH/tools/idf_tools.py --non-interactive install required
55+
$IDF_PATH/tools/idf_tools.py --non-interactive install cmake
56+
echo "Installing Python environment and packages"
57+
$IDF_PATH/tools/idf_tools.py --non-interactive install-python-env
58+
rm -rf $IDF_TOOLS_PATH/dist
59+
shell: bash
60+
- name: Set environment
61+
if: inputs.platform == 'esp'
62+
run: |
63+
source $IDF_PATH/export.sh
64+
echo >> $GITHUB_ENV "IDF_PYTHON_ENV_PATH=$IDF_PYTHON_ENV_PATH"
65+
echo >> $GITHUB_PATH "$PATH"
66+
shell: bash
67+
68+
# riscv
69+
- name: Get riscv toolchain
70+
if: inputs.platform == 'riscv'
71+
run: |
72+
wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
73+
sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
74+
shell: bash
75+
76+
# common
77+
- name: Cache python dependencies
78+
if: inputs.platform != 'esp'
79+
uses: ./.github/actions/deps/python
80+
with:
81+
action: ${{ fromJSON('["restore", "cache"]')[github.job == 'scheduler'] }}
82+
- name: Install python dependencies
83+
run: pip install -r requirements-dev.txt
84+
shell: bash

‎.github/actions/deps/ports/espressif/action.yml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: Fetch espressif port deps
2+
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Set IDF env
7+
run: |
8+
echo >> $GITHUB_ENV "IDF_PATH=$GITHUB_WORKSPACE/ports/espressif/esp-idf"
9+
echo >> $GITHUB_ENV "IDF_TOOLS_PATH=$GITHUB_WORKSPACE/.idf_tools"
10+
shell: bash
11+
12+
- name: Get IDF commit
13+
id: idf-commit
14+
run: |
15+
COMMIT=$(git submodule status ports/espressif/esp-idf | grep -o -P '(?<=^-).*(?= )')
16+
echo "$COMMIT"
17+
echo "commit=$COMMIT" >> $GITHUB_OUTPUT
18+
shell: bash
19+
20+
- name: Cache IDF submodules
21+
uses: actions/cache@v3
22+
with:
23+
path: |
24+
.git/modules/ports/espressif/esp-idf
25+
ports/espressif/esp-idf
26+
key: submodules-idf-${{ steps.idf-commit.outputs.commit }}
27+
28+
- name: Cache IDF tools
29+
uses: actions/cache@v3
30+
with:
31+
path: ${{ env.IDF_TOOLS_PATH }}
32+
key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-idf-${{ steps.idf-commit.outputs.commit }}
33+
34+
- name: Initialize IDF submodules
35+
run: git submodule update --init --depth=1 --recursive $IDF_PATH
36+
shell: bash

‎.github/actions/deps/python/action.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Fetch python deps
2+
3+
inputs:
4+
action:
5+
description: The cache action to use
6+
required: false
7+
default: restore
8+
type: choice
9+
options:
10+
- cache
11+
- restore
12+
13+
runs:
14+
using: composite
15+
steps:
16+
- name: Cache python dependencies
17+
id: cache-python-deps
18+
if: inputs.action == 'cache'
19+
uses: actions/cache@v3
20+
with:
21+
path: .cp_tools
22+
key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
23+
24+
- name: Restore python dependencies
25+
id: restore-python-deps
26+
if: inputs.action == 'restore'
27+
uses: actions/cache/restore@v3
28+
with:
29+
path: .cp_tools
30+
key: ${{ runner.os }}-${{ env.pythonLocation }}-tools-cp-${{ hashFiles('requirements-dev.txt') }}
31+
32+
- name: Set up venv
33+
if: inputs.action == 'cache' && !steps.cache-python-deps.outputs.cache-hit
34+
run: python -m venv .cp_tools
35+
shell: bash
36+
37+
- name: Activate venv
38+
if: inputs.action == 'cache' || (inputs.action == 'restore' && steps.restore-python-deps.outputs.cache-hit)
39+
run: |
40+
source .cp_tools/bin/activate
41+
echo >> $GITHUB_PATH "$PATH"
42+
shell: bash

‎.github/actions/fetch_submodules/action.yml renamed to ‎.github/actions/deps/submodules/action.yml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
11
name: 'Fetch Submodules'
22

33
inputs:
4+
target:
5+
description: 'The target for ci_fetch_deps'
6+
required: false
7+
type: string
8+
49
submodules:
510
description: 'The submodules to cache'
611
required: false
712
default: '["extmod/ulab", "lib/", "tools/"]'
813
type: string
914

10-
cache:
15+
action:
1116
description: 'The cache action to use'
1217
required: false
1318
default: 'restore'
@@ -42,15 +47,15 @@ runs:
4247
shell: bash
4348

4449
- name: Cache submodules
45-
if: ${{ inputs.cache == 'cache' }}
50+
if: ${{ inputs.action == 'cache' }}
4651
uses: actions/cache@v3
4752
with:
4853
path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
4954
key: submodules-common-${{ hashFiles('submodule_status') }}
5055
enableCrossOsArchive: true
5156

5257
- name: Restore submodules
53-
if: ${{ inputs.cache == 'restore' }}
58+
if: ${{ inputs.action == 'restore' }}
5459
uses: actions/cache/restore@v3
5560
with:
5661
path: ".git/modules/\n${{ join(fromJSON(steps.create-submodule-status.outputs.submodules), '\n') }}"
@@ -63,7 +68,7 @@ runs:
6368

6469
- name: CircuitPython dependencies
6570
id: cp-deps
66-
run: python tools/ci_fetch_deps.py ${{ matrix.board || github.job }}
71+
run: python tools/ci_fetch_deps.py ${{ inputs.target || matrix.board || github.job }}
6772
shell: bash
6873

6974
- name: CircuitPython version

‎.github/actions/mpy_cross/action.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
name: Set up mpy-cross
2+
3+
inputs:
4+
download:
5+
required: false
6+
default: true
7+
type: boolean
8+
9+
runs:
10+
using: composite
11+
steps:
12+
- name: Download mpy-cross
13+
id: download-mpy-cross
14+
if: inputs.download == 'true'
15+
continue-on-error: true
16+
uses: actions/download-artifact@v3
17+
with:
18+
name: mpy-cross
19+
path: mpy-cross
20+
21+
- name: Make mpy-cross executable
22+
if: inputs.download == 'true' && steps.download-mpy-cross.outcome == 'success'
23+
run: sudo chmod +x mpy-cross/mpy-cross
24+
shell: bash
25+
26+
- name: Build mpy-cross
27+
if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
28+
run: make -C mpy-cross -j2
29+
shell: bash
30+
31+
- name: Upload mpy-cross
32+
if: inputs.download == 'false' || steps.download-mpy-cross.outcome == 'failure'
33+
continue-on-error: true
34+
uses: actions/upload-artifact@v3
35+
with:
36+
name: mpy-cross
37+
path: mpy-cross/mpy-cross

‎.github/actions/upload_aws/action.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Upload to AWS S3
2+
3+
inputs:
4+
source:
5+
required: true
6+
type: string
7+
8+
destination:
9+
required: false
10+
type: string
11+
12+
AWS_ACCESS_KEY_ID:
13+
required: true
14+
15+
AWS_SECRET_ACCESS_KEY:
16+
required: true
17+
18+
runs:
19+
using: composite
20+
steps:
21+
- name: Upload to S3
22+
if: >-
23+
(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') ||
24+
(github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
25+
run: >-
26+
[ -z "$AWS_ACCESS_KEY_ID" ] ||
27+
aws s3 cp ${{ inputs.source }} s3://adafruit-circuit-python/bin/${{ inputs.destination }} --recursive --no-progress --region us-east-1
28+
env:
29+
AWS_PAGER: ''
30+
AWS_ACCESS_KEY_ID: ${{ inputs.AWS_ACCESS_KEY_ID }}
31+
AWS_SECRET_ACCESS_KEY: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
32+
shell: bash

‎.github/workflows/build-boards.yml

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Build boards
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
platform:
7+
required: true
8+
type: string
9+
10+
boards:
11+
required: true
12+
type: string
13+
14+
cp-version:
15+
required: true
16+
type: string
17+
18+
jobs:
19+
build:
20+
runs-on: ubuntu-22.04
21+
env:
22+
CP_VERSION: ${{ inputs.cp-version }}
23+
strategy:
24+
fail-fast: false
25+
matrix:
26+
board: ${{ fromJSON(inputs.boards) }}
27+
steps:
28+
- name: Set up repository
29+
uses: actions/checkout@v3
30+
with:
31+
submodules: false
32+
fetch-depth: 1
33+
- name: Set up python
34+
uses: actions/setup-python@v4
35+
with:
36+
python-version: 3.x
37+
- name: Set up port
38+
if: inputs.platform == 'esp'
39+
uses: ./.github/actions/deps/ports/espressif
40+
- name: Set up submodules
41+
id: set-up-submodules
42+
uses: ./.github/actions/deps/submodules
43+
- name: Set up external
44+
uses: ./.github/actions/deps/external
45+
with:
46+
platform: ${{ inputs.platform }}
47+
- name: Set up mpy-cross
48+
if: steps.set-up-submodules.outputs.frozen == 'True'
49+
uses: ./.github/actions/mpy_cross
50+
51+
- name: Versions
52+
run: |
53+
gcc --version
54+
python3 --version
55+
cmake --version || true
56+
ninja --version || true
57+
aarch64-none-elf-gcc --version || true
58+
arm-none-eabi-gcc --version || true
59+
xtensa-esp32-elf-gcc --version || true
60+
riscv32-esp-elf-gcc --version || true
61+
riscv64-unknown-elf-gcc --version || true
62+
mkfs.fat --version || true
63+
64+
- name: Set up build failure matcher
65+
run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
66+
- name: Build board
67+
run: python3 -u build_release_files.py
68+
working-directory: tools
69+
env:
70+
BOARDS: ${{ matrix.board }}
71+
72+
- name: Upload artifact
73+
uses: actions/upload-artifact@v3
74+
with:
75+
name: ${{ matrix.board }}
76+
path: bin/${{ matrix.board }}
77+
- name: Upload to S3
78+
uses: ./.github/actions/upload_aws
79+
with:
80+
source: bin/
81+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
82+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

0 commit comments

Comments
 (0)