Skip to content

Commit 4522942

Browse files
dbortdbort
committed
Fork build_wheels_*.yml files from test-infra
This will let us add some options we need to change the wheel-building behavior. Those options have not yet been added to the files, though; this is just a fork with the patches described in README.md. These changes should eventually be pushed upstream, and removed from here.
1 parent ae1a658 commit 4522942

File tree

5 files changed

+507
-2
lines changed

5 files changed

+507
-2
lines changed

‎.github/workflows/README.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Files in this directory whose names have the prefix `test_infra_` are forked
2+
from the pytorch/test-infra repo. Whenever possible, the modifications here
3+
should be pushed to that upstream repo, and these forks should be deleted.
4+
5+
Download and patch files from test-infra so that they can work from this repo:
6+
```
7+
for f in macos linux; do
8+
curl https://raw.githubusercontent.com/pytorch/test-infra/2b257b732b6/.github/workflows/build_wheels_$f.yml \
9+
| sed -e 's,uses: ./\(test-infra/\)\?.github/\(.*\),uses: pytorch/test-infra/.github/\2@release/2.3,' \
10+
> test_infra_build_wheels_$f.yml
11+
done
12+
```
13+
14+
Patch the local build-wheel files to point to the forks:
15+
```
16+
for f in build-wheels-*.yml; do
17+
sed -i -e 's,uses: pytorch/test-infra/.*/\(build_wheels_[^.]*\.yml\).*,uses: ./.github/workflows/test_infra_\1,' $f
18+
done
19+
```

‎.github/workflows/build-wheels-linux.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
smoke-test-script: build/packaging/smoke_test.py
4444
package-name: executorch
4545
name: ${{ matrix.repository }}
46-
uses: pytorch/test-infra/.github/workflows/build_wheels_linux.yml@release/2.3
46+
uses: ./.github/workflows/test_infra_build_wheels_linux.yml
4747
with:
4848
repository: ${{ matrix.repository }}
4949
ref: ""

‎.github/workflows/build-wheels-m1.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
smoke-test-script: build/packaging/smoke_test.py
4444
package-name: executorch
4545
name: ${{ matrix.repository }}
46-
uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@release/2.3
46+
uses: ./.github/workflows/test_infra_build_wheels_macos.yml
4747
with:
4848
repository: ${{ matrix.repository }}
4949
ref: ""

‎.github/workflows/test_infra_build_wheels_linux.yml

Lines changed: 254 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
name: Build Linux Wheels
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
repository:
7+
description: 'Repository to checkout, defaults to ""'
8+
default: ""
9+
type: string
10+
ref:
11+
description: 'Reference to checkout, defaults to "nightly"'
12+
default: "nightly"
13+
type: string
14+
test-infra-repository:
15+
description: "Test infra repository to use"
16+
default: "pytorch/test-infra"
17+
type: string
18+
test-infra-ref:
19+
description: "Test infra reference to use"
20+
default: ""
21+
type: string
22+
build-matrix:
23+
description: "Build matrix to utilize"
24+
default: ""
25+
type: string
26+
pre-script:
27+
description: "Pre script to run prior to build"
28+
default: ""
29+
type: string
30+
post-script:
31+
description: "Post script to run prior to build"
32+
default: ""
33+
type: string
34+
smoke-test-script:
35+
description: "Script for Smoke Test for a specific domain"
36+
default: ""
37+
type: string
38+
env-var-script:
39+
description: "Script that sets Domain-Specific Environment Variables"
40+
default: ""
41+
type: string
42+
package-name:
43+
description: "Name of the actual python package that is imported"
44+
default: ""
45+
type: string
46+
trigger-event:
47+
description: "Trigger Event in caller that determines whether or not to upload"
48+
default: ""
49+
type: string
50+
cache-path:
51+
description: "The path(s) on the runner to cache or restore. The path is relative to repository."
52+
default: ""
53+
type: string
54+
cache-key:
55+
description: "The key created when saving a cache and the key used to search for a cache."
56+
default: ""
57+
type: string
58+
architecture:
59+
description: Architecture to build for x86_64 for default Linux, or aarch64 for Linux aarch64 builds
60+
required: false
61+
type: string
62+
default: x86_64
63+
submodules:
64+
description: Works as stated in actions/checkout, but the default value is recursive
65+
required: false
66+
type: string
67+
default: recursive
68+
setup-miniconda:
69+
description: Set to true if setup-miniconda is needed
70+
required: false
71+
type: boolean
72+
default: true
73+
74+
permissions:
75+
id-token: write
76+
contents: read
77+
78+
jobs:
79+
build:
80+
strategy:
81+
fail-fast: false
82+
matrix: ${{ fromJSON(inputs.build-matrix) }}
83+
env:
84+
PYTHON_VERSION: ${{ matrix.python_version }}
85+
PACKAGE_TYPE: wheel
86+
REPOSITORY: ${{ inputs.repository }}
87+
REF: ${{ inputs.ref }}
88+
CU_VERSION: ${{ matrix.desired_cuda }}
89+
UPLOAD_TO_BASE_BUCKET: ${{ matrix.upload_to_base_bucket }}
90+
ARCH: ${{ inputs.architecture }}
91+
name: ${{ matrix.build_name }}
92+
runs-on: ${{ matrix.validation_runner }}
93+
container:
94+
image: ${{ matrix.container_image }}
95+
options: ${{ matrix.gpu_arch_type == 'cuda' && '--gpus all' || ' ' }}
96+
# If a build is taking longer than 60 minutes on these runners we need
97+
# to have a conversation
98+
timeout-minutes: 120
99+
steps:
100+
- name: Clean workspace
101+
shell: bash -l {0}
102+
run: |
103+
set -euxo pipefail
104+
echo "::group::Cleanup debug output"
105+
rm -rf "${GITHUB_WORKSPACE}"
106+
mkdir -p "${GITHUB_WORKSPACE}"
107+
108+
if [[ "${{ inputs.architecture }}" = "aarch64" ]]; then
109+
rm -rf "${RUNNER_TEMP}/*"
110+
fi
111+
112+
echo "::endgroup::"
113+
- uses: actions/checkout@v3
114+
with:
115+
# Support the use case where we need to checkout someone's fork
116+
repository: ${{ inputs.test-infra-repository }}
117+
ref: ${{ inputs.test-infra-ref }}
118+
path: test-infra
119+
- uses: actions/checkout@v3
120+
if: ${{ env.ARCH == 'aarch64' }}
121+
with:
122+
# Support the use case where we need to checkout someone's fork
123+
repository: "pytorch/builder"
124+
ref: "main"
125+
path: builder
126+
- name: Set linux aarch64 CI
127+
if: ${{ inputs.architecture == 'aarch64' }}
128+
shell: bash -l {0}
129+
env:
130+
DESIRED_PYTHON: ${{ matrix.python_version }}
131+
run: |
132+
set +e
133+
# TODO: This is temporary aarch64 setup script, this should be integrated into aarch64 docker.
134+
${GITHUB_WORKSPACE}/builder/aarch64_linux/aarch64_ci_setup.sh
135+
echo "/opt/conda/bin" >> $GITHUB_PATH
136+
set -e
137+
- uses: pytorch/test-infra/.github/actions/set-channel@release/2.3
138+
- name: Set PYTORCH_VERSION
139+
if: ${{ env.CHANNEL == 'test' }}
140+
run: |
141+
# When building RC, set the version to be the current candidate version,
142+
# otherwise, leave it alone so nightly will pick up the latest
143+
echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
144+
- uses: pytorch/test-infra/.github/actions/setup-binary-builds@release/2.3
145+
env:
146+
PLATFORM: ${{ inputs.architecture == 'aarch64' && 'linux-aarch64' || ''}}
147+
with:
148+
repository: ${{ inputs.repository }}
149+
ref: ${{ inputs.ref }}
150+
submodules: ${{ inputs.submodules }}
151+
setup-miniconda: ${{ inputs.setup-miniconda }}
152+
python-version: ${{ env.PYTHON_VERSION }}
153+
cuda-version: ${{ env.CU_VERSION }}
154+
arch: ${{ env.ARCH }}
155+
- name: Combine Env Var and Build Env Files
156+
if: ${{ inputs.env-var-script != '' }}
157+
working-directory: ${{ inputs.repository }}
158+
run: |
159+
set -euxo pipefail
160+
cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}"
161+
- name: Install torch dependency
162+
run: |
163+
set -euxo pipefail
164+
# shellcheck disable=SC1090
165+
source "${BUILD_ENV_FILE}"
166+
# shellcheck disable=SC2086
167+
${CONDA_RUN} ${PIP_INSTALL_TORCH}
168+
- name: Run Pre-Script with Caching
169+
if: ${{ inputs.pre-script != '' }}
170+
uses: pytorch/test-infra/.github/actions/run-script-with-cache@release/2.3
171+
with:
172+
cache-path: ${{ inputs.cache-path }}
173+
cache-key: ${{ inputs.cache-key }}
174+
repository: ${{ inputs.repository }}
175+
script: ${{ inputs.pre-script }}
176+
- name: Build clean
177+
working-directory: ${{ inputs.repository }}
178+
shell: bash -l {0}
179+
run: |
180+
set -euxo pipefail
181+
source "${BUILD_ENV_FILE}"
182+
${CONDA_RUN} python setup.py clean
183+
- name: Build the wheel (bdist_wheel)
184+
working-directory: ${{ inputs.repository }}
185+
shell: bash -l {0}
186+
run: |
187+
set -euxo pipefail
188+
source "${BUILD_ENV_FILE}"
189+
export PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//' | sed 's/+.\+//')"
190+
${CONDA_RUN} python setup.py bdist_wheel
191+
- name: Run Post-Script
192+
if: ${{ inputs.post-script != '' }}
193+
uses: pytorch/test-infra/.github/actions/run-script-with-cache@release/2.3
194+
with:
195+
repository: ${{ inputs.repository }}
196+
script: ${{ inputs.post-script }}
197+
- name: Smoke Test
198+
shell: bash -l {0}
199+
env:
200+
PACKAGE_NAME: ${{ inputs.package-name }}
201+
SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }}
202+
run: |
203+
set -euxo pipefail
204+
source "${BUILD_ENV_FILE}"
205+
WHEEL_NAME=$(ls "${{ inputs.repository }}/dist/")
206+
echo "$WHEEL_NAME"
207+
208+
${CONDA_RUN} pip install "${{ inputs.repository }}/dist/$WHEEL_NAME"
209+
# Checking that we have a pinned version of torch in our dependency tree
210+
(
211+
pushd "${RUNNER_TEMP}"
212+
unzip -o "${GITHUB_WORKSPACE}/${{ inputs.repository }}/dist/$WHEEL_NAME"
213+
# Ensure that pytorch version is pinned, should output file where it was found
214+
grep "Requires-Dist: torch (==.*)" -r .
215+
)
216+
217+
if [[ (! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT}) ]]; then
218+
echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found"
219+
if [[ "${PACKAGE_NAME}" = "torchrec" ]]; then
220+
# Special case for torchrec temporarily since __version__ does not
221+
# work correctly on main in torchrec. This block will be
222+
# removed once we fix it.
223+
${CONDA_RUN} python -c "import ${PACKAGE_NAME}"
224+
else
225+
${CONDA_RUN} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)"
226+
fi
227+
else
228+
echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
229+
${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
230+
fi
231+
# NB: Only upload to GitHub after passing smoke tests
232+
- name: Upload wheel to GitHub
233+
continue-on-error: true
234+
uses: actions/upload-artifact@v3
235+
with:
236+
name: ${{ env.ARTIFACT_NAME }}
237+
path: ${{ inputs.repository }}/dist/
238+
239+
upload:
240+
needs: build
241+
uses: pytorch/test-infra/.github/workflows/_binary_upload.yml@release/2.3
242+
if: always()
243+
with:
244+
repository: ${{ inputs.repository }}
245+
ref: ${{ inputs.ref }}
246+
test-infra-repository: ${{ inputs.test-infra-repository }}
247+
test-infra-ref: ${{ inputs.test-infra-ref }}
248+
build-matrix: ${{ inputs.build-matrix }}
249+
architecture: ${{ inputs.architecture }}
250+
trigger-event: ${{ inputs.trigger-event }}
251+
252+
concurrency:
253+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}
254+
cancel-in-progress: true

0 commit comments

Comments
 (0)