Skip to content

Commit 5cf96f5

Browse files
dbortdbort
committed
DO NOT LAND: Disable wheel delocation on macos
DO NOT LAND: Update this to point to the real infra repo once the `delocate-wheel` input is available there. The executorch build system will ensure that .dylib/.so files have LC_LOAD_DYLIB and LC_RPATH entries that will work when they're installed. Delocating (i.e., making copies of the .dylibs that ET's libs depend on) will break any libs that depend on the torch libraries if users ever import both `torch` and the executorch library. Both import paths must load exactly the same file, not just a copy of it.
1 parent 780bddd commit 5cf96f5

File tree

2 files changed

+241
-1
lines changed

2 files changed

+241
-1
lines changed

‎.github/workflows/build-wheels-m1.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
smoke-test-script: build/packaging/smoke_test.py
4545
package-name: executorch
4646
name: ${{ matrix.repository }}
47-
uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@release/2.3
47+
uses: ./.github/workflows/test_infra_build_wheels_macos.yml
4848
with:
4949
repository: ${{ matrix.repository }}
5050
ref: ""
@@ -55,6 +55,7 @@ jobs:
5555
# "recursive" default to do less work, and to give the buck daemon fewer
5656
# files to look at.
5757
submodules: true
58+
delocate-wheel: false
5859
env-var-script: build/packaging/env_var_script_m1.sh
5960
pre-script: ${{ matrix.pre-script }}
6061
post-script: ${{ matrix.post-script }}

‎.github/workflows/test_infra_build_wheels_macos.yml

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
name: Build MacOS Wheels
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
repository:
7+
description: 'Repository to checkout, defaults to ""'
8+
default: ""
9+
type: string
10+
ref:
11+
description: 'Reference to checkout, defaults to "nightly"'
12+
default: "nightly"
13+
type: string
14+
test-infra-repository:
15+
description: "Test infra repository to use"
16+
default: "pytorch/test-infra"
17+
type: string
18+
test-infra-ref:
19+
description: "Test infra reference to use"
20+
default: ""
21+
type: string
22+
build-matrix:
23+
description: "Build matrix to utilize"
24+
default: ""
25+
type: string
26+
pre-script:
27+
description: "Pre script to run prior to build"
28+
default: ""
29+
type: string
30+
post-script:
31+
description: "Post script to run prior to build"
32+
default: ""
33+
type: string
34+
runner-type:
35+
description: "Runner environment specified."
36+
default: ""
37+
type: string
38+
package-name:
39+
description: "Name of the actual python package that is imported"
40+
default: ""
41+
type: string
42+
env-var-script:
43+
description: "Script that sets Domain-Specific Environment Variables"
44+
default: ""
45+
type: string
46+
trigger-event:
47+
description: "Trigger Event in caller that determines whether or not to upload"
48+
default: ""
49+
type: string
50+
smoke-test-script:
51+
description: "Script for Smoke Test for a specific domain"
52+
default: ""
53+
type: string
54+
cache-path:
55+
description: "The path(s) on the runner to cache or restore. The path is relative to repository."
56+
default: ""
57+
type: string
58+
cache-key:
59+
description: "The key created when saving a cache and the key used to search for a cache."
60+
default: ""
61+
type: string
62+
submodules:
63+
description: "Works as stated in actions/checkout, but the default value is recursive"
64+
required: false
65+
type: string
66+
default: recursive
67+
delocate-wheel:
68+
description: "Whether to run delocate-wheel after building."
69+
required: false
70+
type: boolean
71+
default: true
72+
73+
permissions:
74+
id-token: write
75+
contents: read
76+
77+
jobs:
78+
build:
79+
strategy:
80+
fail-fast: false
81+
matrix: ${{ fromJSON(inputs.build-matrix) }}
82+
env:
83+
PYTHON_VERSION: ${{ matrix.python_version }}
84+
PACKAGE_TYPE: wheel
85+
REPOSITORY: ${{ inputs.repository }}
86+
REF: ${{ inputs.ref }}
87+
CU_VERSION: ${{ matrix.desired_cuda }}
88+
name: ${{ matrix.build_name }}
89+
runs-on: ${{ inputs.runner-type }}
90+
# If a build is taking longer than 60 minutes on these runners we need
91+
# to have a conversation
92+
timeout-minutes: 60
93+
steps:
94+
- name: Clean workspace
95+
run: |
96+
set -euxo pipefail
97+
echo "::group::Cleanup debug output"
98+
rm -rfv "${GITHUB_WORKSPACE}"
99+
mkdir -p "${GITHUB_WORKSPACE}"
100+
echo "::endgroup::"
101+
- uses: actions/checkout@v3
102+
with:
103+
# Support the use case where we need to checkout someone's fork
104+
repository: ${{ inputs.test-infra-repository }}
105+
ref: ${{ inputs.test-infra-ref }}
106+
path: test-infra
107+
- uses: pytorch/test-infra/.github/actions/set-channel@release/2.3
108+
- name: Set PYTORCH_VERSION
109+
if: ${{ env.CHANNEL == 'test' }}
110+
run: |
111+
# When building RC, set the version to be the current candidate version,
112+
# otherwise, leave it alone so nightly will pick up the latest
113+
echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
114+
- uses: pytorch/test-infra/.github/actions/setup-binary-builds@release/2.3
115+
with:
116+
repository: ${{ inputs.repository }}
117+
ref: ${{ inputs.ref }}
118+
submodules: ${{ inputs.submodules }}
119+
setup-miniconda: false
120+
python-version: ${{ env.PYTHON_VERSION }}
121+
cuda-version: ${{ env.CU_VERSION }}
122+
arch: ${{ env.ARCH }}
123+
- name: Combine Env Var and Build Env Files
124+
if: ${{ inputs.env-var-script != '' }}
125+
working-directory: ${{ inputs.repository }}
126+
run: |
127+
cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}"
128+
- name: Install delocate-wheel
129+
if: ${{ inputs.delocate-wheel }}
130+
run: |
131+
set -euxo pipefail
132+
${CONDA_RUN} python3 -m pip install delocate==0.10.7
133+
- name: Install torch dependency
134+
run: |
135+
set -euxo pipefail
136+
# shellcheck disable=SC1090
137+
source "${BUILD_ENV_FILE}"
138+
# shellcheck disable=SC2086
139+
${CONDA_RUN} ${PIP_INSTALL_TORCH}
140+
- name: Run Pre-Script with Caching
141+
if: ${{ inputs.pre-script != '' }}
142+
uses: pytorch/test-infra/.github/actions/run-script-with-cache@release/2.3
143+
with:
144+
cache-path: ${{ inputs.cache-path }}
145+
cache-key: ${{ inputs.cache-key }}
146+
repository: ${{ inputs.repository }}
147+
script: ${{ inputs.pre-script }}
148+
- name: Build clean
149+
working-directory: ${{ inputs.repository }}
150+
run: |
151+
set -euxo pipefail
152+
# shellcheck disable=SC1090
153+
source "${BUILD_ENV_FILE}"
154+
${CONDA_RUN} python3 setup.py clean
155+
- name: Build the wheel (bdist_wheel)
156+
working-directory: ${{ inputs.repository }}
157+
run: |
158+
set -euxo pipefail
159+
# shellcheck disable=SC1090
160+
source "${BUILD_ENV_FILE}"
161+
162+
if [[ "${{ inputs.package-name }}" = "torchaudio" ]]; then
163+
export USE_OPENMP="0"
164+
fi
165+
PYTORCH_VERSION="$(${CONDA_RUN} pip show torch | grep ^Version: | sed 's/Version: *//')"
166+
export PYTORCH_VERSION
167+
168+
${CONDA_RUN} python3 setup.py bdist_wheel
169+
- name: Delocate wheel
170+
if: ${{ inputs.delocate-wheel }}
171+
working-directory: ${{ inputs.repository }}
172+
run: |
173+
set -euxo pipefail
174+
${CONDA_RUN} DYLD_FALLBACK_LIBRARY_PATH="${CONDA_ENV}/lib" delocate-wheel -v --ignore-missing-dependencies dist/*.whl
175+
- name: Run Post-Script
176+
if: ${{ inputs.post-script != '' }}
177+
uses: pytorch/test-infra/.github/actions/run-script-with-cache@release/2.3
178+
with:
179+
repository: ${{ inputs.repository }}
180+
script: ${{ inputs.post-script }}
181+
- name: Smoke Test
182+
shell: bash -l {0}
183+
env:
184+
PACKAGE_NAME: ${{ inputs.package-name }}
185+
SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }}
186+
run: |
187+
set -euxo pipefail
188+
# shellcheck disable=SC1090
189+
source "${BUILD_ENV_FILE}"
190+
WHEEL_NAME=$(ls "${{ inputs.repository }}/dist/")
191+
echo "$WHEEL_NAME"
192+
# Checking that we have a pinned version of torch in our dependency tree
193+
(
194+
pushd "${RUNNER_TEMP}"
195+
unzip -o "${GITHUB_WORKSPACE}/${{ inputs.repository }}/dist/$WHEEL_NAME"
196+
# Ensure that pytorch version is pinned, should output file where it was found
197+
grep "Requires-Dist: torch (==.*)" -r .
198+
)
199+
export OLD_PATH=${PATH}
200+
export PATH="${CONDA_ENV}/bin:${PATH}"
201+
202+
${CONDA_RUN} pip install "${{ inputs.repository }}/dist/$WHEEL_NAME"
203+
204+
if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then
205+
echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found"
206+
${CONDA_RUN} python -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)"
207+
else
208+
echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
209+
210+
${CONDA_RUN} python3 "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
211+
fi
212+
export PATH=${OLD_PATH}
213+
# NB: Only upload to GitHub after passing smoke tests
214+
- name: Upload wheel to GitHub
215+
continue-on-error: true
216+
uses: actions/upload-artifact@v3
217+
with:
218+
name: ${{ env.ARTIFACT_NAME }}
219+
path: ${{ inputs.repository }}/dist/
220+
- name: Clean up disk space
221+
if: always()
222+
continue-on-error: true
223+
uses: pytorch/test-infra/.github/actions/check-disk-space@release/2.3
224+
225+
upload:
226+
needs: build
227+
uses: pytorch/test-infra/.github/workflows/_binary_upload.yml@release/2.3
228+
if: always()
229+
with:
230+
repository: ${{ inputs.repository }}
231+
ref: ${{ inputs.ref }}
232+
test-infra-repository: ${{ inputs.test-infra-repository }}
233+
test-infra-ref: ${{ inputs.test-infra-ref }}
234+
build-matrix: ${{ inputs.build-matrix }}
235+
trigger-event: ${{ inputs.trigger-event }}
236+
237+
concurrency:
238+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}
239+
cancel-in-progress: true

0 commit comments

Comments
 (0)