Skip to content

Commit f887d72

Browse files
python3kgaepython3kgae
authored
[Build] Implement install_requirements.sh with Python (#4748)
Move the details of install_requirements.sh into install_requirements.py. Then, call install_requirements.py from install_requirements.sh. This will avoid duplication when adding Windows version of installing_requirements. This is for issue #4661. Release Note: Moved the main install_requirements.sh logic into install_requirements.py so that it can also run on Windows. Test Plan: Tested on a mac. Ran `./install_requirements.sh --pybind xnnpack` under python 3.10, and it completed successfully. To make sure that xnnpack was actually installed, I manually ran the steps from the notebook at https://colab.research.google.com/drive/1qpxrXC3YdJQzly3mRg-4ayYiOjC6rue3, and it passed. When running the same in a clean python 3.8 environment, it failed as expected with ``` % ./install_requirements.sh --pybind xnnpack Collecting packaging Using cached packaging-24.1-py3-none-any.whl.metadata (3.2 kB) Using cached packaging-24.1-py3-none-any.whl (53 kB) Installing collected packages: packaging Successfully installed packaging-24.1 ERROR: ExecuTorch does not support python version 3.8.19: must satisfy ">=3.10" ``` Also tested under WSL and native Windows.
1 parent 1f783f5 commit f887d72

File tree

2 files changed

+171
-152
lines changed

2 files changed

+171
-152
lines changed

‎install_requirements.py

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
#
6+
# This source code is licensed under the BSD-style license found in the
7+
# LICENSE file in the root directory of this source tree.
8+
9+
import os
10+
import platform
11+
import re
12+
import subprocess
13+
import sys
14+
15+
# Before doing anything, cd to the directory containing this script.
16+
os.chdir(os.path.dirname(os.path.abspath(__file__)))
17+
18+
19+
def python_is_compatible():
20+
# Scrape the version range from pyproject.toml, which should be in the current directory.
21+
version_specifier = None
22+
with open("pyproject.toml", "r") as file:
23+
for line in file:
24+
if line.startswith("requires-python"):
25+
match = re.search(r'"([^"]*)"', line)
26+
if match:
27+
version_specifier = match.group(1)
28+
break
29+
30+
if not version_specifier:
31+
print(
32+
"WARNING: Skipping python version check: version range not found",
33+
file=sys.stderr,
34+
)
35+
return False
36+
37+
# Install the packaging module if necessary.
38+
try:
39+
import packaging
40+
except ImportError:
41+
subprocess.run(
42+
[sys.executable, "-m", "pip", "install", "packaging"], check=True
43+
)
44+
# Compare the current python version to the range in version_specifier. Exits
45+
# with status 1 if the version is not compatible, or with status 0 if the
46+
# version is compatible or the logic itself fails.
47+
try:
48+
import packaging.specifiers
49+
import packaging.version
50+
51+
python_version = packaging.version.parse(platform.python_version())
52+
version_range = packaging.specifiers.SpecifierSet(version_specifier)
53+
if python_version not in version_range:
54+
print(
55+
f'ERROR: ExecuTorch does not support python version {python_version}: must satisfy "{version_specifier}"',
56+
file=sys.stderr,
57+
)
58+
return False
59+
except Exception as e:
60+
print(f"WARNING: Skipping python version check: {e}", file=sys.stderr)
61+
return True
62+
63+
64+
if not python_is_compatible():
65+
sys.exit(1)
66+
67+
# Parse options.
68+
EXECUTORCH_BUILD_PYBIND = "OFF"
69+
CMAKE_ARGS = os.getenv("CMAKE_ARGS", "")
70+
CMAKE_BUILD_ARGS = os.getenv("CMAKE_BUILD_ARGS", "")
71+
72+
for arg in sys.argv[1:]:
73+
if arg == "--pybind":
74+
EXECUTORCH_BUILD_PYBIND = "ON"
75+
elif arg in ["coreml", "mps", "xnnpack"]:
76+
if EXECUTORCH_BUILD_PYBIND == "ON":
77+
arg_upper = arg.upper()
78+
CMAKE_ARGS += f" -DEXECUTORCH_BUILD_{arg_upper}=ON"
79+
else:
80+
print(f"Error: {arg} must follow --pybind")
81+
sys.exit(1)
82+
else:
83+
print(f"Error: Unknown option {arg}")
84+
sys.exit(1)
85+
86+
# Since ExecuTorch often uses main-branch features of pytorch, only the nightly
87+
# pip versions will have the required features.
88+
#
89+
# NOTE: If a newly-fetched version of the executorch repo changes the value of
90+
# NIGHTLY_VERSION, you should re-run this script to install the necessary
91+
# package versions.
92+
NIGHTLY_VERSION = "dev20240716"
93+
94+
# The pip repository that hosts nightly torch packages.
95+
TORCH_NIGHTLY_URL = "https://download.pytorch.org/whl/nightly/cpu"
96+
97+
# pip packages needed by exir.
98+
EXIR_REQUIREMENTS = [
99+
f"torch==2.5.0.{NIGHTLY_VERSION}",
100+
f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing.
101+
"typing-extensions",
102+
]
103+
104+
# pip packages needed for development.
105+
DEVEL_REQUIREMENTS = [
106+
"cmake", # For building binary targets.
107+
"pip>=23", # For building the pip package.
108+
"pyyaml", # Imported by the kernel codegen tools.
109+
"setuptools>=63", # For building the pip package.
110+
"tomli", # Imported by extract_sources.py when using python < 3.11.
111+
"wheel", # For building the pip package archive.
112+
"zstd", # Imported by resolve_buck.py.
113+
]
114+
115+
# pip packages needed to run examples.
116+
# TODO: Make each example publish its own requirements.txt
117+
EXAMPLES_REQUIREMENTS = [
118+
"timm==1.0.7",
119+
f"torchaudio==2.4.0.{NIGHTLY_VERSION}",
120+
"torchsr==1.0.4",
121+
"transformers==4.42.4",
122+
]
123+
124+
# Assemble the list of requirements to actually install.
125+
# TODO: Add options for reducing the number of requirements.
126+
REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS
127+
128+
# Install the requirements. `--extra-index-url` tells pip to look for package
129+
# versions on the provided URL if they aren't available on the default URL.
130+
subprocess.run(
131+
[
132+
sys.executable,
133+
"-m",
134+
"pip",
135+
"install",
136+
*REQUIREMENTS_TO_INSTALL,
137+
"--extra-index-url",
138+
TORCH_NIGHTLY_URL,
139+
],
140+
check=True,
141+
)
142+
143+
#
144+
# Install executorch pip package. This also makes `flatc` available on the path.
145+
# The --extra-index-url may be necessary if pyproject.toml has a dependency on a
146+
# pre-release or nightly version of a torch package.
147+
#
148+
149+
# Set environment variables
150+
os.environ["EXECUTORCH_BUILD_PYBIND"] = EXECUTORCH_BUILD_PYBIND
151+
os.environ["CMAKE_ARGS"] = CMAKE_ARGS
152+
os.environ["CMAKE_BUILD_ARGS"] = CMAKE_BUILD_ARGS
153+
154+
# Run the pip install command
155+
subprocess.run(
156+
[
157+
sys.executable,
158+
"-m",
159+
"pip",
160+
"install",
161+
".",
162+
"--no-build-isolation",
163+
"-v",
164+
"--extra-index-url",
165+
TORCH_NIGHTLY_URL,
166+
],
167+
check=True,
168+
)

‎install_requirements.sh

Lines changed: 3 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -19,156 +19,7 @@ then
1919
fi
2020
fi
2121

22-
if [[ "$PYTHON_EXECUTABLE" == "python" ]];
23-
then
24-
PIP_EXECUTABLE=pip
25-
else
26-
PIP_EXECUTABLE=pip3
27-
fi
28-
29-
# Returns 0 if the current python version is compatible with the version range
30-
# in pyprojects.toml, or returns 1 if it is not compatible. If the check logic
31-
# itself fails, prints a warning and returns 0.
32-
python_is_compatible() {
33-
# Scrape the version range from pyproject.toml, which should be
34-
# in the current directory.
35-
local version_specifier
36-
version_specifier="$(
37-
grep "^requires-python" pyproject.toml \
38-
| head -1 \
39-
| sed -e 's/[^"]*"//;s/".*//'
40-
)"
41-
if [[ -z ${version_specifier} ]]; then
42-
echo "WARNING: Skipping python version check: version range not found" >& 2
43-
return 0
44-
fi
45-
46-
# Install the packaging module if necessary.
47-
if ! python -c 'import packaging' 2> /dev/null ; then
48-
${PIP_EXECUTABLE} install packaging
49-
fi
50-
51-
# Compare the current python version to the range in version_specifier. Exits
52-
# with status 1 if the version is not compatible, or with status 0 if the
53-
# version is compatible or the logic itself fails.
54-
${PYTHON_EXECUTABLE} <<EOF
55-
import sys
56-
try:
57-
import packaging.version
58-
import packaging.specifiers
59-
import platform
60-
61-
python_version = packaging.version.parse(platform.python_version())
62-
version_range = packaging.specifiers.SpecifierSet("${version_specifier}")
63-
if python_version not in version_range:
64-
print(
65-
"ERROR: ExecuTorch does not support python version "
66-
+ f"{python_version}: must satisfy \"${version_specifier}\"",
67-
file=sys.stderr,
68-
)
69-
sys.exit(1)
70-
except Exception as e:
71-
print(f"WARNING: Skipping python version check: {e}", file=sys.stderr)
72-
sys.exit(0)
73-
EOF
74-
75-
return $?
76-
}
77-
78-
# Fail fast if the wheel build will fail because the current python version
79-
# isn't supported. But don't fail if the check logic itself has problems: the
80-
# wheel build will do a final check before proceeding.
81-
if ! python_is_compatible; then
82-
exit 1
83-
fi
84-
85-
# Parse options.
86-
EXECUTORCH_BUILD_PYBIND=OFF
87-
88-
for arg in "$@"; do
89-
case $arg in
90-
--pybind)
91-
EXECUTORCH_BUILD_PYBIND=ON
92-
;;
93-
coreml|mps|xnnpack)
94-
if [[ "$EXECUTORCH_BUILD_PYBIND" == "ON" ]]; then
95-
arg_upper="$(echo "${arg}" | tr '[:lower:]' '[:upper:]')"
96-
CMAKE_ARGS="$CMAKE_ARGS -DEXECUTORCH_BUILD_${arg_upper}=ON"
97-
else
98-
echo "Error: $arg must follow --pybind"
99-
exit 1
100-
fi
101-
;;
102-
*)
103-
echo "Error: Unknown option $arg"
104-
exit 1
105-
;;
106-
esac
107-
done
108-
109-
#
110-
# Install pip packages used by code in the ExecuTorch repo.
111-
#
112-
113-
# Since ExecuTorch often uses main-branch features of pytorch, only the nightly
114-
# pip versions will have the required features.
115-
#
116-
# NOTE: If a newly-fetched version of the executorch repo changes the value of
117-
# NIGHTLY_VERSION, you should re-run this script to install the necessary
118-
# package versions.
119-
NIGHTLY_VERSION=dev20240716
120-
121-
# The pip repository that hosts nightly torch packages.
122-
TORCH_NIGHTLY_URL="https://download.pytorch.org/whl/nightly/cpu"
123-
124-
# pip packages needed by exir.
125-
EXIR_REQUIREMENTS=(
126-
torch=="2.5.0.${NIGHTLY_VERSION}"
127-
torchvision=="0.20.0.${NIGHTLY_VERSION}" # For testing.
128-
typing-extensions
129-
)
130-
131-
# pip packages needed for development.
132-
DEVEL_REQUIREMENTS=(
133-
cmake # For building binary targets.
134-
"pip>=23" # For building the pip package.
135-
pyyaml # Imported by the kernel codegen tools.
136-
"setuptools>=63" # For building the pip package.
137-
tomli # Imported by extract_sources.py when using python < 3.11.
138-
wheel # For building the pip package archive.
139-
zstd # Imported by resolve_buck.py.
140-
)
141-
142-
# pip packages needed to run examples.
143-
# TODO(dbort): Make each example publish its own requirements.txt
144-
EXAMPLES_REQUIREMENTS=(
145-
timm==1.0.7
146-
torchaudio=="2.4.0.${NIGHTLY_VERSION}"
147-
torchsr==1.0.4
148-
transformers==4.42.4
149-
)
150-
151-
# Assemble the list of requirements to actually install.
152-
# TODO(dbort): Add options for reducing the number of requirements.
153-
REQUIREMENTS_TO_INSTALL=(
154-
"${EXIR_REQUIREMENTS[@]}"
155-
"${DEVEL_REQUIREMENTS[@]}"
156-
"${EXAMPLES_REQUIREMENTS[@]}"
157-
)
158-
159-
# Install the requirements. `--extra-index-url` tells pip to look for package
160-
# versions on the provided URL if they aren't available on the default URL.
161-
$PIP_EXECUTABLE install --extra-index-url "${TORCH_NIGHTLY_URL}" \
162-
"${REQUIREMENTS_TO_INSTALL[@]}"
163-
164-
#
165-
# Install executorch pip package. This also makes `flatc` available on the path.
166-
# The --extra-index-url may be necessary if pyproject.toml has a dependency on a
167-
# pre-release or nightly version of a torch package.
168-
#
22+
$PYTHON_EXECUTABLE ./install_requirements.py "$@"
16923

170-
EXECUTORCH_BUILD_PYBIND="${EXECUTORCH_BUILD_PYBIND}" \
171-
CMAKE_ARGS="${CMAKE_ARGS}" \
172-
CMAKE_BUILD_ARGS="${CMAKE_BUILD_ARGS}" \
173-
$PIP_EXECUTABLE install . --no-build-isolation -v \
174-
--extra-index-url "${TORCH_URL}"
24+
# Exit with the same status as the python script.
25+
exit $?

0 commit comments

Comments
 (0)