Skip to content

Commit aa1c843

Browse files
narendasannarendasan
authored
Merge pull request #2056 from pytorch/pep517_builds
Change python build system to be PEP517 compatible
2 parents aeadd72 + 9b0b8f1 commit aa1c843

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+735
-504
lines changed

‎.bazelversion

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
5.2.0
1+
6.2.1

‎.circleci/config.yml

Lines changed: 135 additions & 85 deletions
Large diffs are not rendered by default.

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,5 @@ bazel-tensorrt
6666
*.cache
6767
*cifar-10-batches-py*
6868
bazel-project
69-
build/
69+
build/
70+
wheelhouse/

‎.pre-commit-config.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ repos:
3030
args:
3131
- --warnings=all
3232
- id: buildifier-lint
33+
- repo: https://github.com/abravalheri/validate-pyproject
34+
rev: v0.13
35+
hooks:
36+
- id: validate-pyproject
3337
- repo: local
3438
hooks:
3539
- id: dont-commit-upstream

‎WORKSPACE

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,36 @@
11
workspace(name = "Torch-TensorRT")
22

3-
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
43
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
54

65
http_archive(
76
name = "rules_python",
8-
sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f",
9-
url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz",
7+
sha256 = "863ba0fa944319f7e3d695711427d9ad80ba92c6edd0b7c7443b84e904689539",
8+
strip_prefix = "rules_python-0.22.0",
9+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.22.0/rules_python-0.22.0.tar.gz",
1010
)
1111

12-
load("@rules_python//python:pip.bzl", "pip_install")
12+
load("@rules_python//python:repositories.bzl", "py_repositories")
13+
14+
py_repositories()
1315

1416
http_archive(
1517
name = "rules_pkg",
16-
sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
18+
sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8",
1719
urls = [
18-
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
19-
"https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
20+
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
21+
"https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
2022
],
2123
)
2224

2325
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
2426

2527
rules_pkg_dependencies()
2628

27-
git_repository(
29+
http_archive(
2830
name = "googletest",
29-
commit = "703bd9caab50b139428cea1aaff9974ebee5742e",
30-
remote = "https://github.com/google/googletest",
31-
shallow_since = "1570114335 -0400",
31+
sha256 = "755f9a39bc7205f5a0c428e920ddad092c33c8a1b46997def3f1d4a82aded6e1",
32+
strip_prefix = "googletest-5ab508a01f9eb089207ee87fd547d290da39d015",
33+
urls = ["https://github.com/google/googletest/archive/5ab508a01f9eb089207ee87fd547d290da39d015.zip"],
3234
)
3335

3436
# External dependency for torch_tensorrt if you already have precompiled binaries.
@@ -127,7 +129,13 @@ http_archive(
127129
# Development Dependencies (optional - comment out on aarch64)
128130
#########################################################################
129131

130-
pip_install(
132+
load("@rules_python//python:pip.bzl", "pip_parse")
133+
134+
pip_parse(
131135
name = "devtools_deps",
132136
requirements = "//:requirements-dev.txt",
133137
)
138+
139+
load("@devtools_deps//:requirements.bzl", "install_deps")
140+
141+
install_deps()

‎docker/Dockerfile

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# syntax=docker/dockerfile:1
2+
13
# Base image starts with CUDA
24
ARG BASE_IMG=nvidia/cuda:12.1.1-devel-ubuntu22.04
35
FROM ${BASE_IMG} as base
@@ -50,7 +52,7 @@ RUN apt-get update
5052
RUN apt-get install -y libnvinfer8=${TENSORRT_VERSION}.* libnvinfer-plugin8=${TENSORRT_VERSION}.* libnvinfer-dev=${TENSORRT_VERSION}.* libnvinfer-plugin-dev=${TENSORRT_VERSION}.* libnvonnxparsers8=${TENSORRT_VERSION}.* libnvonnxparsers-dev=${TENSORRT_VERSION}.* libnvparsers8=${TENSORRT_VERSION}.* libnvparsers-dev=${TENSORRT_VERSION}.*
5153

5254
# Setup Bazel via Bazelisk
53-
RUN wget -q https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64 -O /usr/bin/bazel &&\
55+
RUN wget -q https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-amd64 -O /usr/bin/bazel &&\
5456
chmod a+x /usr/bin/bazel
5557

5658
# Build Torch-TensorRT in an auxillary container
@@ -99,12 +101,12 @@ COPY . /opt/torch_tensorrt
99101
RUN mkdir -p "/opt/python3/" &&\
100102
ln -s "`pyenv which python | xargs dirname | xargs dirname`/lib/python$PYTHON_VERSION/site-packages" "/opt/python3/"
101103

102-
COPY --from=torch-tensorrt-builder /workspace/torch_tensorrt/src/py/dist/ .
104+
COPY --from=torch-tensorrt-builder /workspace/torch_tensorrt/src/dist/ .
103105

104106
RUN cp /opt/torch_tensorrt/docker/WORKSPACE.docker /opt/torch_tensorrt/WORKSPACE
105107
RUN pip install -r /opt/torch_tensorrt/py/requirements.txt
106108
RUN pip install tensorrt==${TENSORRT_VERSION}.*
107-
RUN pip install *.whl && rm -fr /workspace/torch_tensorrt/py/dist/* *.whl
109+
RUN pip install *.whl && rm -fr /workspace/torch_tensorrt/dist/* *.whl
108110

109111
WORKDIR /opt/torch_tensorrt
110112

‎docker/Dockerfile.docs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
FROM nvcr.io/nvidia/tensorrt:22.06-py3
22

3-
ARG BAZEL_VERSION=5.2.0
3+
ARG BAZEL_VERSION=6.2.1
44

55
RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
66
RUN echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list

‎docker/WORKSPACE.docker

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,39 @@
11
workspace(name = "Torch-TensorRT")
22

33
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
4-
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
54

65
http_archive(
76
name = "rules_python",
8-
sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f",
9-
url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz",
7+
sha256 = "863ba0fa944319f7e3d695711427d9ad80ba92c6edd0b7c7443b84e904689539",
8+
strip_prefix = "rules_python-0.22.0",
9+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.22.0/rules_python-0.22.0.tar.gz",
1010
)
1111

12-
load("@rules_python//python:pip.bzl", "pip_install")
12+
load("@rules_python//python:repositories.bzl", "py_repositories")
13+
14+
py_repositories()
1315

1416
http_archive(
1517
name = "rules_pkg",
16-
sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
18+
sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8",
1719
urls = [
18-
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
19-
"https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
20+
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
21+
"https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz",
2022
],
2123
)
2224

2325
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
2426

2527
rules_pkg_dependencies()
2628

27-
git_repository(
29+
http_archive(
2830
name = "googletest",
29-
commit = "703bd9caab50b139428cea1aaff9974ebee5742e",
30-
remote = "https://github.com/google/googletest",
31-
shallow_since = "1570114335 -0400",
31+
sha256 = "755f9a39bc7205f5a0c428e920ddad092c33c8a1b46997def3f1d4a82aded6e1",
32+
strip_prefix = "googletest-5ab508a01f9eb089207ee87fd547d290da39d015",
33+
urls = ["https://github.com/google/googletest/archive/5ab508a01f9eb089207ee87fd547d290da39d015.zip"],
3234
)
3335

36+
3437
# CUDA should be installed on the system locally
3538
new_local_repository(
3639
name = "cuda",
@@ -79,7 +82,14 @@ new_local_repository(
7982
#########################################################################
8083
# Testing Dependencies (optional - comment out on aarch64)
8184
#########################################################################
82-
pip_install(
83-
name = "pylinter_deps",
84-
requirements = "//tools/linter:requirements.txt",
85+
86+
load("@rules_python//python:pip.bzl", "pip_parse")
87+
88+
pip_parse(
89+
name = "devtools_deps",
90+
requirements = "//:requirements-dev.txt",
8591
)
92+
93+
load("@devtools_deps//:requirements.bzl", "install_deps")
94+
95+
install_deps()

‎docker/dist-build.sh

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,22 @@
33
TOP_DIR=$(cd $(dirname $0); pwd)/..
44

55
if [[ -z "${USE_CXX11}" ]]; then
6-
BUILD_CMD="python setup.py bdist_wheel"
6+
BUILD_CMD="python -m pip wheel . --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -w dist"
77
else
8-
BUILD_CMD="python setup.py bdist_wheel --use-cxx11-abi"
8+
BUILD_CMD="python -m pip wheel . --config-setting="--build-option=--use-cxx11-abi" --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -w dist"
99
fi
1010

11+
# TensorRT restricts our pip version
1112
cd ${TOP_DIR} \
12-
&& mkdir -p dist && cd py \
13-
&& pip install -r requirements.txt \
14-
&& pip install wheel
13+
&& python -m pip install "pip<=23.1" wheel \
14+
&& python -m pip install -r py/requirements.txt
1515

1616
# Build Torch-TRT
17-
MAX_JOBS=1 LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 ${BUILD_CMD} $* || exit 1
17+
MAX_JOBS=4 LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 ${BUILD_CMD} $* || exit 1
1818

19-
pip3 install ipywidgets --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host=files.pythonhosted.org
19+
python -m pip install ipywidgets --trusted-host pypi.org --trusted-host pypi.python.org --trusted-host=files.pythonhosted.org
2020
jupyter nbextension enable --py widgetsnbextension
21-
pip3 install timm
21+
python -m pip install timm
2222

2323
# test install
24-
pip3 uninstall -y torch_tensorrt && pip3 install ${TOP_DIR}/py/dist/*.whl
24+
python -m pip uninstall -y torch_tensorrt && python -m pip install ${TOP_DIR}/dist/*.whl

‎py/Dockerfile

Lines changed: 0 additions & 11 deletions
This file was deleted.

‎py/build_whl.sh

Lines changed: 0 additions & 44 deletions
This file was deleted.

‎py/ci/Dockerfile.ci

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
FROM pytorch/manylinux-builder:cuda11.7
1+
FROM pytorch/manylinux-builder:cuda12.1
22
ARG trt_version
33

44
RUN echo -e "Installing with TensorRT ${trt_version}"
55

66
RUN yum install -y ninja-build tensorrt-${trt_version}.*
77

8-
RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
8+
RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-amd64 \
99
&& mv bazelisk-linux-amd64 /usr/bin/bazel \
1010
&& chmod +x /usr/bin/bazel
1111

‎py/ci/build_manifest.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
py38
22
py39
33
py310
4+
py311
45
libtorchtrt_pre_cxx11_abi

0 commit comments

Comments
 (0)