Skip to content

Commit 623e417

Browse files
narendasannarendasan
committed
Adding WORKSPACES per platform
1 parent cd9d75f commit 623e417

File tree

3 files changed

+172
-7
lines changed

3 files changed

+172
-7
lines changed

‎.circleci/config.yml

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,15 @@ commands:
135135
136136
build-py:
137137
description: "Build the torch-tensorrt python release (pre-cxx11-abi)"
138+
parameters:
139+
platform:
140+
type: string
141+
default: "x86_64"
138142
steps:
139143
- run:
140144
name: Build torch-tensorrt python release (pre-cxx11-abi)
141145
command: |
142-
mv WORKSPACE.ci WORKSPACE
146+
mv toolchains/ci_workspaces/WORKSPACE.<< parameters.platform >> WORKSPACE
143147
cd py
144148
python3 -m pip install wheel setuptools
145149
python3 -m pip install pybind11==2.6.2
@@ -150,11 +154,15 @@ commands:
150154
151155
build-py-cxx11-abi:
152156
description: "Build the torch-tensorrt python release (cxx11-abi)"
157+
parameters:
158+
platform:
159+
type: string
160+
default: "x86_64"
153161
steps:
154162
- run:
155163
name: Build torch-tensorrt python release
156164
command: |
157-
mv WORKSPACE.ci WORKSPACE
165+
mv toolchains/ci_workspaces/WORKSPACE.<< parameters.platform >> WORKSPACE
158166
cd py
159167
python3 -m pip install wheel setuptools
160168
python3 -m pip install pybind11==2.6.2
@@ -165,11 +173,15 @@ commands:
165173
166174
build-py-fx-only:
167175
description: "Build the torch-tensorrt python release with only the fx backend"
176+
parameters:
177+
platform:
178+
type: string
179+
default: "x86_64"
168180
steps:
169181
- run:
170182
name: Build torch-tensorrt python release with only the fx backend
171183
command: |
172-
mv WORKSPACE.ci WORKSPACE
184+
mv toolchains/ci_workspaces/WORKSPACE.<< parameters.platform >> WORKSPACE
173185
cd py
174186
python3 -m pip install wheel setuptools
175187
python3 -m pip install pybind11==2.6.2
@@ -211,6 +223,10 @@ commands:
211223
212224
test-ts-core:
213225
description: "Test torchscript backend c++ api"
226+
parameters:
227+
platform:
228+
type: string
229+
default: "x86_64"
214230
steps:
215231
- pull-test-models
216232
- run: mkdir -p /tmp/artifacts
@@ -220,7 +236,7 @@ commands:
220236
LD_LIBRARY_PATH: "/home/circleci/project/bazel-project/external/libtorch_pre_cxx11_abi/lib/:/home/circleci/project/bazel-project/external/tensorrt/lib/:/usr/local/cuda/lib64/:$LD_LIBRARY_PATH"
221237
command: |
222238
set -e
223-
mv WORKSPACE.ci WORKSPACE
239+
mv toolchains/ci_workspaces/WORKSPACE.<< parameters.platform >> WORKSPACE
224240
bazel query 'kind(cc_*, tests(//tests))' --noshow_progress >> /tmp/test_manifest.txt
225241
circleci tests split < /tmp/test_manifest.txt > /tmp/node_test_manifest.txt
226242
bazel test $(cat /tmp/node_test_manifest.txt) --test_arg=--gtest_output=xml:/tmp/artifacts/test_results/ --jobs 4 --config ci_testing --config pre_cxx11_abi --noshow_progress
@@ -346,19 +362,21 @@ jobs:
346362
pyenv global << parameters.python-version >>
347363
- run:
348364
name: Install NGC Torch
349-
environment:
365+
environment:
350366
TORCH_INSTALL: https://developer.download.nvidia.com/compute/redist/jp/v<< parameters.jetpack-version >>/pytorch/<< parameters.torch-build >>
351367
command: |
352368
set -e
353369
python3 -m pip install --upgrade pip; python3 -m pip install setuptools wheel; python3 -m pip install expecttest xmlrunner hypothesis aiohttp numpy=='1.19.4' pyyaml scipy=='1.5.3' ninja cython typing_extensions protobuf; export "LD_LIBRARY_PATH=/usr/lib/llvm-8/lib:$LD_LIBRARY_PATH"; python3 -m pip install --upgrade protobuf; python3 -m pip install --no-cache $TORCH_INSTALL
354370
- when:
355371
condition: << parameters.cxx11-abi >>
356372
steps:
357-
- build-py-cxx11-abi
373+
- build-py-cxx11-abi:
374+
platform: "sbsa"
358375
- unless:
359376
condition: << parameters.cxx11-abi >>
360377
steps:
361-
- build-py
378+
- build-py:
379+
platform: "sbsa"
362380
- run:
363381
name: Move to release dir
364382
command: |

‎toolchains/ci_workspaces/WORKSPACE.sbsa

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
workspace(name = "Torch-TensorRT")
2+
3+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
4+
load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
5+
6+
http_archive(
7+
name = "rules_python",
8+
sha256 = "778197e26c5fbeb07ac2a2c5ae405b30f6cb7ad1f5510ea6fdac03bded96cc6f",
9+
url = "https://github.com/bazelbuild/rules_python/releases/download/0.2.0/rules_python-0.2.0.tar.gz",
10+
)
11+
12+
load("@rules_python//python:pip.bzl", "pip_install")
13+
14+
http_archive(
15+
name = "rules_pkg",
16+
sha256 = "038f1caa773a7e35b3663865ffb003169c6a71dc995e39bf4815792f385d837d",
17+
urls = [
18+
"https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
19+
"https://github.com/bazelbuild/rules_pkg/releases/download/0.4.0/rules_pkg-0.4.0.tar.gz",
20+
],
21+
)
22+
23+
load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
24+
25+
rules_pkg_dependencies()
26+
27+
git_repository(
28+
name = "googletest",
29+
commit = "703bd9caab50b139428cea1aaff9974ebee5742e",
30+
remote = "https://github.com/google/googletest",
31+
shallow_since = "1570114335 -0400",
32+
)
33+
34+
# External dependency for torch_tensorrt if you already have precompiled binaries.
35+
local_repository(
36+
name = "torch_tensorrt",
37+
path = "/opt/circleci/.pyenv/versions/3.8.13/lib/python3.8/site-packages/torch_tensorrt"
38+
)
39+
40+
# CUDA should be installed on the system locally
41+
new_local_repository(
42+
name = "cuda",
43+
build_file = "@//third_party/cuda:BUILD",
44+
path = "/usr/local/cuda/",
45+
)
46+
47+
new_local_repository(
48+
name = "cublas",
49+
build_file = "@//third_party/cublas:BUILD",
50+
path = "/usr",
51+
)
52+
#############################################################################################################
53+
# Tarballs and fetched dependencies (default - use in cases when building from precompiled bin and tarballs)
54+
#############################################################################################################
55+
56+
#http_archive(
57+
# name = "libtorch",
58+
# build_file = "@//third_party/libtorch:BUILD",
59+
# sha256 = "8d9e829ce9478db4f35bdb7943308cf02e8a2f58cf9bb10f742462c1d57bf287",
60+
# strip_prefix = "libtorch",
61+
# urls = ["https://download.pytorch.org/libtorch/cu113/libtorch-cxx11-abi-shared-with-deps-1.11.0%2Bcu113.zip"],
62+
#)
63+
#
64+
#http_archive(
65+
# name = "libtorch_pre_cxx11_abi",
66+
# build_file = "@//third_party/libtorch:BUILD",
67+
# sha256 = "90159ecce3ff451f3ef3f657493b6c7c96759c3b74bbd70c1695f2ea2f81e1ad",
68+
# strip_prefix = "libtorch",
69+
# urls = ["https://download.pytorch.org/libtorch/cu113/libtorch-shared-with-deps-1.11.0%2Bcu113.zip"],
70+
#)
71+
72+
# Download these tarballs manually from the NVIDIA website
73+
# Either place them in the distdir directory in third_party and use the --distdir flag
74+
# or modify the urls to "file:///<PATH TO TARBALL>/<TARBALL NAME>.tar.gz
75+
76+
#http_archive(
77+
# name = "cudnn",
78+
# build_file = "@//third_party/cudnn/archive:BUILD",
79+
# sha256 = "0e5d2df890b9967efa6619da421310d97323565a79f05a1a8cb9b7165baad0d7",
80+
# strip_prefix = "cuda",
81+
# urls = [
82+
# "https://developer.nvidia.com/compute/machine-learning/cudnn/secure/8.2.4/11.4_20210831/cudnn-11.4-linux-x64-v8.2.4.15.tgz",
83+
# ],
84+
#)
85+
#
86+
#http_archive(
87+
# name = "tensorrt",
88+
# build_file = "@//third_party/tensorrt/archive:BUILD",
89+
# sha256 = "826180eaaecdf9a7e76116855b9f1f3400ea9b06e66b06a3f6a0747ba6f863ad",
90+
# strip_prefix = "TensorRT-8.2.4.2",
91+
# urls = [
92+
# "https://developer.nvidia.com/compute/machine-learning/tensorrt/secure/8.2.4/tars/tensorrt-8.2.4.2.linux.x86_64-gnu.cuda-11.4.cudnn8.2.tar.gz",
93+
# ],
94+
#)
95+
96+
####################################################################################
97+
# Locally installed dependencies (use in cases of custom dependencies or aarch64)
98+
####################################################################################
99+
100+
# NOTE: In the case you are using just the pre-cxx11-abi path or just the cxx11 abi path
101+
# with your local libtorch, just point deps at the same path to satisfy bazel.
102+
103+
# NOTE: NVIDIA's aarch64 PyTorch (python) wheel file uses the CXX11 ABI unlike PyTorch's standard
104+
# x86_64 python distribution. If using NVIDIA's version just point to the root of the package
105+
# for both versions here and do not use --config=pre-cxx11-abi
106+
107+
new_local_repository(
108+
name = "libtorch",
109+
path = "/opt/circleci/.pyenv/versions/3.8.13/lib/python3.8/site-packages/torch",
110+
build_file = "third_party/libtorch/BUILD"
111+
)
112+
113+
new_local_repository(
114+
name = "libtorch_pre_cxx11_abi",
115+
path = "/opt/circleci/.pyenv/versions/3.8.13/lib/python3.8/site-packages/torch",
116+
build_file = "third_party/libtorch/BUILD"
117+
)
118+
119+
new_local_repository(
120+
name = "cudnn",
121+
path = "/usr/",
122+
build_file = "@//third_party/cudnn/local:BUILD"
123+
)
124+
125+
new_local_repository(
126+
name = "tensorrt",
127+
path = "/usr/",
128+
build_file = "@//third_party/tensorrt/local:BUILD"
129+
)
130+
131+
# #########################################################################
132+
# # Testing Dependencies (optional - comment out on aarch64)
133+
# #########################################################################
134+
# pip_install(
135+
# name = "torch_tensorrt_py_deps",
136+
# requirements = "//py:requirements.txt",
137+
# )
138+
139+
# pip_install(
140+
# name = "py_test_deps",
141+
# requirements = "//tests/py:requirements.txt",
142+
# )
143+
144+
pip_install(
145+
name = "pylinter_deps",
146+
requirements = "//tools/linter:requirements.txt",
147+
)

‎WORKSPACE.ci renamed to ‎toolchains/ci_workspaces/WORKSPACE.x86_64

File renamed without changes.

0 commit comments

Comments
 (0)