Skip to content

Commit fe7fc94

Browse files
peri044zewenli98narendasan
authored
feat: Implement FP8 functionality (#2763)
Co-authored-by: Evan Li <[email protected]> Co-authored-by: Naren Dasan <[email protected]>
1 parent 856f33d commit fe7fc94

30 files changed

+522
-73
lines changed

‎.github/scripts/install-torch-tensorrt-windows.sh

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,12 @@
1+
#!/usr/bin/env bash
12
set -eou pipefail
2-
source "${BUILD_ENV_FILE}"
3+
# Source conda so it's available to the script environment
4+
source ${BUILD_ENV_FILE}
5+
export EXTRA_INDEX_URL="https://download.pytorch.org/whl/test/${CU_VERSION}"
6+
# Install all the dependencies required for Torch-TensorRT
7+
${CONDA_RUN} pip install --pre -r ${PWD}/tests/py/requirements.txt --use-deprecated=legacy-resolver --extra-index-url=${EXTRA_INDEX_URL}
38

4-
# Install test index version of Torch and Torchvision
5-
${CONDA_RUN} ${PIP_INSTALL_TORCH} torchvision
6-
${CONDA_RUN} pip install pyyaml mpmath==1.3.0
7-
8-
# Install TRT 10 from PyPi
9-
${CONDA_RUN} pip install tensorrt==10.0.0b6 tensorrt-${CU_VERSION::4}-bindings==10.0.0b6 tensorrt-${CU_VERSION::4}-libs==10.0.0b6 --extra-index-url https://pypi.nvidia.com
10-
11-
# Install pre-built Torch-TRT
9+
# Install Torch-TensorRT via pre-built wheels. On windows, the location of wheels is not fixed.
1210
${CONDA_RUN} pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl
1311

14-
echo -e "Running test script";
12+
echo -e "Running test script";

‎.github/scripts/install-torch-tensorrt.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
set -eou pipefail
33
# Source conda so it's available to the script environment
44
source ${BUILD_ENV_FILE}
5-
${CONDA_RUN} ${PIP_INSTALL_TORCH} torchvision
6-
${CONDA_RUN} python -m pip install pyyaml mpmath==1.3.0
7-
export TRT_VERSION=$(${CONDA_RUN} python -c "import versions; versions.tensorrt_version()")
5+
export EXTRA_INDEX_URL="https://download.pytorch.org/whl/test/${CU_VERSION}"
6+
# Install all the dependencies required for Torch-TensorRT
7+
${CONDA_RUN} pip install --pre -r ${PWD}/tests/py/requirements.txt --use-deprecated=legacy-resolver --extra-index-url=${EXTRA_INDEX_URL}
88

9-
# Install Torch-TensorRT
10-
${CONDA_RUN} python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*+${CU_VERSION}*.whl tensorrt~=${TRT_VERSION} --extra-index-url=https://pypi.ngc.nvidia.com
9+
# Install Torch-TensorRT via pre-built wheels. On windows, the location of wheels is not fixed.
10+
${CONDA_RUN} pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl
1111

12-
echo -e "Running test script";
12+
echo -e "Running test script";

‎.github/workflows/build-test-linux.yml

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ jobs:
6666
package-name: torch_tensorrt
6767
pre-script: packaging/pre_build_script.sh
6868
post-script: packaging/post_build_script.sh
69+
smoke-test-script: packaging/smoke_test_script.sh
6970
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
7071
with:
7172
job-name: tests-py-torchscript-fe
@@ -80,13 +81,10 @@ jobs:
8081
export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH
8182
pushd .
8283
cd tests/modules
83-
# Don't use requirements.txt here as it contains tensorrt and torch which should have been installed by now.
84-
${CONDA_RUN} python -m pip install numpy packaging pyyaml transformers==4.39.3 timm==0.9.16 pybind11==2.6.2
8584
${CONDA_RUN} python hub.py
8685
popd
8786
pushd .
8887
cd tests/py/ts
89-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
9088
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_api_test_results.xml api/
9189
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_models_test_results.xml models/
9290
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/ts_integrations_test_results.xml integrations/
@@ -103,6 +101,7 @@ jobs:
103101
package-name: torch_tensorrt
104102
pre-script: packaging/pre_build_script.sh
105103
post-script: packaging/post_build_script.sh
104+
smoke-test-script: packaging/smoke_test_script.sh
106105
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
107106
with:
108107
job-name: tests-py-dynamo-converters
@@ -116,7 +115,6 @@ jobs:
116115
export USE_HOST_DEPS=1
117116
pushd .
118117
cd tests/py/dynamo
119-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
120118
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 10 conversion/
121119
popd
122120
@@ -131,6 +129,7 @@ jobs:
131129
package-name: torch_tensorrt
132130
pre-script: packaging/pre_build_script.sh
133131
post-script: packaging/post_build_script.sh
132+
smoke-test-script: packaging/smoke_test_script.sh
134133
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
135134
with:
136135
job-name: tests-py-dynamo-fe
@@ -144,7 +143,6 @@ jobs:
144143
export USE_HOST_DEPS=1
145144
pushd .
146145
cd tests/py/dynamo
147-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
148146
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_fe_test_results.xml --ir dynamo models/test_models_export.py
149147
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/test_dyn_models.py
150148
popd
@@ -160,6 +158,7 @@ jobs:
160158
package-name: torch_tensorrt
161159
pre-script: packaging/pre_build_script.sh
162160
post-script: packaging/post_build_script.sh
161+
smoke-test-script: packaging/smoke_test_script.sh
163162
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
164163
with:
165164
job-name: tests-py-dynamo-serde
@@ -173,7 +172,6 @@ jobs:
173172
export USE_HOST_DEPS=1
174173
pushd .
175174
cd tests/py/dynamo
176-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
177175
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/export_serde_test_results.xml --ir dynamo models/test_export_serde.py
178176
popd
179177
@@ -188,6 +186,7 @@ jobs:
188186
package-name: torch_tensorrt
189187
pre-script: packaging/pre_build_script.sh
190188
post-script: packaging/post_build_script.sh
189+
smoke-test-script: packaging/smoke_test_script.sh
191190
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
192191
with:
193192
job-name: tests-py-torch-compile-be
@@ -201,7 +200,6 @@ jobs:
201200
export USE_HOST_DEPS=1
202201
pushd .
203202
cd tests/py/dynamo
204-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
205203
${CONDA_RUN} python -m pytest -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/
206204
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_comple_be_e2e_test_results.xml --ir torch_compile models/test_models.py
207205
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_dyn_models_export.xml --ir torch_compile models/test_dyn_models.py
@@ -218,6 +216,7 @@ jobs:
218216
package-name: torch_tensorrt
219217
pre-script: packaging/pre_build_script.sh
220218
post-script: packaging/post_build_script.sh
219+
smoke-test-script: packaging/smoke_test_script.sh
221220
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
222221
with:
223222
job-name: tests-py-dynamo-core
@@ -231,7 +230,6 @@ jobs:
231230
export USE_HOST_DEPS=1
232231
pushd .
233232
cd tests/py/dynamo
234-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
235233
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml runtime/
236234
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/
237235
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/
@@ -247,7 +245,9 @@ jobs:
247245
- repository: pytorch/tensorrt
248246
package-name: torch_tensorrt
249247
pre-script: packaging/pre_build_script.sh
250-
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@main
248+
post-script: packaging/post_build_script.sh
249+
smoke-test-script: packaging/smoke_test_script.sh
250+
uses: pytorch/tensorrt/.github/workflows/linux-test.yml@release/2.3
251251
with:
252252
job-name: tests-py-core
253253
repository: "pytorch/tensorrt"
@@ -260,6 +260,5 @@ jobs:
260260
export USE_HOST_DEPS=1
261261
pushd .
262262
cd tests/py/core
263-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
264263
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_core_test_results.xml .
265264
popd

‎.github/workflows/build-test-windows.yml

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@ jobs:
7272
export USE_HOST_DEPS=1
7373
pushd .
7474
cd tests/py/dynamo
75-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
7675
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 10 conversion/
7776
popd
7877
@@ -98,7 +97,6 @@ jobs:
9897
export USE_HOST_DEPS=1
9998
pushd .
10099
cd tests/py/dynamo
101-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
102100
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_fe_test_results.xml --ir dynamo models/test_models_export.py
103101
${CONDA_RUN} python -m pytest --junitxml=${RUNNER_TEST_RESULTS_DIR}/dyn_models_export.xml --ir dynamo models/test_dyn_models.py
104102
popd
@@ -125,7 +123,6 @@ jobs:
125123
export USE_HOST_DEPS=1
126124
pushd .
127125
cd tests/py/dynamo
128-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
129126
${CONDA_RUN} python -m pytest -n 10 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_compile_be_test_results.xml backend/
130127
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/torch_comple_be_e2e_test_results.xml --ir torch_compile models/test_models.py
131128
popd
@@ -152,7 +149,6 @@ jobs:
152149
export USE_HOST_DEPS=1
153150
pushd .
154151
cd tests/py/dynamo
155-
${CONDA_RUN} python -m pip install --pre -r ../requirements.txt
156152
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_runtime_test_results.xml runtime/
157153
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_partitioning_test_results.xml partitioning/
158154
${CONDA_RUN} python -m pytest -n 4 --junitxml=${RUNNER_TEST_RESULTS_DIR}/tests_py_dynamo_core_lowering_test_results.xml lowering/

‎dev_dep_versions.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@ __version__: "2.3.0"
22
__cuda_version__: "12.1"
33
__cudnn_version__: "8.9"
44
__tensorrt_version__: "10.0.1"
5+
__torch_version__: "2.3.0"
6+
# torchvision version here is not a direct dependency but the one used during testing
7+
__torchvision_version__: "0.18.0"
8+
__index_url__: "https://download.pytorch.org/whl/test/"

‎docsrc/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ Tutorials
111111
tutorials/_rendered_examples/dynamo/torch_compile_transformers_example
112112
tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage
113113
tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion
114+
tutorials/_rendered_examples/dynamo/vgg16_fp8_ptq
114115

115116
Python API Documenation
116117
------------------------

‎examples/dynamo/README.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ a number of ways you can leverage this backend to accelerate inference.
1010
* :ref:`torch_compile_transformer`: Compiling a Transformer model using ``torch.compile``
1111
* :ref:`torch_compile_advanced_usage`: Advanced usage including making a custom backend to use directly with the ``torch.compile`` API
1212
* :ref:`torch_compile_stable_diffusion`: Compiling a Stable Diffusion model using ``torch.compile``
13+
* :ref:`vgg16_fp8_ptq`: Compiling a VGG16 model with FP8 and PTQ using ``torch.compile``

0 commit comments

Comments
 (0)