bump to cuda 12.8 and TRT 10.8

Author: zewenli98

.github/scripts/generate-tensorrt-test-matrix.py

@@ -11,9 +11,9 @@
 # channel: nightly if the future tensorRT version test workflow is triggered from the main branch or your personal branch
 # channel: test if the future tensorRT version test workflow is triggered from the release branch(release/2.5 etc....)
 CUDA_VERSIONS_DICT = {
-    "nightly": ["cu126"],
-    "test": ["cu124", "cu126"],
-    "release": ["cu124", "cu126"],
+    "nightly": ["cu128"],
+    "test": ["cu124", "cu126", "cu128"],
+    "release": ["cu124", "cu126", "cu128"],
 }
 
 # please update the python version you want to test with the future tensorRT version here
@@ -28,14 +28,6 @@
 # please update the future tensorRT version you want to test here
 TENSORRT_VERSIONS_DICT = {
     "windows": {
-        "10.4.0": {
-            "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.4.0/zip/TensorRT-10.4.0.26.Windows.win10.cuda-12.6.zip",
-            "strip_prefix": "TensorRT-10.4.0.26",
-        },
-        "10.5.0": {
-            "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.5.0/zip/TensorRT-10.5.0.18.Windows.win10.cuda-12.6.zip",
-            "strip_prefix": "TensorRT-10.5.0.18",
-        },
         "10.6.0": {
             "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.6.0/zip/TensorRT-10.6.0.26.Windows.win10.cuda-12.6.zip",
             "strip_prefix": "TensorRT-10.6.0.26",
@@ -44,16 +36,12 @@
             "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/zip/TensorRT-10.7.0.23.Windows.win10.cuda-12.6.zip",
             "strip_prefix": "TensorRT-10.7.0.23",
         },
+        "10.8.0": {
+            "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-10.8.0.43.Windows.win10.cuda-12.8.zip",
+            "strip_prefix": "TensorRT-10.8.0.43",
+        },
     },
     "linux": {
-        "10.4.0": {
-            "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.4.0/tars/TensorRT-10.4.0.26.Linux.x86_64-gnu.cuda-12.6.tar.gz",
-            "strip_prefix": "TensorRT-10.4.0.26",
-        },
-        "10.5.0": {
-            "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.5.0/tars/TensorRT-10.5.0.18.Linux.x86_64-gnu.cuda-12.6.tar.gz",
-            "strip_prefix": "TensorRT-10.5.0.18",
-        },
         "10.6.0": {
             "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.6.0/tars/TensorRT-10.6.0.26.Linux.x86_64-gnu.cuda-12.6.tar.gz",
             "strip_prefix": "TensorRT-10.6.0.26",
@@ -62,6 +50,10 @@
             "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz",
             "strip_prefix": "TensorRT-10.7.0.23",
         },
+        "10.8.0": {
+            "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz",
+            "strip_prefix": "TensorRT-10.8.0.43",
+        },
     },
 }
 
@@ -87,7 +79,7 @@ def check_file_availability(url: str) -> bool:
     # calculate the next minor version
     minor = int(list(TENSORRT_VERSIONS_DICT["linux"].keys())[-1].split(".")[1]) + 1
     trt_version = f"{major}.{minor}.0"
-    for patch in range(patch_from, 50):
+    for patch in range(patch_from, 80):
         for cuda_minor in range(4, 11):
             trt_linux_release_url_candidate = f"https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/{trt_version}/tars/TensorRT-{trt_version}.{patch}.Linux.x86_64-gnu.cuda-12.{cuda_minor}.tar.gz"
             if check_file_availability(trt_linux_release_url_candidate):

.github/scripts/generate_binary_build_matrix.py

@@ -24,7 +24,7 @@
     "release": ["3.9", "3.10", "3.11", "3.12"],
 }
 CUDA_ARCHES_DICT = {
-    "nightly": ["11.8", "12.4", "12.6"],
+    "nightly": ["11.8", "12.4", "12.6", "12.8"],
     "test": ["11.8", "12.1", "12.4"],
     "release": ["11.8", "12.1", "12.4"],
 }
@@ -34,12 +34,12 @@
     "release": ["6.1", "6.2"],
 }
 
-CUDA_CUDDN_VERSIONS = {
-    "11.8": {"cuda": "11.8.0", "cudnn": "9"},
-    "12.1": {"cuda": "12.1.1", "cudnn": "9"},
-    "12.4": {"cuda": "12.4.1", "cudnn": "9"},
-    "12.6": {"cuda": "12.6.2", "cudnn": "9"},
-}
+# CUDA_CUDDN_VERSIONS = {
+#     "11.8": {"cuda": "11.8.0", "cudnn": "9"},
+#     "12.1": {"cuda": "12.1.1", "cudnn": "9"},
+#     "12.4": {"cuda": "12.4.1", "cudnn": "9"},
+#     "12.6": {"cuda": "12.6.2", "cudnn": "9"},
+# }
 
 PACKAGE_TYPES = ["wheel", "conda", "libtorch"]
 PRE_CXX11_ABI = "pre-cxx11"
@@ -151,6 +151,7 @@ def initialize_globals(channel: str, build_python_only: bool) -> None:
         "12.1": "pytorch/manylinux2_28-builder:cuda12.1",
         "12.4": "pytorch/manylinux2_28-builder:cuda12.4",
         "12.6": "pytorch/manylinux2_28-builder:cuda12.6",
+        "12.8": "pytorch/manylinux2_28-builder:cuda12.8",
         **{
             gpu_arch: f"pytorch/manylinux2_28-builder:rocm{gpu_arch}"
             for gpu_arch in ROCM_ARCHES
@@ -278,7 +279,7 @@ def get_wheel_install_command(
             return f"{WHL_INSTALL_BASE} {PACKAGES_TO_INSTALL_WHL} --index-url {get_base_download_url_for_repo('whl', channel, gpu_arch_type, desired_cuda)}_pypi_pkg"  # noqa: E501
         else:
             raise ValueError(
-                "Split build is not supported for this configuration. It is only supported for CUDA 11.8, 12.4, 12.6 on Linux nightly builds."  # noqa: E501
+                "Split build is not supported for this configuration. It is only supported for CUDA 11.8, 12.4, 12.6, 12.8 on Linux nightly builds."  # noqa: E501
             )
     if (
         channel == RELEASE

.github/workflows/docgen.yml

@@ -14,10 +14,10 @@ jobs:
         if: ${{ ! contains(github.actor, 'pytorchbot') }}
         environment: pytorchbot-env
         container:
-            image: docker.io/pytorch/manylinux2_28-builder:cuda12.6
+            image: docker.io/pytorch/manylinux2_28-builder:cuda12.8
             options: --gpus all
         env:
-            CUDA_HOME: /usr/local/cuda-12.6
+            CUDA_HOME: /usr/local/cuda-12.8
             VERSION_SUFFIX: cu126
             CU_VERSION: cu126
             CHANNEL: nightly

MODULE.bazel

@@ -36,13 +36,13 @@ new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.
 new_local_repository(
     name = "cuda",
     build_file = "@//third_party/cuda:BUILD",
-    path = "/usr/local/cuda-12.6/",
+    path = "/usr/local/cuda-12.8/",
 )
 
 new_local_repository(
     name = "cuda_win",
     build_file = "@//third_party/cuda:BUILD",
-    path = "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.6/",
+    path = "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.8/",
 )
 
 http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
@@ -79,18 +79,18 @@ http_archive(
 http_archive(
     name = "tensorrt",
     build_file = "@//third_party/tensorrt/archive:BUILD",
-    strip_prefix = "TensorRT-10.7.0.23",
+    strip_prefix = "TensorRT-10.8.0.43",
     urls = [
-        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz",
+        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz",
     ],
 )
 
 http_archive(
     name = "tensorrt_win",
     build_file = "@//third_party/tensorrt/archive:BUILD",
-    strip_prefix = "TensorRT-10.7.0.23",
+    strip_prefix = "TensorRT-10.8.0.43",
     urls = [
-        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/zip/TensorRT-10.7.0.23.Windows.win10.cuda-12.6.zip",
+        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-10.8.0.43.Windows.win10.cuda-12.8.zip",
     ],
 )
 

README.md

@@ -7,7 +7,7 @@ Torch-TensorRT
 [![Documentation](https://img.shields.io/badge/docs-master-brightgreen)](https://nvidia.github.io/Torch-TensorRT/)
 [![pytorch](https://img.shields.io/badge/PyTorch-2.4-green)](https://www.python.org/downloads/release/python-31013/)
 [![cuda](https://img.shields.io/badge/CUDA-12.4-green)](https://developer.nvidia.com/cuda-downloads)
-[![trt](https://img.shields.io/badge/TensorRT-10.7.0-green)](https://github.com/nvidia/tensorrt-llm)
+[![trt](https://img.shields.io/badge/TensorRT-10.8.0-green)](https://github.com/nvidia/tensorrt-llm)
 [![license](https://img.shields.io/badge/license-BSD--3--Clause-blue)](./LICENSE)
 [![linux_tests](https://github.com/pytorch/TensorRT/actions/workflows/build-test-linux.yml/badge.svg)](https://github.com/pytorch/TensorRT/actions/workflows/build-test-linux.yml)
 [![windows_tests](https://github.com/pytorch/TensorRT/actions/workflows/build-test-windows.yml/badge.svg)](https://github.com/pytorch/TensorRT/actions/workflows/build-test-windows.yml)
@@ -119,7 +119,7 @@ These are the following dependencies used to verify the testcases. Torch-TensorR
 - Bazel 6.3.2
 - Libtorch 2.5.0.dev (latest nightly) (built with CUDA 12.4)
 - CUDA 12.4
-- TensorRT 10.7.0.23
+- TensorRT 10.8.0.43
 
 ## Deprecation Policy
 

dev_dep_versions.yml

@@ -1,2 +1,2 @@
-__cuda_version__: "12.6"
-__tensorrt_version__: "10.7.0.post1"
+__cuda_version__: "12.8"
+__tensorrt_version__: "10.8.0"

docker/README.md

@@ -7,7 +7,7 @@
   * [**Optional**] The desired base image be changed by explicitly setting a base image, as in `--build-arg BASE_IMG=nvidia/cuda:11.8.0-devel-ubuntu22.04`, though this is optional.
   * [**Optional**] Additionally, the desired Python version can be changed by explicitly setting a version, as in `--build-arg PYTHON_VERSION=3.10`, though this is optional as well.
 
-* This `Dockerfile` installs `cxx11-abi` versions of Pytorch and builds Torch-TRT using `cxx11-abi` libtorch as well. As of torch 2.7, torch requires `cxx11-abi` for all CUDA 11.8, 12.4, and 12.6.
+* This `Dockerfile` installs `cxx11-abi` versions of Pytorch and builds Torch-TRT using `cxx11-abi` libtorch as well. As of torch 2.7, torch requires `cxx11-abi` for all CUDA 11.8, 12.4, 12.6, and later versions.
 
 Note: By default the container uses the `cxx11-abi` version of Torch + Torch-TRT. If you are using a workflow that requires a build of PyTorch on the PRE CXX11 ABI, please add the Docker build argument: `--build-arg USE_PRE_CXX11_ABI=1`
 
@@ -17,14 +17,14 @@ Note: By default the container uses the `cxx11-abi` version of Torch + Torch-TRT
 
 ### Instructions
 
-- The example below uses TensorRT 10.7.0.23
+- The example below uses TensorRT 10.8.0.43
 - See <a href="https://github.com/pytorch/TensorRT#dependencies">dependencies</a> for a list of current default dependencies.
 
 > From root of Torch-TensorRT repo
 
 Build:
 ```
-DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.7.0 -f docker/Dockerfile -t torch_tensorrt:latest .
+DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.8.0 -f docker/Dockerfile -t torch_tensorrt:latest .
 ```
 
 Run:

examples/dynamo/auto_generate_converters.py

@@ -5,7 +5,7 @@
 ===================================================================
 
 We are going to demonstrate how to automatically generate a converter for a custom kernel using Torch-TensorRT using
-the new Python based plugin system in TensorRT 10.7.
+the new Python based plugin system in TensorRT 10.8.
 
 Torch-TensorRT supports falling back to PyTorch implementations of operations in the case that Torch-TensorRT
 does not know how to compile them in TensorRT. However, this comes at the cost of a graph break and will reduce the performance of the model.
@@ -21,7 +21,7 @@
 the performance and resource overhead from a graph break.
 
 Previously this involved a complex process in not only building a performant kernel but setting it up to run in TensorRT (see: `Using Custom Kernels within TensorRT Engines with Torch-TensorRT <https://pytorch.org/TensorRT/tutorials/_rendered_examples/dynamo/custom_kernel_plugins.html>`_).
-With TensorRT 10.7, there is a new Python native plugin system which greatly streamlines this process. This
+With TensorRT 10.8, there is a new Python native plugin system which greatly streamlines this process. This
 plugin system also allows Torch-TensorRT to automatically generate the necessary conversion code to convert the
 operation in PyTorch to TensorRT.
 """
@@ -99,7 +99,7 @@ def _(x: torch.Tensor, y: torch.Tensor, b: float = 0.2, a: int = 2) -> torch.Ten
 # %%
 # Writing Plugins for TensorRT using the Quick Deploy Plugin system
 # -------------------------------------------------------------------
-# The quick deployment plugin system in TensorRT 10.7 allows for the creation of custom plugins in Python with significantly
+# The quick deployment plugin system in TensorRT 10.8 allows for the creation of custom plugins in Python with significantly
 # less boilerplate. It uses a similar system PyTorch where you define a function that describes the shape and data type transformations
 # that the operator will perform and then define the code to launch the kernel given GPU memory handles.
 #

packaging/pre_build_script.sh

@@ -24,7 +24,7 @@ export TORCH_INSTALL_PATH=$(python -c "import torch, os; print(os.path.dirname(t
 if [[ ${TENSORRT_VERSION} != "" ]]; then
   # this is the upgraded TensorRT version, replace current tensorrt version to the upgrade tensorRT version in the pyproject.toml
   # example: __tensorrt_version__: ">=10.3.0,<=10.6.0"
-  # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.7.0
+  # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.8.0
   current_version=$(cat dev_dep_versions.yml | grep __tensorrt_version__ | sed 's/__tensorrt_version__: //g' | sed 's/"//g')
   sed -i -e "s/tensorrt-cu12${current_version}/tensorrt-cu12==${TENSORRT_VERSION}/g" \
          -e "s/tensorrt-cu12-bindings${current_version}/tensorrt-cu12-bindings==${TENSORRT_VERSION}/g" \

packaging/pre_build_script_windows.sh

@@ -9,7 +9,7 @@ echo TENSORRT_VERSION=${TENSORRT_VERSION}
 if [[ ${TENSORRT_VERSION} != "" ]]; then
   # this is the upgraded TensorRT version, replace current tensorrt version to the upgrade tensorRT version in the pyproject.toml
   # example: __tensorrt_version__: ">=10.3.0,<=10.6.0"
-  # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.7.0
+  # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.8.0
   current_version=$(cat dev_dep_versions.yml | grep __tensorrt_version__ | sed 's/__tensorrt_version__: //g' | sed 's/"//g')
   sed -i -e "s/tensorrt-cu12${current_version}/tensorrt-cu12==${TENSORRT_VERSION}/g" \
          -e "s/tensorrt-cu12-bindings${current_version}/tensorrt-cu12-bindings==${TENSORRT_VERSION}/g" \

packaging/smoke_test_script.sh

@@ -2,5 +2,5 @@
 # The issue was smoke test installs the built torch_tensorrt wheel file and checks `import torch_tensorrt; print(torch_tensorrt.__version__)`
 # Since tensorrt cannot be pip installable in CI, the smoke test will fail.
 # One way we tried to handle it is manually install tensorrt wheel while by extracting from the tarball.
-# However, the TensorRT-10.7.0.23/lib path doesn't seem to show up in LD_LIBRARY_PATH even if we explicitly set it.
+# However, the TensorRT-10.8.0.43/lib path doesn't seem to show up in LD_LIBRARY_PATH even if we explicitly set it.
 # TODO: Implement a custom smoke_test script to verify torch_tensorrt installation.

py/ci/Dockerfile.ci

@@ -1,15 +1,15 @@
-FROM pytorch/manylinux2_28-builder:cuda12.6
+FROM pytorch/manylinux2_28-builder:cuda12.8
 
 RUN yum install -y ninja-build
 
 # download TensorRT tarball
-RUN wget -q https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz \
-&& gunzip TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz \
-&& tar -xvf TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar \
-&& rm TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar
+RUN wget -q https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz \
+&& gunzip TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz \
+&& tar -xvf TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar \
+&& rm TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar
 
-ENV TENSORRT_DIR=/TensorRT-10.7.0.23
-ENV TENSORRT_VERSION=10.7.0
+ENV TENSORRT_DIR=/TensorRT-10.8.0.43
+ENV TENSORRT_VERSION=10.8.0
 
 RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-amd64 \
     && mv bazelisk-linux-amd64 /usr/bin/bazel \

pyproject.toml

@@ -8,7 +8,7 @@ requires = [
     "cffi>=1.15.1",
     "typing-extensions>=4.7.0",
     "future>=0.18.3",
-    "tensorrt-cu12>=10.7.0.post1,<10.8.0",
+    "tensorrt-cu12>=10.8.0,<10.9.0",
     "torch>=2.7.0.dev,<2.8.0",
     "pybind11==2.6.2",
     "numpy",
@@ -55,10 +55,10 @@ keywords = [
 ]
 dependencies = [
     "torch>=2.7.0.dev,<2.8.0",
-    "tensorrt>=10.7.0.post1,<10.8.0",
-    "tensorrt-cu12>=10.7.0.post1,<10.8.0",
-    "tensorrt-cu12-bindings>=10.7.0,<10.8.0",
-    "tensorrt-cu12-libs>=10.7.0,<10.8.0",
+    "tensorrt>=10.8.0,<10.9.0",
+    "tensorrt-cu12>=10.8.0,<10.9.0",
+    "tensorrt-cu12-bindings>=10.8.0,<10.9.0",
+    "tensorrt-cu12-libs>=10.8.0,<10.9.0",
     "packaging>=23",
     "numpy",
     "typing-extensions>=4.7.0",

toolchains/ci_workspaces/MODULE.bazel.tmpl

@@ -67,18 +67,18 @@ http_archive(
 http_archive(
     name = "tensorrt",
     build_file = "@//third_party/tensorrt/archive:BUILD",
-    strip_prefix = "TensorRT-10.7.0.23",
+    strip_prefix = "TensorRT-10.8.0.43",
     urls = [
-        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz",
+        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz",
     ],
 )
 
 http_archive(
     name = "tensorrt_win",
     build_file = "@//third_party/tensorrt/archive:BUILD",
-    strip_prefix = "TensorRT-10.7.0.23",
+    strip_prefix = "TensorRT-10.8.0.43",
     urls = [
-        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/zip/TensorRT-10.7.0.23.Windows.win10.cuda-12.6.zip",
+        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-10.8.0.43.Windows.win10.cuda-12.8.zip",
     ],
 )
 

toolchains/legacy/WORKSPACE.win.release.tmpl

@@ -63,7 +63,7 @@ http_archive(
 
 new_local_repository(
    name = "tensorrt_win",
-   path = "C:/TensorRT-10.7.0.23",
+   path = "C:/TensorRT-10.8.0.43",
    build_file = "@//third_party/tensorrt/local:BUILD"
 )
 

toolchains/legacy/WORKSPACE.x86_64.release.rhel.tmpl

@@ -71,9 +71,9 @@ http_archive(
 http_archive(
     name = "tensorrt",
     build_file = "@//third_party/tensorrt/archive:BUILD",
-    strip_prefix = "TensorRT-10.7.0.23",
+    strip_prefix = "TensorRT-10.8.0.43",
     urls = [
-        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz",
+        "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz",
     ],
 )