diff --git a/backends/nvidia/TARGETS b/backends/nvidia/TARGETS new file mode 100644 index 00000000000..0a42614a385 --- /dev/null +++ b/backends/nvidia/TARGETS @@ -0,0 +1,5 @@ +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets() diff --git a/backends/nvidia/__init__.py b/backends/nvidia/__init__.py new file mode 100644 index 00000000000..4914f3e1f70 --- /dev/null +++ b/backends/nvidia/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""NVIDIA backend package for ExecuTorch.""" diff --git a/backends/nvidia/targets.bzl b/backends/nvidia/targets.bzl new file mode 100644 index 00000000000..dd452dc43c2 --- /dev/null +++ b/backends/nvidia/targets.bzl @@ -0,0 +1,16 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +def define_common_targets(): + """Defines targets that should be shared between fbcode and xplat. + + The directory containing this targets.bzl file should also contain both + TARGETS and BUCK files that call this function. + """ + + runtime.python_library( + name = "nvidia", + srcs = [ + "__init__.py", + ], + visibility = ["PUBLIC"], + ) diff --git a/backends/nvidia/tensorrt/CMakeLists.txt b/backends/nvidia/tensorrt/CMakeLists.txt new file mode 100644 index 00000000000..8856f0e4c3e --- /dev/null +++ b/backends/nvidia/tensorrt/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# TensorRT backend for ExecuTorch +# +# ### Editing this file ### +# +# This file should be formatted with +# ~~~ +# cmake-format -i CMakeLists.txt +# ~~~ +# It should also be cmake-lint clean. +# +# The targets in this file will be built if EXECUTORCH_BUILD_TENSORRT is ON + +cmake_minimum_required(VERSION 3.19) + +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif() + +# Source root directory for executorch. +if(NOT EXECUTORCH_ROOT) + set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..) +endif() + +include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) diff --git a/backends/nvidia/tensorrt/README.md b/backends/nvidia/tensorrt/README.md new file mode 100644 index 00000000000..bbb5dc01cc2 --- /dev/null +++ b/backends/nvidia/tensorrt/README.md @@ -0,0 +1,54 @@ +# ExecuTorch TensorRT Delegate + +This subtree contains the TensorRT Delegate implementation for ExecuTorch. +TensorRT is NVIDIA's high-performance deep learning inference optimizer and +runtime library. The delegate leverages TensorRT to accelerate model execution +on NVIDIA GPUs. + +## Prerequisites + +### TensorRT Installation + +TensorRT is required for both ahead-of-time (AOT) compilation and runtime +execution. The installation method depends on your platform: + +#### NVIDIA Jetson (aarch64) + +TensorRT is **pre-installed** via JetPack SDK. No additional installation is +required. + +To verify your TensorRT installation: +```bash +dpkg -l | grep -i tensorrt +``` + +> **Note:** Ensure you are using JetPack 6.x or later for TensorRT 10.x support. + +#### Linux x86_64 + +Install TensorRT via pip: +```bash +pip install tensorrt>=10.3 +``` + +Alternatively, download and install from the +[NVIDIA TensorRT Download Page](https://developer.nvidia.com/tensorrt). + +#### Windows x86_64 + +Download and install from the +[NVIDIA TensorRT Download Page](https://developer.nvidia.com/tensorrt). + +### Additional Requirements + +- **CUDA Toolkit**: TensorRT requires a compatible CUDA installation +- **cuDNN**: Required for certain layer optimizations +- **NVIDIA GPU**: Compute capability 7.0 or higher recommended + +## Supported Platforms + +| Platform | Architecture | TensorRT Source | +|----------|-------------|-----------------| +| Linux | x86_64 | pip or NVIDIA installer | +| Linux (Jetson) | aarch64 | Pre-installed via JetPack | +| Windows | x86_64 | NVIDIA installer | diff --git a/backends/nvidia/tensorrt/TARGETS b/backends/nvidia/tensorrt/TARGETS new file mode 100644 index 00000000000..0a42614a385 --- /dev/null +++ b/backends/nvidia/tensorrt/TARGETS @@ -0,0 +1,5 @@ +load(":targets.bzl", "define_common_targets") + +oncall("executorch") + +define_common_targets() diff --git a/backends/nvidia/tensorrt/__init__.py b/backends/nvidia/tensorrt/__init__.py new file mode 100644 index 00000000000..8b75df599b1 --- /dev/null +++ b/backends/nvidia/tensorrt/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""TensorRT backend for ExecuTorch. + +This module provides TensorRT delegate support for accelerating +PyTorch models on NVIDIA GPUs. +""" diff --git a/backends/nvidia/tensorrt/targets.bzl b/backends/nvidia/tensorrt/targets.bzl new file mode 100644 index 00000000000..c8a8755f221 --- /dev/null +++ b/backends/nvidia/tensorrt/targets.bzl @@ -0,0 +1,16 @@ +load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") + +def define_common_targets(): + """Defines targets that should be shared between fbcode and xplat. + + The directory containing this targets.bzl file should also contain both + TARGETS and BUCK files that call this function. + """ + + runtime.python_library( + name = "tensorrt", + srcs = [ + "__init__.py", + ], + visibility = ["PUBLIC"], + ) diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 1caf8ea9602..09e1f24a0ef 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -161,6 +161,9 @@ define_overridable_option( define_overridable_option( EXECUTORCH_BUILD_SIZE_TEST "Build the size test" BOOL OFF ) +define_overridable_option( + EXECUTORCH_BUILD_TENSORRT "Build the TensorRT backend" BOOL OFF +) define_overridable_option( EXECUTORCH_BUILD_XNNPACK "Build the XNNPACK backend" BOOL OFF )