Skip to content

Commit 28de1ec

Browse files
tanmayv25tanmayv25
authored
TensorRT backend initial implementation (#1)
* Build and simple model load * Initial implementation * Clean up * Fix internmittent segfaults
1 parent f8fc67b commit 28de1ec

16 files changed

+6040
-0
lines changed

‎.clang-format

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
BasedOnStyle: Google
3+
4+
IndentWidth: 2
5+
ContinuationIndentWidth: 4
6+
UseTab: Never
7+
MaxEmptyLinesToKeep: 2
8+
9+
SortIncludes: true
10+
CompactNamespaces: true
11+
ReflowComments: true
12+
13+
DerivePointerAlignment: false
14+
PointerAlignment: Left
15+
16+
AllowShortIfStatementsOnASingleLine: false
17+
AllowShortBlocksOnASingleLine: false
18+
AllowShortFunctionsOnASingleLine: Inline
19+
20+
AlwaysBreakAfterReturnType: TopLevelDefinitions
21+
AlignAfterOpenBracket: AlwaysBreak
22+
BreakBeforeBraces: Custom
23+
BraceWrapping:
24+
AfterClass: false
25+
AfterControlStatement: false
26+
AfterEnum: false
27+
AfterFunction: true
28+
AfterNamespace: false
29+
AfterStruct: false
30+
AfterUnion: false
31+
BeforeCatch: true
32+
33+
BinPackArguments: true
34+
BinPackParameters: true
35+
ConstructorInitializerAllOnOneLineOrOnePerLine: false
36+
37+
IndentCaseLabels: true

‎CMakeLists.txt

+229
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,229 @@
1+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
cmake_minimum_required(VERSION 3.17)
28+
29+
project(tritontensorrtbackend LANGUAGES C CXX)
30+
31+
#
32+
# Options
33+
#
34+
option(TRITON_ENABLE_GPU "Enable GPU support in backend." ON)
35+
option(TRITON_ENABLE_STATS "Include statistics collections in backend." ON)
36+
set(TRITON_TENSORRT_LIB_PATHS "" CACHE PATH "Paths to TensorRT libraries. Multiple paths may be specified by separating them with a semicolon.")
37+
set(TRITON_TENSORRT_INCLUDE_PATHS "" CACHE PATH "Paths to TensorRT includes. Multiple paths may be specified by separating them with a semicolon.")
38+
39+
set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo.")
40+
set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo.")
41+
set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo.")
42+
43+
if(NOT CMAKE_BUILD_TYPE)
44+
set(CMAKE_BUILD_TYPE Release)
45+
endif()
46+
47+
set(TRITON_TENSORRT_BACKEND_LIBNAME triton_tensorrt)
48+
set(TRITON_TENSORRT_BACKEND_INSTALLDIR ${CMAKE_INSTALL_PREFIX}/backends/tensorrt)
49+
50+
#
51+
# Dependencies
52+
#
53+
# FetchContent's composibility isn't very good. We must include the
54+
# transitive closure of all repos so that we can override the tag.
55+
#
56+
include(FetchContent)
57+
58+
FetchContent_Declare(
59+
repo-common
60+
GIT_REPOSITORY https://github.com/triton-inference-server/common.git
61+
GIT_TAG ${TRITON_COMMON_REPO_TAG}
62+
GIT_SHALLOW ON
63+
)
64+
FetchContent_Declare(
65+
repo-core
66+
GIT_REPOSITORY https://github.com/triton-inference-server/core.git
67+
GIT_TAG ${TRITON_CORE_REPO_TAG}
68+
GIT_SHALLOW ON
69+
)
70+
FetchContent_Declare(
71+
repo-backend
72+
GIT_REPOSITORY https://github.com/triton-inference-server/backend.git
73+
GIT_TAG ${TRITON_BACKEND_REPO_TAG}
74+
GIT_SHALLOW ON
75+
)
76+
FetchContent_MakeAvailable(repo-common repo-core repo-backend)
77+
78+
#
79+
# CUDA
80+
#
81+
if(${TRITON_ENABLE_GPU})
82+
find_package(CUDAToolkit REQUIRED)
83+
message(STATUS "Using CUDA ${CUDA_VERSION}")
84+
set(CUDA_NVCC_FLAGS -std=c++11)
85+
86+
if(CUDA_VERSION VERSION_GREATER "10.1" OR CUDA_VERSION VERSION_EQUAL "10.1")
87+
add_definitions(-DTRITON_ENABLE_CUDA_GRAPH=1)
88+
else()
89+
message(WARNING "CUDA ${CUDA_VERSION} does not support CUDA graphs.")
90+
endif()
91+
else()
92+
message( FATAL_ERROR "TensorRT backend requires TRITON_ENABLE_GPU=1, CMake will exit." )
93+
endif() # TRITON_ENABLE_GPU
94+
95+
#
96+
# Shared library implementing the Triton Backend API
97+
#
98+
configure_file(src/libtriton_tensorrt.ldscript libtriton_tensorrt.ldscript COPYONLY)
99+
100+
add_library(
101+
triton-tensorrt-backend SHARED
102+
src/tensorrt.cc
103+
src/tensorrt_model.cc
104+
src/tensorrt_model.h
105+
src/tensorrt_model_instance.cc
106+
src/tensorrt_model_instance.h
107+
src/tensorrt_utils.cc
108+
src/tensorrt_utils.h
109+
src/loader.cc
110+
src/loader.h
111+
src/logging.cc
112+
src/logging.h
113+
)
114+
115+
add_library(
116+
TritonTensorRTBackend::triton-tensorrt-backend ALIAS triton-tensorrt-backend
117+
)
118+
119+
target_include_directories(
120+
triton-tensorrt-backend
121+
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src
122+
)
123+
124+
target_include_directories(
125+
triton-tensorrt-backend
126+
PRIVATE ${TRITON_TENSORRT_INCLUDE_PATHS}
127+
)
128+
129+
target_compile_features(triton-tensorrt-backend PRIVATE cxx_std_11)
130+
target_compile_options(
131+
triton-tensorrt-backend PRIVATE
132+
$<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
133+
-Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
134+
)
135+
136+
# C/C++ defines that are used directly by this backend.
137+
target_compile_definitions(
138+
triton-tensorrt-backend
139+
PRIVATE TRITON_ENABLE_GPU=1
140+
)
141+
142+
set_target_properties(
143+
triton-tensorrt-backend
144+
PROPERTIES
145+
POSITION_INDEPENDENT_CODE ON
146+
OUTPUT_NAME ${TRITON_TENSORRT_BACKEND_LIBNAME}
147+
SKIP_BUILD_RPATH TRUE
148+
BUILD_WITH_INSTALL_RPATH TRUE
149+
INSTALL_RPATH_USE_LINK_PATH FALSE
150+
INSTALL_RPATH "$\{ORIGIN\}"
151+
LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_tensorrt.ldscript
152+
LINK_FLAGS "-Wl,--version-script libtriton_tensorrt.ldscript"
153+
)
154+
155+
FOREACH(p ${TRITON_TENSORRT_LIB_PATHS})
156+
set(TRITON_TENSORRT_LDFLAGS ${TRITON_TENSORRT_LDFLAGS} "-L${p}")
157+
ENDFOREACH(p)
158+
159+
160+
target_link_libraries(
161+
triton-tensorrt-backend
162+
PRIVATE
163+
triton-core-serverapi # from repo-core
164+
triton-core-serverstub # from repo-core
165+
triton-backend-utils # from repo-backend
166+
-lpthread
167+
)
168+
169+
target_link_libraries(
170+
triton-tensorrt-backend
171+
PRIVATE ${TRITON_TENSORRT_LDFLAGS}
172+
)
173+
174+
target_link_libraries(
175+
triton-tensorrt-backend
176+
PRIVATE
177+
CUDA::cudart
178+
)
179+
180+
181+
#
182+
# Install
183+
#
184+
include(GNUInstallDirs)
185+
set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonTensorRTBackend)
186+
187+
install(
188+
TARGETS
189+
triton-tensorrt-backend
190+
EXPORT
191+
triton-tensorrt-backend-targets
192+
LIBRARY DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
193+
ARCHIVE DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
194+
)
195+
196+
install(
197+
EXPORT
198+
triton-tensorrt-backend-targets
199+
FILE
200+
TritonTensorRTBackendTargets.cmake
201+
NAMESPACE
202+
TritonTensorRTBackend::
203+
DESTINATION
204+
${INSTALL_CONFIGDIR}
205+
)
206+
207+
include(CMakePackageConfigHelpers)
208+
configure_package_config_file(
209+
${CMAKE_CURRENT_LIST_DIR}/cmake/TritonTensorRTBackendConfig.cmake.in
210+
${CMAKE_CURRENT_BINARY_DIR}/TritonTensorRTBackendConfig.cmake
211+
INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
212+
)
213+
214+
install(
215+
FILES
216+
${CMAKE_CURRENT_BINARY_DIR}/TritonTensorRTBackendConfig.cmake
217+
DESTINATION ${INSTALL_CONFIGDIR}
218+
)
219+
220+
#
221+
# Export from build tree
222+
#
223+
export(
224+
EXPORT triton-tensorrt-backend-targets
225+
FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonTensorRTBackendTargets.cmake
226+
NAMESPACE TritonTensorRTBackend::
227+
)
228+
229+
export(PACKAGE TritonTensorRTBackend)

‎README.md

+63
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,70 @@
1+
<!--
2+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
-->
28+
29+
[![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause)
30+
131
# TensorRT Backend
232

333
**WORK IN PROGRESS: This repository tracks the development of TensorRT
434
Backend using new TRITONBACKEND API and is not fit for use yet. The
535
source for current TensorRT backend can be found
636
[here](https://github.com/triton-inference-server/server/tree/master/src/backends/tensorrt).**
737

38+
The Triton backend for [TensorRT](https://github.com/NVIDIA/TensorRT).
39+
You can learn more about Triton backends in the [backend
40+
repo](https://github.com/triton-inference-server/backend). Ask
41+
questions or report problems on the [issues
42+
page](https://github.com/triton-inference-server/server/issues).
43+
This backend is designed to run a serialized [TensorRT engine](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#build_engine_c)
44+
models using the TensorRT C++ API.
45+
46+
Where can I ask general questions about Triton and Triton backends?
47+
Be sure to read all the information below as well as the [general
48+
Triton documentation](https://github.com/triton-inference-server/server#triton-inference-server)
49+
available in the main [server](https://github.com/triton-inference-server/server)
50+
repo. If you don't find your answer there you can ask questions on the
51+
main Triton [issues page](https://github.com/triton-inference-server/server/issues).
52+
53+
## Build the TensorRT Backend
54+
55+
Appropriate version of TensorRT must be installed on the system. Check the support matrix to find the correct version of TensorRT to be installed.
56+
57+
```
58+
$ mkdir build
59+
$ cd build
60+
$ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install ..
61+
$ make install
62+
```
63+
64+
The following required Triton repositories will be pulled and used in
65+
the build. By default the "main" branch/tag will be used for each repo
66+
but the listed CMake argument can be used to override.
67+
68+
* triton-inference-server/backend: -DTRITON_BACKEND_REPO_TAG=[tag]
69+
* triton-inference-server/core: -DTRITON_CORE_REPO_TAG=[tag]
70+
* triton-inference-server/common: -DTRITON_COMMON_REPO_TAG=[tag]

‎cmake/TritonTensorRTBackendConfig.cmake.in

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
27+
include(CMakeFindDependencyMacro)
28+
29+
get_filename_component(
30+
TRITONTENSORRTBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH
31+
)
32+
33+
list(APPEND CMAKE_MODULE_PATH ${TRITONTENSORRTBACKEND_CMAKE_DIR})
34+
35+
if(NOT TARGET TritonTensorRTBackend::triton-tensorrt-backend)
36+
include("${TRITONTENSORRTBACKEND_CMAKE_DIR}/TritonTensorRTBackendTargets.cmake")
37+
endif()
38+
39+
set(TRITONTENSORRTBACKEND_LIBRARIES TritonTensorRTBackend::triton-tensorrt-backend)

0 commit comments

Comments
 (0)