Skip to content

Commit d4ca619

Browse files
tanmayv25tanmayv25
authored
Support features for TRT backend for phase 2 (#6)
* Support features for TRT backend for phase 2 * Fix copyrights
1 parent 28de1ec commit d4ca619

16 files changed

+958
-266
lines changed

‎CMakeLists.txt

+51-23
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions
@@ -130,7 +130,8 @@ target_compile_features(triton-tensorrt-backend PRIVATE cxx_std_11)
130130
target_compile_options(
131131
triton-tensorrt-backend PRIVATE
132132
$<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
133-
-Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror>
133+
-Wall -Wextra -Wno-unused-parameter -Wno-type-limits -Werror -Wno-deprecated-declarations>
134+
$<$<CXX_COMPILER_ID:MSVC>:/Wall /D_WIN32_WINNT=0x0A00 /EHsc>
134135
)
135136

136137
# C/C++ defines that are used directly by this backend.
@@ -139,31 +140,47 @@ target_compile_definitions(
139140
PRIVATE TRITON_ENABLE_GPU=1
140141
)
141142

142-
set_target_properties(
143-
triton-tensorrt-backend
144-
PROPERTIES
145-
POSITION_INDEPENDENT_CODE ON
146-
OUTPUT_NAME ${TRITON_TENSORRT_BACKEND_LIBNAME}
147-
SKIP_BUILD_RPATH TRUE
148-
BUILD_WITH_INSTALL_RPATH TRUE
149-
INSTALL_RPATH_USE_LINK_PATH FALSE
150-
INSTALL_RPATH "$\{ORIGIN\}"
151-
LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_tensorrt.ldscript
152-
LINK_FLAGS "-Wl,--version-script libtriton_tensorrt.ldscript"
153-
)
143+
if (WIN32)
144+
set_target_properties(
145+
triton-tensorrt-backend
146+
PROPERTIES
147+
POSITION_INDEPENDENT_CODE ON
148+
OUTPUT_NAME ${TRITON_TENSORRT_BACKEND_LIBNAME}
149+
SKIP_BUILD_RPATH TRUE
150+
BUILD_WITH_INSTALL_RPATH TRUE
151+
INSTALL_RPATH_USE_LINK_PATH FALSE
152+
INSTALL_RPATH "$\{ORIGIN\}"
153+
)
154+
else ()
155+
set_target_properties(
156+
triton-tensorrt-backend
157+
PROPERTIES
158+
POSITION_INDEPENDENT_CODE ON
159+
OUTPUT_NAME ${TRITON_TENSORRT_BACKEND_LIBNAME}
160+
SKIP_BUILD_RPATH TRUE
161+
BUILD_WITH_INSTALL_RPATH TRUE
162+
INSTALL_RPATH_USE_LINK_PATH FALSE
163+
INSTALL_RPATH "$\{ORIGIN\}"
164+
LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_tensorrt.ldscript
165+
LINK_FLAGS "-Wl,--version-script libtriton_tensorrt.ldscript"
166+
)
167+
endif()
154168

155169
FOREACH(p ${TRITON_TENSORRT_LIB_PATHS})
156170
set(TRITON_TENSORRT_LDFLAGS ${TRITON_TENSORRT_LDFLAGS} "-L${p}")
157171
ENDFOREACH(p)
158172

159-
173+
find_library(NVINFER_LIBRARY NAMES nvinfer)
174+
find_library(NVINFER_PLUGIN_LIBRARY NAMES nvinfer_plugin)
160175
target_link_libraries(
161176
triton-tensorrt-backend
162177
PRIVATE
163178
triton-core-serverapi # from repo-core
164179
triton-core-serverstub # from repo-core
165180
triton-backend-utils # from repo-backend
166181
-lpthread
182+
${NVINFER_LIBRARY}
183+
${NVINFER_PLUGIN_LIBRARY}
167184
)
168185

169186
target_link_libraries(
@@ -184,14 +201,25 @@ target_link_libraries(
184201
include(GNUInstallDirs)
185202
set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonTensorRTBackend)
186203

187-
install(
188-
TARGETS
189-
triton-tensorrt-backend
190-
EXPORT
191-
triton-tensorrt-backend-targets
192-
LIBRARY DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
193-
ARCHIVE DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
194-
)
204+
if (NOT WIN32)
205+
install(
206+
TARGETS
207+
triton-tensorrt-backend
208+
EXPORT
209+
triton-tensorrt-backend-targets
210+
LIBRARY DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
211+
ARCHIVE DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
212+
)
213+
else()
214+
install(
215+
TARGETS
216+
triton-tensorrt-backend
217+
EXPORT
218+
triton-tensorrt-backend-targets
219+
RUNTIME DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
220+
ARCHIVE DESTINATION ${TRITON_TENSORRT_BACKEND_INSTALLDIR}
221+
)
222+
endif() # WIN32
195223

196224
install(
197225
EXPORT

‎LICENSE

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
1+
Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22

33
Redistribution and use in source and binary forms, with or without
44
modification, are permitted provided that the following conditions

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<!--
2-
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
2+
# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions

‎cmake/TritonTensorRTBackendConfig.cmake.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions

‎src/libtriton_tensorrt.ldscript

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions

‎src/loader.cc

+18-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
//
33
// Redistribution and use in source and binary forms, with or without
44
// modification, are permitted provided that the following conditions
@@ -35,8 +35,8 @@ namespace triton { namespace backend { namespace tensorrt {
3535

3636
TRITONSERVER_Error*
3737
LoadPlan(
38-
const std::string& plan_path, nvinfer1::IRuntime** runtime,
39-
nvinfer1::ICudaEngine** engine)
38+
const std::string& plan_path, const int64_t dla_core_id,
39+
nvinfer1::IRuntime** runtime, nvinfer1::ICudaEngine** engine)
4040
{
4141
// Create runtime only if it is not provided
4242
if (*runtime == nullptr) {
@@ -47,6 +47,21 @@ LoadPlan(
4747
}
4848
}
4949

50+
// Report error if 'dla_core_id' >= number of DLA cores
51+
if (dla_core_id != -1) {
52+
auto dla_core_count = (*runtime)->getNbDLACores();
53+
if (dla_core_id < dla_core_count) {
54+
(*runtime)->setDLACore(dla_core_id);
55+
} else {
56+
return TRITONSERVER_ErrorNew(
57+
TRITONSERVER_ERROR_INVALID_ARG,
58+
(std::string("unable to create TensorRT runtime with DLA Core ID: ") +
59+
std::to_string(dla_core_id) +
60+
", available number of DLA cores: " + std::to_string(dla_core_count))
61+
.c_str());
62+
}
63+
}
64+
5065
std::string model_data_str;
5166
RETURN_IF_ERROR(ReadTextFile(plan_path, &model_data_str));
5267
std::vector<char> model_data(model_data_str.begin(), model_data_str.end());

‎src/loader.h

+8-6
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
//
33
// Redistribution and use in source and binary forms, with or without
44
// modification, are permitted provided that the following conditions
@@ -37,14 +37,16 @@ namespace triton { namespace backend { namespace tensorrt {
3737
/// responsibility to destroy any returned runtime or engine object
3838
/// even if an error is returned.
3939
///
40-
/// \param plan_path The path to the model plan file
40+
/// \param plan_path The path to the model plan file.
41+
/// \param dla_core_id The DLA core to use for this runtime. Does not
42+
/// use DLA when set to -1.
4143
/// \param runtime Returns the IRuntime object, or nullptr if failed
42-
/// to create
44+
/// to create.
4345
/// \param engine Returns the ICudaEngine object, or nullptr if failed
44-
/// to create
46+
/// to create.
4547
/// \return Error status.
4648
TRITONSERVER_Error* LoadPlan(
47-
const std::string& plan_path, nvinfer1::IRuntime** runtime,
48-
nvinfer1::ICudaEngine** engine);
49+
const std::string& plan_path, const int64_t dla_core_id,
50+
nvinfer1::IRuntime** runtime, nvinfer1::ICudaEngine** engine);
4951

5052
}}} // namespace triton::backend::tensorrt

‎src/logging.cc

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
//
33
// Redistribution and use in source and binary forms, with or without
44
// modification, are permitted provided that the following conditions
@@ -33,7 +33,7 @@ namespace triton { namespace backend { namespace tensorrt {
3333
TensorRTLogger tensorrt_logger;
3434

3535
void
36-
TensorRTLogger::log(Severity severity, const char* msg)
36+
TensorRTLogger::log(Severity severity, const char* msg) noexcept
3737
{
3838
switch (severity) {
3939
case Severity::kINTERNAL_ERROR:

‎src/logging.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
1+
// Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
//
33
// Redistribution and use in source and binary forms, with or without
44
// modification, are permitted provided that the following conditions
@@ -31,7 +31,7 @@ namespace triton { namespace backend { namespace tensorrt {
3131

3232
// Logger for TensorRT API
3333
class TensorRTLogger : public nvinfer1::ILogger {
34-
void log(Severity severity, const char* msg) override;
34+
void log(Severity severity, const char* msg) noexcept override;
3535
};
3636

3737
extern TensorRTLogger tensorrt_logger;

0 commit comments

Comments
 (0)