ONNX-TensorRT 8.0 GA release (#706)

Signed-off-by: Kevin Chen <[email protected]>

Author: kevinch-nv

CMakeLists.txt

@@ -23,9 +23,9 @@ set(PARSER_LINKER_SCRIPT  ${ONNX2TRT_ROOT}/libnvonnxparser.version)
 #--------------------------------------------------
 # Version information
 #--------------------------------------------------
-set(ONNX2TRT_MAJOR 7)
-set(ONNX2TRT_MINOR 2)
-set(ONNX2TRT_PATCH 2)
+set(ONNX2TRT_MAJOR 8)
+set(ONNX2TRT_MINOR 0)
+set(ONNX2TRT_PATCH 1)
 
 #--------------------------------------------------
 # Build configurations, global to all projects
@@ -36,6 +36,7 @@ set(IMPORTER_SOURCES
   ModelImporter.cpp
   builtin_op_importers.cpp
   onnx2trt_utils.cpp
+  onnxErrorRecorder.cpp
   ShapedWeights.cpp
   ShapeTensor.cpp
   LoopHelpers.cpp
@@ -72,10 +73,6 @@ if (NOT DEFINED BUILD_LIBRARY_ONLY)
   )
 endif()
 
-set(HEADERS
-  NvOnnxParser.h
-)
-
 if (NOT TARGET protobuf::libprotobuf)
   FIND_PACKAGE(Protobuf REQUIRED)
 else()
@@ -102,16 +99,7 @@ find_library(TENSORRT_LIBRARY_INFER nvinfer
 find_library(TENSORRT_LIBRARY_INFER_PLUGIN nvinfer_plugin
   HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
   PATH_SUFFIXES lib lib64 lib/x64)
-if(WIN32)
-  find_library(TENSORRT_LIBRARY_MYELIN myelin64_1
-    HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
-    PATH_SUFFIXES lib lib64 lib/x64)
-else()
-  find_library(TENSORRT_LIBRARY_MYELIN myelin
-    HINTS  ${TENSORRT_ROOT} ${TENSORRT_BUILD} ${CUDA_TOOLKIT_ROOT_DIR}
-    PATH_SUFFIXES lib lib64 lib/x64)
-endif()
-set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN} ${TENSORRT_LIBRARY_MYELIN})
+set(TENSORRT_LIBRARY ${TENSORRT_LIBRARY_INFER} ${TENSORRT_LIBRARY_INFER_PLUGIN})
 MESSAGE(STATUS "Find TensorRT libs at ${TENSORRT_LIBRARY}")
 find_package_handle_standard_args(
   TENSORRT DEFAULT_MSG TENSORRT_INCLUDE_DIR TENSORRT_LIBRARY)
@@ -175,7 +163,6 @@ install(TARGETS
 install(FILES ${HEADERS}
   DESTINATION include
 )
-
 if (NOT DEFINED BUILD_LIBRARY_ONLY)
   install(TARGETS
           onnx2trt

ImporterContext.hpp

@@ -6,88 +6,138 @@
 
 #include "onnx2trt.hpp"
 #include "onnx2trt_utils.hpp"
-
+#include "onnxErrorRecorder.hpp"
+#include "onnx/common/stl_backports.h"
 #include <list>
 #include <unordered_map>
 
 namespace onnx2trt
 {
 
+class ErrorRecorderWrapper
+{
+public:
+    ErrorRecorderWrapper(nvinfer1::INetworkDefinition* network, nvinfer1::ILogger* logger)
+        : mNetwork(network)
+        , mLogger(logger)
+    {
+        if (mNetwork)
+        {
+            mUserErrorRecorder = mNetwork->getErrorRecorder();
+            mOnnxErrorRecorder = ONNXParserErrorRecorder::create(logger, mUserErrorRecorder);
+            if (mOnnxErrorRecorder)
+            {
+                if (mUserErrorRecorder)
+                {
+                    mUserErrorRecorder->incRefCount();
+                }
+                mNetwork->setErrorRecorder(mOnnxErrorRecorder);
+            }
+        }
+    }
+
+    ~ErrorRecorderWrapper()
+    {
+        if (mNetwork && mOnnxErrorRecorder)
+        {
+            mNetwork->setErrorRecorder(mUserErrorRecorder);
+            if (mUserErrorRecorder)
+            {
+                mUserErrorRecorder->decRefCount();
+            }
+            ONNXParserErrorRecorder::destroy(mOnnxErrorRecorder);
+        }
+    }
+
+    bool hasError() const
+    {
+        return mOnnxErrorRecorder != nullptr && mOnnxErrorRecorder->getNbErrors() != 0;
+    }
+
+    //! Return recorder used by hasError().
+    nvinfer1::IErrorRecorder* getErrorRecorder() const
+    {
+        return mOnnxErrorRecorder ? mOnnxErrorRecorder : nullptr;
+    }
+private:
+    nvinfer1::INetworkDefinition* mNetwork{nullptr};
+    nvinfer1::ILogger* mLogger{nullptr};
+    ONNXParserErrorRecorder* mOnnxErrorRecorder{nullptr};
+    nvinfer1::IErrorRecorder* mUserErrorRecorder{nullptr};
+};
+
 class ImporterContext final : public IImporterContext
 {
-    nvinfer1::INetworkDefinition* _network;
-    nvinfer1::ILogger* _logger;
-    std::list<std::vector<uint8_t>> _temp_bufs;
-    StringMap<nvinfer1::ITensor*> _user_inputs;
-    StringMap<nvinfer1::ITensor**> _user_outputs;
-    StringMap<int64_t> _opsets;
+    nvinfer1::INetworkDefinition* mNetwork;
+    nvinfer1::ILogger* mLogger;
+    std::list<std::vector<uint8_t>> mTempBufs;
+    StringMap<nvinfer1::ITensor*> mUserInputs;
+    StringMap<nvinfer1::ITensor**> mUserOutputs;
+    StringMap<int64_t> mOpsets;
     StringMap<TensorOrWeights> mTensors; // All tensors in the graph mapped to their names.
     StringMap<nvinfer1::TensorLocation> mTensorLocations;
     StringMap<float> mTensorRangeMins;
     StringMap<float> mTensorRangeMaxes;
     StringMap<nvinfer1::DataType> mLayerPrecisions;
     std::set<std::string> mTensorNames; // Keep track of how many times a tensor name shows up, to avoid duplicate naming in TRT.
     std::set<std::string> mLayerNames; // Keep track of how many times a tensor name shows up, to avoid duplicate naming in TRT.
-    int64_t mSuffixCounter = 0; // increasing suffix counter used to uniquify layer names.
+    int64_t mSuffixCounter{0}; // increasing suffix counter used to uniquify layer names.
     std::unordered_set<std::string> mUnsupportedShapeTensors; // Container to hold output tensor names of layers that produce shape tensor outputs but do not natively support them.
     StringMap<std::string> mLoopTensors; // Container to map subgraph tensors to their original outer graph names.
     std::string mOnnxFileLocation; // Keep track of the directory of the parsed ONNX file
-    std::list<std::string> mInitializerNames; // Keep track of unique names of any initializers
-    RefitMap_t* mRefitMap; // Keep track of names of ONNX refittable weights with their corresponding TRT layer and role
+    std::unique_ptr<ErrorRecorderWrapper> mErrorWrapper; // error recorder to control TRT errors
 
 public:
-    ImporterContext(nvinfer1::INetworkDefinition* network, nvinfer1::ILogger* logger, RefitMap_t* refitMap)
-        : _network(network)
-        , _logger(logger)
-        , mRefitMap(refitMap)
+    ImporterContext(nvinfer1::INetworkDefinition* network, nvinfer1::ILogger* logger)
+        : mNetwork(network)
+        , mLogger(logger)
+        // Disable ErrorRecorder for now due to incompatibilities with ONNXRT.
+        // , mErrorWrapper(ONNX_NAMESPACE::make_unique<ErrorRecorderWrapper>(mNetwork, logger))
+        , mErrorWrapper(nullptr)
     {
     }
-    virtual nvinfer1::INetworkDefinition* network() override
+    nvinfer1::INetworkDefinition* network() override
     {
-        return _network;
+        return mNetwork;
     }
-    virtual StringMap<TensorOrWeights>& tensors() override
+    StringMap<TensorOrWeights>& tensors() override
     {
         return mTensors;
     }
-    virtual StringMap<nvinfer1::TensorLocation>& tensorLocations() override
+    StringMap<nvinfer1::TensorLocation>& tensorLocations() override
     {
         return mTensorLocations;
     }
-    virtual StringMap<float>& tensorRangeMins() override
+    StringMap<float>& tensorRangeMins() override
     {
         return mTensorRangeMins;
     }
-    virtual StringMap<float>& tensorRangeMaxes() override
+    StringMap<float>& tensorRangeMaxes() override
     {
         return mTensorRangeMaxes;
     }
-    virtual StringMap<nvinfer1::DataType>& layerPrecisions() override
+    StringMap<nvinfer1::DataType>& layerPrecisions() override
     {
         return mLayerPrecisions;
     }
-    virtual std::unordered_set<std::string>& unsupportedShapeTensors() override
+    std::unordered_set<std::string>& unsupportedShapeTensors() override
     {
         return mUnsupportedShapeTensors;
     }
-    virtual StringMap<std::string>& loopTensors() override
+    StringMap<std::string>& loopTensors() override
     {
         return mLoopTensors;
     }
-    virtual void setOnnxFileLocation(std::string location) override
+    void setOnnxFileLocation(std::string location) override
     {
         mOnnxFileLocation = location;
     }
-    virtual std::string getOnnxFileLocation() override
+    std::string getOnnxFileLocation() override
     {
         return mOnnxFileLocation;
     }
-    virtual void insertRefitMap(std::string weightsName, std::string layerName, nvinfer1::WeightsRole role) override
-    {
-        mRefitMap->insert({weightsName, WeightsPair_t{layerName, role}});
-    }
     // This actually handles weights as well, but is named this way to be consistent with the tensors()
-    virtual void registerTensor(TensorOrWeights tensor, const std::string& basename) override
+    void registerTensor(TensorOrWeights tensor, const std::string& basename) override
     {
         // TRT requires unique tensor names.
         const std::string uniqueName = generateUniqueName(mTensorNames, basename);
@@ -103,22 +153,22 @@ class ImporterContext final : public IImporterContext
             }
             else if (tensor.is_weights())
             {
-                mInitializerNames.push_back(uniqueName);
                 const auto& weights = tensor.weights();
                 if (tensor.weights().type == ::ONNX_NAMESPACE::TensorProto::INT64)
                 {
                     tensor = ShapedWeights{::ONNX_NAMESPACE::TensorProto::INT32,
                         convertINT64(reinterpret_cast<int64_t*>(weights.values), weights.shape, ctx), weights.shape};
                 }
-                tensor.weights().setName(mInitializerNames.back().c_str());
+                tensor.weights().setName(basename.c_str());
             }
+
         }
         // Overwrite previous tensors registered with the same name (this only happens when there are subgraphs,
         // and in that case, overwriting is the desired behavior).
         this->tensors()[basename] = std::move(tensor);
     }
 
-    virtual void registerLayer(nvinfer1::ILayer* layer, const std::string& basename) override
+    void registerLayer(nvinfer1::ILayer* layer, const std::string& basename) override
     {
         // No layer will be added for Constant nodes in ONNX.
         if (layer)
@@ -127,99 +177,102 @@ class ImporterContext final : public IImporterContext
             const std::string uniqueName = generateUniqueName(mLayerNames, name);
 
             auto* ctx = this; // To enable logging.
-            if (layer->getType() == nvinfer1::LayerType::kCONSTANT)
-            {
-                LOG_VERBOSE("Registering constant layer: " << uniqueName << " for ONNX initializer: " << basename);
-            }
-            else
-            {
-                LOG_VERBOSE("Registering layer: " << uniqueName << " for ONNX node: " << basename);
-            }
+            LOG_VERBOSE("Registering layer: " << uniqueName << " for ONNX node: " << basename);
+
             layer->setName(uniqueName.c_str());
         }
     }
 
-    virtual nvinfer1::ILogger& logger() override
+    nvinfer1::ILogger& logger() override
     {
-        return *_logger;
+        return *mLogger;
     }
 
-    virtual ShapedWeights createTempWeights(ShapedWeights::DataType type, nvinfer1::Dims shape) override
+    ShapedWeights createTempWeights(ShapedWeights::DataType type, nvinfer1::Dims shape, uint8_t value = 0) override
     {
         ShapedWeights weights(type, nullptr, shape);
         // Need special logic for handling scalars.
         if (shape.nbDims == 0)
         {
-            _temp_bufs.push_back(std::vector<uint8_t>(getDtypeSize(type)));
+            mTempBufs.push_back(std::vector<uint8_t>(getDtypeSize(type), value));
         }
         else
         {
-            _temp_bufs.push_back(std::vector<uint8_t>(weights.size_bytes()));
+            mTempBufs.push_back(std::vector<uint8_t>(weights.size_bytes(), value));
         }
-        weights.values = _temp_bufs.back().data();
+        weights.values = mTempBufs.back().data();
         return weights;
     }
 
     bool setUserInput(const char* name, nvinfer1::ITensor* input)
     {
-        _user_inputs[name] = input;
+        mUserInputs[name] = input;
         return true;
     }
     bool setUserOutput(const char* name, nvinfer1::ITensor** output)
     {
-        _user_outputs[name] = output;
+        mUserOutputs[name] = output;
         return true;
     }
     nvinfer1::ITensor* getUserInput(const char* name)
     {
-        if (!_user_inputs.count(name))
+        if (!mUserInputs.count(name))
         {
             return nullptr;
         }
         else
         {
-            return _user_inputs.at(name);
+            return mUserInputs.at(name);
         }
     }
     nvinfer1::ITensor** getUserOutput(const char* name)
     {
-        if (!_user_outputs.count(name))
+        if (!mUserOutputs.count(name))
         {
             return nullptr;
         }
         else
         {
-            return _user_outputs.at(name);
+            return mUserOutputs.at(name);
         }
     }
     StringMap<nvinfer1::ITensor**> const& getUserOutputs() const
     {
-        return _user_outputs;
+        return mUserOutputs;
     }
     void clearOpsets()
     {
-        _opsets.clear();
+        mOpsets.clear();
     }
     void addOpset(std::string domain, int64_t version)
     {
-        _opsets.emplace(domain, version);
+        mOpsets.emplace(domain, version);
     }
-    virtual int64_t getOpsetVersion(const char* domain = "") const override
+    int64_t getOpsetVersion(const char* domain = "") const override
     {
-        if (_opsets.empty())
+        if (mOpsets.empty())
         {
             return 1;
         }
-        else if (_opsets.size() == 1)
+        else if (mOpsets.size() == 1)
         {
-            return _opsets.begin()->second;
+            return mOpsets.begin()->second;
         }
         else
         {
-            assert(_opsets.count(domain));
-            return _opsets.at(domain);
+            assert(mOpsets.count(domain));
+            return mOpsets.at(domain);
         }
     }
+    bool hasError() const noexcept override
+    {
+        return mErrorWrapper != nullptr && mErrorWrapper->hasError();
+    }
+
+    nvinfer1::IErrorRecorder* getErrorRecorder() const noexcept override
+    {
+        return mErrorWrapper ? mErrorWrapper->getErrorRecorder() : nullptr;
+    }
 private:
     std::string generateUniqueName(std::set<std::string>& namesSet, const std::string& basename)
     {

ModelImporter.cpp

@@ -8,7 +8,6 @@
 #include "NvInferPlugin.h"
 #include "NvOnnxParser.h"
 #include "builtin_op_importers.hpp"
-#include "onnx_utils.hpp"
 #include "utils.hpp"
 
 namespace onnx2trt
@@ -20,24 +19,21 @@ class ModelImporter : public nvonnxparser::IParser
 {
 protected:
     string_map<NodeImporter> _op_importers;
-    virtual Status importModel(::ONNX_NAMESPACE::ModelProto const& model, uint32_t weight_count,
-        onnxTensorDescriptorV1 const* weight_descriptors);
+    virtual Status importModel(::ONNX_NAMESPACE::ModelProto const& model);
 
 private:
     ImporterContext _importer_ctx;
-    RefitMap_t mRefitMap;
     std::list<::ONNX_NAMESPACE::ModelProto> _onnx_models; // Needed for ownership of weights
     int _current_node;
     std::vector<Status> _errors;
 
 public:
     ModelImporter(nvinfer1::INetworkDefinition* network, nvinfer1::ILogger* logger)
         : _op_importers(getBuiltinOpImporterMap())
-        , _importer_ctx(network, logger, &mRefitMap)
+        , _importer_ctx(network, logger)
     {
     }
-    bool parseWithWeightDescriptors(void const* serialized_onnx_model, size_t serialized_onnx_model_size,
-        uint32_t weight_count, onnxTensorDescriptorV1 const* weight_descriptors) override;
+    bool parseWithWeightDescriptors(void const* serialized_onnx_model, size_t serialized_onnx_model_size) override;
     bool parse(void const* serialized_onnx_model, size_t serialized_onnx_model_size, const char* model_path = nullptr) override;
     bool supportsModel(void const* serialized_onnx_model, size_t serialized_onnx_model_size,
         SubGraphCollection_t& sub_graph_collection, const char* model_path = nullptr) override;
@@ -68,27 +64,7 @@ class ModelImporter : public nvonnxparser::IParser
     {
         _errors.clear();
     }
-    virtual int getRefitMap(const char** weightNames, const char** layerNames, nvinfer1::WeightsRole* roles) override
-    {
-        int count = 0;
-        for (const auto& entry: mRefitMap)
-        {
-            if (weightNames != nullptr)
-            {
-                weightNames[count] = entry.first.c_str();
-            }
-            if (layerNames != nullptr)
-            {
-                layerNames[count] = entry.second.first.c_str();
-            }
-            if (roles != nullptr)
-            {
-                roles[count] = entry.second.second;
-            }
-            ++count;
-        }
-        return mRefitMap.size();
-    }
+
     //...LG: Move the implementation to .cpp
     bool parseFromFile(const char* onnxModelFile, int verbosity) override;
 };

ModelImporter.hpp

@@ -26,16 +26,15 @@ static const int NV_ONNX_PARSER_VERSION = ((NV_ONNX_PARSER_MAJOR * 10000) + (NV_
 //! \brief The data structure containing the parsing capability of
 //! a set of nodes in an ONNX graph.
 //!
-using SubGraph_t = std::pair<std::vector<size_t>, bool>;
+typedef std::pair<std::vector<size_t>, bool> SubGraph_t;
 
 //! \typedef SubGraphCollection_t
 //!
 //! \brief The data structure containing all SubGraph_t partitioned
 //! out of an ONNX graph.
 //!
-using SubGraphCollection_t = std::vector<SubGraph_t>;
+typedef std::vector<SubGraph_t> SubGraphCollection_t;
 
-class onnxTensorDescriptorV1;
 //!
 //! \namespace nvonnxparser
 //!
@@ -108,7 +107,7 @@ class IParser
 {
 public:
     /** \brief Parse a serialized ONNX model into the TensorRT network.
-     *         This method has very limited diagnostic. If parsing the serialized model
+     *         This method has very limited diagnostics. If parsing the serialized model
      *         fails for any reason (e.g. unsupported IR version, unsupported opset, etc.)
      *         it the user responsibility to intercept and report the error.
      *         To obtain a better diagnostic, use the parseFromFile method below.
@@ -125,7 +124,7 @@ class IParser
                        const char* model_path = nullptr)
         = 0;
 
-    /** \brief Parse an onnx model file, can be a binary protobuf or a text onnx model
+    /** \brief Parse an onnx model file, which can be a binary protobuf or a text onnx model
      *         calls parse method inside.
      *
      * \param File name
@@ -157,15 +156,11 @@ class IParser
      * \param serialized_onnx_model Pointer to the serialized ONNX model
      * \param serialized_onnx_model_size Size of the serialized ONNX model
      *        in bytes
-     * \param weight_count number of user provided weights
-     * \param weight_descriptors pointer to user provided weight array
      * \return true if the model was parsed successfully
      * \see getNbErrors() getError()
      */
     virtual bool parseWithWeightDescriptors(
-        void const* serialized_onnx_model, size_t serialized_onnx_model_size,
-        uint32_t weight_count,
-        onnxTensorDescriptorV1 const* weight_descriptors)
+        void const* serialized_onnx_model, size_t serialized_onnx_model_size)
         = 0;
 
     /** \brief Returns whether the specified operator may be supported by the
@@ -178,8 +173,10 @@ class IParser
      */
     virtual bool supportsOperator(const char* op_name) const = 0;
     /** \brief destroy this object
+     *
+     * \warning deprecated and planned on being removed in TensorRT 10.0
      */
-    virtual void destroy() = 0;
+    TRT_DEPRECATED virtual void destroy() = 0;
     /** \brief Get the number of errors that occurred during prior calls to
      *         \p parse
      *
@@ -197,25 +194,7 @@ class IParser
      */
     virtual void clearErrors() = 0;
 
-    /** \brief Get description of all ONNX weights that can be refitted.
-     *
-     * \param weightsNames Where to write the weight names to
-     * \param layerNames Where to write the layer names to
-     * \param roles Where to write the roles to
-     *
-     * \return The number of weights from the ONNX model that can be refitted
-     *
-     * If weightNames or layerNames != nullptr, each written pointer points to a string owned by
-     * the parser, and becomes invalid when the parser is destroyed
-     *
-     * If the same weight is used in multiple TRT layers it will be represented as a new
-     * entry in weightNames with name <weightName>_x, with x being the number of times the weight
-     * has been used before the current layer
-     */
-    virtual int getRefitMap(const char** weightNames, const char** layerNames, nvinfer1::WeightsRole* roles) = 0;
-
-protected:
-    virtual ~IParser() {}
+    virtual ~IParser() noexcept = default;
 };
 
 } // namespace nvonnxparser
@@ -226,15 +205,6 @@ extern "C" TENSORRTAPI int getNvOnnxParserVersion();
 namespace nvonnxparser
 {
 
-#ifdef SWIG
-inline IParser* createParser(nvinfer1::INetworkDefinition* network,
-                             nvinfer1::ILogger* logger)
-{
-    return static_cast<IParser*>(
-        createNvOnnxParser_INTERNAL(network, logger, NV_ONNX_PARSER_VERSION));
-}
-#endif // SWIG
-
 namespace
 {
 
@@ -243,18 +213,17 @@ namespace
  * \param network The network definition that the parser will write to
  * \param logger The logger to use
  * \return a new parser object or NULL if an error occurred
+ *
+ * Any input dimensions that are constant should not be changed after parsing,
+ * because correctness of the translation may rely on those constants.
+ * Changing a dynamic input dimension, i.e. one that translates to -1 in
+ * TensorRT, to a constant is okay if the constant is consistent with the model.
+ *
  * \see IParser
  */
-#ifdef _MSC_VER
-TENSORRTAPI IParser* createParser(nvinfer1::INetworkDefinition& network,
-                                  nvinfer1::ILogger& logger)
-#else
-inline IParser* createParser(nvinfer1::INetworkDefinition& network,
-                             nvinfer1::ILogger& logger)
-#endif
+inline IParser* createParser(nvinfer1::INetworkDefinition& network, nvinfer1::ILogger& logger)
 {
-    return static_cast<IParser*>(
-        createNvOnnxParser_INTERNAL(&network, &logger, NV_ONNX_PARSER_VERSION));
+    return static_cast<IParser*>(createNvOnnxParser_INTERNAL(&network, &logger, NV_ONNX_PARSER_VERSION));
 }
 
 } // namespace

NvOnnxParser.h

@@ -303,14 +303,72 @@ nvinfer1::MatrixOperation OnnxAttrs::get<nvinfer1::MatrixOperation>(const std::s
 template <>
 nvinfer1::ResizeMode OnnxAttrs::get<nvinfer1::ResizeMode>(const std::string& key) const
 {
-    std::string mode = this->get<std::string>(key);
-    if (mode == std::string("nearest"))
+    const auto& mode = this->get<std::string>(key);
+    if (mode == "nearest")
     {
         return nvinfer1::ResizeMode::kNEAREST;
     }
-    if (mode == std::string("linear"))
+    if (mode == "linear")
     {
         return nvinfer1::ResizeMode::kLINEAR;
     }
     throw std::runtime_error("Unknown ResizeMode: " + mode);
 }
+
+template <>
+nvinfer1::ResizeCoordinateTransformation OnnxAttrs::get<nvinfer1::ResizeCoordinateTransformation>(
+    const std::string& key) const
+{
+    const auto& transformation = this->get<std::string>(key);
+    if (transformation == "align_corners")
+    {
+        return nvinfer1::ResizeCoordinateTransformation::kALIGN_CORNERS;
+    }
+    if (transformation == "asymmetric")
+    {
+        return nvinfer1::ResizeCoordinateTransformation::kASYMMETRIC;
+    }
+    if (transformation == "half_pixel")
+    {
+        return nvinfer1::ResizeCoordinateTransformation::kHALF_PIXEL;
+    }
+    throw std::runtime_error("Unknown ResizeCoordinateTransformation: " + transformation);
+}
+
+template <>
+nvinfer1::ResizeSelector OnnxAttrs::get<nvinfer1::ResizeSelector>(const std::string& key) const
+{
+    const auto& selector = this->get<std::string>(key);
+    if (selector == "formula")
+    {
+        return nvinfer1::ResizeSelector::kFORMULA;
+    }
+    if (selector == "upper")
+    {
+        return nvinfer1::ResizeSelector::kUPPER;
+    }
+    throw std::runtime_error("Unknown ResizeSelector: " + selector);
+}
+
+template <>
+nvinfer1::ResizeRoundMode OnnxAttrs::get<nvinfer1::ResizeRoundMode>(const std::string& key) const
+{
+    const auto& roundMode = this->get<std::string>(key);
+    if (roundMode == "half_up")
+    {
+        return nvinfer1::ResizeRoundMode::kHALF_UP;
+    }
+    if (roundMode == "half_down")
+    {
+        return nvinfer1::ResizeRoundMode::kHALF_DOWN;
+    }
+    if (roundMode == "floor")
+    {
+        return nvinfer1::ResizeRoundMode::kFLOOR;
+    }
+    if (roundMode == "ceil")
+    {
+        return nvinfer1::ResizeRoundMode::kCEIL;
+    }
+    throw std::runtime_error("Unknown ResizeRoundMode: " + roundMode);
+}

OnnxAttrs.cpp

@@ -16,7 +16,7 @@ For press and other inquiries, please contact Hector Marinez at hmarinez@nvidia.
 
 ## Supported TensorRT Versions
 
-Development on the Master branch is for the latest version of [TensorRT 7.2.3.4](https://developer.nvidia.com/nvidia-tensorrt-download) with full-dimensions and dynamic shape support.
+Development on the Master branch is for the latest version of [TensorRT 8.0.1.6](https://developer.nvidia.com/nvidia-tensorrt-download) with full-dimensions and dynamic shape support.
 
 For previous versions of TensorRT, refer to their respective branches.
 
@@ -48,12 +48,12 @@ Current supported ONNX operators are found in the [operator support matrix](docs
 ### Dependencies
 
  - [Protobuf >= 3.0.x](https://github.com/google/protobuf/releases)
- - [TensorRT 7.2.3.4](https://developer.nvidia.com/tensorrt)
- - [TensorRT 7.2.3.4 open source libaries (master branch)](https://github.com/NVIDIA/TensorRT/)
+ - [TensorRT 8.0.1.6](https://developer.nvidia.com/tensorrt)
+ - [TensorRT 8.0.1.6 open source libaries (master branch)](https://github.com/NVIDIA/TensorRT/)
 
 ### Building
 
-For building within docker, we recommend using and setting up the docker containers as instructed in the main (TensorRT repository)[https://github.com/NVIDIA/TensorRT#setting-up-the-build-environment] to build the onnx-tensorrt library.
+For building within docker, we recommend using and setting up the docker containers as instructed in the main [TensorRT repository](https://github.com/NVIDIA/TensorRT#setting-up-the-build-environment) to build the onnx-tensorrt library.
 
 Once you have cloned the repository, you can build the parser libraries and executables by running:
 
@@ -65,6 +65,11 @@ Once you have cloned the repository, you can build the parser libraries and exec
 
 For building only the libraries, append `-DBUILD_LIBRARY_ONLY=1` to the CMake build command. If your model has Gather or GatherElements operations with negative indices, add `-DSUPPORT_NEGATIVE_GATHER` to the build command. Note that enabling negative-indices gather will have a performance impact on gathers with non-negative indices.
 
+### Experimental Ops
+All experimental operators will be considered unsupported by the ONNX-TRT's `supportsModel()` function.
+
+`NonMaxSuppression` is available as an experimental operator in TensorRT 8. It has the limitation that the output shape is always padded to length [`max_output_boxes_per_class`, 3], therefore some post processing is required to extract the valid indices.
+
 ## Executable Usage
 
 ONNX models can be converted to serialized TensorRT engines using the `onnx2trt` executable:
@@ -92,9 +97,9 @@ See more usage information by running:
 
 Python bindings for the ONNX-TensorRT parser are packaged in the shipped `.whl` files. Install them with
 
-    python3 -m pip install <tensorrt_install_dir>/python/tensorrt-7.x.x.x-cp<python_ver>-none-linux_x86_64.whl
+    python3 -m pip install <tensorrt_install_dir>/python/tensorrt-8.x.x.x-cp<python_ver>-none-linux_x86_64.whl
 
-TensorRT 7.2.3.4 supports ONNX release 1.6.0. Install it with:
+TensorRT 8.0.1.6 supports ONNX release 1.6.0. Install it with:
 
     python3 -m pip install onnx==1.6.0
 

README.md

@@ -6,12 +6,15 @@
 #include "TensorOrWeights.hpp"
 #include "onnx2trt_utils.hpp"
 #include <algorithm>
-#include <cassert>
 #include <functional>
 
 namespace onnx2trt
 {
 
+//! If true, tolerate bug where scalar constant of type FLOAT is missing its value,
+//! and a shape tensor is expected.
+static const bool gTolerateTRT_12408 = true;
+
 ShapeTensor::ShapeTensor(int rank_, std::vector<int64_t>&& values_)
     : mDepth(0)
     , mAllValuesKnown(true)
@@ -23,7 +26,7 @@ ShapeTensor::ShapeTensor(int rank_, std::vector<int64_t>&& values_)
     assert(rank_ > 0 || mValues.size() == 1);
 }
 
-ShapeTensor::ShapeTensor(TensorOrWeights& t)
+ShapeTensor::ShapeTensor(IImporterContext* ctx, TensorOrWeights& t)
     : mDepth(0)
 {
     if (t.is_tensor())
@@ -33,11 +36,24 @@ ShapeTensor::ShapeTensor(TensorOrWeights& t)
     else
     {
         const nvinfer1::Dims d = t.shape();
+         const auto& weights = t.weights();
+        if (gTolerateTRT_12408 && weights.type == ::ONNX_NAMESPACE::TensorProto::FLOAT && d.nbDims == 0 && weights.count() == 0)
+        {
+            LOG_WARNING("Scalar constant of type FLOAT with no value encountered where ONNX specification requires tensor describing a shape. Assuming it's an INT64 empty vector.");
+            mRank = 1;
+            mSize = 0;
+            mAllValuesKnown = true;
+            return;
+        }
         assert(0 <= d.nbDims);
         assert(d.nbDims <= 1 && "shape tensor must be 0D or 1D");
         mRank = d.nbDims;
         mSize = d.nbDims == 0 ? 1 : d.d[0];
-        weightsToVector(t.weights(), &mValues);
+        auto status = weightsToVector(weights, &mValues);
+        if (status.code() != ErrorCode::kSUCCESS)
+        {
+            throw std::runtime_error("constant " + t.getName() + " is not a valid shape tensor");
+        }
         mAllValuesKnown = true;
     }
 }
@@ -142,7 +158,7 @@ nvinfer1::ITensor& ShapeTensor::tensor(IImporterContext* ctx) const
         if (allValuesKnown())
         {
             // Create constant
-            const nvinfer1::Dims dims{rank(), {size()}, {}};
+            const nvinfer1::Dims dims{rank(), {size()}};
             const nvinfer1::Weights w{nvinfer1::DataType::kINT32, convertINT64(mValues.data(), dims, ctx), size()};
             mTensor = ctx->network()->addConstant(dims, w)->getOutput(0);
             mDepth = 0;
@@ -328,11 +344,8 @@ ShapeTensor shapeOf(TensorOrWeights& t)
     {
         return shapeOf(t.tensor());
     }
-    else
-    {
-        const nvinfer1::Dims& d = t.weights().shape;
-        return ShapeTensor(1, std::vector<int64_t>(d.d, d.d + d.nbDims));
-    }
+    const nvinfer1::Dims& d = t.weights().shape;
+    return ShapeTensor(1, std::vector<int64_t>(d.d, d.d + d.nbDims));
 }
 
 ShapeTensor shapeOf(const ShapeTensor& t)
@@ -342,15 +355,12 @@ ShapeTensor shapeOf(const ShapeTensor& t)
     {
         return ShapeTensor(*t.mTensor, t.mDepth + 1);
     }
-    else
-    {
-        assert(t.rankKnown());
-        assert(t.sizeKnown());
-        // ShapeTensor is either a scalar or vector.
-        // shape of a scalar is an empty tensor.
-        // shape of a vector is a one-element tensor containing the length of the vector.
-        return t.rank() == 0 ? ShapeTensor(0, {}) : ShapeTensor(1, {t.size()});
-    }
+    assert(t.rankKnown());
+    assert(t.sizeKnown());
+    // ShapeTensor is either a scalar or vector.
+    // shape of a scalar is an empty tensor.
+    // shape of a vector is a one-element tensor containing the length of the vector.
+    return t.rank() == 0 ? ShapeTensor(0, {}) : ShapeTensor(1, {t.size()});
 }
 
 ShapeTensor convertTo1D(IImporterContext* ctx, const ShapeTensor& tensor)
@@ -364,17 +374,30 @@ ShapeTensor convertTo1D(IImporterContext* ctx, const ShapeTensor& tensor)
     return ShapeTensor(*addShuffle(ctx, tensor.tensor(ctx), shapeVector(1))->getOutput(0));
 }
 
-//! If all values of x are known, return Dims with those values.
+//! If all values of x are known, return Dims with those values,
+//! but throw exception if any value is outside specified bounds.
 //! Otherwise return Dims with zeros.
-static nvinfer1::Dims toDims(const ShapeTensor& x)
+//!
+//! The string that should describe the context of the dimensions,
+//! e.g. "reshape" or "fill output".
+static nvinfer1::Dims toDims(const ShapeTensor& x, const char* what, int32_t minAllowed, int32_t maxAllowed)
 {
-    nvinfer1::Dims d{-1, {}, {}};
+    nvinfer1::Dims d{-1, {}};
     if (x.sizeKnown())
     {
         d.nbDims = x.size();
         if (x.allValuesKnown())
         {
             assert(x.size() <= nvinfer1::Dims::MAX_DIMS);
+            for (const auto& dim : x)
+            {
+                if (dim < minAllowed || dim > maxAllowed)
+                {
+                    std::ostringstream msg;
+                    msg << what << " dimensions have value " << dim << " beyond allowed bounds." << std::endl;
+                    throw std::runtime_error(msg.str());
+                }
+            }
             std::copy(x.begin(), x.end(), d.d);
         }
     }
@@ -417,7 +440,7 @@ nvinfer1::IShuffleLayer* addShuffle(
     nvinfer1::IShuffleLayer* shuffle = ctx->network()->addShuffle(data);
     if (reshapeDims.allValuesKnown())
     {
-        shuffle->setReshapeDimensions(toDims(reshapeDims));
+        shuffle->setReshapeDimensions(toDims(reshapeDims, "reshape", -1, std::numeric_limits<int32_t>::max()));
     }
     else
     {
@@ -430,7 +453,10 @@ nvinfer1::IShuffleLayer* addShuffle(
 nvinfer1::ISliceLayer* addSlice(IImporterContext* ctx, nvinfer1::ITensor& data, const ShapeTensor& starts,
     const ShapeTensor& sizes, const ShapeTensor& strides)
 {
-    nvinfer1::ISliceLayer* slice = ctx->network()->addSlice(data, toDims(starts), toDims(sizes), toDims(strides));
+    constexpr int32_t minDim = std::numeric_limits<int32_t>::min();
+    constexpr int32_t maxDim = std::numeric_limits<int32_t>::max();
+    nvinfer1::ISliceLayer* slice = ctx->network()->addSlice(data, toDims(starts, "slice start", 0, maxDim),
+        toDims(sizes, "slice size", 0, maxDim), toDims(strides, "slide strides", minDim, maxDim));
     setShapeInputIfDynamic(ctx, slice, 1, starts);
     setShapeInputIfDynamic(ctx, slice, 2, sizes);
     setShapeInputIfDynamic(ctx, slice, 3, strides);
@@ -439,7 +465,8 @@ nvinfer1::ISliceLayer* addSlice(IImporterContext* ctx, nvinfer1::ITensor& data,
 
 nvinfer1::IFillLayer* addFill(IImporterContext* ctx, const ShapeTensor& shape, nvinfer1::FillOperation op)
 {
-    nvinfer1::IFillLayer* fill = ctx->network()->addFill(toDims(shape), op);
+    nvinfer1::IFillLayer* fill
+        = ctx->network()->addFill(toDims(shape, "fill output", 0, std::numeric_limits<int32_t>::max()), op);
     setShapeInputIfDynamic(ctx, fill, 0, shape);
     return fill;
 }

ShapeTensor.cpp

@@ -26,7 +26,7 @@ class ShapeTensor
     ShapeTensor(int rank_, std::vector<int64_t>&& values_);
 
     //! Create ShapeTensor representing value of TensorOrWeights.
-    ShapeTensor(TensorOrWeights& t);
+    ShapeTensor(IImporterContext* ctx, TensorOrWeights& t);
 
     //! Construct ShapeTensor equivalent to applying IShapeLayer depth times.
     //! The depth may be in [0,3].
@@ -133,7 +133,7 @@ class ShapeTensor
     //! When mAllValuesKnown==true, all the values in mValues are correct
     //! and mValues.size() == mSize.
     //! When mAllValuesKnown==false, only the non-negative values in mValues
-    //! are guranteed to be correct, and only so if mValues.size() == mSize.
+    //! are guaranteed to be correct, and only so if mValues.size() == mSize.
     std::vector<int64_t> mValues;
 };
 

ShapeTensor.hpp

@@ -50,16 +50,6 @@ size_t ShapedWeights::size_bytes() const
     return this->count() * getDtypeSize(this->type);
 }
 
-const char* ShapedWeights::getName() const
-{
-    return this->name;
-}
-
-void ShapedWeights::setName(const char* name)
-{
-    this->name = name;
-}
-
 ShapedWeights::operator bool() const
 {
     return (bool) this->values;
@@ -76,6 +66,16 @@ ShapedWeights::operator nvinfer1::Weights() const
     return w;
 }
 
+const char* ShapedWeights::getName() const
+{
+    return this->name;
+}
+
+void ShapedWeights::setName(const char* name)
+{
+    this->name = name;
+}
+
 template <typename DType>
 void transpose4DWeights(ShapedWeights const& weights, nvinfer1::Permutation const perm, ShapedWeights* result)
 {
@@ -88,7 +88,7 @@ void transpose4DWeights(ShapedWeights const& weights, nvinfer1::Permutation cons
     nvinfer1::Dims expanded_original_shape{4, {1, 1, 1, 1}};
     nvinfer1::Dims expanded_new_shape{4, {1, 1, 1, 1}};
     nvinfer1::Permutation expanded_perm{0, 1, 2, 3};
-
+    
     int pad = 4 - nbDims;
     for (int i = 0; i < nbDims; ++i)
     {
@@ -97,14 +97,15 @@ void transpose4DWeights(ShapedWeights const& weights, nvinfer1::Permutation cons
         expanded_perm.order[pad + i] = perm.order[i] + pad;
     }
 
+
     int src_strides[4] = {1, 1, 1, 1};
     int dst_strides[4] = {1, 1, 1, 1};
-
+    
     for (int i = 2; i >= 0; --i)
     {
         src_strides[i] = expanded_original_shape.d[i + 1] * src_strides[i + 1];
         dst_strides[i] = expanded_new_shape.d[i + 1] * dst_strides[i + 1];
-    }
+    } 
 
     for (int n = 0; n < expanded_original_shape.d[0]; ++n)
     {
@@ -131,7 +132,7 @@ void transpose4DWeights(ShapedWeights const& weights, nvinfer1::Permutation cons
     }
 }
 
-bool transposeWeights(ShapedWeights const& weights, nvinfer1::Permutation const& perm, ShapedWeights* result)
+bool transposeWeights(ShapedWeights const& weights, nvinfer1::Permutation const& perm, ShapedWeights* result, IImporterContext* ctx)
 {
     nvinfer1::Dims shape = weights.shape;
     int nbDims = shape.nbDims;
@@ -164,6 +165,12 @@ bool transposeWeights(ShapedWeights const& weights, nvinfer1::Permutation const&
         // Unsupported weights transpose
         return false;
     }
+    nvinfer1::Dims permDims{nbDims, {}};
+    std::copy_n(perm.order, nbDims, permDims.d);
+    LOG_WARNING("Weights "
+        << weights.getName() << " has been transposed with permutation of " << permDims
+        << "! If you plan on overwriting the weights with the Refitter API, the new weights must be pre-transposed.");
+    result->setName(weights.getName());
     return true;
 }
 

ShapedWeights.cpp

@@ -14,21 +14,47 @@ class ShapedWeights
 {
 public:
     using DataType = int32_t;
-    DataType type;
-    void* values;
-    nvinfer1::Dims shape;
-    const char* name = nullptr;
+
     static ShapedWeights empty(DataType type);
+
     ShapedWeights();
+
     explicit ShapedWeights(DataType type, void* values, nvinfer1::Dims shape_);
+
     size_t count() const;
+
     size_t size_bytes() const;
+
     const char* getName() const;
+
     void setName(const char* name);
+
     explicit operator bool() const;
+
     operator nvinfer1::Weights() const;
+
+    template <typename T>
+    T& at(size_t index)
+    {
+        assert(index >= 0 && (index * sizeof(T)) < size_bytes());
+        return static_cast<T*>(values)[index];
+    }
+
+    template <typename T>
+    const T& at(size_t index) const
+    {
+        assert(index >= 0 && (index * sizeof(T)) < size_bytes());
+        return static_cast<const T*>(values)[index];
+    }
+
+public:
+    DataType type;
+    void* values;
+    nvinfer1::Dims shape;
+    const char* name{};
 };
 
-bool transposeWeights(ShapedWeights const& weights, nvinfer1::Permutation const& perm, ShapedWeights* result);
+class IImporterContext;
+bool transposeWeights(ShapedWeights const& weights, nvinfer1::Permutation const& perm, ShapedWeights* result, IImporterContext* ctx);
 
 } // namespace onnx2trt

ShapedWeights.hpp

@@ -9,7 +9,7 @@
 #include <cassert>
 #include <string>
 
-// Used to strip out Eris build path information from debug prints
+// Used to strip out build path information from debug prints
 #if defined(SOURCE_LENGTH)
 #define __FILENAME__ (__FILE__ + SOURCE_LENGTH)
 #else
@@ -61,7 +61,7 @@
         }                                                                                                              \
     } while (0)
 
-#define TRT_CHECK(call)                                                                                                \
+#define CHECK(call)                                                                                                    \
     do                                                                                                                 \
     {                                                                                                                  \
         Status status = call;                                                                                          \

Status.hpp

@@ -93,6 +93,35 @@ class TensorOrWeights
     {
         return is_tensor() ? _tensor->getName() : _weights.getName();
     }
+    std::string getType() const
+    {
+        if (is_tensor())
+        {
+            switch(_tensor->getType())
+            {
+                case nvinfer1::DataType::kFLOAT:return "FLOAT";
+                case nvinfer1::DataType::kHALF: return "HALF";
+                case nvinfer1::DataType::kINT8: return "INT8";
+                case nvinfer1::DataType::kINT32: return "INT32";
+                case nvinfer1::DataType::kBOOL: return "BOOL";
+                default: return "UNKNOWN TYPE";
+            }
+        }
+        else
+        {
+            switch(_weights.type)
+            {
+                case ::ONNX_NAMESPACE::TensorProto::DOUBLE: return "DOUBLE -> FLOAT";
+                case ::ONNX_NAMESPACE::TensorProto::FLOAT: return "FLOAT";
+                case ::ONNX_NAMESPACE::TensorProto::INT8: return "INT8";
+                case ::ONNX_NAMESPACE::TensorProto::FLOAT16: return "HALF";
+                case ::ONNX_NAMESPACE::TensorProto::BOOL: return "BOOL";
+                case ::ONNX_NAMESPACE::TensorProto::INT32: return "INT32";
+                case ::ONNX_NAMESPACE::TensorProto::INT64: return "INT64 -> INT32";
+                default: return "UNKNOWN TYPE";
+            }
+        }
+    }
 };
 
 } // namespace onnx2trt

TensorOrWeights.hpp

@@ -2,9 +2,25 @@
 
 # ONNX-TensorRT Changelog
 
-## 21.05 Container Release - 2021-05-19
+## TensorRT 8.0 Release - 2021-07-02
+### Added
+ - Rehauled resize operator, now fully supporting the following modes:
+    - Coordinate Transformation modes: `half_pixel`, `pytorch_half_pixel`, `tf_half_pixel_for_nn`, `asymmetric`, and `align_corners`
+    - Modes: `nearest`, `linear`
+    - Nearest Modes: `floor`, `ceil`, `round_prefer_floor`, `round_prefer_ceil`
+ - QuantizeLinear/DequantizeLinear updates:
+   - Added support for tensor scales
+   - Added support for per-axis quantization
+ - Added support for multi-input ConvTranpose
+ - Added support for generic 2D padding
+ - Added experimental support for `NonMaxSuppression`
+
+### Updated
+ - Moved `RefitMap` API to core TensorRT.
+ - Added Datatype column to [operators.md](https://github.com/onnx/onnx-tensorrt/blob/master/docs/operators.md)
+
+## 21.05 Container Release - 2021-05-17
 ### Added
-- Added support for InstanceNormalization on 5D tensors
 - Added library only build target [#659](https://github.com/onnx/onnx-tensorrt/pull/659)
 - Added support for negative gather indices [#681](https://github.com/onnx/onnx-tensorrt/pull/681)
 - Added support for `DOUBLE`-typed inputs and weights through downcast to float [#674](https://github.com/onnx/onnx-tensorrt/pull/674)

builtin_op_importers.cpp

@@ -2,61 +2,4 @@
 
 # Contributing
 
-Contributions are always welcome to improve the onnx-tensorrt parser. For those looking to contribute, please follow the PR process as outlined in the [TensorRT Open Source Software repository](https://github.com/NVIDIA/TensorRT/blob/master/CONTRIBUTING.md).
-
-#### Signing Your Work
-
-* We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license.
-
-  * Any contribution which contains commits that are not Signed-Off will not be accepted.
-
-* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes:
-  ```bash
-  $ git commit -s -m "Add cool feature."
-  ```
-  This will append the following to your commit message:
-  ```
-  Signed-off-by: Your Name <your@email.com>
-  ```
-
-* Full text of the DCO:
-
-  ```
-    Developer Certificate of Origin
-    Version 1.1
-    
-    Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
-    1 Letterman Drive
-    Suite D4700
-    San Francisco, CA, 94129
-    
-    Everyone is permitted to copy and distribute verbatim copies of this
-    license document, but changing it is not allowed.
-    
-    
-    Developer's Certificate of Origin 1.1
-    
-    By making a contribution to this project, I certify that:
-    
-    (a) The contribution was created in whole or in part by me and I
-        have the right to submit it under the open source license
-        indicated in the file; or
-    
-    (b) The contribution is based upon previous work that, to the best
-        of my knowledge, is covered under an appropriate open source
-        license and I have the right under that license to submit that
-        work with modifications, whether created in whole or in part
-        by me, under the same open source license (unless I am
-        permitted to submit under a different license), as indicated
-        in the file; or
-    
-    (c) The contribution was provided directly to me by some other
-        person who certified (a), (b) or (c) and I have not modified
-        it.
-    
-    (d) I understand and agree that this project and the contribution
-        are public and that a record of the contribution (including all
-        personal information I submit with it, including my sign-off) is
-        maintained indefinitely and may be redistributed consistent with
-        this project or the open source license(s) involved.
-  ```
+Contributions are always welcome to improve the onnx-tensorrt parser. For those looking to contribute, please follow the PR process as outlined in [the main TensorRT open source repository](https://github.com/NVIDIA/TensorRT/blob/master/CONTRIBUTING.md).

docs/Changelog.md

@@ -87,7 +87,6 @@ int main(int argc, char* argv[]) {
     common::TRT_Logger trt_logger((nvinfer1::ILogger::Severity)verbosity);
 
     auto trt_builder = common::infer_object(nvinfer1::createInferBuilder(trt_logger));
-
     auto trt_network = common::infer_object(trt_builder->createNetworkV2(1U << static_cast<uint32_t>(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH)));
     auto trt_parser  = common::infer_object(nvonnxparser::createParser(*trt_network, trt_logger));
 
@@ -129,14 +128,15 @@ int main(int argc, char* argv[]) {
     // Note we do not call trt_parser->parse() here since it's already done above in parser->supportsModel()
     if( !engine_filename.empty() ) {
         trt_builder->setMaxBatchSize(max_batch_size);
-        trt_builder->setMaxWorkspaceSize(max_workspace_size);
+        auto builder_config = common::infer_object(trt_builder->createBuilderConfig());
+        builder_config->setMaxWorkspaceSize(max_workspace_size);
 
         cout << "input name: " << trt_network->getInput(0)->getName() << endl;
         cout << "output name: " << trt_network->getOutput(0)->getName() << endl;
         cout << "num layers: " << trt_network->getNbLayers() << endl;
         cout << "outputs: " << trt_network->getNbOutputs() << endl;
 
-        auto trt_engine = common::infer_object(trt_builder->buildCudaEngine(*trt_network.get()));
+        auto trt_engine = common::infer_object(trt_builder->buildEngineWithConfig(*trt_network.get(), *builder_config.get()));
 
         if( verbosity >= (int)nvinfer1::ILogger::Severity::kWARNING ) {
             cout << "Writing TensorRT engine to " << engine_filename << endl;

docs/contributing.md

@@ -34,7 +34,6 @@ void print_usage() {
        << "                [-O passes] (optimize onnx model. Argument is a semicolon-separated list of passes)" << "\n"
        << "                [-p] (list available optimization passes and exit)" << "\n"
        << "                [-l] (list layers and their shapes)" << "\n"
-       << "                [-g] (debug mode)" << "\n"
        << "                [-F] (optimize onnx model in fixed mode)" << "\n"
        << "                [-v] (increase verbosity)" << "\n"
        << "                [-q] (decrease verbosity)" << "\n"
@@ -58,7 +57,6 @@ int main(int argc, char* argv[]) {
   bool optimize_model_fixed = false;
   bool print_optimization_passes_info = false;
   bool print_layer_info = false;
-  bool debug_builder = false;
 
   int arg = 0;
   while( (arg = ::getopt(argc, argv, "o:b:w:t:T:m:d:O:plgFvqVh")) != -1 ) {
@@ -90,7 +88,6 @@ int main(int argc, char* argv[]) {
       else { cerr << "ERROR: -O flag requires argument" << endl; return -1; }
     case 'p': print_optimization_passes_info = true; break;
     case 'l': print_layer_info = true; break;
-    case 'g': debug_builder = true; break;
     case 'F': optimize_model_fixed = true; optimize_model = true; break;
     case 'v': ++verbosity; break;
     case 'q': --verbosity; break;
@@ -273,18 +270,17 @@ int main(int argc, char* argv[]) {
       cout << "    Max batch size:     " << max_batch_size << endl;
       cout << "    Max workspace size: " << max_workspace_size / (1024. * 1024) << " MiB" << endl;
     }
-    trt_builder->setMaxBatchSize(max_batch_size);
-    trt_builder->setMaxWorkspaceSize(max_workspace_size);
+    auto builder_config = common::infer_object(trt_builder->createBuilderConfig());
+    builder_config->setMaxWorkspaceSize(max_workspace_size);
     if( fp16 && model_dtype == nvinfer1::DataType::kHALF) {
-      trt_builder->setHalf2Mode(true);
+      builder_config->setFlag(nvinfer1::BuilderFlag::kFP16);
     } else if( model_dtype == nvinfer1::DataType::kINT8 ) {
       // TODO: Int8 support
       //trt_builder->setInt8Mode(true);
       cerr << "ERROR: Int8 mode not yet supported" << endl;
       return -5;
     }
-    trt_builder->setDebugSync(debug_builder);
-    auto trt_engine = common::infer_object(trt_builder->buildCudaEngine(*trt_network.get()));
+    auto trt_engine = common::infer_object(trt_builder->buildEngineWithConfig(*trt_network.get(), *builder_config.get()));
 
     auto engine_plan = common::infer_object(trt_engine->serialize());
     std::ofstream engine_file(engine_filename.c_str());

docs/operators.md

@@ -12,16 +12,11 @@
 #include <NvInfer.h>
 #include <functional>
 #include <onnx/onnx_pb.h>
-#include <map>
 #include <unordered_map>
 #include <unordered_set>
 #include <fstream>
 #include <vector>
 
-using WeightsPair_t = std::pair<std::string, nvinfer1::WeightsRole>;
-
-using RefitMap_t = std::multimap<std::string, WeightsPair_t>;
-
 namespace onnx2trt
 {
 
@@ -54,10 +49,11 @@ class IImporterContext
     virtual std::string getOnnxFileLocation() = 0;
     virtual void registerTensor(TensorOrWeights tensor, const std::string& basename) = 0;
     virtual void registerLayer(nvinfer1::ILayer* layer, const std::string& basename) = 0;
-    virtual ShapedWeights createTempWeights(ShapedWeights::DataType type, nvinfer1::Dims shape) = 0;
+    virtual ShapedWeights createTempWeights(ShapedWeights::DataType type, nvinfer1::Dims shape, uint8_t value = 0) = 0;
     virtual int64_t getOpsetVersion(const char* domain = "") const = 0;
     virtual nvinfer1::ILogger& logger() = 0;
-    virtual void insertRefitMap(std::string weightsName, std::string layerName, nvinfer1::WeightsRole role) = 0;
+    virtual bool hasError() const = 0;
+    virtual nvinfer1::IErrorRecorder* getErrorRecorder() const = 0;
 
 protected:
     virtual ~IImporterContext()

getSupportedAPITest.cpp

@@ -22,21 +22,29 @@ enum class PluginFormat : uint8_t
 class IPluginExt : public IPlugin
 {
 public:
-    virtual int getTensorRTVersion() const
+    virtual int getTensorRTVersion() const noexcept
     {
         return NV_TENSORRT_VERSION;
     }
-    virtual bool supportsFormat(DataType type, PluginFormat format) const = 0;
+    virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept = 0;
     virtual void configureWithFormat(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs,
-        DataType type, PluginFormat format, int maxBatchSize)
+        DataType type, PluginFormat format, int maxBatchSize) noexcept
         = 0;
 
 protected:
-    void configure(const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) final
+    void configure(
+        const Dims* inputDims, int nbInputs, const Dims* outputDims, int nbOutputs, int maxBatchSize) noexcept final
     {
-        DataType type = nvinfer1::DataType::kFLOAT;
-        PluginFormat format = nvinfer1::PluginFormat::kNCHW;
-        return this->configureWithFormat(inputDims, nbInputs, outputDims, nbOutputs, type, format, maxBatchSize);
+        try
+        {
+            DataType type = nvinfer1::DataType::kFLOAT;
+            PluginFormat format = nvinfer1::PluginFormat::kLINEAR;
+            return this->configureWithFormat(inputDims, nbInputs, outputDims, nbOutputs, type, format, maxBatchSize);
+        }
+        catch (const std::exception& e)
+        {
+            nvinfer1::getLogger()->log(nvinfer1::ILogger::Severity::kERROR, e.what().c_str());
+        }
     }
     virtual ~IPluginExt()
     {

main.cpp

@@ -5,18 +5,18 @@
 #pragma once
 
 #include "ShapedWeights.hpp"
+#include "ShapeTensor.hpp"
 #include "Status.hpp"
 #include "trt_utils.hpp"
 
 #include <NvInfer.h>
 #include <onnx/onnx_pb.h>
-#include <onnx/onnxifi.h>
 
 #include <cstring> // For std::memcpy
 #include <iostream>
-#include <limits>
 #include <numeric>
 #include <sstream>
+#include <limits>
 
 #define LOG(msg, severity)                                                                                             \
     do                                                                                                                 \
@@ -79,7 +79,10 @@ static std::ostream& operator<<(std::ostream& stream, const nvinfer1::DataType&
 namespace onnx2trt
 {
 
-class ShapeTensor;
+struct PluginDeleter
+{
+    void operator()(nvinfer1::IPluginV2* t);
+};
 
 // Helper function to calculate the volume of a Dims object
 int64_t volume(const nvinfer1::Dims& dims);
@@ -138,6 +141,9 @@ Status broadcastTensors(IImporterContext* ctx, nvinfer1::ITensor*& t1, nvinfer1:
 // Helper function to broadcast three tensors to the largest one's shape
 Status broadcastTensors(IImporterContext* ctx, nvinfer1::ITensor*& t1, nvinfer1::ITensor*& t2, nvinfer1::ITensor*& t3);
 
+// Helper funtion to check that two shapes conform to the broadcasting rules
+Status isBroadcastValid(IImporterContext* ctx, const nvinfer1::Dims& firstShape, const nvinfer1::Dims& secondShape);
+
 // Helper function to calculate the bias tensor for GatherElements.
 std::vector<int32_t> calculateBias(
     const nvinfer1::Dims& daDims, const nvinfer1::Dims& idxDims, const std::vector<int32_t>& pitches, int32_t axis);
@@ -152,7 +158,8 @@ bool canUseLinearResize(const size_t scaleSize, const float* scaleFactors);
 nvinfer1::ITensor* castHelper(IImporterContext* ctx, nvinfer1::ITensor* input, nvinfer1::DataType dtype);
 
 // Helper function for constantOfShape operator. Input shape must be a shape tensor
-nvinfer1::ITensor* constantOfShape(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, nvinfer1::ITensor* constant, nvinfer1::ITensor* shape);
+nvinfer1::ITensor* constantOfShape(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node,
+    nvinfer1::ITensor* constant, nvinfer1::ITensor* shape);
 
 // Helper function to convert an ONNX axis into a TRT axis
 Status convertAxis(int& axis, int nbDims);
@@ -163,12 +170,14 @@ bool convertDtype(int32_t onnx_dtype, nvinfer1::DataType* trt_dtype);
 // Helper function to convert INT64 weight values into INT32
 int32_t* convertINT64(const int64_t* weightValues, nvinfer1::Dims shape, IImporterContext* ctx);
 
-// Helper function to convert negative gather indices into positive ones
+// Helper function to convert negative gather indices into non-negative indices.
 nvinfer1::ITensor* convertGatherIndices(IImporterContext* ctx, nvinfer1::ITensor* data, nvinfer1::ITensor* indices, int32_t axis);
 
-// Helper function to convert ONNX padding into TRT padding
-bool convertOnnxPadding(
-    const std::vector<int64_t>& onnxPadding, nvinfer1::Dims2* begPadding, nvinfer1::Dims2* endPadding);
+// Helper function to convert ONNX padding into TRT padding. Will update begPadding, endPadding, firstPerm, and secondPerm by reference
+bool convertOnnxPadding(std::vector<int64_t>& onnxPadding, nvinfer1::Dims2& begPadding, nvinfer1::Dims2& endPadding, nvinfer1::Permutation& firstPerm, nvinfer1::Permutation& secondPerm);
+
+// Helper function to check if all of the values in the shift tensor are zeros
+bool shiftIsAllZeros(const ShapedWeights& shiftInt8);
 
 // Helper function to create zero shifts for QuantizeLinear/DequantizeLinear ops
 onnx2trt::ShapedWeights createZeroShifts(const onnx2trt::ShapedWeights& shiftInt8, int32_t type, IImporterContext* ctx);
@@ -180,9 +189,9 @@ nvinfer1::ITensor* createZeroTensor(IImporterContext* ctx, nvinfer1::ITensor* da
 bool convertOnnxWeights(
     const ::ONNX_NAMESPACE::TensorProto& onnxTensor, onnx2trt::ShapedWeights* weights, IImporterContext* ctx);
 
-// Helper function to convert multi input convolution
-NodeImportResult convMultiInput(
-    IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, std::vector<TensorOrWeights>& inputs);
+// Helper function to convert multi input convolution/deconvolution
+NodeImportResult convDeconvMultiInput(
+    IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, std::vector<TensorOrWeights>& inputs, bool isConv);
 
 // Helper function to convert a 1D tensor into a scalar
 nvinfer1::ITensor* convertToScalar(IImporterContext* ctx, nvinfer1::ITensor* inpTensor);
@@ -193,10 +202,6 @@ nvinfer1::ITensor& convertToTensor(TensorOrWeights& input, IImporterContext* ctx
 // Helper function to convert a ShapedWeights object into a scalar
 nvinfer1::ITensor* convertToScalar(TensorOrWeights& input, IImporterContext* ctx);
 
-// Helper function to convert an ONNX weight descriptor into a ShapedWeights object
-bool convertWeightDescriptor(
-    onnxTensorDescriptorV1 const& desc, onnx2trt::ShapedWeights* weights, IImporterContext* ctx);
-
 // Helper function to provide a ceiling-rounding division between two integers
 int divCeil(int n, int d);
 
@@ -242,9 +247,6 @@ void getKernelParams(IImporterContext* ctx, ::ONNX_NAMESPACE::NodeProto const& o
     nvinfer1::PaddingMode& paddingMode, bool& count_exclude_padding, nvinfer1::Dims* dilations = nullptr,
     nvinfer1::Dims* output_padding = nullptr, const bool poolingCeilMode = false);
 
-// Helper function to convert ONNX node name. If no node name is provided, use the name of the first output.
-const std::string getNodeName(const ::ONNX_NAMESPACE::NodeProto& node);
-
 // Helper function to get the scaling mode for TRT's scale layer
 nvinfer1::ScaleMode getScaleMode(nvinfer1::Dims const& weights_shape, nvinfer1::Dims const& tensor_shape);
 
@@ -258,10 +260,11 @@ bool isDynamic(const nvinfer1::Dims& shape);
 bool isOnnxTensorEmpty(const ::ONNX_NAMESPACE::TensorProto& onnxTensor);
 
 // Helper function to load a creator from the registry
-nvinfer1::IPluginCreator* importPluginCreator(const std::string& pluginName, const std::string& pluginVersion, const std::string& pluginNamespace="");
+nvinfer1::IPluginCreator* importPluginCreator(
+    const std::string& pluginName, const std::string& pluginVersion, const std::string& pluginNamespace = "");
 
 // Helper function to get a plugin from the PluginRegistry
-nvinfer1::IPluginV2* createPlugin(const std::string& name,
+std::unique_ptr<nvinfer1::IPluginV2, PluginDeleter> createPlugin(const std::string& name,
     nvinfer1::IPluginCreator* pluginCreator, const std::vector<nvinfer1::PluginField>& pluginFields);
 
 // Helper function to determine if a transpose is required
@@ -275,7 +278,7 @@ NodeImportResult lstmLegacyImporter(
 nvinfer1::Dims makeDims(int nbDims, int val);
 
 // Helper function to read weights from an external file
-bool parseExternalWeights(IImporterContext* ctx, std::string file, std::string path, int offset, int length,
+bool parseExternalWeights(IImporterContext* ctx, std::string file, std::string path, int64_t offset, int64_t length,
     std::vector<char>& weightsBuf, size_t& size);
 
 // Helper function to map various ONNX pooling ops into TensorRT.
@@ -284,7 +287,7 @@ NodeImportResult poolingHelper(IImporterContext* ctx, ::ONNX_NAMESPACE::NodeProt
 
 // Helper function to import reduce ops into TRT
 NodeImportResult reduceTensor(IImporterContext* ctx, ::ONNX_NAMESPACE::NodeProto const& node, TensorOrWeights input,
-    nvinfer1::ReduceOperation operation);
+    nvinfer1::ReduceOperation operation, TensorOrWeights inputAxes = TensorOrWeights());
 
 // Helper function to shape a Tensor given a new shape
 nvinfer1::ITensor* reshapeTensor(IImporterContext* ctx, nvinfer1::ITensor& tensor, nvinfer1::Dims shape);
@@ -298,24 +301,59 @@ NodeImportResult scaleHelper(IImporterContext* ctx, const ::ONNX_NAMESPACE::Node
 void setAttr(
     nvinfer1::Dims* trtAttr, ::ONNX_NAMESPACE::AttributeProto const* onnxAttr, int nbSpatialDims, int defaultVal);
 
+// Helper function to slice away elements on a given axis dimension
+nvinfer1::ITensor* sliceAcrossAxis(
+    IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, nvinfer1::ITensor* data, const int axis);
+
 // Helper function to filter out shape tensor outputs for layers that do not support it
-bool supportsShapeTensor(nvinfer1::LayerType type, nvinfer1::ElementWiseOperation eleOp, nvinfer1::ReduceOperation redOp);
+bool supportsShapeTensor(nvinfer1::LayerType type, nvinfer1::ElementWiseOperation eleOp,
+    nvinfer1::ReduceOperation redOp, nvinfer1::FillOperation fillOp);
 
 // Helper function to squeeze a tensor on a given set of axes
 nvinfer1::ITensor* squeezeTensor(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, nvinfer1::ITensor& tensor, const std::vector<int>& axes, bool regLayer = false);
 
 // Helper function to transpose a tensor given a permutation
-nvinfer1::ITensor* transposeTensor(
-    IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, nvinfer1::ITensor& tensor, nvinfer1::Permutation const& perm, bool permute_dim_types = true);
+nvinfer1::ITensor* transposeTensor(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node,
+    nvinfer1::ITensor& tensor, nvinfer1::Permutation const& perm);
 
 // Helper function to import ONNX unary ops into TRT
-NodeImportResult unaryHelper(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, TensorOrWeights& input, nvinfer1::UnaryOperation op);
+NodeImportResult unaryHelper(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, TensorOrWeights& input,
+    nvinfer1::UnaryOperation op);
 
 // Helper function to unsqueeze tensors on a given set of axes
-nvinfer1::ITensor* unsqueezeTensor(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node, nvinfer1::ITensor& tensor, const std::vector<int>& axes, bool regLayer = false);
+nvinfer1::ITensor* unsqueezeTensor(IImporterContext* ctx, const ::ONNX_NAMESPACE::NodeProto& node,
+    nvinfer1::ITensor& tensor, const std::vector<int>& axes, bool regLayer = false);
 
 // Helper function to convert a ShapedWeights object into a vector
-Status weightsToVector(TensorOrWeights weights, std::vector<int64_t>* weightVector);
+template <typename WeightType>
+Status weightsToVector(TensorOrWeights weights, std::vector<WeightType>* weightVector)
+{
+    ASSERT(weights.is_weights(), ErrorCode::kUNSUPPORTED_NODE);
+    ASSERT((weights.weights().type == ::ONNX_NAMESPACE::TensorProto::INT32)
+            || (weights.weights().type == ::ONNX_NAMESPACE::TensorProto::INT64)
+            || (weights.weights().type == ::ONNX_NAMESPACE::TensorProto::BOOL),
+        ErrorCode::kINVALID_NODE);
+    weightVector->resize(weights.weights().count());
+    if (weights.weights().type == ::ONNX_NAMESPACE::TensorProto::INT64)
+    {
+        auto array_start = static_cast<int64_t*>(weights.weights().values);
+        std::copy(array_start, array_start + weights.weights().count(), weightVector->begin());
+    }
+    else if (weights.weights().type == ::ONNX_NAMESPACE::TensorProto::INT32)
+    {
+        auto array_start = static_cast<int32_t*>(weights.weights().values);
+        std::copy(array_start, array_start + weights.weights().count(), weightVector->begin());
+    }
+    else if (weights.weights().type == ::ONNX_NAMESPACE::TensorProto::BOOL)
+    {
+        auto array_start = static_cast<bool*>(weights.weights().values);
+        std::copy(array_start, array_start + weights.weights().count(), weightVector->begin());
+    }
+    return Status(ErrorCode::kSUCCESS);
+}
+
+// Helper function to convert ONNX node name. If no node name, using name of first output.
+const std::string getNodeName(const ::ONNX_NAMESPACE::NodeProto& node);
 
 //! Decode in place the starts and ends indices according to ONNX Slice rules.
 void decodeOnnxStartsAndEnds(IImporterContext* ctx, const ShapeTensor& inputDims, const ShapeTensor& steps, ShapeTensor& starts, ShapeTensor& ends);

onnx2trt.hpp

@@ -0,0 +1,121 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "onnxErrorRecorder.hpp"
+#include <exception>
+
+namespace onnx2trt
+{
+
+
+ONNXParserErrorRecorder* ONNXParserErrorRecorder::create(
+    nvinfer1::ILogger* logger, nvinfer1::IErrorRecorder* otherRecorder)
+{
+    try
+    {
+        auto recorder = new ONNXParserErrorRecorder(logger, otherRecorder);
+        if (recorder)
+        {
+            recorder->incRefCount();
+        }
+        return recorder;
+    }
+    catch (const std::exception& e)
+    {
+        logError(logger, e.what());
+        return nullptr;
+    }
+}
+
+void ONNXParserErrorRecorder::destroy(ONNXParserErrorRecorder*& recorder)
+{
+    if (recorder)
+    {
+        recorder->decRefCount();
+        recorder = nullptr;
+    }
+}
+
+void ONNXParserErrorRecorder::logError(nvinfer1::ILogger* logger, const char* str)
+{
+    if (logger)
+    {
+        logger->log(ILogger::Severity::kERROR, str);
+    }
+}
+
+ONNXParserErrorRecorder::ONNXParserErrorRecorder(
+    nvinfer1::ILogger* logger, nvinfer1::IErrorRecorder* otherRecorder)
+    : mUserRecorder(otherRecorder)
+    , mLogger(logger)
+{
+    if (mUserRecorder)
+    {
+        mUserRecorder->incRefCount();
+    }
+}
+
+ONNXParserErrorRecorder::~ONNXParserErrorRecorder() noexcept
+{
+    if (mUserRecorder)
+    {
+        mUserRecorder->decRefCount();
+    }
+}
+
+void ONNXParserErrorRecorder::clear() noexcept
+{
+    try
+    {
+        // grab a lock so that there is no addition while clearing.
+        std::lock_guard<std::mutex> guard(mStackLock);
+        mErrorStack.clear();
+    }
+    catch (const std::exception& e)
+    {
+        logError(mLogger, e.what());
+    }
+};
+
+bool ONNXParserErrorRecorder::reportError(
+    nvinfer1::ErrorCode val, nvinfer1::IErrorRecorder::ErrorDesc desc) noexcept
+{
+    try
+    {
+        std::lock_guard<std::mutex> guard(mStackLock);
+        mErrorStack.push_back(errorPair(val, desc));
+        if (mUserRecorder)
+        {
+            mUserRecorder->reportError(val, desc);
+        }
+        else
+        {
+            logError(mLogger, desc);
+        }
+    }
+    catch (const std::exception& e)
+    {
+        logError(mLogger, e.what());
+    }
+    // All errors are considered fatal.
+    return true;
+}
+
+nvinfer1::IErrorRecorder::RefCount ONNXParserErrorRecorder::incRefCount() noexcept
+{
+    // Atomically increment or decrement the ref counter.
+    return ++mRefCount;
+}
+
+nvinfer1::IErrorRecorder::RefCount ONNXParserErrorRecorder::decRefCount() noexcept
+{
+    auto newVal = --mRefCount;
+    if (newVal == 0)
+    {
+        delete this;
+    }
+    return newVal;
+}
+
+} // namespace onnx2trt

onnx2trt_common.hpp

@@ -0,0 +1,106 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include "NvInferRuntimeCommon.h"
+#include "onnx2trt_utils.hpp"
+#include <atomic>
+#include <cstdint>
+#include <exception>
+#include <mutex>
+#include <vector>
+
+namespace onnx2trt
+{
+
+//!
+//! A simple implementation of the IErrorRecorder interface for
+//! use by ONNX importer.
+//! ONNX-importer Error recorder is based on a vector that pairs the error
+//! code and the error string into a single element. It also uses
+//! standard mutex and atomics in order to make sure that the code
+//! works in a multi-threaded environment.
+//!
+class ONNXParserErrorRecorder : public nvinfer1::IErrorRecorder
+{
+    using RefCount       = nvinfer1::IErrorRecorder::RefCount;
+    using ErrorDesc      = nvinfer1::IErrorRecorder::ErrorDesc;
+    using ErrorCode      = nvinfer1::ErrorCode;
+    using IErrorRecorder = nvinfer1::IErrorRecorder;
+    using ILogger        = nvinfer1::ILogger;
+
+    using errorPair      = std::pair<ErrorCode, std::string>;
+    using errorStack     = std::vector<errorPair>;
+
+public:
+    static ONNXParserErrorRecorder* create(
+        ILogger* logger, IErrorRecorder* otherRecorder = nullptr);
+
+    static void destroy(ONNXParserErrorRecorder*& recorder);
+
+    void     clear()       noexcept final;
+    RefCount incRefCount() noexcept final;
+    RefCount decRefCount() noexcept final;
+    bool     reportError(ErrorCode val, ErrorDesc desc) noexcept final;
+
+    int32_t getNbErrors() const noexcept final
+    {
+        return mErrorStack.size();
+    }
+
+    ErrorCode getErrorCode(int32_t errorIdx) const noexcept final
+    {
+        return invalidIndexCheck(errorIdx) ? ErrorCode::kINVALID_ARGUMENT : (*this)[errorIdx].first;
+    }
+
+    ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept final
+    {
+        return invalidIndexCheck(errorIdx) ? "errorIdx out of range." : (*this)[errorIdx].second.c_str();
+    }
+
+    bool hasOverflowed() const noexcept final
+    {
+        // This class can never overflow since we have dynamic resize via std::vector usage.
+        return false;
+    }
+
+protected:
+    ONNXParserErrorRecorder(ILogger* logger, IErrorRecorder* otherRecorder = nullptr);
+
+    virtual ~ONNXParserErrorRecorder() noexcept;
+
+    static void logError(ILogger* logger, const char* str);
+
+    // Simple helper functions.
+    const errorPair& operator[](size_t index) const noexcept
+    {
+        return mErrorStack[index];
+    }
+
+    bool invalidIndexCheck(int32_t index) const noexcept
+    {
+        // By converting signed to unsigned, we only need a single check since
+        // negative numbers turn into large positive greater than the size.
+        size_t sIndex = index;
+        return sIndex >= mErrorStack.size();
+    }
+    // Mutex to hold when locking mErrorStack.
+    std::mutex mStackLock;
+
+    // Reference count of the class. Destruction of the class when mRefCount
+    // is not zero causes undefined behavior.
+    std::atomic<int32_t> mRefCount{0};
+
+    // The error stack that holds the errors recorded by TensorRT.
+    errorStack mErrorStack;
+
+    // Original error recorder (set by user)
+    IErrorRecorder* mUserRecorder{nullptr};
+
+    // logger
+    ILogger* mLogger{nullptr};
+}; // class ONNXParserErrorRecorder
+
+} // namespace onnx2trt

onnx2trt_utils.cpp

@@ -33,6 +33,7 @@
 backend_test.include(r'.*test_AvgPool.*')
 backend_test.include(r'.*test_BatchNorm.*eval.*')
 backend_test.include(r'.*test_ceil.*')
+backend_test.include(r'.*test_celu.*')
 backend_test.include(r'.*test_clip.*')
 backend_test.include(r'.*test_concat.*')
 backend_test.include(r'.*test_constant.*')