Skip to content

Commit 67c1cd2

Browse files
AllentDanAllentDan
authored
Add YOLOv5 support for RV1126 device. (#1321)
* support pose simcc * fix yolov5 create_input * add yolov5 post process and update mmcls.yml * add letter resize * add yolov5 export info * fix * add pose face config * pick 4dd4d48 * fix yolov5 head * fix ut * refactor mmpose config * pass output_names outside * rknn batch size * lint * add input names to wrapper * update according to open-mmlab/mmyolo#305 * add pre_compile option * update doc and fix typo * fix padding * fix typo * use throw_exception
1 parent 6d376af commit 67c1cd2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+853
-133
lines changed

‎configs/_base_/backends/rknn.py

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
backend_config = dict(
22
type='rknn',
33
common_config=dict(
4-
mean_values=None, # [[103.53, 116.28, 123.675]],
5-
std_values=None, # [[57.375, 57.12, 58.395]],
64
target_platform='rv1126', # 'rk3588'
75
optimization_level=1),
8-
quantization_config=dict(do_quantization=False, dataset=None))
6+
quantization_config=dict(
7+
do_quantization=True,
8+
dataset=None,
9+
pre_compile=False,
10+
rknn_batch_size=-1))

‎configs/mmcls/classification_rknn-fp16_static-224x224.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
_base_ = ['./classification_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[224, 224])
4+
codebase_config = dict(model_type='end2end')
5+
backend_config = dict(
6+
input_size_list=[[3, 224, 224]],
7+
quantization_config=dict(do_quantization=False))

‎configs/mmcls/classification_rknn_static-224x224.py renamed to ‎configs/mmcls/classification_rknn-int8_static-224x224.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
_base_ = ['./classification_static.py', '../_base_/backends/rknn.py']
22

33
onnx_config = dict(input_shape=[224, 224])
4-
codebase_config = dict(model_type='rknn')
4+
codebase_config = dict(model_type='end2end')
55
backend_config = dict(input_size_list=[[3, 224, 224]])

‎configs/mmdet/detection/detection_rknn-fp16_static-320x320.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
_base_ = ['../_base_/base_static.py', '../../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[320, 320])
4+
5+
codebase_config = dict(model_type='rknn')
6+
7+
backend_config = dict(
8+
input_size_list=[[3, 320, 320]],
9+
quantization_config=dict(do_quantization=False))
10+
11+
# # yolov3, yolox for rknn-toolkit and rknn-toolkit2
12+
# partition_config = dict(
13+
# type='rknn', # the partition policy name
14+
# apply_marks=True, # should always be set to True
15+
# partition_cfg=[
16+
# dict(
17+
# save_file='model.onnx', # name to save the partitioned onnx
18+
# start=['detector_forward:input'], # [mark_name:input, ...]
19+
# end=['yolo_head:input'], # [mark_name:output, ...]
20+
# output_names=[f'pred_maps.{i}' for i in range(3)]) # out names
21+
# ])
22+
23+
# # retinanet, ssd, fsaf for rknn-toolkit2
24+
# partition_config = dict(
25+
# type='rknn', # the partition policy name
26+
# apply_marks=True,
27+
# partition_cfg=[
28+
# dict(
29+
# save_file='model.onnx',
30+
# start='detector_forward:input',
31+
# end=['BaseDenseHead:output'],
32+
# output_names=[f'BaseDenseHead.cls.{i}' for i in range(5)] +
33+
# [f'BaseDenseHead.loc.{i}' for i in range(5)])
34+
# ])

‎configs/mmdet/detection/detection_rknn_static-320x320.py renamed to ‎configs/mmdet/detection/detection_rknn-int8_static-320x320.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,27 @@
66

77
backend_config = dict(input_size_list=[[3, 320, 320]])
88

9-
# yolov3, yolox
9+
# # yolov3, yolox for rknn-toolkit and rknn-toolkit2
1010
# partition_config = dict(
1111
# type='rknn', # the partition policy name
1212
# apply_marks=True, # should always be set to True
1313
# partition_cfg=[
1414
# dict(
1515
# save_file='model.onnx', # name to save the partitioned onnx
1616
# start=['detector_forward:input'], # [mark_name:input, ...]
17-
# end=['yolo_head:input']) # [mark_name:output, ...]
17+
# end=['yolo_head:input'], # [mark_name:output, ...]
18+
# output_names=[f'pred_maps.{i}' for i in range(3)]) # out names
1819
# ])
1920

20-
# # retinanet, ssd, fsaf
21+
# # retinanet, ssd, fsaf for rknn-toolkit2
2122
# partition_config = dict(
2223
# type='rknn', # the partition policy name
2324
# apply_marks=True,
2425
# partition_cfg=[
2526
# dict(
2627
# save_file='model.onnx',
2728
# start='detector_forward:input',
28-
# end=['BaseDenseHead:output'])
29+
# end=['BaseDenseHead:output'],
30+
# output_names=[f'BaseDenseHead.cls.{i}' for i in range(5)] +
31+
# [f'BaseDenseHead.loc.{i}' for i in range(5)])
2932
# ])

‎configs/mmdet/detection/yolov3_partition_onnxruntime_static.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@
88
dict(
99
save_file='yolov3.onnx',
1010
start=['detector_forward:input'],
11-
end=['yolo_head:input'])
11+
end=['yolo_head:input'],
12+
output_names=[f'pred_maps.{i}' for i in range(3)])
1213
])

‎configs/mmpose/pose-detection_rknn-fp16_static-256x192.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
_base_ = ['./pose-detection_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[192, 256])
4+
5+
codebase_config = dict(model_type='end2end')
6+
7+
backend_config = dict(
8+
input_size_list=[[3, 256, 192]],
9+
quantization_config=dict(do_quantization=False),
10+
common_config=dict(target_platform='rv1126'))

‎configs/mmpose/pose-detection_rknn-fp16_static-256x256.py

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
_base_ = ['./pose-detection_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[256, 256])
4+
5+
codebase_config = dict(model_type='end2end')
6+
7+
backend_config = dict(
8+
input_size_list=[[3, 256, 256]],
9+
quantization_config=dict(do_quantization=False),
10+
common_config=dict(target_platform='rv1126'))

‎configs/mmpose/pose-detection_rknn-int8_static-256x192.py

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
_base_ = ['./pose-detection_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[192, 256])
4+
5+
codebase_config = dict(model_type='end2end')
6+
7+
backend_config = dict(
8+
input_size_list=[[3, 256, 192]],
9+
common_config=dict(target_platform='rv1126'))

‎configs/mmpose/pose-detection_rknn-int8_static-256x256.py

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
_base_ = ['./pose-detection_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[256, 256])
4+
5+
codebase_config = dict(model_type='end2end')
6+
7+
backend_config = dict(
8+
input_size_list=[[3, 256, 256]],
9+
common_config=dict(target_platform='rv1126'))

‎configs/mmpose/pose-detection_simcc_rknn-fp16_static-256x192.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
_base_ = ['./pose-detection_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[192, 256], output_names=['simcc_x', 'simcc_y'])
4+
5+
backend_config = dict(
6+
input_size_list=[[3, 256, 192]],
7+
quantization_config=dict(do_quantization=False))

‎configs/mmpose/pose-detection_simcc_rknn-int8_static-256x192.py

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
_base_ = ['./pose-detection_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[192, 256], output_names=['simcc_x', 'simcc_y'])
4+
5+
backend_config = dict(input_size_list=[[3, 256, 192]])

‎configs/mmseg/segmentation_rknn-fp16_static-320x320.py

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
_base_ = ['./segmentation_static.py', '../_base_/backends/rknn.py']
2+
3+
onnx_config = dict(input_shape=[320, 320])
4+
5+
codebase_config = dict(model_type='rknn')
6+
7+
backend_config = dict(
8+
input_size_list=[[3, 320, 320]],
9+
quantization_config=dict(do_quantization=False))

‎configs/mmseg/segmentation_rknn_static-320x320.py renamed to ‎configs/mmseg/segmentation_rknn-int8_static-320x320.py

File renamed without changes.

‎csrc/mmdeploy/codebase/mmdet/base_dense_head.cpp

+2-1
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ Result<Detections> BaseDenseHead::GetBBoxes(const Value& prep_res, const Tensor&
8181

8282
MMDEPLOY_DEBUG("{}-th box: ({}, {}, {}, {}), {}, {}", i, x1, y1, x2, y2, label_id, score);
8383

84-
auto rect = MapToOriginImage(x1, y1, x2, y2, scale_factor.data(), 0, 0, ori_width, ori_height);
84+
auto rect =
85+
MapToOriginImage(x1, y1, x2, y2, scale_factor.data(), 0, 0, ori_width, ori_height, 0, 0);
8586
if (rect[2] - rect[0] < min_bbox_size_ || rect[3] - rect[1] < min_bbox_size_) {
8687
MMDEPLOY_DEBUG("ignore small bbox with width '{}' and height '{}", rect[2] - rect[0],
8788
rect[3] - rect[1]);

‎csrc/mmdeploy/codebase/mmdet/object_detection.cpp

+9-10
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "mmdeploy/core/utils/device_utils.h"
77
#include "mmdeploy/core/utils/formatter.h"
88
#include "mmdeploy/experimental/module_adapter.h"
9+
#include "utils.h"
910

1011
using namespace std;
1112

@@ -127,6 +128,13 @@ Result<Detections> ResizeBBox::GetBBoxes(const Value& prep_res, const Tensor& de
127128
scale_factor = {1.f, 1.f, 1.f, 1.f};
128129
}
129130

131+
int top_padding = 0;
132+
int left_padding = 0;
133+
if (prep_res.contains("pad_param")) {
134+
top_padding = prep_res["pad_param"][0].get<int>();
135+
left_padding = prep_res["pad_param"][1].get<int>();
136+
}
137+
130138
float w_offset = 0.f;
131139
float h_offset = 0.f;
132140
if (prep_res.contains("border")) {
@@ -153,7 +161,7 @@ Result<Detections> ResizeBBox::GetBBoxes(const Value& prep_res, const Tensor& de
153161
MMDEPLOY_DEBUG("ori left {}, top {}, right {}, bottom {}, label {}", left, top, right, bottom,
154162
*labels_ptr);
155163
auto rect = MapToOriginImage(left, top, right, bottom, scale_factor.data(), w_offset, h_offset,
156-
ori_width, ori_height);
164+
ori_width, ori_height, top_padding, left_padding);
157165
if (rect[2] - rect[0] < min_bbox_size_ || rect[3] - rect[1] < min_bbox_size_) {
158166
MMDEPLOY_DEBUG("ignore small bbox with width '{}' and height '{}", rect[2] - rect[0],
159167
rect[3] - rect[1]);
@@ -170,15 +178,6 @@ Result<Detections> ResizeBBox::GetBBoxes(const Value& prep_res, const Tensor& de
170178
}
171179
return objs;
172180
}
173-
std::array<float, 4> ResizeBBox::MapToOriginImage(float left, float top, float right, float bottom,
174-
const float* scale_factor, float x_offset,
175-
float y_offset, int ori_width, int ori_height) {
176-
left = std::max(left / scale_factor[0] + x_offset, 0.f);
177-
top = std::max(top / scale_factor[1] + y_offset, 0.f);
178-
right = std::min(right / scale_factor[2] + x_offset, (float)ori_width - 1.f);
179-
bottom = std::min(bottom / scale_factor[3] + y_offset, (float)ori_height - 1.f);
180-
return {left, top, right, bottom};
181-
}
182181

183182
MMDEPLOY_REGISTER_CODEBASE_COMPONENT(MMDetection, ResizeBBox);
184183

‎csrc/mmdeploy/codebase/mmdet/object_detection.h

-4
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,6 @@ class ResizeBBox : public MMDetection {
2222
template <typename T>
2323
Result<Detections> GetBBoxes(const Value& prep_res, const Tensor& dets, const Tensor& labels);
2424

25-
std::array<float, 4> MapToOriginImage(float left, float top, float right, float bottom,
26-
const float* scale_factor, float x_offset, float y_offset,
27-
int ori_width, int ori_height);
28-
2925
std::vector<Tensor> GetDetsLabels(const Value& prep_res, const Value& infer_res);
3026

3127
protected:

‎csrc/mmdeploy/codebase/mmdet/utils.cpp

+6-5
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,12 @@ namespace mmdeploy::mmdet {
1010

1111
std::array<float, 4> MapToOriginImage(float left, float top, float right, float bottom,
1212
const float* scale_factor, float x_offset, float y_offset,
13-
int ori_width, int ori_height) {
14-
left = std::max(left / scale_factor[0] + x_offset, 0.f);
15-
top = std::max(top / scale_factor[1] + y_offset, 0.f);
16-
right = std::min(right / scale_factor[2] + x_offset, (float)ori_width - 1.f);
17-
bottom = std::min(bottom / scale_factor[3] + y_offset, (float)ori_height - 1.f);
13+
int ori_width, int ori_height, int top_padding,
14+
int left_padding) {
15+
left = std::max((left - left_padding) / scale_factor[0] + x_offset, 0.f);
16+
top = std::max((top - top_padding) / scale_factor[1] + y_offset, 0.f);
17+
right = std::min((right - left_padding) / scale_factor[2] + x_offset, (float)ori_width - 1.f);
18+
bottom = std::min((bottom - top_padding) / scale_factor[3] + y_offset, (float)ori_height - 1.f);
1819
return {left, top, right, bottom};
1920
}
2021

‎csrc/mmdeploy/codebase/mmdet/utils.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
namespace mmdeploy::mmdet {
1212
std::array<float, 4> MapToOriginImage(float left, float top, float right, float bottom,
1313
const float* scale_factor, float x_offset, float y_offset,
14-
int ori_width, int ori_height);
14+
int ori_width, int ori_height, int top_padding,
15+
int left_padding);
1516
// @brief Filter results using score threshold and topk candidates.
1617
// scores (Tensor): The scores, shape (num_bboxes, K).
1718
// probs: The scores after being filtered

0 commit comments

Comments
 (0)