diff --git a/fastdeploy/libs/__init__.py b/fastdeploy/libs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/fastdeploy/version.py b/fastdeploy/version.py deleted file mode 100644 index 16e152c9104..00000000000 --- a/fastdeploy/version.py +++ /dev/null @@ -1,7 +0,0 @@ -# This file is generated by setup.py. DO NOT EDIT! -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals -version = '0.3.0' -git_version = 'b69f13b26846f2e13f6ad3c81f1d4ad3ad81fdbb' diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index d6331e357c1..ece0973c0c9 100644 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -59,31 +59,6 @@ void DetectionResult::Resize(int size) { label_ids.resize(size); } -void DetectionResult::Sort() { - for (size_t i = 0; i < scores.size(); ++i) { - float max_score = scores[i]; - float index = i; - for (size_t j = i + 1; j < scores.size(); ++j) { - if (max_score < scores[j]) { - max_score = scores[j]; - index = j; - } - } - if (i == index) { - continue; - } - float tmp_score = scores[i]; - scores[i] = scores[index]; - scores[index] = tmp_score; - int32_t tmp_label_id = label_ids[i]; - label_ids[i] = label_ids[index]; - label_ids[index] = tmp_label_id; - std::array tmp_box = boxes[i]; - boxes[i] = boxes[index]; - boxes[index] = tmp_box; - } -} - std::string DetectionResult::Str() { std::string out; out = "DetectionResult: [xmin, ymin, xmax, ymax, score, label_id]\n"; @@ -97,5 +72,5 @@ std::string DetectionResult::Str() { return out; } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index 29e7322062d..22227a26cbd 100644 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -53,8 +53,6 @@ struct FASTDEPLOY_DECL DetectionResult : public BaseResult { void Resize(int size); - void Sort(); - std::string Str(); }; diff --git a/fastdeploy/vision/ultralytics/__init__.py b/fastdeploy/vision/ultralytics/__init__.py index ebfa0e1cec8..3a5446e0f22 100644 --- a/fastdeploy/vision/ultralytics/__init__.py +++ b/fastdeploy/vision/ultralytics/__init__.py @@ -51,6 +51,10 @@ def padding_value(self): def is_no_pad(self): return self.model.is_no_pad + @property + def is_mini_pad(self): + return self.model.is_mini_pad + @property def is_scale_up(self): return self.model.is_scale_up @@ -59,14 +63,16 @@ def is_scale_up(self): def stride(self): return self.model.stride + @property + def max_wh(self): + return self.model.max_wh + @size.setter def size(self, wh): - assert isinstance(wh, [ - list, tuple - ]), "The value to set `size` must be type of tuple or list." - assert len( - wh - ) == 2, "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( + assert isinstance(wh, [list, tuple]),\ + "The value to set `size` must be type of tuple or list." + assert len(wh) == 2,\ + "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( len(wh)) self.model.size = wh @@ -83,6 +89,13 @@ def is_no_pad(self, value): value, bool), "The value to set `is_no_pad` must be type of bool." self.model.is_no_pad = value + @is_mini_pad.setter + def is_mini_pad(self, value): + assert isinstance( + value, + bool), "The value to set `is_mini_pad` must be type of bool." + self.model.is_mini_pad = value + @is_scale_up.setter def is_scale_up(self, value): assert isinstance( @@ -95,3 +108,9 @@ def stride(self, value): assert isinstance( value, int), "The value to set `stride` must be type of int." self.model.stride = value + + @max_wh.setter + def max_wh(self, value): + assert isinstance( + value, float), "The value to set `max_wh` must be type of float." + self.model.max_wh = value diff --git a/fastdeploy/vision/ultralytics/ultralytics_pybind.cc b/fastdeploy/vision/ultralytics/ultralytics_pybind.cc index bfb81b0c24f..3b73b586fee 100644 --- a/fastdeploy/vision/ultralytics/ultralytics_pybind.cc +++ b/fastdeploy/vision/ultralytics/ultralytics_pybind.cc @@ -34,6 +34,8 @@ void BindUltralytics(pybind11::module& m) { &vision::ultralytics::YOLOv5::padding_value) .def_readwrite("is_mini_pad", &vision::ultralytics::YOLOv5::is_mini_pad) .def_readwrite("is_no_pad", &vision::ultralytics::YOLOv5::is_no_pad) - .def_readwrite("is_scale_up", &vision::ultralytics::YOLOv5::stride); + .def_readwrite("is_scale_up", &vision::ultralytics::YOLOv5::is_scale_up) + .def_readwrite("stride", &vision::ultralytics::YOLOv5::stride) + .def_readwrite("max_wh", &vision::ultralytics::YOLOv5::max_wh); } } // namespace fastdeploy diff --git a/fastdeploy/vision/ultralytics/yolov5.cc b/fastdeploy/vision/ultralytics/yolov5.cc index 79158dd60d1..632c825e598 100644 --- a/fastdeploy/vision/ultralytics/yolov5.cc +++ b/fastdeploy/vision/ultralytics/yolov5.cc @@ -64,8 +64,9 @@ bool YOLOv5::Initialize() { padding_value = {114.0, 114.0, 114.0}; is_mini_pad = false; is_no_pad = false; - is_scale_up = true; + is_scale_up = false; stride = 32; + max_wh = 7680.0; if (!InitRuntime()) { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; @@ -76,6 +77,18 @@ bool YOLOv5::Initialize() { bool YOLOv5::Preprocess(Mat* mat, FDTensor* output, std::map>* im_info) { + // process after image load + double ratio = (size[0] * 1.0) / std::max(static_cast(mat->Height()), + static_cast(mat->Width())); + if (ratio != 1.0) { + int interp = cv::INTER_AREA; + if (ratio > 1.0) { + interp = cv::INTER_LINEAR; + } + int resize_h = int(mat->Height() * ratio); + int resize_w = int(mat->Width() * ratio); + Resize::Run(mat, resize_w, resize_h, -1, -1, interp); + } // yolov5's preprocess steps // 1. letterbox // 2. BGR->RGB @@ -119,11 +132,14 @@ bool YOLOv5::Postprocess( if (confidence <= conf_threshold) { continue; } + int32_t label_id = std::distance(data + s + 5, max_class_score); // convert from [x, y, w, h] to [x1, y1, x2, y2] result->boxes.emplace_back(std::array{ - data[s] - data[s + 2] / 2, data[s + 1] - data[s + 3] / 2, - data[s + 0] + data[s + 2] / 2, data[s + 1] + data[s + 3] / 2}); - result->label_ids.push_back(std::distance(data + s + 5, max_class_score)); + data[s] - data[s + 2] / 2.0f + label_id * max_wh, + data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, + data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, + data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); + result->label_ids.push_back(label_id); result->scores.push_back(confidence); } utils::NMS(result, nms_iou_threshold); @@ -141,8 +157,12 @@ bool YOLOv5::Postprocess( for (size_t i = 0; i < result->boxes.size(); ++i) { float pad_h = (out_h - ipt_h * scale) / 2; float pad_w = (out_w - ipt_w * scale) / 2; - + int32_t label_id = (result->label_ids)[i]; // clip box + result->boxes[i][0] = result->boxes[i][0] - max_wh * label_id; + result->boxes[i][1] = result->boxes[i][1] - max_wh * label_id; + result->boxes[i][2] = result->boxes[i][2] - max_wh * label_id; + result->boxes[i][3] = result->boxes[i][3] - max_wh * label_id; result->boxes[i][0] = std::max((result->boxes[i][0] - pad_w) / scale, 0.0f); result->boxes[i][1] = std::max((result->boxes[i][1] - pad_h) / scale, 0.0f); result->boxes[i][2] = std::max((result->boxes[i][2] - pad_w) / scale, 0.0f); @@ -183,13 +203,11 @@ bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, #endif input_tensors[0].name = InputInfoOfRuntime(0).name; - std::vector output_tensors; if (!Infer(input_tensors, &output_tensors)) { FDERROR << "Failed to inference." << std::endl; return false; } - #ifdef FASTDEPLOY_DEBUG TIMERECORD_END(1, "Inference") TIMERECORD_START(2) diff --git a/fastdeploy/vision/ultralytics/yolov5.h b/fastdeploy/vision/ultralytics/yolov5.h index 7dd901db88b..fab44a6e837 100644 --- a/fastdeploy/vision/ultralytics/yolov5.h +++ b/fastdeploy/vision/ultralytics/yolov5.h @@ -79,6 +79,8 @@ class FASTDEPLOY_DECL YOLOv5 : public FastDeployModel { bool is_scale_up; // padding stride, for is_mini_pad int stride; + // for offseting the boxes by classes when using NMS + float max_wh; }; } // namespace ultralytics } // namespace vision diff --git a/fastdeploy/vision/utils/nms.cc b/fastdeploy/vision/utils/nms.cc index 9f2c1c7bcad..d0cd1d59efd 100644 --- a/fastdeploy/vision/utils/nms.cc +++ b/fastdeploy/vision/utils/nms.cc @@ -22,13 +22,13 @@ namespace utils { // The implementation refers to // https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.4/deploy/cpp/src/utils.cc void NMS(DetectionResult* result, float iou_threshold) { - result->Sort(); + utils::SortDetectionResult(result); std::vector area_of_boxes(result->boxes.size()); std::vector suppressed(result->boxes.size(), 0); for (size_t i = 0; i < result->boxes.size(); ++i) { - area_of_boxes[i] = (result->boxes[i][2] - result->boxes[i][0] + 1) * - (result->boxes[i][3] - result->boxes[i][1] + 1); + area_of_boxes[i] = (result->boxes[i][2] - result->boxes[i][0]) * + (result->boxes[i][3] - result->boxes[i][1]); } for (size_t i = 0; i < result->boxes.size(); ++i) { @@ -43,12 +43,11 @@ void NMS(DetectionResult* result, float iou_threshold) { float ymin = std::max(result->boxes[i][1], result->boxes[j][1]); float xmax = std::min(result->boxes[i][2], result->boxes[j][2]); float ymax = std::min(result->boxes[i][3], result->boxes[j][3]); - float overlap_w = std::max(0.0f, xmax - xmin + 1); - float overlap_h = std::max(0.0f, ymax - ymin + 1); + float overlap_w = std::max(0.0f, xmax - xmin); + float overlap_h = std::max(0.0f, ymax - ymin); float overlap_area = overlap_w * overlap_h; float overlap_ratio = - overlap_area / - (area_of_boxes[i] + area_of_boxes[j] - overlap_area + 1e-06); + overlap_area / (area_of_boxes[i] + area_of_boxes[j] - overlap_area); if (overlap_ratio > iou_threshold) { suppressed[j] = 1; } @@ -67,6 +66,6 @@ void NMS(DetectionResult* result, float iou_threshold) { } } -} // namespace utils -} // namespace vision -} // namespace fastdeploy +} // namespace utils +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/utils/sort_det_res.cc b/fastdeploy/vision/utils/sort_det_res.cc new file mode 100644 index 00000000000..e4a0db97614 --- /dev/null +++ b/fastdeploy/vision/utils/sort_det_res.cc @@ -0,0 +1,77 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace utils { + +void Merge(DetectionResult* result, size_t low, size_t mid, size_t high) { + std::vector>& boxes = result->boxes; + std::vector& scores = result->scores; + std::vector& label_ids = result->label_ids; + std::vector> temp_boxes(boxes); + std::vector temp_scores(scores); + std::vector temp_label_ids(label_ids); + size_t i = low; + size_t j = mid + 1; + size_t k = i; + for (; i <= mid && j <= high; k++) { + if (temp_scores[i] >= temp_scores[j]) { + scores[k] = temp_scores[i]; + label_ids[k] = temp_label_ids[i]; + boxes[k] = temp_boxes[i]; + i++; + } else { + scores[k] = temp_scores[j]; + label_ids[k] = temp_label_ids[j]; + boxes[k] = temp_boxes[j]; + j++; + } + } + while (i <= mid) { + scores[k] = temp_scores[i]; + label_ids[k] = temp_label_ids[i]; + boxes[k] = temp_boxes[i]; + k++; + i++; + } + while (j <= high) { + scores[k] = temp_scores[j]; + label_ids[k] = temp_label_ids[j]; + boxes[k] = temp_boxes[j]; + k++; + j++; + } +} + +void MergeSort(DetectionResult* result, size_t low, size_t high) { + if (low < high) { + size_t mid = (high - low) / 2 + low; + MergeSort(result, low, mid); + MergeSort(result, mid + 1, high); + Merge(result, low, mid, high); + } +} + +void SortDetectionResult(DetectionResult* result) { + size_t low = 0; + size_t high = result->scores.size() - 1; + MergeSort(result, low, high); +} + +} // namespace utils +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/utils/utils.h b/fastdeploy/vision/utils/utils.h index 3e333297c7c..79ece458c80 100644 --- a/fastdeploy/vision/utils/utils.h +++ b/fastdeploy/vision/utils/utils.h @@ -14,11 +14,11 @@ #pragma once +#include +#include #include "fastdeploy/core/fd_tensor.h" #include "fastdeploy/utils/utils.h" #include "fastdeploy/vision/common/result.h" -#include -#include namespace fastdeploy { namespace vision { @@ -53,6 +53,9 @@ std::vector TopKIndices(const T* array, int array_size, int topk) { void NMS(DetectionResult* output, float iou_threshold = 0.5); -} // namespace utils -} // namespace vision -} // namespace fastdeploy +// MergeSort +void SortDetectionResult(DetectionResult* output); + +} // namespace utils +} // namespace vision +} // namespace fastdeploy