[Other] Add accuracy evaluation scripts (#1034)

* add accuracy scripts

* add accuracy scripts

* Add FlyCV doc

* fix conflict

* fix conflict

* fix conflict

Author: yunyaoXYY

tests/acc_eval/README.md

@@ -0,0 +1,29 @@
+# 模型精度批量验证脚本
+
+本目录下的Python脚本可以在CPU/GPU/昆仑芯/昇腾,以及后续的新增硬件上, 完成对高优模型的精度批量验证.
+各模型的精度测试代码是基于Python部署demo修改而成, 当后续有新增硬件或者新增模型时，用户可以通过同样的方式(新增option和模型),添加新的Python代码来完成精度验证.
+
+
+## 用法
+
+### 1.准备数据集
+- 分类模型需要ImageNet验证集以及标签
+- 检测模型需要COCO2017验证集以及标签
+- 分割模型需要Cityscape验证集以及标签
+- PP-OCRv2/v3的数据集在准备脚本中会自行下载.
+
+请将准备好的数据集解压至dataset目录中使用
+
+### 2.精度验证
+分类/检测/分割/OCR四个场景的精度验证启用方式是一样的.
+其中分类, 检测和分割模型会返回预测精度, OCR模型会返回与GPU预测结果的差异.
+
+```bash
+# 进入分类模型目录下
+cd classification
+# 执行prepare.sh脚本,自动下载并解压模型至models文件夹下
+bash prepare.sh
+# 首先修改run.sh中的TARGET_DEVICE为想测试的硬件,之后执行run.sh脚本
+bash run.sh
+# 验证完毕的输出以及精度数据,会保存至log文件夹下,用户自行查看
+```

tests/acc_eval/classification/eval.py

@@ -0,0 +1,66 @@
+import fastdeploy as fd
+import cv2
+import os
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model", required=True, help="Path of PaddleClas model.")
+    parser.add_argument(
+        "--image", type=str, required=True, help="Path of test image file.")
+    parser.add_argument(
+        "--topk", type=int, default=1, help="Return topk results.")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Type of inference device, support 'cpu' or 'gpu' or 'ipu' or 'kunlunxin' or 'ascend' ."
+    )
+    parser.add_argument(
+        "--use_trt",
+        type=ast.literal_eval,
+        default=False,
+        help="Wether to use tensorrt.")
+    return parser.parse_args()
+
+
+def build_option(args):
+    option = fd.RuntimeOption()
+
+    if args.device.lower() == "gpu":
+        option.use_gpu()
+
+    if args.device.lower() == "ipu":
+        option.use_ipu()
+
+    if args.device.lower() == "kunlunxin":
+        option.use_kunlunxin()
+
+    if args.device.lower() == "ascend":
+        option.use_ascend()
+
+    if args.use_trt:
+        option.use_trt_backend()
+    return option
+
+
+args = parse_arguments()
+
+# 配置runtime，加载模型
+runtime_option = build_option(args)
+
+model_file = os.path.join(args.model, "inference.pdmodel")
+params_file = os.path.join(args.model, "inference.pdiparams")
+config_file = os.path.join(args.model, "inference_cls.yaml")
+model = fd.vision.classification.PaddleClasModel(
+    model_file, params_file, config_file, runtime_option=runtime_option)
+
+res = fd.vision.evaluation.eval_classify(
+    model=model,
+    image_file_path="../dataset/imagenet/",
+    label_file_path="../dataset/imagenet/val_list.txt",
+    topk=1)
+print(res)

tests/acc_eval/classification/prepare.sh

@@ -0,0 +1,28 @@
+mkdir models
+cd models
+
+wget https://bj.bcebos.com/paddlehub/fastdeploy/PPLCNet_x1_0_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/PPLCNetV2_base_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/EfficientNetB7_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/EfficientNetB0_small_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/GhostNet_x1_3_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/GhostNet_x0_5_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV1_x0_25_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV1_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV2_x0_25_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV2_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV3_small_x0_35_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/MobileNetV3_large_x1_0_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/ShuffleNetV2_x0_25_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/ShuffleNetV2_x2_0_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/SqueezeNet1_1_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/InceptionV3_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/PPHGNet_tiny_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/PPHGNet_base_ssld_infer.tgz
+wget https://bj.bcebos.com/paddlehub/fastdeploy/ResNet50_vd_infer.tgz
+
+ls *.tgz | xargs -n1 tar xzvf
+
+rm -rf *.tgz
+
+cd ..

tests/acc_eval/classification/run.sh

@@ -0,0 +1,8 @@
+TARGET_DEVICE=ascend
+
+model_dir=`ls ./models/`
+
+for MODEL_NAME in $model_dir
+do
+    python infer.py --model ./models/$MODEL_NAME  --image None --device $TARGET_DEVICE 2>&1 | tee ./log/${MODEL_NAME}_acc.log
+done

tests/acc_eval/detection/eval_faster_rcnn.py

@@ -0,0 +1,69 @@
+import fastdeploy as fd
+import cv2
+import os
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_dir",
+        default=None,
+        help="Path of PaddleDetection model directory")
+    parser.add_argument(
+        "--image", default=None, help="Path of test image file.")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Type of inference device, support 'cpu' or 'gpu'.")
+    parser.add_argument(
+        "--use_trt",
+        type=ast.literal_eval,
+        default=False,
+        help="Wether to use tensorrt.")
+    return parser.parse_args()
+
+
+def build_option(args):
+    option = fd.RuntimeOption()
+
+    if args.device.lower() == "gpu":
+        option.use_gpu()
+
+    if args.device.lower() == "kunlunxin":
+        option.use_kunlunxin()
+
+    if args.device.lower() == "ascend":
+        option.use_ascend()
+
+    if args.use_trt:
+        option.use_trt_backend()
+        option.set_trt_input_shape("image", [1, 3, 640, 640])
+        option.set_trt_input_shape("scale_factor", [1, 2])
+    return option
+
+
+args = parse_arguments()
+
+if args.model_dir is None:
+    model_dir = fd.download_model(name='faster_rcnn_r50_vd_fpn_2x_coco')
+else:
+    model_dir = args.model_dir
+
+model_file = os.path.join(model_dir, "model.pdmodel")
+params_file = os.path.join(model_dir, "model.pdiparams")
+config_file = os.path.join(model_dir, "infer_cfg.yml")
+
+# 配置runtime，加载模型
+runtime_option = build_option(args)
+model = fd.vision.detection.FasterRCNN(
+    model_file, params_file, config_file, runtime_option=runtime_option)
+
+image_file_path = "../dataset/coco/val2017"
+annotation_file_path = "../dataset/coco/annotations/instances_val2017.json"
+
+res = fd.vision.evaluation.eval_detection(model, image_file_path,
+                                          annotation_file_path)
+print(res)

tests/acc_eval/detection/eval_mask_rcnn.py

@@ -0,0 +1,76 @@
+import fastdeploy as fd
+import cv2
+import os
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_dir",
+        default=None,
+        help="Path of PaddleDetection model directory")
+    parser.add_argument(
+        "--image", default=None, help="Path of test image file.")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Type of inference device, support 'cpu' or 'gpu'.")
+    parser.add_argument(
+        "--use_trt",
+        type=ast.literal_eval,
+        default=False,
+        help="Wether to use tensorrt.")
+    return parser.parse_args()
+
+
+def build_option(args):
+    option = fd.RuntimeOption()
+
+    if args.device.lower() == "gpu":
+        # option.use_gpu()
+        print(
+            """GPU inference with Backend::Paddle in python has not been supported yet. \
+          \nWill ignore this option.""")
+
+    if args.use_trt:
+        # TODO(qiuyanjun): may remove TRT option
+        # Backend::TRT has not been supported yet.
+        print(
+            """Backend::TRT has not been supported yet, will ignore this option.\
+              \nPaddleDetection/MaskRCNN has only support Backend::Paddle now."""
+        )
+
+    if args.device.lower() == "kunlunxin":
+        option.use_kunlunxin()
+
+    if args.device.lower() == "ascend":
+        option.use_ascend()
+
+    return option
+
+
+args = parse_arguments()
+
+if args.model_dir is None:
+    model_dir = fd.download_model(name='mask_rcnn_r50_1x_coco')
+else:
+    model_dir = args.model_dir
+
+model_file = os.path.join(model_dir, "model.pdmodel")
+params_file = os.path.join(model_dir, "model.pdiparams")
+config_file = os.path.join(model_dir, "infer_cfg.yml")
+
+# 配置runtime，加载模型
+runtime_option = build_option(args)
+model = fd.vision.detection.MaskRCNN(
+    model_file, params_file, config_file, runtime_option=runtime_option)
+
+image_file_path = "../dataset/coco/val2017"
+annotation_file_path = "../dataset/coco/annotations/instances_val2017.json"
+
+res = fd.vision.evaluation.eval_detection(model, image_file_path,
+                                          annotation_file_path)
+print(res)

tests/acc_eval/detection/eval_picodet.py

@@ -0,0 +1,67 @@
+import fastdeploy as fd
+import cv2
+import os
+
+
+def parse_arguments():
+    import argparse
+    import ast
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_dir",
+        default=None,
+        help="Path of PaddleDetection model directory")
+    parser.add_argument(
+        "--image", default=None, help="Path of test image file.")
+    parser.add_argument(
+        "--device",
+        type=str,
+        default='cpu',
+        help="Type of inference device, support 'cpu' or 'gpu'.")
+    parser.add_argument(
+        "--use_trt",
+        type=ast.literal_eval,
+        default=False,
+        help="Wether to use tensorrt.")
+    return parser.parse_args()
+
+
+def build_option(args):
+    option = fd.RuntimeOption()
+
+    if args.device.lower() == "gpu":
+        option.use_gpu()
+
+    if args.device.lower() == "kunlunxin":
+        option.use_kunlunxin()
+
+    if args.device.lower() == "ascend":
+        option.use_ascend()
+
+    if args.use_trt:
+        option.use_trt_backend()
+    return option
+
+
+args = parse_arguments()
+
+if args.model_dir is None:
+    model_dir = fd.download_model(name='picodet_l_320_coco_lcnet')
+else:
+    model_dir = args.model_dir
+
+model_file = os.path.join(model_dir, "model.pdmodel")
+params_file = os.path.join(model_dir, "model.pdiparams")
+config_file = os.path.join(model_dir, "infer_cfg.yml")
+
+# 配置runtime，加载模型
+runtime_option = build_option(args)
+model = fd.vision.detection.PicoDet(
+    model_file, params_file, config_file, runtime_option=runtime_option)
+
+image_file_path = "../dataset/coco/val2017"
+annotation_file_path = "../dataset/coco/annotations/instances_val2017.json"
+
+res = fd.vision.evaluation.eval_detection(model, image_file_path,
+                                          annotation_file_path)
+print(res)