diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py index 2770fe18212..d248b1b1810 100644 --- a/tests/release_task/compare_with_gt.py +++ b/tests/release_task/compare_with_gt.py @@ -1,6 +1,39 @@ import numpy as np import re +diff_score_threshold = { + "linux-x64": { + "label_diff": 0, + "score_diff": 1e-4, + "boxes_diff_ratio": 1e-4, + "boxes_diff": 1e-3 + }, + "linux-aarch64": { + "label_diff": 0, + "score_diff": 1e-4, + "boxes_diff_ratio": 1e-4, + "boxes_diff": 1e-3 + }, + "osx-x86_64": { + "label_diff": 0, + "score_diff": 1e-4, + "boxes_diff_ratio": 2e-4, + "boxes_diff": 1e-3 + }, + "osx-arm64": { + "label_diff": 0, + "score_diff": 1e-4, + "boxes_diff_ratio": 2e-4, + "boxes_diff": 1e-3 + }, + "win-x64": { + "label_diff": 0, + "score_diff": 5e-4, + "boxes_diff_ratio": 1e-3, + "boxes_diff": 1e-3 + } +} + def parse_arguments(): import argparse @@ -59,6 +92,7 @@ def save_numpy_result(file_path, error_msg): def check_result(gt_result, infer_result, args): + platform = args.platform if len(gt_result) != len(infer_result): infer_result = infer_result[-len(gt_result):] diff = np.abs(gt_result - infer_result) @@ -68,17 +102,19 @@ def check_result(gt_result, infer_result, args): boxes_diff_ratio = boxes_diff / (infer_result[:, :-2] + 1e-6) is_diff = False backend = args.result_path.split(".")[0] - if (label_diff > 0).any(): + if (label_diff > diff_score_threshold[platform]["label_diff"]).any(): print(args.platform, args.device, "label diff ", label_diff) is_diff = True label_diff_bool_file = args.platform + "_" + backend + "_" + "label_diff_bool.txt" save_numpy_result(label_diff_bool_file, label_diff > 0) - if (score_diff > 2e-4).any(): + if (score_diff > diff_score_threshold[platform]["score_diff"]).any(): print(args.platform, args.device, "score diff ", score_diff) is_diff = True score_diff_bool_file = args.platform + "_" + backend + "_" + "score_diff_bool.txt" save_numpy_result(score_diff_bool_file, score_diff > 1e-4) - if (boxes_diff_ratio > 1e-4).any() and (boxes_diff > 1e-3).any(): + if (boxes_diff_ratio > diff_score_threshold[platform]["boxes_diff_ratio"] + ).any() and ( + boxes_diff > diff_score_threshold[platform]["boxes_diff"]).any(): print(args.platform, args.device, "boxes diff ", boxes_diff_ratio) is_diff = True boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt" diff --git a/tests/release_task/cpp_run.bat b/tests/release_task/cpp_run.bat index f79a0c01e53..2c8a8aed628 100644 --- a/tests/release_task/cpp_run.bat +++ b/tests/release_task/cpp_run.bat @@ -17,7 +17,7 @@ if "%DEVICE%" == "gpu" ( set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%DEVICE%-%VERSION% set RUN_CASES=ort paddle trt ) else ( - set CPP_FASTDEPLOY_PACKAGE=fastdeploy-python + set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%VERSION% set RUN_CASES=ort paddle openvino ) @@ -36,7 +36,7 @@ set COMPARE_SHELL=%CURRENT_DIR%\compare_with_gt.py python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/dev/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" mkdir build && cd build -cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2" -DCMAKE_CXX_COMPILER=%CMAKE_CXX_COMPILER% +cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2" msbuild infer_demo.sln /m:4 /p:Configuration=Release /p:Platform=x64 @@ -47,7 +47,6 @@ echo "FASTDEPLOY_HOME" %FASTDEPLOY_HOME% copy /Y %FASTDEPLOY_HOME%\third_libs\install\onnxruntime\lib\onnxruntime* Release\ set PATH=%FASTDEPLOY_HOME%\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\onnxruntime\lib;%PATH% -set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv-win-x64-3.4.16\build\x64\vc15\bin;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\paddle\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mkldnn\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mklml\lib;%PATH% @@ -56,9 +55,9 @@ set PATH=%FASTDEPLOY_HOME%\third_libs\install\tensorrt\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\third_party\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\yaml-cpp\lib;%PATH% -set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\bin;%PATH% -set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\3rdparty\tbb\bin;%PATH% - +set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv\build\x64\vc15\bin;%PATH% +set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\bin;%PATH% +set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\3rdparty\tbb\bin;%PATH% echo "set path done" cd %cd%\Release for %%b in (%RUN_CASES%) do ( @@ -71,11 +70,9 @@ for %%b in (%RUN_CASES%) do ( if %%b == trt ( infer_ppyoloe_demo.exe --model_dir=%MODEL_PATH% --image_file=%IMAGE_PATH% --device=gpu --backend=%%b >> cpp_%%b_trt_result.txt python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path cpp_%%b_trt_result.txt --platform %PLATFORM% --device trt --conf_threshold 0.5 - ) else if %%b == ort ( + ) else ( infer_ppyoloe_demo.exe --model_dir=%MODEL_PATH% --image_file=%IMAGE_PATH% --device=gpu --backend=%%b >> cpp_%%b_gpu_result.txt python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path cpp_%%b_gpu_result.txt --platform %PLATFORM% --device gpu --conf_threshold 0.5 - ) else if %%b == paddle ( - echo "Temporarily skip paddle gpu case in windows for Inaccurate inference precision" ) ) ) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 0c14904ca71..21f1df8bfb5 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -12,7 +12,8 @@ fi echo $CPP_FASTDEPLOY_PACKAGE LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') -LINUX_AARCH_CPU_CASE=('ort' 'openvino') +#LINUX_AARCH_CPU_CASE=('ort' 'openvino') +LINUX_AARCH_CPU_CASE=('ort') MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz @@ -25,6 +26,7 @@ GROUND_TRUTH_PATH=$CURRENT_DIR/release_task_groud_truth_result.txt COMPARE_SHELL=$CURRENT_DIR/compare_with_gt.py RUN_CASE=() +CONF_THRESHOLD=0 if [ "$DEVICE" = "gpu" ] && [ "$PLATFORM" = "linux-x64" ];then RUN_CASE=(${LINUX_X64_GPU_CASE[*]}) elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "linux-x64" ];then @@ -35,6 +37,7 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-x86_64" ];then RUN_CASE=(${MACOS_INTEL_CPU_CASE[*]}) elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) + CONF_THRESHOLD=0.5 fi case_number=${#RUN_CASE[@]} @@ -52,16 +55,16 @@ do echo "Cpp Backend:" $backend if [ "$backend" != "trt" ];then ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_cpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD fi if [ "$DEVICE" = "gpu" ];then if [ "$backend" = "trt" ];then ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_trt_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt --conf_threshold $CONF_THRESHOLD else ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_gpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD fi fi done @@ -78,4 +81,3 @@ else cat $res_file exit -1 fi - diff --git a/tests/release_task/py_run.bat b/tests/release_task/py_run.bat index b2e179d1b6c..5be6475494a 100644 --- a/tests/release_task/py_run.bat +++ b/tests/release_task/py_run.bat @@ -46,12 +46,10 @@ for %%b in (%RUN_CASES%) do ( if %%b == trt ( python infer_ppyoloe.py --model_dir=%MODEL_PATH% --image=%IMAGE_PATH% --device=gpu --backend=%%b >> py_%%b_trt_result.txt python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path py_%%b_trt_result.txt --platform %PLATFORM% --device trt --conf_threshold 0.5 - ) else if %%b == ort ( + ) else ( python infer_ppyoloe.py --model_dir=%MODEL_PATH% --image=%IMAGE_PATH% --device=gpu --backend=%%b >> py_%%b_gpu_result.txt python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path py_%%b_gpu_result.txt --platform %PLATFORM% --device gpu --conf_threshold 0.5 - ) else if %%b == paddle ( - echo "Temporarily skip paddle gpu case in windows for Inaccurate inference precision" - ) + ) ) ) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 2856bc4da3c..36624246306 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -12,7 +12,8 @@ fi echo $PY_FASTDEPLOY_PACKAGE LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') -LINUX_AARCH_CPU_CASE=('ort' 'openvino') +#LINUX_AARCH_CPU_CASE=('ort' 'openvino') +LINUX_AARCH_CPU_CASE=('ort') MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz @@ -60,7 +61,7 @@ do fi done -ret=$? +ret=$? res_file="result.txt" if [ ! -f $res_file ];then