From 5e7386fe46213eccf7fee3e14c3b308869ca6f36 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 16:35:54 +0100 Subject: [PATCH 01/11] OpenVINO export --- export.py | 28 +++++++++++++++++++++++++--- requirements.txt | 1 + 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/export.py b/export.py index 7feb525711e8..dc5cca9d449a 100644 --- a/export.py +++ b/export.py @@ -8,6 +8,7 @@ TorchScript | yolov5s.torchscript | `torchscript` ONNX | yolov5s.onnx | `onnx` CoreML | yolov5s.mlmodel | `coreml` +OpenVINO | yolov5s.xml | `openvino` TensorFlow SavedModel | yolov5s_saved_model/ | `saved_model` TensorFlow GraphDef | yolov5s.pb | `pb` TensorFlow Lite | yolov5s.tflite | `tflite` @@ -15,13 +16,14 @@ TensorRT | yolov5s.engine | `engine` Usage: - $ python path/to/export.py --weights yolov5s.pt --include torchscript onnx coreml saved_model pb tflite tfjs + $ python path/to/export.py --weights yolov5s.pt --include torchscript onnx coreml openvino saved_model tflite tfjs Inference: $ python path/to/detect.py --weights yolov5s.pt yolov5s.torchscript yolov5s.onnx yolov5s.mlmodel (under development) + yolov5s.xml (under development) yolov5s_saved_model yolov5s.pb yolov5s.tflite @@ -144,6 +146,24 @@ def export_coreml(model, im, file, prefix=colorstr('CoreML:')): return ct_model +def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): + # YOLOv5 OpenVINO export + ct_model = None + try: + check_requirements(('openvino',)) # https://pypi.org/project/openvino-dev/ + import openvino.inference_engine as ie + + LOGGER.info(f'\n{prefix} starting export with OpenVINO {ie.__version__}...') + f = file.with_suffix('.xml') + + cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {file.parent}" + subprocess.check_output(cmd, shell=True, timeout=5).decode() + + LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') + except Exception as e: + LOGGER.info(f'\n{prefix} export failure: {e}') + + def export_saved_model(model, im, file, dynamic, tf_nms=False, agnostic_nms=False, topk_per_class=100, topk_all=100, iou_thres=0.45, conf_thres=0.25, prefix=colorstr('TensorFlow saved_model:')): @@ -317,7 +337,7 @@ def run(data=ROOT / 'data/coco128.yaml', # 'dataset.yaml path' imgsz=(640, 640), # image (height, width) batch_size=1, # batch size device='cpu', # cuda device, i.e. 0 or 0,1,2,3 or cpu - include=('torchscript', 'onnx', 'coreml'), # include formats + include=('torchscript', 'onnx'), # include formats half=False, # FP16 half-precision export inplace=False, # set YOLOv5 Detect() inplace=True train=False, # model.train() mode @@ -372,12 +392,14 @@ def run(data=ROOT / 'data/coco128.yaml', # 'dataset.yaml path' # Exports if 'torchscript' in include: export_torchscript(model, im, file, optimize) - if 'onnx' in include: + if ('onnx' in include) or ('openvino' in include): # OpenVINO requires ONNX export_onnx(model, im, file, opset, train, dynamic, simplify) if 'engine' in include: export_engine(model, im, file, train, half, simplify, workspace, verbose) if 'coreml' in include: export_coreml(model, im, file) + if 'openvino' in include: + export_openvino(model, im, file) # TensorFlow Exports if any(tf_exports): diff --git a/requirements.txt b/requirements.txt index 22b51fc490e3..96fc9d1a1f32 100755 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ seaborn>=0.11.0 # scikit-learn==0.19.2 # CoreML quantization # tensorflow>=2.4.1 # TFLite export # tensorflowjs>=3.9.0 # TF.js export +# openvino-dev # OpenVINO export # Extras -------------------------------------- # albumentations>=1.0.3 From d734745f94b01f47a3108c7de1cf35c858c0d726 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 16:51:25 +0100 Subject: [PATCH 02/11] Remove timeout --- export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export.py b/export.py index dc5cca9d449a..f4ff1f37c6a1 100644 --- a/export.py +++ b/export.py @@ -157,7 +157,7 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): f = file.with_suffix('.xml') cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {file.parent}" - subprocess.check_output(cmd, shell=True, timeout=5).decode() + subprocess.check_output(cmd, shell=True).decode() LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') except Exception as e: From 5fb7db81d5e39a36c7a7071a9e79d1366ef746f8 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 16:53:42 +0100 Subject: [PATCH 03/11] Add 3 files --- export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/export.py b/export.py index f4ff1f37c6a1..e1c5f87d3464 100644 --- a/export.py +++ b/export.py @@ -154,12 +154,12 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): import openvino.inference_engine as ie LOGGER.info(f'\n{prefix} starting export with OpenVINO {ie.__version__}...') - f = file.with_suffix('.xml') + f = [file.with_suffix('.xml'), file.with_suffix('.bin'), file.with_suffix('.mapping')] cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {file.parent}" subprocess.check_output(cmd, shell=True).decode() - LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') + LOGGER.info(f'{prefix} export success, saved as {f} ({sum(file_size(f) for f in f):.1f} MB)') except Exception as e: LOGGER.info(f'\n{prefix} export failure: {e}') From 75e97c187ae8eaf9140904bcbaddf6619829acdb Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 16:55:10 +0100 Subject: [PATCH 04/11] str --- export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export.py b/export.py index e1c5f87d3464..5d1faa9b1e43 100644 --- a/export.py +++ b/export.py @@ -154,7 +154,7 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): import openvino.inference_engine as ie LOGGER.info(f'\n{prefix} starting export with OpenVINO {ie.__version__}...') - f = [file.with_suffix('.xml'), file.with_suffix('.bin'), file.with_suffix('.mapping')] + f = [str(f) for f in (file.with_suffix('.xml'), file.with_suffix('.bin'), file.with_suffix('.mapping'))] cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {file.parent}" subprocess.check_output(cmd, shell=True).decode() From 73e416c967873a734f4a2d9bcb3f4057352f581a Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 17:01:13 +0100 Subject: [PATCH 05/11] Constrain opset to 12 --- export.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/export.py b/export.py index 5d1faa9b1e43..f446763fb104 100644 --- a/export.py +++ b/export.py @@ -153,7 +153,7 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): check_requirements(('openvino',)) # https://pypi.org/project/openvino-dev/ import openvino.inference_engine as ie - LOGGER.info(f'\n{prefix} starting export with OpenVINO {ie.__version__}...') + LOGGER.info(f'\n{prefix} starting export with openvino {ie.__version__}...') f = [str(f) for f in (file.with_suffix('.xml'), file.with_suffix('.bin'), file.with_suffix('.mapping'))] cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {file.parent}" @@ -345,7 +345,7 @@ def run(data=ROOT / 'data/coco128.yaml', # 'dataset.yaml path' int8=False, # CoreML/TF INT8 quantization dynamic=False, # ONNX/TF: dynamic axes simplify=False, # ONNX: simplify model - opset=14, # ONNX: opset version + opset=12, # ONNX: opset version verbose=False, # TensorRT: verbose log workspace=4, # TensorRT: workspace size (GB) nms=False, # TF: add NMS to model @@ -358,9 +358,12 @@ def run(data=ROOT / 'data/coco128.yaml', # 'dataset.yaml path' t = time.time() include = [x.lower() for x in include] tf_exports = list(x in include for x in ('saved_model', 'pb', 'tflite', 'tfjs')) # TensorFlow exports - imgsz *= 2 if len(imgsz) == 1 else 1 # expand file = Path(url2file(weights) if str(weights).startswith(('http:/', 'https:/')) else weights) + # Checks + imgsz *= 2 if len(imgsz) == 1 else 1 # expand + opset = 12 if ('openvino' in include) else opset # OpenVINO requires opset <= 12 + # Load PyTorch model device = select_device(device) assert not (device.type == 'cpu' and half), '--half only compatible with GPU export, i.e. use --device 0' From f812536a48173c50802d29601bfaef77801e1c66 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 17:01:31 +0100 Subject: [PATCH 06/11] Default ONNX opset to 12 --- export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export.py b/export.py index f446763fb104..64c94177312c 100644 --- a/export.py +++ b/export.py @@ -438,7 +438,7 @@ def parse_opt(): parser.add_argument('--int8', action='store_true', help='CoreML/TF INT8 quantization') parser.add_argument('--dynamic', action='store_true', help='ONNX/TF: dynamic axes') parser.add_argument('--simplify', action='store_true', help='ONNX: simplify model') - parser.add_argument('--opset', type=int, default=14, help='ONNX: opset version') + parser.add_argument('--opset', type=int, default=12, help='ONNX: opset version') parser.add_argument('--verbose', action='store_true', help='TensorRT: verbose log') parser.add_argument('--workspace', type=int, default=4, help='TensorRT: workspace size (GB)') parser.add_argument('--nms', action='store_true', help='TF: add NMS to model') From 2ffb33d2c49b7c83f504c6a9c6872208066f1245 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 17:18:07 +0100 Subject: [PATCH 07/11] Make dir --- export.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/export.py b/export.py index 64c94177312c..9dfba8426d4e 100644 --- a/export.py +++ b/export.py @@ -156,10 +156,11 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): LOGGER.info(f'\n{prefix} starting export with openvino {ie.__version__}...') f = [str(f) for f in (file.with_suffix('.xml'), file.with_suffix('.bin'), file.with_suffix('.mapping'))] - cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {file.parent}" + f = str(file.with_suffix('')) + '_openvino' + cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f}" subprocess.check_output(cmd, shell=True).decode() - LOGGER.info(f'{prefix} export success, saved as {f} ({sum(file_size(f) for f in f):.1f} MB)') + LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') except Exception as e: LOGGER.info(f'\n{prefix} export failure: {e}') From f0c8006d55cf121bfbc090807613d79a9bec73e8 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 17:24:42 +0100 Subject: [PATCH 08/11] Make dir --- export.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/export.py b/export.py index 9dfba8426d4e..4bdad8f928ec 100644 --- a/export.py +++ b/export.py @@ -8,7 +8,7 @@ TorchScript | yolov5s.torchscript | `torchscript` ONNX | yolov5s.onnx | `onnx` CoreML | yolov5s.mlmodel | `coreml` -OpenVINO | yolov5s.xml | `openvino` +OpenVINO | yolov5s_openvino_model/ | `openvino` TensorFlow SavedModel | yolov5s_saved_model/ | `saved_model` TensorFlow GraphDef | yolov5s.pb | `pb` TensorFlow Lite | yolov5s.tflite | `tflite` @@ -23,7 +23,7 @@ yolov5s.torchscript yolov5s.onnx yolov5s.mlmodel (under development) - yolov5s.xml (under development) + yolov5s_openvino_model (under development) yolov5s_saved_model yolov5s.pb yolov5s.tflite @@ -154,9 +154,8 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): import openvino.inference_engine as ie LOGGER.info(f'\n{prefix} starting export with openvino {ie.__version__}...') - f = [str(f) for f in (file.with_suffix('.xml'), file.with_suffix('.bin'), file.with_suffix('.mapping'))] + f = str(file.with_suffix('')) + '_openvino_model' + os.sep - f = str(file.with_suffix('')) + '_openvino' cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f}" subprocess.check_output(cmd, shell=True).decode() From 3e20cc9e4de037daf4155b6df17e57b140db3940 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 17:29:50 +0100 Subject: [PATCH 09/11] Cleanup --- export.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/export.py b/export.py index 4bdad8f928ec..4da770e626be 100644 --- a/export.py +++ b/export.py @@ -148,13 +148,12 @@ def export_coreml(model, im, file, prefix=colorstr('CoreML:')): def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): # YOLOv5 OpenVINO export - ct_model = None try: - check_requirements(('openvino',)) # https://pypi.org/project/openvino-dev/ + check_requirements(('openvino',)) # requires openvino-dev: https://pypi.org/project/openvino-dev/ import openvino.inference_engine as ie LOGGER.info(f'\n{prefix} starting export with openvino {ie.__version__}...') - f = str(file.with_suffix('')) + '_openvino_model' + os.sep + f = str(file).replace('.pt', '_openvino_model' + os.sep) cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f}" subprocess.check_output(cmd, shell=True).decode() From 22d6aa53c3899c19fd7b1efc619e7b0e39415c94 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Tue, 21 Dec 2021 17:31:04 +0100 Subject: [PATCH 10/11] Cleanup --- export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export.py b/export.py index 4da770e626be..d7aebe0567af 100644 --- a/export.py +++ b/export.py @@ -156,7 +156,7 @@ def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): f = str(file).replace('.pt', '_openvino_model' + os.sep) cmd = f"mo --input_model {file.with_suffix('.onnx')} --output_dir {f}" - subprocess.check_output(cmd, shell=True).decode() + subprocess.check_output(cmd, shell=True) LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') except Exception as e: From 30a9e3f5fe244653656efee6eab260b61726136d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Wed, 22 Dec 2021 11:28:07 +0100 Subject: [PATCH 11/11] check_requirements(('openvino-dev',)) --- export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/export.py b/export.py index d7aebe0567af..600e0c318f33 100644 --- a/export.py +++ b/export.py @@ -149,7 +149,7 @@ def export_coreml(model, im, file, prefix=colorstr('CoreML:')): def export_openvino(model, im, file, prefix=colorstr('OpenVINO:')): # YOLOv5 OpenVINO export try: - check_requirements(('openvino',)) # requires openvino-dev: https://pypi.org/project/openvino-dev/ + check_requirements(('openvino-dev',)) # requires openvino-dev: https://pypi.org/project/openvino-dev/ import openvino.inference_engine as ie LOGGER.info(f'\n{prefix} starting export with openvino {ie.__version__}...')