From 133167c85eb4ea811cc1cf77cef8ad5bcd624c01 Mon Sep 17 00:00:00 2001 From: SingleZombie Date: Wed, 26 Jan 2022 14:36:08 +0800 Subject: [PATCH 1/4] Add log file --- mmdeploy/codebase/base/task.py | 6 ++++- .../codebase/mmcls/deploy/classification.py | 24 ++++++++++++------- .../codebase/mmdet/deploy/object_detection.py | 15 ++++++++---- .../mmedit/deploy/super_resolution.py | 14 +++++++---- .../codebase/mmocr/deploy/text_detection.py | 17 ++++++++----- .../codebase/mmocr/deploy/text_recognition.py | 17 ++++++++----- .../codebase/mmseg/deploy/segmentation.py | 15 ++++++++---- mmdeploy/utils/timer.py | 24 +++++++------------ tools/test.py | 11 ++++----- 9 files changed, 84 insertions(+), 59 deletions(-) diff --git a/mmdeploy/codebase/base/task.py b/mmdeploy/codebase/base/task.py index 3ef1b582c7..6d0cdaaf82 100644 --- a/mmdeploy/codebase/base/task.py +++ b/mmdeploy/codebase/base/task.py @@ -230,6 +230,7 @@ def evaluate_outputs(model_cfg, out: Optional[str] = None, metric_options: Optional[dict] = None, format_only: bool = False, + log_file: Optional[str] = None, **kwargs): """Perform post-processing to predictions of model. @@ -244,13 +245,16 @@ def evaluate_outputs(model_cfg, for single label dataset, and "mAP", "CP", "CR", "CF1", "OP", "OR", "OF1" for multi-label dataset in mmcls. Defaults is `None`. - out (str): Output result file in pickle format, defaults to `None`. + out (str): Output inference results in pickle format, defaults to + `None`. metric_options (dict): Custom options for evaluation, will be kwargs for dataset.evaluate() function. Defaults to `None`. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to `False`. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ pass diff --git a/mmdeploy/codebase/mmcls/deploy/classification.py b/mmdeploy/codebase/mmcls/deploy/classification.py index ad94e22eb0..a586c4065c 100644 --- a/mmdeploy/codebase/mmcls/deploy/classification.py +++ b/mmdeploy/codebase/mmcls/deploy/classification.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. +import logging from typing import Any, Dict, Optional, Sequence, Tuple, Union import mmcv @@ -208,7 +209,8 @@ def evaluate_outputs(model_cfg: mmcv.Config, metrics: Optional[str] = None, out: Optional[str] = None, metric_options: Optional[dict] = None, - format_only: bool = False) -> None: + format_only: bool = False, + log_file: Optional[str] = None) -> None: """Perform post-processing to predictions of model. Args: @@ -224,13 +226,19 @@ def evaluate_outputs(model_cfg: mmcv.Config, evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Default: False. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ import warnings + from mmcv.utils import get_logger + logger = get_logger('test', log_file=log_file, log_level=logging.INFO) + print(log_file) + print(logger) if metrics: results = dataset.evaluate(outputs, metrics, metric_options) for k, v in results.items(): - print(f'\n{k} : {v:.2f}') + logger.info(f'{k} : {v:.2f}') else: warnings.warn('Evaluation metrics are not specified.') scores = np.vstack(outputs) @@ -243,13 +251,13 @@ def evaluate_outputs(model_cfg: mmcv.Config, 'pred_class': pred_class } if not out: - print('\nthe predicted result for the first element is ' - f'pred_score = {pred_score[0]:.2f}, ' - f'pred_label = {pred_label[0]} ' - f'and pred_class = {pred_class[0]}. ' - 'Specify --out to save all results to files.') + logger.info('the predicted result for the first element is ' + f'pred_score = {pred_score[0]:.2f}, ' + f'pred_label = {pred_label[0]} ' + f'and pred_class = {pred_class[0]}. ' + 'Specify --out to save all results to files.') if out: - print(f'\nwriting results to {out}') + logger.debug(f'writing results to {out}') mmcv.dump(results, out) def get_preprocess(self) -> Dict: diff --git a/mmdeploy/codebase/mmdet/deploy/object_detection.py b/mmdeploy/codebase/mmdet/deploy/object_detection.py index f6f17e8aba..dc8e74cd30 100644 --- a/mmdeploy/codebase/mmdet/deploy/object_detection.py +++ b/mmdeploy/codebase/mmdet/deploy/object_detection.py @@ -7,7 +7,7 @@ from mmcv.parallel import DataContainer from torch.utils.data import Dataset -from mmdeploy.utils import Task, get_root_logger +from mmdeploy.utils import Task from mmdeploy.utils.config_utils import get_input_shape, is_dynamic_shape from ...base import BaseTask from .mmdetection import MMDET_TASK @@ -235,7 +235,8 @@ def evaluate_outputs(model_cfg: mmcv.Config, metrics: Optional[str] = None, out: Optional[str] = None, metric_options: Optional[dict] = None, - format_only: bool = False): + format_only: bool = False, + log_file: Optional[str] = None): """Perform post-processing to predictions of model. Args: @@ -252,10 +253,14 @@ def evaluate_outputs(model_cfg: mmcv.Config, evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to `False`. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ + from mmcv.utils import get_logger + logger = get_logger('test', log_file=log_file) + if out: - logger = get_root_logger() - logger.info(f'\nwriting results to {out}') + logger.debug(f'writing results to {out}') mmcv.dump(outputs, out) kwargs = {} if metric_options is None else metric_options if format_only: @@ -269,7 +274,7 @@ def evaluate_outputs(model_cfg: mmcv.Config, ]: eval_kwargs.pop(key, None) eval_kwargs.update(dict(metric=metrics, **kwargs)) - print(dataset.evaluate(outputs, **eval_kwargs)) + logger.info(dataset.evaluate(outputs, **eval_kwargs)) def get_preprocess(self) -> Dict: """Get the preprocess information for SDK. diff --git a/mmdeploy/codebase/mmedit/deploy/super_resolution.py b/mmdeploy/codebase/mmedit/deploy/super_resolution.py index 3b0e80f36e..8d9140683d 100644 --- a/mmdeploy/codebase/mmedit/deploy/super_resolution.py +++ b/mmdeploy/codebase/mmedit/deploy/super_resolution.py @@ -10,7 +10,7 @@ from mmdeploy.codebase.base import BaseTask from mmdeploy.codebase.mmedit.deploy.mmediting import MMEDIT_TASK -from mmdeploy.utils import Task, get_input_shape, get_root_logger, load_config +from mmdeploy.utils import Task, get_input_shape, load_config def process_model_config(model_cfg: mmcv.Config, @@ -249,6 +249,7 @@ def evaluate_outputs(model_cfg, out: Optional[str] = None, metric_options: Optional[dict] = None, format_only: bool = False, + log_file: Optional[str] = None, **kwargs) -> None: """Evaluation function implemented in mmedit. @@ -265,17 +266,20 @@ def evaluate_outputs(model_cfg, evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to `False`. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ + from mmcv.utils import get_logger + logger = get_logger('test', log_file=log_file) + if out: - logger = get_root_logger() - logger.info(f'\nwriting results to {out}') + logger.debug(f'writing results to {out}') mmcv.dump(outputs, out) # The Dataset doesn't need metrics - print('\n') # print metrics stats = dataset.evaluate(outputs) for stat in stats: - print('Eval-{}: {}'.format(stat, stats[stat])) + logger.info('Eval-{}: {}'.format(stat, stats[stat])) def get_preprocess(self) -> Dict: """Get the preprocess information for SDK. diff --git a/mmdeploy/codebase/mmocr/deploy/text_detection.py b/mmdeploy/codebase/mmocr/deploy/text_detection.py index af5c9b9a66..bdc37a6c21 100644 --- a/mmdeploy/codebase/mmocr/deploy/text_detection.py +++ b/mmdeploy/codebase/mmocr/deploy/text_detection.py @@ -10,7 +10,7 @@ from torch.utils.data import Dataset from mmdeploy.codebase.base import BaseTask -from mmdeploy.utils import Task, get_input_shape, get_root_logger +from mmdeploy.utils import Task, get_input_shape from .mmocr import MMOCR_TASK @@ -246,7 +246,8 @@ def evaluate_outputs(model_cfg, metrics: Optional[str] = None, out: Optional[str] = None, metric_options: Optional[dict] = None, - format_only: bool = False): + format_only: bool = False, + log_file: Optional[str] = None): """Perform post-processing to predictions of model. Args: @@ -254,7 +255,7 @@ def evaluate_outputs(model_cfg, dataset (Dataset): Input dataset to run test. model_cfg (mmcv.Config): The model config. metrics (str): Evaluation metrics, which depends on - the codebase and the dataset, e.g., e.g., "acc" for text + the codebase and the dataset, e.g., "acc" for text recognition, and "hmean-iou" for text detection. out (str): Output result file in pickle format, defaults to `None`. metric_options (dict): Custom options for evaluation, will be @@ -263,10 +264,14 @@ def evaluate_outputs(model_cfg, evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to `False`. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ + from mmcv.utils import get_logger + logger = get_logger('test', log_file=log_file) + if out: - logger = get_root_logger() - logger.info(f'\nwriting results to {out}') + logger.debug(f'writing results to {out}') mmcv.dump(outputs, out) kwargs = {} if metric_options is None else metric_options if format_only: @@ -280,7 +285,7 @@ def evaluate_outputs(model_cfg, ]: eval_kwargs.pop(key, None) eval_kwargs.update(dict(metric=metrics, **kwargs)) - print(dataset.evaluate(outputs, **eval_kwargs)) + logger.info(dataset.evaluate(outputs, **eval_kwargs)) def get_preprocess(self) -> Dict: """Get the preprocess information for SDK. diff --git a/mmdeploy/codebase/mmocr/deploy/text_recognition.py b/mmdeploy/codebase/mmocr/deploy/text_recognition.py index f813024571..6e7eeb0994 100644 --- a/mmdeploy/codebase/mmocr/deploy/text_recognition.py +++ b/mmdeploy/codebase/mmocr/deploy/text_recognition.py @@ -10,7 +10,7 @@ from torch.utils.data import Dataset from mmdeploy.codebase.base import BaseTask -from mmdeploy.utils import Task, get_input_shape, get_root_logger +from mmdeploy.utils import Task, get_input_shape from .mmocr import MMOCR_TASK @@ -259,7 +259,8 @@ def evaluate_outputs(model_cfg: mmcv.Config, metrics: Optional[str] = None, out: Optional[str] = None, metric_options: Optional[dict] = None, - format_only: bool = False): + format_only: bool = False, + log_file: Optional[str] = None): """Perform post-processing to predictions of model. Args: @@ -267,7 +268,7 @@ def evaluate_outputs(model_cfg: mmcv.Config, outputs (list): A list of predictions of model inference. dataset (Dataset): Input dataset to run test. metrics (str): Evaluation metrics, which depends on - the codebase and the dataset, e.g., e.g., "acc" for text + the codebase and the dataset, e.g., "acc" for text recognition, and "hmean-iou" for text detection. out (str): Output result file in pickle format, defaults to `None`. metric_options (dict): Custom options for evaluation, will be @@ -276,10 +277,14 @@ def evaluate_outputs(model_cfg: mmcv.Config, evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to `False`. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ + from mmcv.utils import get_logger + logger = get_logger('test', log_file=log_file) + if out: - logger = get_root_logger() - logger.info(f'\nwriting results to {out}') + logger.debug(f'writing results to {out}') mmcv.dump(outputs, out) kwargs = {} if metric_options is None else metric_options if format_only: @@ -293,7 +298,7 @@ def evaluate_outputs(model_cfg: mmcv.Config, ]: eval_kwargs.pop(key, None) eval_kwargs.update(dict(metric=metrics, **kwargs)) - print(dataset.evaluate(outputs, **eval_kwargs)) + logger.info(dataset.evaluate(outputs, **eval_kwargs)) def get_preprocess(self) -> Dict: """Get the preprocess information for SDK. diff --git a/mmdeploy/codebase/mmseg/deploy/segmentation.py b/mmdeploy/codebase/mmseg/deploy/segmentation.py index a81244abd7..955562a580 100644 --- a/mmdeploy/codebase/mmseg/deploy/segmentation.py +++ b/mmdeploy/codebase/mmseg/deploy/segmentation.py @@ -7,7 +7,7 @@ from torch.utils.data import Dataset from mmdeploy.codebase.base import BaseTask -from mmdeploy.utils import Task, get_input_shape, get_root_logger +from mmdeploy.utils import Task, get_input_shape from .mmsegmentation import MMSEG_TASK @@ -206,7 +206,8 @@ def evaluate_outputs(model_cfg, metrics: Optional[str] = None, out: Optional[str] = None, metric_options: Optional[dict] = None, - format_only: bool = False): + format_only: bool = False, + log_file: Optional[str] = None): """Perform post-processing to predictions of model. Args: @@ -223,16 +224,20 @@ def evaluate_outputs(model_cfg, evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to `False`. + log_file (str | None): The file to write the evaluation results. + Defaults to `None` and the results will only print on stdout. """ + from mmcv.utils import get_logger + logger = get_logger('test', log_file=log_file) + if out: - logger = get_root_logger() - logger.info(f'\nwriting results to {out}') + logger.debug(f'writing results to {out}') mmcv.dump(outputs, out) kwargs = {} if metric_options is None else metric_options if format_only: dataset.format_results(outputs, **kwargs) if metrics: - dataset.evaluate(outputs, metrics, **kwargs) + dataset.evaluate(outputs, metrics, logger=logger, **kwargs) def get_preprocess(self) -> Dict: """Get the preprocess information for SDK. diff --git a/mmdeploy/utils/timer.py b/mmdeploy/utils/timer.py index c0f82b360a..d29a7d8b70 100644 --- a/mmdeploy/utils/timer.py +++ b/mmdeploy/utils/timer.py @@ -1,18 +1,16 @@ # Copyright (c) OpenMMLab. All rights reserved. -import io -import sys import time import warnings from contextlib import contextmanager -from typing import Union +from typing import Optional import torch +from mmcv.utils import get_logger class TimeCounter: """A tool for counting inference time of backends.""" names = dict() - file = sys.stdout # Avoid instantiating every time @classmethod @@ -76,10 +74,7 @@ def fun(*args, **kwargs): msg = f'[{func.__name__}]-{count} times per count: '\ f'{times_per_count:.2f} ms, '\ f'{1000/times_per_count:.2f} FPS' - if cls.file != sys.stdout: - msg += '\n' - cls.file.write(msg) - cls.file.flush() + cls.logger.info(msg) return result @@ -94,7 +89,7 @@ def activate(cls, warmup: int = 1, log_interval: int = 1, with_sync: bool = False, - file: Union[str, io.TextIOWrapper] = sys.stdout): + file: Optional[str] = None): """Activate the time counter. Args: @@ -104,13 +99,12 @@ def activate(cls, log_interval (int): Interval between each log, default 1. with_sync (bool): Whether use cuda synchronize for time counting, default False. - file (str | io.TextIOWrapper): A file or file-like object to save - output messages. The default is `sys.stdout`. + file (str | None): The file to save output messages. The default + is `None`. """ assert warmup >= 1 - if file != sys.stdout: - file = open(file, 'w+') - cls.file = file + logger = get_logger('mmdeploy', log_file=file) + cls.logger = logger if func_name is not None: warnings.warn('func_name must be globally unique if you call ' 'activate multiple times') @@ -127,8 +121,6 @@ def activate(cls, cls.names[name]['with_sync'] = with_sync cls.names[name]['enable'] = True yield - if file != sys.stdout: - cls.file.close() if func_name is not None: cls.names[func_name]['enable'] = False else: diff --git a/tools/test.py b/tools/test.py index c191585afb..a8ee762b3c 100644 --- a/tools/test.py +++ b/tools/test.py @@ -1,6 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. import argparse -import sys from mmcv import DictAction from mmcv.parallel import MMDataParallel @@ -72,7 +71,8 @@ def parse_args(): parser.add_argument( '--log2file', type=str, - help='log speed in file format, require speed-test first') + help='log evaluation results and speed to file', + default=None) args = parser.parse_args() return args @@ -117,15 +117,12 @@ def main(): model.CLASSES = model.module.CLASSES if args.speed_test: with_sync = not is_device_cpu - output_file = sys.stdout - if args.log2file: - output_file = args.log2file with TimeCounter.activate( warmup=args.warmup, log_interval=args.log_interval, with_sync=with_sync, - file=output_file): + file=args.log2file): outputs = task_processor.single_gpu_test(model, data_loader, args.show, args.show_dir) else: @@ -133,7 +130,7 @@ def main(): args.show_dir) task_processor.evaluate_outputs(model_cfg, outputs, dataset, args.metrics, args.out, args.metric_options, - args.format_only) + args.format_only, args.log2file) if __name__ == '__main__': From c1564f9b9e6c90427ef76a7d81ac28c8f292e070 Mon Sep 17 00:00:00 2001 From: SingleZombie Date: Wed, 26 Jan 2022 15:26:36 +0800 Subject: [PATCH 2/4] Delete debug code --- mmdeploy/codebase/mmcls/deploy/classification.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mmdeploy/codebase/mmcls/deploy/classification.py b/mmdeploy/codebase/mmcls/deploy/classification.py index a586c4065c..fb4461ff9f 100644 --- a/mmdeploy/codebase/mmcls/deploy/classification.py +++ b/mmdeploy/codebase/mmcls/deploy/classification.py @@ -232,8 +232,6 @@ def evaluate_outputs(model_cfg: mmcv.Config, import warnings from mmcv.utils import get_logger logger = get_logger('test', log_file=log_file, log_level=logging.INFO) - print(log_file) - print(logger) if metrics: results = dataset.evaluate(outputs, metrics, metric_options) From 03f7bd3f4c5d06c69e4a549e88aae1d3bdfd2e81 Mon Sep 17 00:00:00 2001 From: SingleZombie Date: Wed, 26 Jan 2022 19:25:40 +0800 Subject: [PATCH 3/4] Rename logger --- mmdeploy/utils/timer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdeploy/utils/timer.py b/mmdeploy/utils/timer.py index d29a7d8b70..2c7702ec3d 100644 --- a/mmdeploy/utils/timer.py +++ b/mmdeploy/utils/timer.py @@ -103,7 +103,7 @@ def activate(cls, is `None`. """ assert warmup >= 1 - logger = get_logger('mmdeploy', log_file=file) + logger = get_logger('test', log_file=file) cls.logger = logger if func_name is not None: warnings.warn('func_name must be globally unique if you call ' From 7183d496b5cd7d1921bbe3c036c80537e25ec870 Mon Sep 17 00:00:00 2001 From: SingleZombie Date: Wed, 26 Jan 2022 19:51:19 +0800 Subject: [PATCH 4/4] resolve comments --- docs/en/tutorials/how_to_evaluate_a_model.md | 2 ++ .../tutorials/how_to_measure_performance_of_models.md | 2 +- tools/test.py | 10 +++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/en/tutorials/how_to_evaluate_a_model.md b/docs/en/tutorials/how_to_evaluate_a_model.md index 9fc0ce96bd..83f67b6fd2 100644 --- a/docs/en/tutorials/how_to_evaluate_a_model.md +++ b/docs/en/tutorials/how_to_evaluate_a_model.md @@ -22,6 +22,7 @@ ${MODEL_CFG} \ --device ${DEVICE} \ [--cfg-options ${CFG_OPTIONS}] \ [--metric-options ${METRIC_OPTIONS}] +[--log2file work_dirs/output.txt] ``` ## Description of all arguments @@ -39,6 +40,7 @@ ${MODEL_CFG} \ * `--cfg-options`: Extra or overridden settings that will be merged into the current deploy config. * `--metric-options`: Custom options for evaluation. The key-value pair in xxx=yyy format will be kwargs for dataset.evaluate() function. +* `--log2file`: log evaluation results (and speed) to file. \* Other arguments in `tools/test.py` are used for speed test. They have no concern with evaluation. diff --git a/docs/en/tutorials/how_to_measure_performance_of_models.md b/docs/en/tutorials/how_to_measure_performance_of_models.md index 0addd7dd27..380bcb6a05 100644 --- a/docs/en/tutorials/how_to_measure_performance_of_models.md +++ b/docs/en/tutorials/how_to_measure_performance_of_models.md @@ -24,10 +24,10 @@ ${MODEL_CFG} \ * `deploy_cfg`: The config for deployment. * `model_cfg`: The config of the model in OpenMMLab codebases. * `--model`: The backend model files. For example, if we convert a model to ncnn, we need to pass a ".param" file and a ".bin" file. If we convert a model to TensorRT, we need to pass the model file with ".engine" suffix. +* `--log2file`: log evaluation results and speed to file. * `--speed-test`: Whether to activate speed test. * `--warmup`: warmup before counting inference elapse, require setting speed-test first. * `--log-interval`: The interval between each log, require setting speed-test first. -* `--log2file`: Log speed test result in file format, need speed-test first. \* Other arguments in `tools/test.py` are used for performance test. They have no concern with speed test. diff --git a/tools/test.py b/tools/test.py index a8ee762b3c..50ae79ca44 100644 --- a/tools/test.py +++ b/tools/test.py @@ -54,6 +54,11 @@ def parse_args(): action=DictAction, help='custom options for evaluation, the key-value pair in xxx=yyy ' 'format will be kwargs for dataset.evaluate() function') + parser.add_argument( + '--log2file', + type=str, + help='log evaluation results and speed to file', + default=None) parser.add_argument( '--speed-test', action='store_true', help='activate speed test') parser.add_argument( @@ -68,11 +73,6 @@ def parse_args(): help='the interval between each log, require setting ' 'speed-test first', default=100) - parser.add_argument( - '--log2file', - type=str, - help='log evaluation results and speed to file', - default=None) args = parser.parse_args() return args