From 16974da594e179e4466f2f808edc79cb2b318b87 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 10 Aug 2020 10:14:54 +0000 Subject: [PATCH 01/15] Refine paddle/metrics --- .../incubate/hapi/tests/test_metrics.py | 4 +- python/paddle/metric/__init__.py | 17 +- python/paddle/metric/metrics.py | 256 ++++++++++++++++++ 3 files changed, 263 insertions(+), 14 deletions(-) create mode 100644 python/paddle/metric/metrics.py diff --git a/python/paddle/incubate/hapi/tests/test_metrics.py b/python/paddle/incubate/hapi/tests/test_metrics.py index 3d25a275d5f1c5..b7e24f48559daf 100644 --- a/python/paddle/incubate/hapi/tests/test_metrics.py +++ b/python/paddle/incubate/hapi/tests/test_metrics.py @@ -35,7 +35,7 @@ def accuracy(pred, label, topk=(1, )): res = [] for k in topk: correct_k = correct[:, :k].sum() - res.append(correct_k / batch_size) + res.append(float(correct_k) / batch_size) return res @@ -71,7 +71,7 @@ def test_main(self): label_var = to_variable(label) pred_var = to_variable(pred) state = to_list(acc.add_metric_op(pred_var, label_var)) - acc.update(* [s.numpy() for s in state]) + acc.update(*[s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) assert np.all(np.isclose(np.array(res_m, dtype='float64'), np.array(res_f, dtype='float64'), rtol=1e-3)), \ diff --git a/python/paddle/metric/__init__.py b/python/paddle/metric/__init__.py index e03336f6dbab7b..eb81ec1dc06636 100644 --- a/python/paddle/metric/__init__.py +++ b/python/paddle/metric/__init__.py @@ -12,17 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: define the functions to calculate metric in this directory -__all__ = [ - 'Accuracy', 'Auc', 'ChunkEvaluator', 'CompositeMetric', 'DetectionMAP', - 'EditDistance', 'Precision', 'Recall', 'accuracy', 'auc', 'chunk_eval', - 'cos_sim', 'mean_iou' -] - - - -from ..fluid.metrics import Accuracy, Auc, ChunkEvaluator, CompositeMetric, DetectionMAP, EditDistance, \ - Precision, Recall - from ..fluid.layers.metric_op import accuracy, auc from ..fluid.layers.nn import chunk_eval, cos_sim, mean_iou + +__all__ = [ + 'Accuracy', 'Auc', 'ChunkEvaluator', 'Precision', 'Recall', 'accuracy', + 'auc', 'chunk_eval', 'cos_sim', 'mean_iou' +] diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py new file mode 100644 index 00000000000000..f7180e107999d4 --- /dev/null +++ b/python/paddle/metric/metrics.py @@ -0,0 +1,256 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['Metric', 'Accuracy'] + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import six +import abc +#import numpy as np +import paddle.fluid as fluid + + +@six.add_metaclass(abc.ABCMeta) +class Metric(object): + """ + Base class for metric, encapsulates metric logic and APIs + Usage: + + m = SomeMetric() + for prediction, label in ...: + m.update(prediction, label) + m.accumulate() + + Advanced usage for :code:`compute`: + + Metric calculation can be accelerated by calculating metric states + from model outputs and labels by build-in operators not by Python/Numpy + in :code:`compute`, metric states will be fetched as numpy array and + call :code:`update` with states in numpy format. + Metric calculated as follows (operations in Model and Metric are + indicated with curly brackets, while data nodes not): + inputs & labels || ------------------ + | || + {model} || + | || + outputs & labels || + | || tensor data + {Metric.compute} || + | || + metric states(tensor) || + | || + {fetch as numpy} || ------------------ + | || + metric states(numpy) || numpy data + | || + {Metric.update} \/ ------------------ + Examples: + + For :code:`Accuracy` metric, which takes :code:`pred` and :code:`label` + as inputs, we can calculate the correct prediction matrix between + :code:`pred` and :code:`label` in :code:`compute`. + For examples, prediction results contains 10 classes, while :code:`pred` + shape is [N, 10], :code:`label` shape is [N, 1], N is mini-batch size, + and we only need to calculate accurary of top-1 and top-5, we could + calculate the correct prediction matrix of the top-5 scores of the + prediction of each sample like follows, while the correct prediction + matrix shape is [N, 5]. + + .. code-block:: python + def compute(pred, label): + # sort prediction and slice the top-5 scores + pred = fluid.layers.argsort(pred, descending=True)[1][:, :5] + # calculate whether the predictions are correct + correct = pred == label + return fluid.layers.cast(correct, dtype='float32') + + With the :code:`compute`, we split some calculations to OPs (which + may run on GPU devices, will be faster), and only fetch 1 tensor with + shape as [N, 5] instead of 2 tensors with shapes as [N, 10] and [N, 1]. + :code:`update` can be define as follows: + + .. code-block:: python + def update(self, correct): + accs = [] + for i, k in enumerate(self.topk): + num_corrects = correct[:, :k].sum() + num_samples = len(correct) + accs.append(float(num_corrects) / num_samples) + self.total[i] += num_corrects + self.count[i] += num_samples + return accs + """ + + def __init__(self): + pass + + @abc.abstractmethod + def reset(self): + """ + Reset states and result + """ + raise NotImplementedError("function 'reset' not implemented in {}.". + format(self.__class__.__name__)) + + @abc.abstractmethod + def update(self, *args): + """ + Update states for metric + + Inputs of :code:`update` is the outputs of :code:`Metric.compute`, + if :code:`compute` is not defined, the inputs of :code:`update` + will be flatten arguments of **output** of mode and **label** from data: + :code:`update(output1, output2, ..., label1, label2,...)` + + see :code:`Metric.compute` + """ + raise NotImplementedError("function 'update' not implemented in {}.". + format(self.__class__.__name__)) + + @abc.abstractmethod + def accumulate(self): + """ + Accumulates statistics, computes and returns the metric value + """ + raise NotImplementedError( + "function 'accumulate' not implemented in {}.".format( + self.__class__.__name__)) + + @abc.abstractmethod + def name(self): + """ + Returns metric name + """ + raise NotImplementedError("function 'name' not implemented in {}.". + format(self.__class__.__name__)) + + def compute(self, *args): + """ + This API is advanced usage to accelerate metric calculating, calulations + from outputs of model to the states which should be updated by Metric can + be defined here, where Paddle OPs is also supported. Outputs of this API + will be the inputs of "Metric.update". + + If :code:`compute` is defined, it will be called with **outputs** + of model and **labels** from data as arguments, all outputs and labels + will be concatenated and flatten and each filed as a separate argument + as follows: + :code:`compute(output1, output2, ..., label1, label2,...)` + + If :code:`compute` is not defined, default behaviour is to pass + input to output, so output format will be: + :code:`return output1, output2, ..., label1, label2,...` + + see :code:`Metric.update` + """ + return args + + +class Accuracy(Metric): + """ + Encapsulates accuracy metric logic + + Example by standalone: + + .. code-block:: python + + import numpy as np + import paddle + + paddle.enable_imperative() + + x = paddle.imperative.to_variable(np.array([ + [0.1, 0.2, 0.3, 0.4], + [0.1, 0.4, 0.3, 0.2], + [0.1, 0.2, 0.4, 0.3], + [0.1, 0.2, 0.3, 0.4]])) + y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) + + m = paddle.incubate.hapi.metrics.Accuracy() + correct = m.compute(x, y) + m.update(correct) + res = m.accumulate() + print(res) + + + Example with Model API: + + .. code-block:: python + + import paddle + import paddle.fluid as fluid + import paddle.incubate.hapi as hapi + + fluid.enable_dygraph() + + train_dataset = hapi.datasets.MNIST(mode='train') + + model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) + optim = fluid.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters()) + model.prepare( + optim, + loss_function=paddle.nn.CrossEntropyLoss(), + metrics=paddle.metrics.Accuracy()) + + model.fit(train_dataset, batch_size=64) + + """ + + def __init__(self, topk=(1, ), name=None, *args, **kwargs): + super(Accuracy, self).__init__(*args, **kwargs) + self.topk = topk + self.maxk = max(topk) + self._init_name(name) + self.reset() + + def compute(self, pred, label, *args): + pred = fluid.layers.argsort(pred, descending=True)[1][:, :self.maxk] + correct = pred == label + return fluid.layers.cast(correct, dtype='float32') + + def update(self, correct, *args): + if isinstance(correct, fluid.core.VarBase): + correct = correct.numpy() + accs = [] + for i, k in enumerate(self.topk): + num_corrects = correct[:, :k].sum() + num_samples = len(correct) + accs.append(float(num_corrects) / num_samples) + self.total[i] += num_corrects + self.count[i] += num_samples + return accs + + def reset(self): + self.total = [0.] * len(self.topk) + self.count = [0] * len(self.topk) + + def accumulate(self): + res = [] + for t, c in zip(self.total, self.count): + res.append(float(t) / c) + return res + + def _init_name(self, name): + name = name or 'acc' + if self.maxk != 1: + self._name = ['{}_top{}'.format(name, k) for k in self.topk] + else: + self._name = [name] + + def name(self): + return self._name From 43d8b64bab77a3c309830bd7aeca80eaba583375 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 11 Aug 2020 10:41:14 +0000 Subject: [PATCH 02/15] Move paddle/incubate/hapi/metrics to paddle/metric test=develop --- python/CMakeLists.txt | 1 + .../fluid/tests/unittests/test_layers.py | 27 ++ .../fluid/tests/unittests/test_metrics.py | 49 ---- python/paddle/incubate/hapi/__init__.py | 2 - python/paddle/incubate/hapi/callbacks.py | 4 +- python/paddle/incubate/hapi/metrics.py | 233 ------------------ python/paddle/incubate/hapi/model.py | 19 +- .../hapi/tests/dist_hapi_mnist_dynamic.py | 2 +- .../hapi/tests/dist_hapi_mnist_static.py | 2 +- .../paddle/incubate/hapi/tests/test_model.py | 2 +- python/paddle/metric/__init__.py | 11 +- python/paddle/metric/metrics.py | 61 ++++- python/paddle/tests/CMakeLists.txt | 6 + .../{incubate/hapi => }/tests/test_metrics.py | 50 +++- 14 files changed, 152 insertions(+), 317 deletions(-) delete mode 100644 python/paddle/fluid/tests/unittests/test_metrics.py delete mode 100644 python/paddle/incubate/hapi/metrics.py create mode 100644 python/paddle/tests/CMakeLists.txt rename python/paddle/{incubate/hapi => }/tests/test_metrics.py (72%) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 59dfc5c9d03113..83d1975487f049 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -93,6 +93,7 @@ set(PADDLE_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/) if (WITH_TESTING) add_subdirectory(paddle/reader/tests) add_subdirectory(paddle/dataset/tests) + add_subdirectory(paddle/tests) add_subdirectory(paddle/fluid/tests) add_subdirectory(paddle/fluid/contrib/tests) add_subdirectory(paddle/fluid/contrib/slim/tests) diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 9da70e85f01c0a..86fca98332c02e 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -3686,5 +3686,32 @@ def test_basic_gru(self): batch_first=batch_first) +class TestMetricsDetectionMap(unittest.TestCase): + def test_detection_map(self): + program = fluid.Program() + with program_guard(program): + detect_res = fluid.layers.data( + name='detect_res', + shape=[10, 6], + append_batch_size=False, + dtype='float32') + label = fluid.layers.data( + name='label', + shape=[10, 1], + append_batch_size=False, + dtype='float32') + box = fluid.layers.data( + name='bbox', + shape=[10, 4], + append_batch_size=False, + dtype='float32') + map_eval = fluid.metrics.DetectionMAP( + detect_res, label, box, class_num=21) + cur_map, accm_map = map_eval.get_map_var() + self.assertIsNotNone(cur_map) + self.assertIsNotNone(accm_map) + print(str(program)) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_metrics.py b/python/paddle/fluid/tests/unittests/test_metrics.py deleted file mode 100644 index ec27884cae2b04..00000000000000 --- a/python/paddle/fluid/tests/unittests/test_metrics.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -import paddle.fluid as fluid -from paddle.fluid.framework import Program, program_guard - - -class TestMetricsDetectionMap(unittest.TestCase): - def test_detection_map(self): - program = fluid.Program() - with program_guard(program): - detect_res = fluid.layers.data( - name='detect_res', - shape=[10, 6], - append_batch_size=False, - dtype='float32') - label = fluid.layers.data( - name='label', - shape=[10, 1], - append_batch_size=False, - dtype='float32') - box = fluid.layers.data( - name='bbox', - shape=[10, 4], - append_batch_size=False, - dtype='float32') - map_eval = fluid.metrics.DetectionMAP( - detect_res, label, box, class_num=21) - cur_map, accm_map = map_eval.get_map_var() - self.assertIsNotNone(cur_map) - self.assertIsNotNone(accm_map) - print(str(program)) - - -if __name__ == '__main__': - unittest.main() diff --git a/python/paddle/incubate/hapi/__init__.py b/python/paddle/incubate/hapi/__init__.py index a6b5faef57ca95..c0361fa33246ff 100644 --- a/python/paddle/incubate/hapi/__init__.py +++ b/python/paddle/incubate/hapi/__init__.py @@ -20,7 +20,6 @@ from . import model from .model import * -from . import metrics from . import datasets from . import distributed from . import vision @@ -39,7 +38,6 @@ 'datasets', 'distributed', 'download', - 'metrics', 'vision', 'text', 'utils', diff --git a/python/paddle/incubate/hapi/callbacks.py b/python/paddle/incubate/hapi/callbacks.py index 741552511f9fdc..ed19abe64feb2f 100644 --- a/python/paddle/incubate/hapi/callbacks.py +++ b/python/paddle/incubate/hapi/callbacks.py @@ -306,7 +306,7 @@ class ProgBarLogger(Callback): optim = fluid.optimizer.Adam(0.001) model.prepare(optimizer=optim, loss_function=paddle.nn.CrossEntropyLoss(), - metrics=hapi.metrics.Accuracy()) + metrics=paddle.metric.Accuracy()) callback = hapi.callbacks.ProgBarLogger(log_freq=10) model.fit(train_dataset, batch_size=64, callbacks=callback) @@ -442,7 +442,7 @@ class ModelCheckpoint(Callback): optim = fluid.optimizer.Adam(0.001) model.prepare(optimizer=optim, loss_function=paddle.nn.CrossEntropyLoss(), - metrics=hapi.metrics.Accuracy()) + metrics=paddle.metric.Accuracy()) callback = hapi.callbacks.ModelCheckpoint(save_dir='./temp') model.fit(train_dataset, batch_size=64, callbacks=callback) diff --git a/python/paddle/incubate/hapi/metrics.py b/python/paddle/incubate/hapi/metrics.py deleted file mode 100644 index 9e9a2e78524022..00000000000000 --- a/python/paddle/incubate/hapi/metrics.py +++ /dev/null @@ -1,233 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import six -import abc -import numpy as np -import paddle.fluid as fluid - -import logging - -FORMAT = '%(asctime)s-%(levelname)s: %(message)s' -logging.basicConfig(level=logging.INFO, format=FORMAT) -logger = logging.getLogger(__name__) - -__all__ = ['Metric', 'Accuracy'] - - -@six.add_metaclass(abc.ABCMeta) -class Metric(object): - """ - Base class for metric, encapsulates metric logic and APIs - Usage: - - m = SomeMetric() - for prediction, label in ...: - m.update(prediction, label) - m.accumulate() - - Advanced usage for :code:`add_metric_op` - Metric calculation can be accelerated by calculating metric states - from model outputs and labels by Paddle OPs in :code:`add_metric_op`, - metric states will be fetch as numpy array and call :code:`update` - with states in numpy format. - Metric calculated as follows (operations in Model and Metric are - indicated with curly brackets, while data nodes not): - inputs & labels || ------------------ - | || - {model} || - | || - outputs & labels || - | || tensor data - {Metric.add_metric_op} || - | || - metric states(tensor) || - | || - {fetch as numpy} || ------------------ - | || - metric states(numpy) || numpy data - | || - {Metric.update} \/ ------------------ - Examples: - - For :code:`Accuracy` metric, which takes :code:`pred` and :code:`label` - as inputs, we can calculate the correct prediction matrix between - :code:`pred` and :code:`label` in :code:`add_metric_op`. - For examples, prediction results contains 10 classes, while :code:`pred` - shape is [N, 10], :code:`label` shape is [N, 1], N is mini-batch size, - and we only need to calculate accurary of top-1 and top-5, we could - calculated the correct prediction matrix of the top-5 scores of the - prediction of each sample like follows, while the correct prediction - matrix shape is [N, 5]. - .. code-block:: python - def add_metric_op(pred, label): - # sort prediction and slice the top-5 scores - pred = fluid.layers.argsort(pred, descending=True)[1][:, :5] - # calculate whether the predictions are correct - correct = pred == label - return fluid.layers.cast(correct, dtype='float32') - With the :code:`add_metric_op`, we split some calculations to OPs(which - may run on GPU devices, will be faster), and only fetch 1 tensor with - shape as [N, 5] instead of 2 tensors with shapes as [N, 10] and [N, 1]. - :code:`update` can be define as follows: - .. code-block:: python - def update(self, correct): - accs = [] - for i, k in enumerate(self.topk): - num_corrects = correct[:, :k].sum() - num_samples = len(correct) - accs.append(float(num_corrects) / num_samples) - self.total[i] += num_corrects - self.count[i] += num_samples - return accs - """ - - def __init__(self): - pass - - @abc.abstractmethod - def reset(self): - """ - Reset states and result - """ - raise NotImplementedError("function 'reset' not implemented in {}.". - format(self.__class__.__name__)) - - @abc.abstractmethod - def update(self, *args): - """ - Update states for metric - - Inputs of :code:`update` is the outputs of :code:`Metric.add_metric_op`, - if :code:`add_metric_op` is not defined, the inputs of :code:`update` - will be flatten arguments of **output** of mode and **label** from data: - :code:`update(output1, output2, ..., label1, label2,...)` - - see :code:`Metric.add_metric_op` - """ - raise NotImplementedError("function 'update' not implemented in {}.". - format(self.__class__.__name__)) - - @abc.abstractmethod - def accumulate(self): - """ - Accumulates statistics, computes and returns the metric value - """ - raise NotImplementedError( - "function 'accumulate' not implemented in {}.".format( - self.__class__.__name__)) - - @abc.abstractmethod - def name(self): - """ - Returns metric name - """ - raise NotImplementedError("function 'name' not implemented in {}.". - format(self.__class__.__name__)) - - def add_metric_op(self, *args): - """ - This API is advanced usage to accelerate metric calculating, calulations - from outputs of model to the states which should be updated by Metric can - be defined here, where Paddle OPs is also supported. Outputs of this API - will be the inputs of "Metric.update". - - If :code:`add_metric_op` is defined, it will be called with **outputs** - of model and **labels** from data as arguments, all outputs and labels - will be concatenated and flatten and each filed as a separate argument - as follows: - :code:`add_metric_op(output1, output2, ..., label1, label2,...)` - - If :code:`add_metric_op` is not defined, default behaviour is to pass - input to output, so output format will be: - :code:`return output1, output2, ..., label1, label2,...` - - see :code:`Metric.update` - """ - return args - - -class Accuracy(Metric): - """ - Encapsulates accuracy metric logic - - Examples: - - .. code-block:: python - - import paddle - import paddle.fluid as fluid - import paddle.incubate.hapi as hapi - - fluid.enable_dygraph() - - train_dataset = hapi.datasets.MNIST(mode='train') - - model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) - optim = fluid.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) - model.prepare( - optim, - loss_function=paddle.nn.CrossEntropyLoss(), - metrics=hapi.metrics.Accuracy()) - - model.fit(train_dataset, batch_size=64) - - """ - - def __init__(self, topk=(1, ), name=None, *args, **kwargs): - super(Accuracy, self).__init__(*args, **kwargs) - self.topk = topk - self.maxk = max(topk) - self._init_name(name) - self.reset() - - def add_metric_op(self, pred, label, *args): - pred = fluid.layers.argsort(pred, descending=True)[1][:, :self.maxk] - correct = pred == label - return fluid.layers.cast(correct, dtype='float32') - - def update(self, correct, *args): - accs = [] - for i, k in enumerate(self.topk): - num_corrects = correct[:, :k].sum() - num_samples = len(correct) - accs.append(float(num_corrects) / num_samples) - self.total[i] += num_corrects - self.count[i] += num_samples - return accs - - def reset(self): - self.total = [0.] * len(self.topk) - self.count = [0] * len(self.topk) - - def accumulate(self): - res = [] - for t, c in zip(self.total, self.count): - res.append(float(t) / c) - return res - - def _init_name(self, name): - name = name or 'acc' - if self.maxk != 1: - self._name = ['{}_top{}'.format(name, k) for k in self.topk] - else: - self._name = [name] - - def name(self): - return self._name diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 0b12987b10a051..3644faf81f0b4a 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -34,9 +34,9 @@ from paddle.fluid.incubate.fleet.collective import fleet, DistributedStrategy from paddle.fluid.incubate.fleet.base import role_maker from paddle.io import DataLoader, Dataset +from paddle.metric import Metric from .distributed import DistributedBatchSampler, _all_gather, prepare_distributed_context, _parallel_context_initialized -from .metrics import Metric from .callbacks import config_callbacks from .utils import to_list, to_numpy, flatten_list, restore_flatten_list, extract_args from .device import _get_device @@ -404,7 +404,7 @@ def _make_program(self, mode): if mode != 'test': for metric in self.model._metrics: metrics.append( - to_list(metric.add_metric_op(*(outputs + labels)))) + to_list(metric.compute(*(outputs + labels)))) if mode == 'train' and self.model._optimizer: self._loss_endpoint = fluid.layers.sum(losses) @@ -527,7 +527,7 @@ def train_batch(self, inputs, labels=None): self.model.network.clear_gradients() metrics = [] for metric in self.model._metrics: - metric_outs = metric.add_metric_op(*(to_list(outputs) + labels)) + metric_outs = metric.compute(*(to_list(outputs) + labels)) m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) @@ -571,7 +571,7 @@ def eval_batch(self, inputs, labels=None): self._merge_count[self.mode + '_total'] += samples self._merge_count[self.mode + '_batch'] = samples - metric_outs = metric.add_metric_op(*(to_list(outputs) + labels)) + metric_outs = metric.compute(*(to_list(outputs) + labels)) m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) @@ -712,7 +712,7 @@ def forward(self, x): parameter_list=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss(), - hapi.metrics.Accuracy()) + paddle.metric.Accuracy()) mnist_data = hapi.datasets.MNIST(mode='train', chw_format=False) model.fit(mnist_data, epochs=2, batch_size=32, verbose=1) @@ -1204,7 +1204,7 @@ def fit( model.prepare( optim, paddle.nn.CrossEntropyLoss(), - hapi.metrics.Accuracy(topk=(1, 2))) + paddle.metric.Accuracy(topk=(1, 2))) model.fit(train_dataset, val_dataset, epochs=2, @@ -1241,7 +1241,7 @@ def fit( model.prepare( optim, paddle.nn.CrossEntropyLoss(), - hapi.metrics.Accuracy(topk=(1, 2))) + paddle.metric.Accuracy(topk=(1, 2))) model.fit(train_loader, val_loader, epochs=2, @@ -1353,6 +1353,7 @@ def evaluate( Examples: .. code-block:: python + import paddle import paddle.fluid as fluid import paddle.incubate.hapi as hapi @@ -1362,7 +1363,7 @@ def evaluate( input = hapi.Input('image', [-1, 1, 28, 28], 'float32') label = hapi.Input('label', [None, 1], 'int64') model = hapi.Model(hapi.vision.LeNet(), input, label) - model.prepare(metrics=hapi.metrics.Accuracy()) + model.prepare(metrics=paddle.metric.Accuracy()) result = model.evaluate(val_dataset, batch_size=64) print(result) @@ -1370,7 +1371,7 @@ def evaluate( # imperative mode fluid.enable_dygraph() model = hapi.Model(hapi.vision.LeNet()) - model.prepare(metrics=hapi.metrics.Accuracy()) + model.prepare(metrics=paddle.metric.Accuracy()) result = model.evaluate(val_dataset, batch_size=64) print(result) diff --git a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py index b338f3310b4c79..21fe73aea6592b 100644 --- a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py +++ b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_dynamic.py @@ -25,7 +25,7 @@ from paddle.incubate.hapi import Model, Input, set_device from paddle.nn.layer.loss import CrossEntropyLoss from paddle.incubate.hapi.vision.models import LeNet -from paddle.incubate.hapi.metrics import Accuracy +from paddle.metric import Accuracy from paddle.incubate.hapi.callbacks import ProgBarLogger from paddle.incubate.hapi.datasets import MNIST diff --git a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py index 1484620a4efdff..63903f45993c35 100644 --- a/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py +++ b/python/paddle/incubate/hapi/tests/dist_hapi_mnist_static.py @@ -25,7 +25,7 @@ from paddle.incubate.hapi import Model, Input, set_device from paddle.nn.layer.loss import CrossEntropyLoss from paddle.incubate.hapi.vision.models import LeNet -from paddle.incubate.hapi.metrics import Accuracy +from paddle.metric import Accuracy from paddle.incubate.hapi.callbacks import ProgBarLogger from paddle.incubate.hapi.datasets import MNIST diff --git a/python/paddle/incubate/hapi/tests/test_model.py b/python/paddle/incubate/hapi/tests/test_model.py index f8be2e242568de..21a64a07660e98 100644 --- a/python/paddle/incubate/hapi/tests/test_model.py +++ b/python/paddle/incubate/hapi/tests/test_model.py @@ -29,7 +29,7 @@ import paddle.incubate.hapi as hapi from paddle.incubate.hapi import Model, Input from paddle.nn.layer.loss import CrossEntropyLoss -from paddle.incubate.hapi.metrics import Accuracy +from paddle.metric import Accuracy from paddle.incubate.hapi.datasets import MNIST from paddle.incubate.hapi.vision.models import LeNet from paddle.incubate.hapi.distributed import DistributedBatchSampler, prepare_distributed_context diff --git a/python/paddle/metric/__init__.py b/python/paddle/metric/__init__.py index eb81ec1dc06636..4891dce11e0c55 100644 --- a/python/paddle/metric/__init__.py +++ b/python/paddle/metric/__init__.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +from .metrics import * + from ..fluid.layers.metric_op import accuracy, auc from ..fluid.layers.nn import chunk_eval, cos_sim, mean_iou __all__ = [ - 'Accuracy', 'Auc', 'ChunkEvaluator', 'Precision', 'Recall', 'accuracy', - 'auc', 'chunk_eval', 'cos_sim', 'mean_iou' + 'Accuracy', + 'Auc', + 'accuracy', + 'auc', + 'chunk_eval', + 'cos_sim', + 'mean_iou', ] diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index f7180e107999d4..2093aba38bebe8 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -12,17 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ['Metric', 'Accuracy'] - from __future__ import absolute_import from __future__ import division from __future__ import print_function import six import abc -#import numpy as np import paddle.fluid as fluid +__all__ = ['Metric', 'Accuracy'] + @six.add_metaclass(abc.ABCMeta) class Metric(object): @@ -38,9 +37,9 @@ class Metric(object): Advanced usage for :code:`compute`: Metric calculation can be accelerated by calculating metric states - from model outputs and labels by build-in operators not by Python/Numpy - in :code:`compute`, metric states will be fetched as numpy array and - call :code:`update` with states in numpy format. + from model outputs and labels by build-in operators not by Python/NumPy + in :code:`compute`, metric states will be fetched as NumPy array and + call :code:`update` with states in NumPy format. Metric calculated as follows (operations in Model and Metric are indicated with curly brackets, while data nodes not): inputs & labels || ------------------ @@ -49,7 +48,7 @@ class Metric(object): | || outputs & labels || | || tensor data - {Metric.compute} || + {Metric.compute} || | || metric states(tensor) || | || @@ -162,7 +161,15 @@ def compute(self, *args): class Accuracy(Metric): """ - Encapsulates accuracy metric logic + Encapsulates accuracy metric logic. + def __init__(self, topk=(1, ), name='acc', *args, **kwargs): + + + Args: + topk (int|tuple(int)): Number of top elements to look at + for computing accuracy. Default is (1,). + name (str, optional): String name of the metric instance. Default + is `acc`. Example by standalone: @@ -180,7 +187,7 @@ class Accuracy(Metric): [0.1, 0.2, 0.3, 0.4]])) y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) - m = paddle.incubate.hapi.metrics.Accuracy() + m = paddle.metric.Accuracy() correct = m.compute(x, y) m.update(correct) res = m.accumulate() @@ -205,7 +212,7 @@ class Accuracy(Metric): model.prepare( optim, loss_function=paddle.nn.CrossEntropyLoss(), - metrics=paddle.metrics.Accuracy()) + metrics=paddle.metric.Accuracy()) model.fit(train_dataset, batch_size=64) @@ -219,11 +226,34 @@ def __init__(self, topk=(1, ), name=None, *args, **kwargs): self.reset() def compute(self, pred, label, *args): + """ + Compute the top-k (maxinum value in `topk`) indices. + + Args: + pred (Tensor): The predicted value is a Tensor wit type + float32 or float64. + label (Tensor): The ground truth value is a 2D Tensor, its + shape is [batch_size, 1] and type is int64. + + Return: + Tensor: Correct mask, a tensor with shape [batch_size, topk]. + """ pred = fluid.layers.argsort(pred, descending=True)[1][:, :self.maxk] correct = pred == label return fluid.layers.cast(correct, dtype='float32') def update(self, correct, *args): + """ + Update the metrics states (correct count and total count), in order to + calculate cumulative accuracy of all instances. This function also + returns the accuracy of current step. + + Args: + correct: Correct mask, a tensor with shape [batch_size, topk]. + + Return: + Tensor: the accuracy of current step. + """ if isinstance(correct, fluid.core.VarBase): correct = correct.numpy() accs = [] @@ -233,16 +263,24 @@ def update(self, correct, *args): accs.append(float(num_corrects) / num_samples) self.total[i] += num_corrects self.count[i] += num_samples + accs = accs[0] if len(self.topk) == 1 else accs return accs def reset(self): + """ + Resets all of the metric state. + """ self.total = [0.] * len(self.topk) self.count = [0] * len(self.topk) def accumulate(self): + """ + Computes and returns the accumulated metric. + """ res = [] for t, c in zip(self.total, self.count): res.append(float(t) / c) + res = res[0] if len(self.topk) == 1 else res return res def _init_name(self, name): @@ -253,4 +291,7 @@ def _init_name(self, name): self._name = [name] def name(self): + """ + Return name of metric instance. + """ return self._name diff --git a/python/paddle/tests/CMakeLists.txt b/python/paddle/tests/CMakeLists.txt new file mode 100644 index 00000000000000..79bec8c4ad34d6 --- /dev/null +++ b/python/paddle/tests/CMakeLists.txt @@ -0,0 +1,6 @@ +file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") +string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") + +foreach(src ${TEST_OPS}) + py_test(${src} SRCS ${src}.py) +endforeach() diff --git a/python/paddle/incubate/hapi/tests/test_metrics.py b/python/paddle/tests/test_metrics.py similarity index 72% rename from python/paddle/incubate/hapi/tests/test_metrics.py rename to python/paddle/tests/test_metrics.py index b7e24f48559daf..f45a8fb320346a 100644 --- a/python/paddle/incubate/hapi/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -19,10 +19,10 @@ import unittest import numpy as np +import paddle import paddle.fluid as fluid from paddle.fluid.dygraph.base import to_variable -from paddle.incubate.hapi.metrics import * from paddle.incubate.hapi.utils import to_list @@ -46,6 +46,41 @@ def convert_to_one_hot(y, C): return oh +class TestAccuracy(unittest.TestCase): + def test_acc(self): + paddle.enable_imperative() + + x = paddle.imperative.to_variable( + np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], + [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) + y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) + + m = paddle.metric.Accuracy(name='my_acc') + + # check name + self.assertEqual(m.name(), ['my_acc']) + + correct = m.compute(x, y) + # check results + self.assertEqual(m.update(correct), 0.75) + self.assertEqual(m.accumulate(), 0.75) + + x = paddle.imperative.to_variable( + np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.3, 0.4, 0.2], + [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) + y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) + correct = m.compute(x, y) + # check results + self.assertEqual(m.update(correct), 0.5) + self.assertEqual(m.accumulate(), 0.625) + + # check reset + m.reset() + self.assertEqual(m.total[0], 0.0) + self.assertEqual(m.count[0], 0.0) + paddle.disable_imperative() + + class TestAccuracyDynamic(unittest.TestCase): def setUp(self): self.topk = (1, ) @@ -65,17 +100,18 @@ def random_pred_label(self): def test_main(self): with fluid.dygraph.guard(fluid.CPUPlace()): - acc = Accuracy(topk=self.topk, name=self.name) + acc = paddle.metric.Accuracy(topk=self.topk, name=self.name) for _ in range(10): label, pred = self.random_pred_label() label_var = to_variable(label) pred_var = to_variable(pred) - state = to_list(acc.add_metric_op(pred_var, label_var)) + state = to_list(acc.compute(pred_var, label_var)) acc.update(*[s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m, dtype='float64'), np.array(res_f, dtype='float64'), rtol=1e-3)), \ - "Accuracy precision error: {} != {}".format(res_m, res_f) + assert np.all(np.isclose(np.array(res_m, dtype='float64'), + np.array(res_f, dtype='float64'), rtol=1e-3)), \ + "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 assert np.sum(acc.count) == 0 @@ -97,8 +133,8 @@ def test_main(self): pred = fluid.data( name='pred', shape=[None, self.class_num], dtype='float32') label = fluid.data(name='label', shape=[None, 1], dtype='int64') - acc = Accuracy(topk=self.topk, name=self.name) - state = acc.add_metric_op(pred, label) + acc = paddle.metric.Accuracy(topk=self.topk, name=self.name) + state = acc.compute(pred, label) exe = fluid.Executor(fluid.CPUPlace()) compiled_main_prog = fluid.CompiledProgram(main_prog) From ad3256840c82425f68e3def9621f8f1a349de658 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Sat, 15 Aug 2020 10:19:33 +0000 Subject: [PATCH 03/15] Add Precision, Recall and Auc metric --- python/paddle/incubate/hapi/callbacks.py | 4 +- python/paddle/incubate/hapi/model.py | 43 +- .../paddle/incubate/hapi/tests/test_model.py | 20 +- python/paddle/metric/metrics.py | 447 +++++++++++++++++- python/paddle/tests/test_metrics.py | 77 ++- 5 files changed, 551 insertions(+), 40 deletions(-) diff --git a/python/paddle/incubate/hapi/callbacks.py b/python/paddle/incubate/hapi/callbacks.py index ed19abe64feb2f..54ec859269e4d6 100644 --- a/python/paddle/incubate/hapi/callbacks.py +++ b/python/paddle/incubate/hapi/callbacks.py @@ -305,7 +305,7 @@ class ProgBarLogger(Callback): optim = fluid.optimizer.Adam(0.001) model.prepare(optimizer=optim, - loss_function=paddle.nn.CrossEntropyLoss(), + loss=paddle.nn.CrossEntropyLoss(), metrics=paddle.metric.Accuracy()) callback = hapi.callbacks.ProgBarLogger(log_freq=10) @@ -441,7 +441,7 @@ class ModelCheckpoint(Callback): optim = fluid.optimizer.Adam(0.001) model.prepare(optimizer=optim, - loss_function=paddle.nn.CrossEntropyLoss(), + loss=paddle.nn.CrossEntropyLoss(), metrics=paddle.metric.Accuracy()) callback = hapi.callbacks.ModelCheckpoint(save_dir='./temp') diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 3644faf81f0b4a..6e89ad179494dc 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -393,8 +393,8 @@ def _make_program(self, mode): self._label_vars[mode] = labels outputs = to_list(self.model.network.forward(*inputs)) - if mode != 'test' and self.model._loss_function: - losses = self.model._loss_function(*(outputs + labels)) + if mode != 'test' and self.model._loss: + losses = self.model._loss(*(outputs + labels)) if self._nranks > 1 and mode != 'train': outputs = [_all_gather(o, self._nranks) for o in outputs] @@ -509,7 +509,7 @@ def train_batch(self, inputs, labels=None): if self._nranks > 1: outputs = self.ddp_model.forward(* [to_variable(x) for x in inputs]) - losses = self.model._loss_function(*(to_list(outputs) + labels)) + losses = self.model._loss(*(to_list(outputs) + labels)) losses = to_list(losses) final_loss = fluid.layers.sum(losses) final_loss = self.ddp_model.scale_loss(final_loss) @@ -518,7 +518,7 @@ def train_batch(self, inputs, labels=None): else: outputs = self.model.network.forward( * [to_variable(x) for x in inputs]) - losses = self.model._loss_function(*(to_list(outputs) + labels)) + losses = self.model._loss(*(to_list(outputs) + labels)) losses = to_list(losses) final_loss = fluid.layers.sum(losses) final_loss.backward() @@ -542,8 +542,8 @@ def eval_batch(self, inputs, labels=None): labels = [to_variable(l) for l in to_list(labels)] outputs = self.model.network.forward(* [to_variable(x) for x in inputs]) - if self.model._loss_function: - losses = self.model._loss_function(*(to_list(outputs) + labels)) + if self.model._loss: + losses = self.model._loss(*(to_list(outputs) + labels)) losses = to_list(losses) if self._nranks > 1: @@ -575,9 +575,9 @@ def eval_batch(self, inputs, labels=None): m = metric.update(* [to_numpy(m) for m in to_list(metric_outs)]) metrics.append(m) - if self.model._loss_function and len(metrics): + if self.model._loss and len(metrics): return [to_numpy(l) for l in losses], metrics - elif self.model._loss_function: + elif self.model._loss: return [to_numpy(l) for l in losses] else: return metrics @@ -679,7 +679,7 @@ class Model(object): inputs must be set. For dynamic graph, it could be None. labels (Input|list|None): `labels`, entry points of network, could be a Input layer or lits of Input layers, or None. - For static graph, if labels is required in loss_function, + For static graph, if labels is required in loss, labels must be set. Otherwise, it could be None. @@ -724,7 +724,7 @@ def __init__(self, network, inputs=None, labels=None): self.network = network self._inputs = None self._labels = None - self._loss_function = None + self._loss = None self._loss_weights = None self._optimizer = None self._optimizer = None @@ -884,7 +884,7 @@ def forward(self, x): model = hapi.Model(MyNet()) model.prepare() data = np.random.random(size=(4,784)).astype(np.float32) - out = model.eval_batch([data]) + out = model.test_batch([data]) print(out) """ return self._adapter.test_batch(inputs) @@ -1059,7 +1059,7 @@ def forward(self, x): """ return self._adapter.parameters() - def prepare(self, optimizer=None, loss_function=None, metrics=None): + def prepare(self, optimizer=None, loss=None, metrics=None): """ Configures the model before runing. @@ -1067,7 +1067,7 @@ def prepare(self, optimizer=None, loss_function=None, metrics=None): optimizer (Optimizer|None): Optimizer must be set in training and should be a Optimizer instance. It can be None in eval and test mode. - loss_function (Loss|callable function|None): Loss function can + loss (Loss|callable function|None): Loss function can be a `fluid.dygraph.Layer` instance or any callable function taken the predicted values and ground truth values as input. It can be None when there is no loss. @@ -1099,12 +1099,11 @@ def prepare(self, optimizer=None, loss_function=None, metrics=None): _parallel_context_initialized = True self._optimizer = optimizer - if loss_function: - if not isinstance(loss_function, fluid.dygraph.Layer) or \ - not callable(loss_function): - raise TypeError("'loss_function' must be sub classes of \ - `fluid.dygraph.Layer` or any callable function.") - self._loss_function = loss_function + if loss is not None: + if not isinstance(loss, fluid.dygraph.Layer) and not callable(loss): + raise TypeError("'loss' must be sub classes of " \ + "`fluid.dygraph.Layer` or any callable function.") + self._loss = loss metrics = metrics or [] for metric in to_list(metrics): @@ -1602,9 +1601,9 @@ def _run_one_epoch(self, data_loader, callbacks, mode, logs={}): if mode != 'test': outs = getattr(self, mode + '_batch')(data[:len(self._inputs)], data[len(self._inputs):]) - if self._metrics and self._loss_function: + if self._metrics and self._loss: metrics = [[l[0] for l in outs[0]]] - elif self._loss_function: + elif self._loss: metrics = [[l[0] for l in outs]] else: metrics = [] @@ -1645,7 +1644,7 @@ def _reset_metrics(self): metric.reset() def _metrics_name(self): - metrics_name = ['loss'] if self._loss_function else [] + metrics_name = ['loss'] if self._loss else [] for m in self._metrics: metrics_name.extend(to_list(m.name())) return metrics_name diff --git a/python/paddle/incubate/hapi/tests/test_model.py b/python/paddle/incubate/hapi/tests/test_model.py index 21a64a07660e98..c96379e6c4ea45 100644 --- a/python/paddle/incubate/hapi/tests/test_model.py +++ b/python/paddle/incubate/hapi/tests/test_model.py @@ -196,7 +196,7 @@ def fit(self, dynamic): model = Model(net, inputs=self.inputs, labels=self.labels) model.prepare( optim_new, - loss_function=CrossEntropyLoss(reduction="sum"), + loss=CrossEntropyLoss(reduction="sum"), metrics=Accuracy()) model.fit(self.train_dataset, batch_size=64, shuffle=False) @@ -319,8 +319,7 @@ def get_expect(): inputs = [Input('x', [None, dim], 'float32')] labels = [Input('label', [None, 1], 'int64')] model = Model(net, inputs, labels) - model.prepare( - optim2, loss_function=CrossEntropyLoss(reduction="sum")) + model.prepare(optim2, loss=CrossEntropyLoss(reduction="sum")) loss, = model.train_batch([data], [label]) np.testing.assert_allclose(loss.flatten(), ref.flatten()) @@ -365,8 +364,7 @@ def test_save_load(self): parameter_list=net.parameters()) model = Model(net, inputs, labels) model.prepare( - optimizer=optim, - loss_function=CrossEntropyLoss(reduction="sum")) + optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path + '/test') model.load(path + '/test') shutil.rmtree(path) @@ -380,8 +378,7 @@ def test_dynamic_save_static_load(self): model = Model(MyModel(classifier_activation=None)) optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=model.parameters()) - model.prepare( - optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) + model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path + '/test') fluid.disable_dygraph() @@ -390,8 +387,7 @@ def test_dynamic_save_static_load(self): model = Model(MyModel(classifier_activation=None), inputs, labels) optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=model.parameters()) - model.prepare( - optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) + model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.load(path + '/test') shutil.rmtree(path) @@ -404,8 +400,7 @@ def test_static_save_dynamic_load(self): optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=net.parameters()) model = Model(net, inputs, labels) - model.prepare( - optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) + model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.save(path + '/test') device = hapi.set_device('cpu') @@ -417,8 +412,7 @@ def test_static_save_dynamic_load(self): optim = fluid.optimizer.SGD(learning_rate=0.001, parameter_list=net.parameters()) model = Model(net, inputs, labels) - model.prepare( - optimizer=optim, loss_function=CrossEntropyLoss(reduction="sum")) + model.prepare(optimizer=optim, loss=CrossEntropyLoss(reduction="sum")) model.load(path + '/test') shutil.rmtree(path) fluid.disable_dygraph() diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 2093aba38bebe8..881e677be67911 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -18,9 +18,14 @@ import six import abc +import numpy as np import paddle.fluid as fluid -__all__ = ['Metric', 'Accuracy'] +__all__ = ['Metric', 'Accuracy', 'Precision', 'Recall', 'Auc'] + + +def _is_numpy_(var): + return isinstance(var, (np.ndarray, np.generic)) @six.add_metaclass(abc.ABCMeta) @@ -211,7 +216,7 @@ def __init__(self, topk=(1, ), name='acc', *args, **kwargs): learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, - loss_function=paddle.nn.CrossEntropyLoss(), + loss=paddle.nn.CrossEntropyLoss(), metrics=paddle.metric.Accuracy()) model.fit(train_dataset, batch_size=64) @@ -295,3 +300,441 @@ def name(self): Return name of metric instance. """ return self._name + + +class Precision(Metric): + """ + Precision (also called positive predictive value) is the fraction of + relevant instances among the retrieved instances. Refer to + https://en.wikipedia.org/wiki/Evaluation_of_binary_classifiers + + Noted that this class manages the precision score only for binary + classification task. + + Args: + name (str, optional): String name of the metric instance. + Default is `precision`. + + Example by standalone: + + .. code-block:: python + + import numpy as np + import paddle + + paddle.enable_imperative() + + x = [0.1, 0.5, 0.6, 0.7] + y = [0, 1, 1, 1] + + m = paddle.metric.Precision() + m.update(x, y) + res = m.accumulate() + print(res) + + + Example with Model API: + + .. code-block:: python + + import numpy as np + + import paddle + import paddle.nn as nn + import paddle.incubate.hapi as hapi + + class Data(paddle.io.Dataset): + def __init__(self): + super(Data, self).__init__() + self.n = 1024 + self.x = np.random.randn(self.n, 10).astype('float32') + self.y = np.random.randint(2, size=(self.n, 1)).astype('int64') + + def __getitem__(self, idx): + return self.x[idx], self.y[idx] + + def __len__(self): + return self.n + + paddle.enable_imperative() + model = hapi.Model(nn.Sequential( + nn.Linear(10, 1), + nn.Sigmoid() + )) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters()) + model.prepare( + optim, + loss=nn.BCELoss(), + metrics=paddle.metric.Precision()) + + data = Data() + model.fit(data, batch_size=16) + """ + + def __init__(self, name='precision', *args, **kwargs): + super(Precision, self).__init__(*args, **kwargs) + self.tp = 0 # true positive + self.fp = 0 # false positive + self._name = name + + def update(self, preds, labels): + """ + Update the states based on the current mini-batch prediction results. + + Args: + preds (numpy.ndarray): The prediction result, usually the output + of two-class sigmoid function. It should be a vector (column + vector or row vector) with data type: 'float64' or 'float32'. + labels (numpy.ndarray): The ground truth (labels), + the shape should keep the same as preds. + The data type is 'int32' or 'int64'. + """ + if isinstance(preds, fluid.core.VarBase): + preds = preds.numpy() + elif not _is_numpy_(preds): + raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") + + if isinstance(labels, fluid.core.VarBase): + labels = preds.numpy() + elif not _is_numpy_(labels): + raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") + + sample_num = labels.shape[0] + preds = np.rint(preds).astype("int32") + + for i in range(sample_num): + pred = preds[i] + label = labels[i] + if pred == 1: + if pred == label: + self.tp += 1 + else: + self.fp += 1 + + def reset(self): + """ + Resets all of the metric state. + """ + self.tp = 0 + self.fp = 0 + + def accumulate(self): + """ + Calculate the final precision. + + Returns: + A scaler float: results of the calculated precision. + """ + ap = self.tp + self.fp + return float(self.tp) / ap if ap != 0 else .0 + + def name(self): + """ + Returns metric name + """ + return self._name + + +class Recall(Metric): + """ + Recall (also known as sensitivity) is the fraction of + relevant instances that have been retrieved over the + total amount of relevant instances + + Refer to: + https://en.wikipedia.org/wiki/Precision_and_recall + + Noted that this class manages the recall score only for + binary classification task. + + Args: + name (str, optional): String name of the metric instance. + Default is `precision`. + + Example by standalone: + + .. code-block:: python + + import numpy as np + import paddle + + paddle.enable_imperative() + + x = [0.1, 0.5, 0.6, 0.7] + y = [1, 0, 1, 1] + + m = paddle.metric.Recall() + m.update(x, y) + res = m.accumulate() + print(res) + + + Example with Model API: + + .. code-block:: python + + import numpy as np + + import paddle + import paddle.nn as nn + import paddle.incubate.hapi as hapi + + paddle.enable_imperative() + + class Data(paddle.io.Dataset): + def __init__(self): + super(Data, self).__init__() + self.n = 1024 + self.x = np.random.randn(self.n, 10).astype('float32') + self.y = np.random.randint(2, size=(self.n, 1)).astype('int64') + + def __getitem__(self, idx): + return self.x[idx], self.y[idx] + + def __len__(self): + return self.n + + model = hapi.Model(nn.Sequential( + nn.Linear(10, 1), + nn.Sigmoid() + )) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters()) + model.prepare( + optim, + loss=nn.BCELoss(), + metrics=paddle.metric.Recall()) + + data = Data() + model.fit(data, batch_size=16) + """ + + def __init__(self, name='recall', *args, **kwargs): + super(Recall, self).__init__(*args, **kwargs) + self.tp = 0 # true positive + self.fn = 0 # false negative + self._name = name + + def update(self, preds, labels): + """ + Update the states based on the current mini-batch prediction results. + + Args: + preds(numpy.array): prediction results of current mini-batch, + the output of two-class sigmoid function. + Shape: [batch_size, 1]. Dtype: 'float64' or 'float32'. + labels(numpy.array): ground truth (labels) of current mini-batch, + the shape should keep the same as preds. + Shape: [batch_size, 1], Dtype: 'int32' or 'int64'. + """ + if isinstance(preds, fluid.core.VarBase): + preds = preds.numpy() + elif not _is_numpy_(preds): + raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") + + if isinstance(labels, fluid.core.VarBase): + labels = preds.numpy() + elif not _is_numpy_(labels): + raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") + + sample_num = labels.shape[0] + preds = np.rint(preds).astype("int32") + + for i in range(sample_num): + pred = preds[i] + label = labels[i] + if label == 1: + if pred == label: + self.tp += 1 + else: + self.fn += 1 + + def accumulate(self): + """ + Calculate the final recall. + + Returns: + A scaler float: results of the calculated Recall. + """ + recall = self.tp + self.fn + return float(self.tp) / recall if recall != 0 else .0 + + def reset(self): + """ + Resets all of the metric state. + """ + self.tp = 0 + self.fn = 0 + + def name(self): + """ + Returns metric name + """ + return self._name + + +class Auc(Metric): + """ + The auc metric is for binary classification. + Refer to https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve. + Please notice that the auc metric is implemented with python, which may be a little bit slow. + If you concern the speed, please use the fluid.layers.auc instead. + + The `auc` function creates four local variables, `true_positives`, + `true_negatives`, `false_positives` and `false_negatives` that are used to + compute the AUC. To discretize the AUC curve, a linearly spaced set of + thresholds is used to compute pairs of recall and precision values. The area + under the ROC-curve is therefore computed using the height of the recall + values by the false positive rate, while the area under the PR-curve is the + computed using the height of the precision values by the recall. + + Args: + name (str, optional): String name of the metric instance. Default + is `acc`. + curve (str): Specifies the mode of the curve to be computed, + 'ROC' or 'PR' for the Precision-Recall-curve. Default is 'ROC'. + + "NOTE: only implement the ROC curve type via Python now." + + Example by standalone: + .. code-block:: python + + import numpy as np + import paddle + + m = fluid.metrics.Auc() + + n = 8 + class0_preds = np.random.random(size = (n, 1)) + class1_preds = 1 - class0_preds + + preds = np.concatenate((class0_preds, class1_preds), axis=1) + labels = np.random.randint(2, size = (n, 1)) + + m.update(preds=preds, labels=labels) + res = m.accumulate() + + + Example with Model API: + + .. code-block:: python + + import numpy as np + import paddle + import paddle.nn as nn + import paddle.incubate.hapi as hapi + + paddle.enable_imperative() + + class Data(paddle.io.Dataset): + def __init__(self): + super(Data, self).__init__() + self.n = 1024 + self.x = np.random.randn(self.n, 10).astype('float32') + self.y = np.random.randint(2, size=(self.n, 1)).astype('int64') + + def __getitem__(self, idx): + return self.x[idx], self.y[idx] + + def __len__(self): + return self.n + + model = hapi.Model(nn.Sequential( + nn.Linear(10, 2, act='softmax'), + )) + optim = paddle.optimizer.Adam( + learning_rate=0.001, parameter_list=model.parameters()) + + def loss(x, y): + return fluid.layers.cross_entropy(x, y) + + model.prepare( + optim, + loss=loss, + metrics=paddle.metric.Auc()) + data = Data() + model.fit(data, batch_size=16) + """ + + def __init__(self, + curve='ROC', + num_thresholds=4095, + name='auc', + *args, + **kwargs): + super(Auc, self).__init__(*args, **kwargs) + self._curve = curve + self._num_thresholds = num_thresholds + + _num_pred_buckets = num_thresholds + 1 + self._stat_pos = np.zeros(_num_pred_buckets) + self._stat_neg = np.zeros(_num_pred_buckets) + self._name = name + + def update(self, preds, labels): + """ + Update the auc curve with the given predictions and labels. + + Args: + preds (numpy.array): An numpy array in the shape of + (batch_size, 2), preds[i][j] denotes the probability of + classifying the instance i into the class j. + labels (numpy.array): an numpy array in the shape of + (batch_size, 1), labels[i] is either o or 1, + representing the label of the instance i. + """ + if not _is_numpy_(labels): + raise ValueError("The 'labels' must be a numpy ndarray.") + if not _is_numpy_(preds): + raise ValueError("The 'predictions' must be a numpy ndarray.") + + for i, lbl in enumerate(labels): + value = preds[i, 1] + bin_idx = int(value * self._num_thresholds) + assert bin_idx <= self._num_thresholds + if lbl: + self._stat_pos[bin_idx] += 1.0 + else: + self._stat_neg[bin_idx] += 1.0 + + @staticmethod + def trapezoid_area(x1, x2, y1, y2): + return abs(x1 - x2) * (y1 + y2) / 2.0 + + def accumulate(self): + """ + Return the area (a float score) under auc curve + + Return: + float: the area under auc curve + """ + tot_pos = 0.0 + tot_neg = 0.0 + auc = 0.0 + + idx = self._num_thresholds + while idx >= 0: + tot_pos_prev = tot_pos + tot_neg_prev = tot_neg + tot_pos += self._stat_pos[idx] + tot_neg += self._stat_neg[idx] + auc += self.trapezoid_area(tot_neg, tot_neg_prev, tot_pos, + tot_pos_prev) + idx -= 1 + + return auc / tot_pos / tot_neg if tot_pos > 0.0 and tot_neg > 0.0 else 0.0 + + def reset(self): + """ + Reset states and result + """ + _num_pred_buckets = self._num_thresholds + 1 + self._stat_pos = np.zeros(_num_pred_buckets) + self._stat_neg = np.zeros(_num_pred_buckets) + + def name(self): + """ + Returns metric name + """ + return self._name diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index f45a8fb320346a..254b35c3ef7d91 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -149,7 +149,8 @@ def test_main(self): acc.update(*state_ret) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m, dtype='float64'), np.array(res_f, dtype='float64'), rtol=1e-3)), \ + assert np.all(np.isclose(np.array(res_m, dtype='float64'), + np.array(res_f, dtype='float64'), rtol=1e-3)), \ "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 @@ -164,5 +165,79 @@ def setUp(self): self.name = "accuracy" +class TestPrecision(unittest.TestCase): + def test_1d(self): + paddle.enable_imperative() + + x = np.array([0.1, 0.5, 0.6, 0.7]) + y = np.array([1, 0, 1, 1]) + + m = paddle.metric.Recall() + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 2. / 3.) + + x = np.array([0.1, 0.5, 0.6, 0.7, 0.2]) + y = np.array([1, 0, 1, 1, 1]) + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 4. / 7.) + + paddle.disable_imperative() + + def test_2d(self): + paddle.enable_imperative() + + x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) + y = np.array([1, 0, 1, 1]).reshape(-1, 1) + + m = paddle.metric.Recall() + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 2. / 3.) + + x = np.array([0.1, 0.5, 0.6, 0.7, 0.2]).reshape(-1, 1) + y = np.array([1, 0, 1, 1, 1]).reshape(-1, 1) + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 4. / 7.) + + paddle.disable_imperative() + + +class TestRecall(unittest.TestCase): + def test_1d(self): + paddle.enable_imperative() + + x = np.array([0.1, 0.5, 0.6, 0.7]) + y = np.array([1, 0, 1, 1]) + + m = paddle.metric.Recall() + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 2. / 3.) + + x = np.array([0.1, 0.5, 0.6, 0.7]) + y = np.array([1, 0, 0, 1]) + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 3. / 5.) + + paddle.disable_imperative() + + +class TestAuc(unittest.TestCase): + def test_auc(self): + paddle.enable_imperative() + x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], + [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) + y = np.array([[0], [1], [1], [0], [1], [0], [0], [1]]) + m = paddle.metric.Auc() + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 0.8125) + paddle.disable_imperative() + + if __name__ == '__main__': unittest.main() From 20b4936e0573cc8a861afd74dd64cec40a376b5f Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 17 Aug 2020 07:32:55 +0000 Subject: [PATCH 04/15] Update unit testing --- python/paddle/tests/test_metrics.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index 254b35c3ef7d91..27f90a5fa70066 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -48,7 +48,7 @@ def convert_to_one_hot(y, C): class TestAccuracy(unittest.TestCase): def test_acc(self): - paddle.enable_imperative() + fluid.enable_dygrap() x = paddle.imperative.to_variable( np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], @@ -78,7 +78,7 @@ def test_acc(self): m.reset() self.assertEqual(m.total[0], 0.0) self.assertEqual(m.count[0], 0.0) - paddle.disable_imperative() + fluid.disable_imperative() class TestAccuracyDynamic(unittest.TestCase): @@ -129,6 +129,8 @@ class TestAccuracyStatic(TestAccuracyDynamic): def test_main(self): main_prog = fluid.Program() startup_prog = fluid.Program() + main_prog.random_seed = 1024 + startup_prog.random_seed = 1024 with fluid.program_guard(main_prog, startup_prog): pred = fluid.data( name='pred', shape=[None, self.class_num], dtype='float32') @@ -161,13 +163,13 @@ class TestAccuracyStaticMultiTopk(TestAccuracyStatic): def setUp(self): self.topk = (1, 5) self.class_num = 10 - self.sample_num = 1000 + self.sample_num = 100 self.name = "accuracy" class TestPrecision(unittest.TestCase): def test_1d(self): - paddle.enable_imperative() + fluid.enable_dygrap() x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -183,10 +185,10 @@ def test_1d(self): r = m.accumulate() self.assertAlmostEqual(r, 4. / 7.) - paddle.disable_imperative() + fluid.disable_imperative() def test_2d(self): - paddle.enable_imperative() + fluid.enable_dygrap() x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) y = np.array([1, 0, 1, 1]).reshape(-1, 1) @@ -202,12 +204,12 @@ def test_2d(self): r = m.accumulate() self.assertAlmostEqual(r, 4. / 7.) - paddle.disable_imperative() + fluid.disable_imperative() class TestRecall(unittest.TestCase): def test_1d(self): - paddle.enable_imperative() + fluid.enable_dygrap() x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -223,12 +225,12 @@ def test_1d(self): r = m.accumulate() self.assertAlmostEqual(r, 3. / 5.) - paddle.disable_imperative() + fluid.disable_imperative() class TestAuc(unittest.TestCase): def test_auc(self): - paddle.enable_imperative() + fluid.enable_dygrap() x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) y = np.array([[0], [1], [1], [0], [1], [0], [0], [1]]) @@ -236,7 +238,7 @@ def test_auc(self): m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 0.8125) - paddle.disable_imperative() + fluid.disable_imperative() if __name__ == '__main__': From 6b9e9d70ef0d68d05225b56f1623e1677475802f Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 17 Aug 2020 07:53:43 +0000 Subject: [PATCH 05/15] Update code to latest develop test=develop --- python/paddle/metric/__init__.py | 5 ++-- python/paddle/metric/metrics.py | 43 ++++++++++++----------------- python/paddle/tests/test_metrics.py | 21 +++++++------- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/python/paddle/metric/__init__.py b/python/paddle/metric/__init__.py index 4891dce11e0c55..6e197881fc0bcb 100644 --- a/python/paddle/metric/__init__.py +++ b/python/paddle/metric/__init__.py @@ -13,13 +13,12 @@ # limitations under the License. from .metrics import * +from . import metrics from ..fluid.layers.metric_op import accuracy, auc from ..fluid.layers.nn import chunk_eval, cos_sim, mean_iou -__all__ = [ - 'Accuracy', - 'Auc', +__all__ = metrics.__all__ + [ 'accuracy', 'auc', 'chunk_eval', diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 881e677be67911..cf867ab8145206 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -183,20 +183,19 @@ def __init__(self, topk=(1, ), name='acc', *args, **kwargs): import numpy as np import paddle - paddle.enable_imperative() - - x = paddle.imperative.to_variable(np.array([ + paddle.disable_static() + x = paddle.to_variable(np.array([ [0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) - y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) + y = paddle.to_variable(np.array([[0], [1], [2], [3]])) m = paddle.metric.Accuracy() correct = m.compute(x, y) m.update(correct) res = m.accumulate() - print(res) + print(res) # 0.75 Example with Model API: @@ -207,8 +206,7 @@ def __init__(self, topk=(1, ), name='acc', *args, **kwargs): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - fluid.enable_dygraph() - + paddle.disable_static() train_dataset = hapi.datasets.MNIST(mode='train') model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) @@ -322,15 +320,13 @@ class Precision(Metric): import numpy as np import paddle - paddle.enable_imperative() - - x = [0.1, 0.5, 0.6, 0.7] - y = [0, 1, 1, 1] + x = np.array([0.1, 0.5, 0.6, 0.7]) + y = np.array([0, 1, 1, 1]) m = paddle.metric.Precision() m.update(x, y) res = m.accumulate() - print(res) + print(res) # 1.0 Example with Model API: @@ -348,7 +344,7 @@ def __init__(self): super(Data, self).__init__() self.n = 1024 self.x = np.random.randn(self.n, 10).astype('float32') - self.y = np.random.randint(2, size=(self.n, 1)).astype('int64') + self.y = np.random.randint(2, size=(self.n, 1)).astype('float32') def __getitem__(self, idx): return self.x[idx], self.y[idx] @@ -356,7 +352,7 @@ def __getitem__(self, idx): def __len__(self): return self.n - paddle.enable_imperative() + paddle.disable_static() model = hapi.Model(nn.Sequential( nn.Linear(10, 1), nn.Sigmoid() @@ -459,15 +455,13 @@ class Recall(Metric): import numpy as np import paddle - paddle.enable_imperative() - - x = [0.1, 0.5, 0.6, 0.7] - y = [1, 0, 1, 1] + x = np.array([0.1, 0.5, 0.6, 0.7]) + y = np.array([1, 0, 1, 1]) m = paddle.metric.Recall() m.update(x, y) res = m.accumulate() - print(res) + print(res) # 2.0 / 3.0 Example with Model API: @@ -480,14 +474,12 @@ class Recall(Metric): import paddle.nn as nn import paddle.incubate.hapi as hapi - paddle.enable_imperative() - class Data(paddle.io.Dataset): def __init__(self): super(Data, self).__init__() self.n = 1024 self.x = np.random.randn(self.n, 10).astype('float32') - self.y = np.random.randint(2, size=(self.n, 1)).astype('int64') + self.y = np.random.randint(2, size=(self.n, 1)).astype('float32') def __getitem__(self, idx): return self.x[idx], self.y[idx] @@ -495,6 +487,7 @@ def __getitem__(self, idx): def __len__(self): return self.n + paddle.disable_static() model = hapi.Model(nn.Sequential( nn.Linear(10, 1), nn.Sigmoid() @@ -603,7 +596,7 @@ class Auc(Metric): import numpy as np import paddle - m = fluid.metrics.Auc() + m = paddle.metric.Auc() n = 8 class0_preds = np.random.random(size = (n, 1)) @@ -622,11 +615,10 @@ class Auc(Metric): import numpy as np import paddle + import paddle.fluid as fluid import paddle.nn as nn import paddle.incubate.hapi as hapi - paddle.enable_imperative() - class Data(paddle.io.Dataset): def __init__(self): super(Data, self).__init__() @@ -640,6 +632,7 @@ def __getitem__(self, idx): def __len__(self): return self.n + paddle.disable_static() model = hapi.Model(nn.Sequential( nn.Linear(10, 2, act='softmax'), )) diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index 27f90a5fa70066..9814af9d95c109 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -48,12 +48,12 @@ def convert_to_one_hot(y, C): class TestAccuracy(unittest.TestCase): def test_acc(self): - fluid.enable_dygrap() + fluid.enable_dygraph() - x = paddle.imperative.to_variable( + x = paddle.to_variable( np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) - y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) + y = paddle.to_variable(np.array([[0], [1], [2], [3]])) m = paddle.metric.Accuracy(name='my_acc') @@ -65,10 +65,10 @@ def test_acc(self): self.assertEqual(m.update(correct), 0.75) self.assertEqual(m.accumulate(), 0.75) - x = paddle.imperative.to_variable( + x = paddle.to_variable( np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.3, 0.4, 0.2], [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) - y = paddle.imperative.to_variable(np.array([[0], [1], [2], [3]])) + y = paddle.to_variable(np.array([[0], [1], [2], [3]])) correct = m.compute(x, y) # check results self.assertEqual(m.update(correct), 0.5) @@ -151,8 +151,7 @@ def test_main(self): acc.update(*state_ret) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) - assert np.all(np.isclose(np.array(res_m, dtype='float64'), - np.array(res_f, dtype='float64'), rtol=1e-3)), \ + assert np.all(np.isclose(np.array(res_m), np.array(res_f), rtol=1e-3)), \ "Accuracy precision error: {} != {}".format(res_m, res_f) acc.reset() assert np.sum(acc.total) == 0 @@ -169,7 +168,7 @@ def setUp(self): class TestPrecision(unittest.TestCase): def test_1d(self): - fluid.enable_dygrap() + fluid.enable_dygraph() x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -188,7 +187,7 @@ def test_1d(self): fluid.disable_imperative() def test_2d(self): - fluid.enable_dygrap() + fluid.enable_dygraph() x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) y = np.array([1, 0, 1, 1]).reshape(-1, 1) @@ -209,7 +208,7 @@ def test_2d(self): class TestRecall(unittest.TestCase): def test_1d(self): - fluid.enable_dygrap() + fluid.enable_dygraph() x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -230,7 +229,7 @@ def test_1d(self): class TestAuc(unittest.TestCase): def test_auc(self): - fluid.enable_dygrap() + fluid.enable_dygraph() x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) y = np.array([[0], [1], [1], [0], [1], [0], [0], [1]]) From a6235a4b1cbe951d0fe69b6ef199e3e910eb9379 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 17 Aug 2020 09:48:59 +0000 Subject: [PATCH 06/15] Follow comments test=develop --- python/paddle/metric/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index cf867ab8145206..d6ffa7f403dccc 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -53,7 +53,7 @@ class Metric(object): | || outputs & labels || | || tensor data - {Metric.compute} || + {Metric.compute} || | || metric states(tensor) || | || From 362d9140c01d89b95e9e762196a6d04b54ea7152 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 18 Aug 2020 04:19:04 +0000 Subject: [PATCH 07/15] Update API to 2.0 test=develop --- python/paddle/incubate/hapi/model.py | 52 ++++++++++++++-------------- python/paddle/metric/metrics.py | 27 ++++++++------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 6e89ad179494dc..32d069c525616f 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -47,7 +47,7 @@ ] -class Input(fluid.dygraph.Layer): +class Input(paddle.nn.Layer): """ Define inputs the model. @@ -665,14 +665,14 @@ class Model(object): """ An Model object is network with training and inference features. Dynamic graph and static graph are supported at the same time, - switched by `fluid.enable_dygraph()`. The usage is as follows. + switched by `paddle.disable_static()`. The usage is as follows. But note, the switching between dynamic and static should be before instantiating a Model. The input description, i.e, hapi.Input, must be required for static graph. Args: - network (fluid.dygraph.Layer): The network is an instance of - fluid.dygraph.Layer. + network (paddle.nn.Layer): The network is an instance of + paddle.nn.Layer. inputs (Input|list|dict|None): `inputs`, entry points of network, could be a Input layer, or lits of Input layers, or dict (name: Input), or None. For static graph, @@ -690,7 +690,7 @@ class Model(object): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self, classifier_act=None): super(MyNet, self).__init__() self._fc1 = fluid.dygraph.Linear(784, 200, act=classifier_act) @@ -701,7 +701,7 @@ def forward(self, x): device = hapi.set_device('gpu') # if use static graph, do not set - fluid.enable_dygraph(device) + paddle.disable_static(device) # inputs and labels are not required for dynamic graph. input = hapi.Input('x', [None, 784], 'float32') @@ -775,7 +775,7 @@ def train_batch(self, inputs, labels=None): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self, classifier_act=None): super(MyNet, self).__init__() self._fc = fluid.dygraph.Linear(784, 10, act=classifier_act) @@ -785,7 +785,7 @@ def forward(self, x): return y device = hapi.set_device('gpu') - fluid.enable_dygraph(device) + paddle.disable_static(device) input = hapi.Input('x', [None, 784], 'float32') label = hapi.Input('label', [None, 1], 'int64') @@ -824,7 +824,7 @@ def eval_batch(self, inputs, labels=None): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self, classifier_act=None): super(MyNet, self).__init__() self._fc = fluid.dygraph.Linear(784, 10, act=classifier_act) @@ -834,7 +834,7 @@ def forward(self, x): return y device = hapi.set_device('gpu') - fluid.enable_dygraph(device) + paddle.disable_static(device) input = hapi.Input('x', [None, 784], 'float32') label = hapi.Input('label', [None, 1], 'int64') @@ -870,7 +870,7 @@ def test_batch(self, inputs): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() self._fc = fluid.dygraph.Linear(784, 1, act='softmax') @@ -879,7 +879,7 @@ def forward(self, x): return y device = hapi.set_device('gpu') - fluid.enable_dygraph(device) + paddle.disable_static(device) model = hapi.Model(MyNet()) model.prepare() @@ -918,7 +918,7 @@ def save(self, path): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() self._fc = fluid.dygraph.Linear(784, 1, act='softmax') @@ -927,7 +927,7 @@ def forward(self, x): return y device = hapi.set_device('cpu') - fluid.enable_dygraph(device) + paddle.disable_static(device) model = hapi.Model(MyNet()) model.save('checkpoint/test') """ @@ -970,7 +970,7 @@ def load(self, path, skip_mismatch=False, reset_optimizer=False): import paddle.fluid as fluid import paddle.incubate.hapi as hapi - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() self._fc = fluid.dygraph.Linear(784, 1, act='softmax') @@ -979,7 +979,7 @@ def forward(self, x): return y device = hapi.set_device('cpu') - fluid.enable_dygraph(device) + paddle.disable_static(device) model = hapi.Model(MyNet()) model.load('checkpoint/test') """ @@ -1045,7 +1045,7 @@ def parameters(self, *args, **kwargs): import paddle.fluid as fluid from paddle.incubate.hapi import Model - class MyNet(fluid.dygraph.Layer): + class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() self._fc = fluid.dygraph.Linear(20, 10, act='softmax') @@ -1053,7 +1053,7 @@ def forward(self, x): y = self._fc(x) return y - fluid.enable_dygraph() + paddle.disable_static() model = Model(MyNet()) params = model.parameters() """ @@ -1068,7 +1068,7 @@ def prepare(self, optimizer=None, loss=None, metrics=None): and should be a Optimizer instance. It can be None in eval and test mode. loss (Loss|callable function|None): Loss function can - be a `fluid.dygraph.Layer` instance or any callable function + be a `paddle.nn.Layer` instance or any callable function taken the predicted values and ground truth values as input. It can be None when there is no loss. metrics (Metric|list of Metric|None): If metrics is set, all @@ -1087,7 +1087,7 @@ def prepare(self, optimizer=None, loss=None, metrics=None): startup_prog_seed = fluid.default_startup_program( ).random_seed fluid.disable_dygraph() - fluid.enable_dygraph(self._place) + paddle.disable_static(self._place) # enable_dygraph would create and switch to a new program, # thus also copy seed to the new program fluid.default_main_program().random_seed = main_prog_seed @@ -1100,9 +1100,9 @@ def prepare(self, optimizer=None, loss=None, metrics=None): self._optimizer = optimizer if loss is not None: - if not isinstance(loss, fluid.dygraph.Layer) and not callable(loss): + if not isinstance(loss, paddle.nn.Layer) and not callable(loss): raise TypeError("'loss' must be sub classes of " \ - "`fluid.dygraph.Layer` or any callable function.") + "`paddle.nn.Layer` or any callable function.") self._loss = loss metrics = metrics or [] @@ -1188,7 +1188,7 @@ def fit( dynamic = True device = hapi.set_device('gpu') - fluid.enable_dygraph(device) if dynamic else None + paddle.disable_static(device) if dynamic else None train_dataset = hapi.datasets.MNIST(mode='train') val_dataset = hapi.datasets.MNIST(mode='test') @@ -1221,7 +1221,7 @@ def fit( dynamic = True device = hapi.set_device('gpu') - fluid.enable_dygraph(device) if dynamic else None + paddle.disable_static(device) if dynamic else None train_dataset = hapi.datasets.MNIST(mode='train') train_loader = fluid.io.DataLoader(train_dataset, @@ -1368,7 +1368,7 @@ def evaluate( print(result) # imperative mode - fluid.enable_dygraph() + paddle.disable_static() model = hapi.Model(hapi.vision.LeNet()) model.prepare(metrics=paddle.metric.Accuracy()) result = model.evaluate(val_dataset, batch_size=64) @@ -1475,7 +1475,7 @@ def __len__(self): # imperative mode device = hapi.set_device('cpu') - fluid.enable_dygraph(device) + paddle.disable_static(device) model = hapi.Model(hapi.vision.LeNet()) model.prepare() result = model.predict(test_dataset, batch_size=64) diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index d6ffa7f403dccc..20d6b32d38baa2 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -77,10 +77,10 @@ class Metric(object): .. code-block:: python def compute(pred, label): # sort prediction and slice the top-5 scores - pred = fluid.layers.argsort(pred, descending=True)[1][:, :5] + pred = paddle.argsort(pred, descending=True)[1][:, :5] # calculate whether the predictions are correct correct = pred == label - return fluid.layers.cast(correct, dtype='float32') + return paddle.cast(correct, dtype='float32') With the :code:`compute`, we split some calculations to OPs (which may run on GPU devices, will be faster), and only fetch 1 tensor with @@ -184,12 +184,12 @@ def __init__(self, topk=(1, ), name='acc', *args, **kwargs): import paddle paddle.disable_static() - x = paddle.to_variable(np.array([ + x = paddle.to_tensor(np.array([ [0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) - y = paddle.to_variable(np.array([[0], [1], [2], [3]])) + y = paddle.to_tensor(np.array([[0], [1], [2], [3]])) m = paddle.metric.Accuracy() correct = m.compute(x, y) @@ -203,14 +203,13 @@ def __init__(self, topk=(1, ), name='acc', *args, **kwargs): .. code-block:: python import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi paddle.disable_static() train_dataset = hapi.datasets.MNIST(mode='train') model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) - optim = fluid.optimizer.Adam( + optim = paddle.optimizer.Adam( learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, @@ -241,9 +240,9 @@ def compute(self, pred, label, *args): Return: Tensor: Correct mask, a tensor with shape [batch_size, topk]. """ - pred = fluid.layers.argsort(pred, descending=True)[1][:, :self.maxk] + pred = paddle.argsort(pred, descending=True)[1][:, :self.maxk] correct = pred == label - return fluid.layers.cast(correct, dtype='float32') + return paddle.cast(correct, dtype='float32') def update(self, correct, *args): """ @@ -446,7 +445,7 @@ class Recall(Metric): Args: name (str, optional): String name of the metric instance. - Default is `precision`. + Default is `recall`. Example by standalone: @@ -583,10 +582,13 @@ class Auc(Metric): computed using the height of the precision values by the recall. Args: - name (str, optional): String name of the metric instance. Default - is `acc`. curve (str): Specifies the mode of the curve to be computed, 'ROC' or 'PR' for the Precision-Recall-curve. Default is 'ROC'. + num_thresholds (int): The number of thresholds to use when + discretizing the roc curve. Default is 4095. + 'ROC' or 'PR' for the Precision-Recall-curve. Default is 'ROC'. + name (str, optional): String name of the metric instance. Default + is `auc`. "NOTE: only implement the ROC curve type via Python now." @@ -615,7 +617,6 @@ class Auc(Metric): import numpy as np import paddle - import paddle.fluid as fluid import paddle.nn as nn import paddle.incubate.hapi as hapi @@ -640,7 +641,7 @@ def __len__(self): learning_rate=0.001, parameter_list=model.parameters()) def loss(x, y): - return fluid.layers.cross_entropy(x, y) + return nn.functional.nll_loss(paddle.log(x), y) model.prepare( optim, From 562bd144c327cf3c79e3e1f459fae2d925f461e0 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 18 Aug 2020 04:27:33 +0000 Subject: [PATCH 08/15] --amend --- python/paddle/incubate/hapi/model.py | 3 +-- python/paddle/metric/metrics.py | 2 -- python/paddle/tests/test_metrics.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 32d069c525616f..53447f0b9c299b 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -403,8 +403,7 @@ def _make_program(self, mode): if mode != 'test': for metric in self.model._metrics: - metrics.append( - to_list(metric.compute(*(outputs + labels)))) + metrics.append(to_list(metric.compute(*(outputs + labels)))) if mode == 'train' and self.model._optimizer: self._loss_endpoint = fluid.layers.sum(losses) diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 20d6b32d38baa2..5bbcca202eeb7b 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -167,8 +167,6 @@ def compute(self, *args): class Accuracy(Metric): """ Encapsulates accuracy metric logic. - def __init__(self, topk=(1, ), name='acc', *args, **kwargs): - Args: topk (int|tuple(int)): Number of top elements to look at diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index 9814af9d95c109..b88bb7d0bdda53 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -106,7 +106,7 @@ def test_main(self): label_var = to_variable(label) pred_var = to_variable(pred) state = to_list(acc.compute(pred_var, label_var)) - acc.update(*[s.numpy() for s in state]) + acc.update(* [s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) assert np.all(np.isclose(np.array(res_m, dtype='float64'), From e2dd416d65024a0febc42139241979461dab8a97 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 18 Aug 2020 06:15:49 +0000 Subject: [PATCH 09/15] Fix unit testing after updating API test=develop --- python/paddle/incubate/hapi/model.py | 1 + python/paddle/metric/metrics.py | 18 +++++++-------- python/paddle/tests/test_metrics.py | 33 ++++++++++++++-------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 53447f0b9c299b..9c2bf73d284de8 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -24,6 +24,7 @@ import warnings from collections import Iterable +import paddle from paddle import fluid from paddle.fluid.framework import in_dygraph_mode, Variable from paddle.fluid.executor import global_scope diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 5bbcca202eeb7b..21ce27fddaec52 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -19,7 +19,8 @@ import six import abc import numpy as np -import paddle.fluid as fluid + +import paddle __all__ = ['Metric', 'Accuracy', 'Precision', 'Recall', 'Auc'] @@ -77,7 +78,7 @@ class Metric(object): .. code-block:: python def compute(pred, label): # sort prediction and slice the top-5 scores - pred = paddle.argsort(pred, descending=True)[1][:, :5] + pred = paddle.argsort(pred, descending=True)[:, :5] # calculate whether the predictions are correct correct = pred == label return paddle.cast(correct, dtype='float32') @@ -238,7 +239,7 @@ def compute(self, pred, label, *args): Return: Tensor: Correct mask, a tensor with shape [batch_size, topk]. """ - pred = paddle.argsort(pred, descending=True)[1][:, :self.maxk] + pred = paddle.argsort(pred, descending=True)[:, :self.maxk] correct = pred == label return paddle.cast(correct, dtype='float32') @@ -254,7 +255,7 @@ def update(self, correct, *args): Return: Tensor: the accuracy of current step. """ - if isinstance(correct, fluid.core.VarBase): + if isinstance(correct, paddle.Tensor): correct = correct.numpy() accs = [] for i, k in enumerate(self.topk): @@ -383,12 +384,12 @@ def update(self, preds, labels): the shape should keep the same as preds. The data type is 'int32' or 'int64'. """ - if isinstance(preds, fluid.core.VarBase): + if isinstance(preds, paddle.Tensor): preds = preds.numpy() elif not _is_numpy_(preds): raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") - if isinstance(labels, fluid.core.VarBase): + if isinstance(labels, paddle.Tensor): labels = preds.numpy() elif not _is_numpy_(labels): raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") @@ -518,12 +519,12 @@ def update(self, preds, labels): the shape should keep the same as preds. Shape: [batch_size, 1], Dtype: 'int32' or 'int64'. """ - if isinstance(preds, fluid.core.VarBase): + if isinstance(preds, paddle.Tensor): preds = preds.numpy() elif not _is_numpy_(preds): raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") - if isinstance(labels, fluid.core.VarBase): + if isinstance(labels, paddle.Tensor): labels = preds.numpy() elif not _is_numpy_(labels): raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") @@ -569,7 +570,6 @@ class Auc(Metric): The auc metric is for binary classification. Refer to https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve. Please notice that the auc metric is implemented with python, which may be a little bit slow. - If you concern the speed, please use the fluid.layers.auc instead. The `auc` function creates four local variables, `true_positives`, `true_negatives`, `false_positives` and `false_negatives` that are used to diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index b88bb7d0bdda53..219ed4d51e2944 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -21,7 +21,6 @@ import paddle import paddle.fluid as fluid -from paddle.fluid.dygraph.base import to_variable from paddle.incubate.hapi.utils import to_list @@ -48,12 +47,12 @@ def convert_to_one_hot(y, C): class TestAccuracy(unittest.TestCase): def test_acc(self): - fluid.enable_dygraph() + paddle.disable_static() - x = paddle.to_variable( + x = paddle.to_tensor( np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.4, 0.3, 0.2], [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) - y = paddle.to_variable(np.array([[0], [1], [2], [3]])) + y = paddle.to_tensor(np.array([[0], [1], [2], [3]])) m = paddle.metric.Accuracy(name='my_acc') @@ -65,10 +64,10 @@ def test_acc(self): self.assertEqual(m.update(correct), 0.75) self.assertEqual(m.accumulate(), 0.75) - x = paddle.to_variable( + x = paddle.to_tensor( np.array([[0.1, 0.2, 0.3, 0.4], [0.1, 0.3, 0.4, 0.2], [0.1, 0.2, 0.4, 0.3], [0.1, 0.2, 0.3, 0.4]])) - y = paddle.to_variable(np.array([[0], [1], [2], [3]])) + y = paddle.to_tensor(np.array([[0], [1], [2], [3]])) correct = m.compute(x, y) # check results self.assertEqual(m.update(correct), 0.5) @@ -78,7 +77,7 @@ def test_acc(self): m.reset() self.assertEqual(m.total[0], 0.0) self.assertEqual(m.count[0], 0.0) - fluid.disable_imperative() + paddle.enable_static() class TestAccuracyDynamic(unittest.TestCase): @@ -103,8 +102,8 @@ def test_main(self): acc = paddle.metric.Accuracy(topk=self.topk, name=self.name) for _ in range(10): label, pred = self.random_pred_label() - label_var = to_variable(label) - pred_var = to_variable(pred) + label_var = to_tensor(label) + pred_var = to_tensor(pred) state = to_list(acc.compute(pred_var, label_var)) acc.update(* [s.numpy() for s in state]) res_m = acc.accumulate() @@ -168,7 +167,7 @@ def setUp(self): class TestPrecision(unittest.TestCase): def test_1d(self): - fluid.enable_dygraph() + paddle.disable_static() x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -184,10 +183,10 @@ def test_1d(self): r = m.accumulate() self.assertAlmostEqual(r, 4. / 7.) - fluid.disable_imperative() + paddle.enable_static() def test_2d(self): - fluid.enable_dygraph() + paddle.disable_static() x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) y = np.array([1, 0, 1, 1]).reshape(-1, 1) @@ -203,12 +202,12 @@ def test_2d(self): r = m.accumulate() self.assertAlmostEqual(r, 4. / 7.) - fluid.disable_imperative() + paddle.enable_static() class TestRecall(unittest.TestCase): def test_1d(self): - fluid.enable_dygraph() + paddle.disable_static() x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) @@ -224,12 +223,12 @@ def test_1d(self): r = m.accumulate() self.assertAlmostEqual(r, 3. / 5.) - fluid.disable_imperative() + paddle.enable_static() class TestAuc(unittest.TestCase): def test_auc(self): - fluid.enable_dygraph() + paddle.disable_static() x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) y = np.array([[0], [1], [1], [0], [1], [0], [0], [1]]) @@ -237,7 +236,7 @@ def test_auc(self): m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 0.8125) - fluid.disable_imperative() + paddle.enable_static() if __name__ == '__main__': From 180a95f838f32c5d6a43003da08955d4965ddec1 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 19 Aug 2020 03:17:22 +0000 Subject: [PATCH 10/15] Update test_metrics --- python/paddle/metric/metrics.py | 3 +- python/paddle/tests/test_metrics.py | 43 +++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 21ce27fddaec52..2448b2dbdb6feb 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -280,7 +280,8 @@ def accumulate(self): """ res = [] for t, c in zip(self.total, self.count): - res.append(float(t) / c) + r = float(t) / c if c > 0 else 0. + res.append(r) res = res[0] if len(self.topk) == 1 else res return res diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index ad267fd98937db..915e062fa878fc 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -106,7 +106,7 @@ def test_main(self): label_var = paddle.to_tensor(label) pred_var = paddle.to_tensor(pred) state = to_list(acc.compute(pred_var, label_var)) - acc.update(* [s.numpy() for s in state]) + acc.update(*[s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) assert np.all(np.isclose(np.array(res_m, dtype='float64'), @@ -178,8 +178,8 @@ def test_1d(self): r = m.accumulate() self.assertAlmostEqual(r, 2. / 3.) - x = np.array([0.1, 0.5, 0.6, 0.7, 0.2]) - y = np.array([1, 0, 1, 1, 1]) + x = paddle.to_tensor(np.array([0.1, 0.5, 0.6, 0.7, 0.2])) + y = paddle.to_tensor(np.array([1, 0, 1, 1, 1])) m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 4. / 7.) @@ -203,6 +203,12 @@ def test_2d(self): r = m.accumulate() self.assertAlmostEqual(r, 4. / 7.) + # check reset + m.reset() + self.assertEqual(m.tp, 0.0) + self.assertEqual(m.fp, 0.0) + self.assertEqual(m.accumulate(), 0.0) + paddle.enable_static() @@ -218,17 +224,22 @@ def test_1d(self): r = m.accumulate() self.assertAlmostEqual(r, 2. / 3.) - x = np.array([0.1, 0.5, 0.6, 0.7]) - y = np.array([1, 0, 0, 1]) + x = paddle.to_tensor(np.array([0.1, 0.5, 0.6, 0.7])) + y = paddle.to_tensor(np.array([1, 0, 0, 1])) m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 3. / 5.) + # check reset + m.reset() + self.assertEqual(m.tp, 0.0) + self.assertEqual(m.fn, 0.0) + self.assertEqual(m.accumulate(), 0.0) paddle.enable_static() class TestAuc(unittest.TestCase): - def test_auc(self): + def test_auc_numpy(self): paddle.disable_static() x = np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]]) @@ -237,6 +248,26 @@ def test_auc(self): m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 0.8125) + + m.reset() + self.assertEqual(m.accumulate(), 0.0) + + paddle.enable_static() + + def test_auc_tensor(self): + paddle.disable_static() + x = paddle.to_tensor( + np.array([[0.78, 0.22], [0.62, 0.38], [0.55, 0.45], [0.30, 0.70], + [0.14, 0.86], [0.59, 0.41], [0.91, 0.08], [0.16, 0.84]])) + y = paddle.to_tensor(np.array([[0], [1], [1], [0], [1], [0], [0], [1]])) + m = paddle.metric.Auc() + m.update(x, y) + r = m.accumulate() + self.assertAlmostEqual(r, 0.8125) + + m.reset() + self.assertEqual(m.accumulate(), 0.0) + paddle.enable_static() From 08308fd786612ec0c7d2861c716328d7dcb0e9f6 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 19 Aug 2020 06:35:18 +0000 Subject: [PATCH 11/15] Enhance unit testing test=develop --- python/paddle/metric/metrics.py | 19 ++++++++++++------- python/paddle/tests/test_metrics.py | 8 ++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 2448b2dbdb6feb..6575c86700073a 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -391,12 +391,12 @@ def update(self, preds, labels): raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") if isinstance(labels, paddle.Tensor): - labels = preds.numpy() + labels = labels.numpy() elif not _is_numpy_(labels): raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") sample_num = labels.shape[0] - preds = np.rint(preds).astype("int32") + preds = np.floor(preds + 0.5).astype("int32") for i in range(sample_num): pred = preds[i] @@ -526,7 +526,7 @@ def update(self, preds, labels): raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") if isinstance(labels, paddle.Tensor): - labels = preds.numpy() + labels = labels.numpy() elif not _is_numpy_(labels): raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") @@ -677,10 +677,15 @@ def update(self, preds, labels): (batch_size, 1), labels[i] is either o or 1, representing the label of the instance i. """ - if not _is_numpy_(labels): - raise ValueError("The 'labels' must be a numpy ndarray.") - if not _is_numpy_(preds): - raise ValueError("The 'predictions' must be a numpy ndarray.") + if isinstance(labels, paddle.Tensor): + labels = labels.numpy() + elif not _is_numpy_(labels): + raise ValueError("The 'labels' must be a numpy ndarray or Tensor.") + + if isinstance(preds, paddle.Tensor): + preds = preds.numpy() + elif not _is_numpy_(preds): + raise ValueError("The 'preds' must be a numpy ndarray or Tensor.") for i, lbl in enumerate(labels): value = preds[i, 1] diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index 915e062fa878fc..d83dae56ff434f 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -173,7 +173,7 @@ def test_1d(self): x = np.array([0.1, 0.5, 0.6, 0.7]) y = np.array([1, 0, 1, 1]) - m = paddle.metric.Recall() + m = paddle.metric.Precision() m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 2. / 3.) @@ -182,7 +182,7 @@ def test_1d(self): y = paddle.to_tensor(np.array([1, 0, 1, 1, 1])) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 4. / 7.) + self.assertAlmostEqual(r, 4. / 6.) paddle.enable_static() @@ -192,7 +192,7 @@ def test_2d(self): x = np.array([0.1, 0.5, 0.6, 0.7]).reshape(-1, 1) y = np.array([1, 0, 1, 1]).reshape(-1, 1) - m = paddle.metric.Recall() + m = paddle.metric.Precision() m.update(x, y) r = m.accumulate() self.assertAlmostEqual(r, 2. / 3.) @@ -201,7 +201,7 @@ def test_2d(self): y = np.array([1, 0, 1, 1, 1]).reshape(-1, 1) m.update(x, y) r = m.accumulate() - self.assertAlmostEqual(r, 4. / 7.) + self.assertAlmostEqual(r, 4. / 6.) # check reset m.reset() From 52b9f204193e465231d80fcb2e18a63457255650 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 19 Aug 2020 09:59:18 +0000 Subject: [PATCH 12/15] Fix code style test=develop --- python/paddle/tests/test_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/tests/test_metrics.py b/python/paddle/tests/test_metrics.py index d83dae56ff434f..2272a81b3f602e 100644 --- a/python/paddle/tests/test_metrics.py +++ b/python/paddle/tests/test_metrics.py @@ -106,7 +106,7 @@ def test_main(self): label_var = paddle.to_tensor(label) pred_var = paddle.to_tensor(pred) state = to_list(acc.compute(pred_var, label_var)) - acc.update(*[s.numpy() for s in state]) + acc.update(* [s.numpy() for s in state]) res_m = acc.accumulate() res_f = accuracy(pred, label, self.topk) assert np.all(np.isclose(np.array(res_m, dtype='float64'), From 24c3849e6ebc90c4224f0de43b79adad963393d2 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 19 Aug 2020 12:58:15 +0000 Subject: [PATCH 13/15] Fix sample code test=develop --- python/paddle/incubate/hapi/model.py | 45 ++++++++++++---------------- tools/wlist.json | 19 ++++++++++-- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 9c2bf73d284de8..41a97a7c36916f 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -687,13 +687,12 @@ class Model(object): .. code-block:: python import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi class MyNet(paddle.nn.Layer): def __init__(self, classifier_act=None): super(MyNet, self).__init__() - self._fc1 = fluid.dygraph.Linear(784, 200, act=classifier_act) + self._fc1 = paddle.nn.Linear(784, 200, act=classifier_act) def forward(self, x): y = self._fc1(x) @@ -708,7 +707,7 @@ def forward(self, x): label = hapi.Input('label', [None, 1], 'int64') model = hapi.Model(MyNet(), input, label) - optim = fluid.optimizer.SGD(learning_rate=1e-3, + optim = paddle.optimizer.SGD(learning_rate=1e-3, parameter_list=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss(), @@ -772,13 +771,12 @@ def train_batch(self, inputs, labels=None): import numpy as np import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi class MyNet(paddle.nn.Layer): def __init__(self, classifier_act=None): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 10, act=classifier_act) + self._fc = paddle.nn.Linear(784, 10, act=classifier_act) def forward(self, x): y = self._fc(x) @@ -790,7 +788,7 @@ def forward(self, x): input = hapi.Input('x', [None, 784], 'float32') label = hapi.Input('label', [None, 1], 'int64') model = hapi.Model(MyNet(), input, label) - optim = fluid.optimizer.SGD(learning_rate=1e-3, + optim = paddle.optimizer.SGD(learning_rate=1e-3, parameter_list=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss()) data = np.random.random(size=(4,784)).astype(np.float32) @@ -821,13 +819,12 @@ def eval_batch(self, inputs, labels=None): import numpy as np import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi class MyNet(paddle.nn.Layer): def __init__(self, classifier_act=None): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 10, act=classifier_act) + self._fc = paddle.nn.Linear(784, 10, act=classifier_act) def forward(self, x): y = self._fc(x) @@ -839,7 +836,7 @@ def forward(self, x): input = hapi.Input('x', [None, 784], 'float32') label = hapi.Input('label', [None, 1], 'int64') model = hapi.Model(MyNet(), input, label) - optim = fluid.optimizer.SGD(learning_rate=1e-3, + optim = paddle.optimizer.SGD(learning_rate=1e-3, parameter_list=model.parameters()) model.prepare(optim, paddle.nn.CrossEntropyLoss()) @@ -867,13 +864,13 @@ def test_batch(self, inputs): .. code-block:: python import numpy as np - import paddle.fluid as fluid + import paddle import paddle.incubate.hapi as hapi class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 1, act='softmax') + self._fc = paddle.nn.Linear(784, 1, act='softmax') def forward(self, x): y = self._fc(x) return y @@ -915,13 +912,13 @@ def save(self, path): .. code-block:: python - import paddle.fluid as fluid + import paddle import paddle.incubate.hapi as hapi class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 1, act='softmax') + self._fc = paddle.nn.Linear(784, 1, act='softmax') def forward(self, x): y = self._fc(x) return y @@ -967,13 +964,13 @@ def load(self, path, skip_mismatch=False, reset_optimizer=False): .. code-block:: python - import paddle.fluid as fluid + import paddle import paddle.incubate.hapi as hapi class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(784, 1, act='softmax') + self._fc = paddle.nn.Linear(784, 1, act='softmax') def forward(self, x): y = self._fc(x) return y @@ -1042,13 +1039,13 @@ def parameters(self, *args, **kwargs): .. code-block:: python - import paddle.fluid as fluid + import paddle from paddle.incubate.hapi import Model class MyNet(paddle.nn.Layer): def __init__(self): super(MyNet, self).__init__() - self._fc = fluid.dygraph.Linear(20, 10, act='softmax') + self._fc = paddle.nn.Linear(20, 10, act='softmax') def forward(self, x): y = self._fc(x) return y @@ -1183,7 +1180,6 @@ def fit( .. code-block:: python import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi dynamic = True @@ -1198,7 +1194,7 @@ def fit( model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), input, label) - optim = fluid.optimizer.Adam( + optim = paddle.optimizer.Adam( learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, @@ -1216,7 +1212,6 @@ def fit( .. code-block:: python import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi dynamic = True @@ -1224,10 +1219,10 @@ def fit( paddle.disable_static(device) if dynamic else None train_dataset = hapi.datasets.MNIST(mode='train') - train_loader = fluid.io.DataLoader(train_dataset, + train_loader = paddle.io.DataLoader(train_dataset, places=device, batch_size=64) val_dataset = hapi.datasets.MNIST(mode='test') - val_loader = fluid.io.DataLoader(val_dataset, + val_loader = paddle.io.DataLoader(val_dataset, places=device, batch_size=64) input = hapi.Input('image', [None, 1, 28, 28], 'float32') @@ -1235,7 +1230,7 @@ def fit( model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), input, label) - optim = fluid.optimizer.Adam( + optim = paddle.optimizer.Adam( learning_rate=0.001, parameter_list=model.parameters()) model.prepare( optim, @@ -1353,7 +1348,6 @@ def evaluate( .. code-block:: python import paddle - import paddle.fluid as fluid import paddle.incubate.hapi as hapi # declarative mode @@ -1446,7 +1440,7 @@ def predict(self, .. code-block:: python import numpy as np - import paddle.fluid as fluid + import paddle import paddle.incubate.hapi as hapi class MnistDataset(hapi.datasets.MNIST): @@ -1545,7 +1539,6 @@ def save_inference_model(self, Examples: .. code-block:: python - import paddle.fluid as fluid import paddle.incubate.hapi as hapi input = hapi.Input('image', [-1, 1, 28, 28], 'float32') diff --git a/tools/wlist.json b/tools/wlist.json index 6989882504eded..7109a70a79f7e7 100644 --- a/tools/wlist.json +++ b/tools/wlist.json @@ -107,12 +107,27 @@ "Metric.update", "Metric.accumulate", "Metric.name", - "Metric.add_metric_op", + "Metric.compute", "Accuracy.reset", "Accuracy.update", "Accuracy.accumulate", "Accuracy.name", - "Accuracy.add_metric_op", + "Accuracy.compute", + "Precision.reset", + "Precision.update", + "Precision.accumulate", + "Precision.name", + "Precision.compute", + "Recall.reset", + "Recall.update", + "Recall.accumulate", + "Recall.name", + "Recall.compute", + "Auc.reset", + "Auc.update", + "Auc.accumulate", + "Auc.name", + "Auc.compute", "Callback.set_params", "Callback.on_train_begin", "Callback.on_train_end", From 553cb0178d357d899522531d2cf67cf3f6e6fc34 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Fri, 21 Aug 2020 03:05:16 +0000 Subject: [PATCH 14/15] Update code sample in metric.Recall test=develop --- python/paddle/metric/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index 6575c86700073a..b5fe1be47a854e 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -496,7 +496,7 @@ def __len__(self): model.prepare( optim, loss=nn.BCELoss(), - metrics=paddle.metric.Recall()) + metrics=[paddle.metric.Precision(), paddle.metric.Recall()]) data = Data() model.fit(data, batch_size=16) From 0fb112c0f645afe712f48ba7222b6dedef28bc35 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 24 Aug 2020 03:29:12 +0000 Subject: [PATCH 15/15] Fix code sample about Adam after develop code is updated test=develop --- python/paddle/incubate/hapi/model.py | 4 ++-- python/paddle/metric/metrics.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/paddle/incubate/hapi/model.py b/python/paddle/incubate/hapi/model.py index 10516a28f73ce3..c12df569790f7c 100644 --- a/python/paddle/incubate/hapi/model.py +++ b/python/paddle/incubate/hapi/model.py @@ -1155,7 +1155,7 @@ def fit( model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), input, label) optim = paddle.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) + learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, paddle.nn.CrossEntropyLoss(), @@ -1191,7 +1191,7 @@ def fit( model = hapi.Model(hapi.vision.LeNet(classifier_activation=None), input, label) optim = paddle.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) + learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, paddle.nn.CrossEntropyLoss(), diff --git a/python/paddle/metric/metrics.py b/python/paddle/metric/metrics.py index b5fe1be47a854e..110a62c300559b 100644 --- a/python/paddle/metric/metrics.py +++ b/python/paddle/metric/metrics.py @@ -209,7 +209,7 @@ class Accuracy(Metric): model = hapi.Model(hapi.vision.LeNet(classifier_activation=None)) optim = paddle.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) + learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, loss=paddle.nn.CrossEntropyLoss(), @@ -357,7 +357,7 @@ def __len__(self): nn.Sigmoid() )) optim = paddle.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) + learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, loss=nn.BCELoss(), @@ -492,7 +492,7 @@ def __len__(self): nn.Sigmoid() )) optim = paddle.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) + learning_rate=0.001, parameters=model.parameters()) model.prepare( optim, loss=nn.BCELoss(), @@ -637,7 +637,7 @@ def __len__(self): nn.Linear(10, 2, act='softmax'), )) optim = paddle.optimizer.Adam( - learning_rate=0.001, parameter_list=model.parameters()) + learning_rate=0.001, parameters=model.parameters()) def loss(x, y): return nn.functional.nll_loss(paddle.log(x), y)