[BC-breaking] added ROUGE calculation on batch input (#2259)

* added ROUGE calculation on batch input

* updated typehints

Author: Ishan-Kumar2

ignite/metrics/nlp/rouge.py

@@ -149,18 +149,19 @@ def reset(self) -> None:
         self._num_examples = 0
 
     @reinit__is_reduced
-    def update(self, output: Tuple[Sequence[Any], Sequence[Sequence[Any]]]) -> None:
-        candidate, references = output[0], output[1]
-        multiref_scores = [self._compute_score(candidate=candidate, reference=reference,) for reference in references]
-        score = self._mutliref_reducer(multiref_scores)
-        precision = score.precision()
-        recall = score.recall()
-        self._precision += precision
-        self._recall += recall
-        precision_recall = precision * recall
-        if precision_recall > 0:  # avoid zero division
-            self._fmeasure += precision_recall / ((1 - self._alpha) * precision + self._alpha * recall)
-        self._num_examples += 1
+    def update(self, output: Tuple[Sequence[Sequence[Any]], Sequence[Sequence[Sequence[Any]]]]) -> None:
+        candidates, references = output
+        for _candidate, _reference in zip(candidates, references):
+            multiref_scores = [self._compute_score(candidate=_candidate, reference=_ref,) for _ref in _reference]
+            score = self._mutliref_reducer(multiref_scores)
+            precision = score.precision()
+            recall = score.recall()
+            self._precision += precision
+            self._recall += recall
+            precision_recall = precision * recall
+            if precision_recall > 0:  # avoid zero division
+                self._fmeasure += precision_recall / ((1 - self._alpha) * precision + self._alpha * recall)
+            self._num_examples += 1
 
     @sync_all_reduce("_precision", "_recall", "_fmeasure", "_num_examples")
     def compute(self) -> Mapping:
@@ -192,8 +193,8 @@ class RougeN(_BaseRouge):
     __ https://www.aclweb.org/anthology/W04-1013.pdf
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
-    - `y_pred` must be a sequence of tokens.
-    - `y` must be a list of sequence of tokens.
+    - `y_pred` (list(list(str))) must be a sequence of tokens.
+    - `y` (list(list(list(str))) must be a list of sequence of tokens.
 
     Args:
         ngram: ngram order (default: 4).
@@ -222,7 +223,7 @@ class RougeN(_BaseRouge):
                 "there is a cat on the mat".split()
             ]
 
-            m.update((candidate, references))
+            m.update(([candidate], [references]))
 
             m.compute()
             # {'Rouge-2-P': 0.5, 'Rouge-2-R': 0.4, 'Rouge-2-F': 0.4}
@@ -260,8 +261,8 @@ class RougeL(_BaseRouge):
     __ https://www.aclweb.org/anthology/W04-1013.pdf
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
-    - `y_pred` must be a sequence of tokens.
-    - `y` must be a list of sequence of tokens.
+    - `y_pred` (list(list(str))) must be a sequence of tokens.
+    - `y` (list(list(list(str))) must be a list of sequence of tokens.
 
     Args:
         multiref: reduces scores for multi references. Valid values are "best" and "average" (default: "average").
@@ -288,7 +289,7 @@ class RougeL(_BaseRouge):
                 "there is a cat on the mat".split()
             ]
 
-            m.update((candidate, references))
+            m.update(([candidate], [references]))
 
            m.compute()
            # {'Rouge-L-P': 0.6, 'Rouge-L-R': 0.5, 'Rouge-L-F': 0.5}
@@ -320,8 +321,8 @@ class Rouge(Metric):
     __ https://www.aclweb.org/anthology/W04-1013.pdf
 
     - ``update`` must receive output of the form ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``.
-    - `y_pred` must be a sequence of tokens.
-    - `y` must be a list of sequence of tokens.
+    - `y_pred` (list(list(str))) must be a sequence of tokens.
+    - `y` (list(list(list(str))) must be a list of sequence of tokens.
 
     Args:
         variants: set of metrics computed. Valid inputs are "L" and integer 1 <= n <= 9.
@@ -349,13 +350,15 @@ class Rouge(Metric):
                 "there is a cat on the mat".split()
             ]
 
-            m.update((candidate, references))
+            m.update(([candidate], [references]))
 
             m.compute()
             # {'Rouge-L-P': 0.6, 'Rouge-L-R': 0.5, 'Rouge-L-F': 0.5, 'Rouge-2-P': 0.5, 'Rouge-2-R': 0.4,
             # 'Rouge-2-F': 0.4}
 
     .. versionadded:: 0.4.5
+    .. versionchanged:: 0.5.0
+        Changed input type to work on batch of inputs
     """
 
     def __init__(
@@ -388,7 +391,7 @@ def reset(self) -> None:
             m.reset()
 
     @reinit__is_reduced
-    def update(self, output: Tuple[Sequence[Any], Sequence[Sequence[Any]]]) -> None:
+    def update(self, output: Tuple[Sequence[Sequence[Any]], Sequence[Sequence[Sequence[Any]]]]) -> None:
         for m in self.internal_metrics:
             m.update(output)
 

tests/ignite/metrics/nlp/test_rouge.py

@@ -73,7 +73,7 @@ def test_wrong_inputs():
 def test_rouge_n_alpha(ngram, candidate, reference, expected):
     for alpha in [0, 1, 0.3, 0.5, 0.8]:
         rouge = RougeN(ngram=ngram, alpha=alpha)
-        rouge.update((candidate, [reference]))
+        rouge.update(([candidate], [[reference]]))
         results = rouge.compute()
         assert results[f"Rouge-{ngram}-P"] == expected[0]
         assert results[f"Rouge-{ngram}-R"] == expected[1]
@@ -110,8 +110,7 @@ def test_rouge_metrics(candidates, references):
         lower_split_candidates = [candidate.lower().split() for candidate in candidates]
 
         m = Rouge(variants=[1, 2, 4, "L"], multiref=multiref, alpha=0.5)
-        for candidate, references_per_candidate in zip(lower_split_candidates, lower_split_references):
-            m.update((candidate, references_per_candidate))
+        m.update((lower_split_candidates, lower_split_references))
         results = m.compute()
 
         for key in ["1", "2", "4", "L"]:
@@ -136,7 +135,7 @@ def update(_, i):
         candidate, references = data[i + size * rank]
         lower_split_references = [reference.lower().split() for reference in references[0]]
         lower_split_candidate = candidate[0].lower().split()
-        return lower_split_candidate, lower_split_references
+        return [lower_split_candidate], [lower_split_references]
 
     def _test(metric_device):
         engine = Engine(update)
@@ -158,11 +157,10 @@ def _test(metric_device):
         )
         rouge_1_f, rouge_2_f, rouge_l_f = (0, 0, 0)
         for candidate, references in data:
-            scores = evaluator.get_scores([candidate[0]], [references[0]])
+            scores = evaluator.get_scores(candidate, references)
             rouge_1_f += scores["rouge-1"]["f"]
             rouge_2_f += scores["rouge-2"]["f"]
             rouge_l_f += scores["rouge-l"]["f"]
-
         assert pytest.approx(engine.state.metrics["Rouge-1-F"], abs=1e-4) == rouge_1_f / len(data)
         assert pytest.approx(engine.state.metrics["Rouge-2-F"], abs=1e-4) == rouge_2_f / len(data)
         assert pytest.approx(engine.state.metrics["Rouge-L-F"], abs=1e-4) == rouge_l_f / len(data)