[GSK-2871] Fix TextPerturbationDetector failed with error from num2word under "it" or "fa" language code

giskard/scanner/robustness/text_transformations.py

@@ -285,6 +285,13 @@ def make_perturbation(self, row):
         if pd.isna(value):
             return value
         lang = row["language__gsk__meta"] if not pd.isna(row["language__gsk__meta"]) else "en"
+
+        if lang == "fa" or lang == "id":
+            # In num2words, the convertor of "fa" and "id" are buggy,
+            # see https://github.com/savoirfairelinux/num2words/issues/476
+            # Give up doing this now, wait for merging https://github.com/savoirfairelinux/num2words/pull/524
+            return value
+
         return self._regex.sub(lambda x: num2words(x.group(), lang=lang), value)
 
 

tests/scan/test_text_transformations.py

@@ -114,6 +114,37 @@ def test_punctuation_strip_transformation():
     assert transformed_text[5] == "comma separated list"
 
 
+def test_number_to_words_transformation_exception():
+    datasets = [
+        _dataset_from_dict(
+            {
+                "text": [
+                    "Negara seperti Italia, yang diikuti tanda baca",
+                ],
+                "language__gsk__meta": "id",
+            }
+        ),
+        _dataset_from_dict(
+            {
+                "text": [
+                    "کشورهایی مانند ایتالیا که با علائم نگارشی دنبال می شوند",
+                ],
+                "language__gsk__meta": "fa",
+            }
+        ),
+    ]
+
+    from giskard.scanner.robustness.text_transformations import TextNumberToWordTransformation
+
+    t = TextNumberToWordTransformation(column="text")
+
+    for dataset in datasets:
+        # No more exception here now
+        transformed = dataset.transform(t)
+        # Nothing should have been changed
+        assert transformed.df.text.values[0] == dataset.df.text.values[0]
+
+
 def test_number_to_words_transformation():
     dataset = _dataset_from_dict(
         {