add space separator

Author: lhoestq

src/datasets/data_files.py

@@ -27,28 +27,28 @@ class Url(str):
 SPLIT_PATTERN_SHARDED = "data/{split}-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*"
 
 DEFAULT_PATTERNS_SPLIT_IN_FILENAME = {
-    str(Split.TRAIN): ["**[-._/]train[-._]*", "train[-._]*", "**[-._/]training[-._]*", "training[-._]*"],
-    str(Split.TEST): ["**[-._/]test[-._]*", "test[-._]*", "**[-._/]eval[-._]*", "eval[-._]*"],
+    str(Split.TRAIN): ["**[-._ /]train[-._ ]*", "train[-._ ]*", "**[-._ /]training[-._ ]*", "training[-._ ]*"],
+    str(Split.TEST): ["**[-._ /]test[-._ ]*", "test[-._ ]*", "**[-._ /]eval[-._ ]*", "eval[-._ ]*"],
     str(Split.VALIDATION): [
-        "**[-._/]dev[-._]*",
-        "dev[-._]*",
-        "**[-._/]valid[-._]*",
-        "valid[-._]*",
-        "**[-._/]validation[-._]*",
-        "validation[-._]*",
+        "**[-._ /]dev[-._ ]*",
+        "dev[-._ ]*",
+        "**[-._ /]valid[-._ ]*",
+        "valid[-._ ]*",
+        "**[-._ /]validation[-._ ]*",
+        "validation[-._ ]*",
     ],
 }
 
 DEFAULT_PATTERNS_SPLIT_IN_DIR_NAME = {
-    str(Split.TRAIN): ["train[-._/]**", "**[-._/]train[-._/]**", "training[-._/]**", "**[-._/]training[-._/]**"],
-    str(Split.TEST): ["test[-._/]**", "**[-._/]test[-._/]**", "eval[-._/]**", "**[-._/]eval[-._/]**"],
+    str(Split.TRAIN): ["train[-._ /]**", "**[-._ /]train[-._ /]**", "training[-._ /]**", "**[-._ /]training[-._ /]**"],
+    str(Split.TEST): ["test[-._ /]**", "**[-._ /]test[-._ /]**", "eval[-._ /]**", "**[-._ /]eval[-._ /]**"],
     str(Split.VALIDATION): [
-        "dev[-._/]**",
-        "**[-._/]dev[-._/]**",
-        "valid[-._/]**",
-        "**[-._/]valid[-._/]**",
-        "validation[-._/]**",
-        "**[-._/]validation[-._/]**",
+        "dev[-._ /]**",
+        "**[-._ /]dev[-._ /]**",
+        "valid[-._ /]**",
+        "**[-._ /]valid[-._ /]**",
+        "validation[-._ /]**",
+        "**[-._ /]validation[-._ /]**",
     ],
 }
 

tests/test_data_files.py

@@ -609,6 +609,11 @@ def _open(
         # With "dev" or "eval" without separators
         {"train": "developers_list.txt"},
         {"train": "data/seqeval_results.txt"},
+        # With supported separators
+        {"test": "my.test.file.txt"},
+        {"test": "my-test-file.txt"},
+        {"test": "my_test_file.txt"},
+        {"test": "my test file.txt"},
     ],
 )
 def test_get_data_files_patterns(data_file_per_split):