better error message when using the wrong load_from_disk

Author: lhoestq

src/datasets/arrow_dataset.py

@@ -674,6 +674,12 @@ def load_from_disk(dataset_path: str, fs=None, keep_in_memory: Optional[bool] =
             tmp_dir = tempfile.TemporaryDirectory()
             dataset_path = Path(tmp_dir.name, src_dataset_path)
             fs.download(src_dataset_path, dataset_path.as_posix(), recursive=True)
+        dataset_dict_json_path = Path(dataset_path, config.DATASETDICT_JSON_FILENAME).as_posix()
+        dataset_info_path = Path(dataset_path, config.DATASET_INFO_FILENAME).as_posix()
+        if not fs.isfile(dataset_info_path) and fs.isfile(dataset_dict_json_path):
+            raise FileNotFoundError(
+                f"No such file or directory: '{dataset_info_path}'. Looks like you tried to load a DatasetDict object, not a Dataset. Please use DatasetDict.load_from_disk instead."
+            )
 
         with open(
             Path(dataset_path, config.DATASET_STATE_JSON_FILENAME).as_posix(), "r", encoding="utf-8"

src/datasets/config.py

@@ -157,6 +157,7 @@
 DATASETDICT_INFOS_FILENAME = "dataset_infos.json"
 LICENSE_FILENAME = "LICENSE"
 METRIC_INFO_FILENAME = "metric_info.json"
+DATASETDICT_JSON_FILENAME = "dataset_dict.json"
 
 MODULE_NAME_FOR_DYNAMIC_MODULES = "datasets_modules"
 

src/datasets/dataset_dict.py

@@ -11,6 +11,7 @@
 
 from datasets.utils.doc_utils import is_documented_by
 
+from . import config
 from .arrow_dataset import Dataset
 from .features import Features
 from .filesystems import extract_path_from_uri, is_remote_filesystem
@@ -673,7 +674,7 @@ def save_to_disk(self, dataset_dict_path: str, fs=None):
 
         json.dump(
             {"splits": list(self)},
-            fs.open(Path(dest_dataset_dict_path, "dataset_dict.json").as_posix(), "w", encoding="utf-8"),
+            fs.open(Path(dest_dataset_dict_path, config.DATASETDICT_JSON_FILENAME).as_posix(), "w", encoding="utf-8"),
         )
         for k, dataset in self.items():
             dataset.save_to_disk(Path(dest_dataset_dict_path, k).as_posix(), fs)
@@ -706,8 +707,14 @@ def load_from_disk(dataset_dict_path: str, fs=None, keep_in_memory: Optional[boo
         else:
             fs = fsspec.filesystem("file")
             dest_dataset_dict_path = dataset_dict_path
+        dataset_dict_json_path = Path(dest_dataset_dict_path, config.DATASETDICT_JSON_FILENAME).as_posix()
+        dataset_info_path = Path(dest_dataset_dict_path, config.DATASET_INFO_FILENAME).as_posix()
+        if fs.isfile(dataset_info_path) and not fs.isfile(dataset_dict_json_path):
+            raise FileNotFoundError(
+                f"No such file or directory: '{dataset_dict_json_path}'. Looks like you tried to load a Dataset object, not a DatasetDict. Please use Dataset.load_from_disk instead."
+            )
         for k in json.load(
-            fs.open(Path(dest_dataset_dict_path, "dataset_dict.json").as_posix(), "r", encoding="utf-8")
+            fs.open(Path(dest_dataset_dict_path, config.DATASET_STATE_JSON_FILENAME).as_posix(), "r", encoding="utf-8")
         )["splits"]:
             dataset_dict_split_path = (
                 dataset_dict_path.split("://")[0] + "://" + Path(dest_dataset_dict_path, k).as_posix()

src/datasets/load.py

@@ -797,9 +797,9 @@ def load_from_disk(dataset_path: str, fs=None, keep_in_memory: Optional[bool] =
 
     if not fs.exists(dest_dataset_path):
         raise FileNotFoundError("Directory {} not found".format(dataset_path))
-    if fs.isfile(Path(dest_dataset_path, "dataset_info.json").as_posix()):
+    if fs.isfile(Path(dest_dataset_path, config.DATASET_INFO_FILENAME).as_posix()):
         return Dataset.load_from_disk(dataset_path, fs, keep_in_memory=keep_in_memory)
-    elif fs.isfile(Path(dest_dataset_path, "dataset_dict.json").as_posix()):
+    elif fs.isfile(Path(dest_dataset_path, config.DATASETDICT_JSON_FILENAME).as_posix()):
         return DatasetDict.load_from_disk(dataset_path, fs, keep_in_memory=keep_in_memory)
     else:
         raise FileNotFoundError(