From 779fc06db5b672eaff7a8cf0bb694995ccbc899c Mon Sep 17 00:00:00 2001 From: VijayKalmath Date: Tue, 28 Jun 2022 21:40:15 +0000 Subject: [PATCH] Generalize `meta_path` json creation in `load.py` In function `_copy_script_and_other_resources_in_importable_dir`, using string split when generating `meta_path` throws error in the edge case raised in #4540. Using `os.path.splitext` generalizes json path creation. --- src/datasets/load.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/datasets/load.py b/src/datasets/load.py index 434f746c60a..c6511854401 100644 --- a/src/datasets/load.py +++ b/src/datasets/load.py @@ -241,7 +241,6 @@ def _copy_script_and_other_resources_in_importable_dir( # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together importable_subdirectory = os.path.join(importable_directory_path, subdirectory_name) importable_local_file = os.path.join(importable_subdirectory, name + ".py") - # Prevent parallel disk operations lock_path = importable_directory_path + ".lock" with FileLock(lock_path): @@ -267,9 +266,9 @@ def _copy_script_and_other_resources_in_importable_dir( # Copy dataset.py file in hash folder if needed if not os.path.exists(importable_local_file): shutil.copyfile(original_local_path, importable_local_file) - # Record metadata associating original dataset path with local unique folder - meta_path = importable_local_file.split(".py")[0] + ".json" + # Use os.path.splitext to split extenstion from importable_local_file + meta_path = os.path.splitext(importable_local_file)[0] + ".json" if not os.path.exists(meta_path): meta = {"original file path": original_local_path, "local file path": importable_local_file} # the filename is *.py in our case, so better rename to filenam.json instead of filename.py.json