integrate file_lock in the lib for better logging control

Author: lhoestq

src/datasets/builder.py

@@ -26,8 +26,6 @@
 from functools import partial
 from typing import Dict, List, Optional, Union
 
-from filelock import FileLock
-
 from . import utils
 from .arrow_dataset import Dataset
 from .arrow_reader import HF_GCP_BASE_URL, ArrowReader, DatasetNotOnHfGcs, MissingFilesOnHfGcs
@@ -45,6 +43,7 @@
 from .naming import camelcase_to_snakecase, filename_prefix_for_split
 from .splits import Split, SplitDict, SplitGenerator
 from .utils.download_manager import DownloadManager, GenerateMode
+from .utils.filelock import FileLock
 from .utils.file_utils import HF_DATASETS_CACHE, DownloadConfig, is_remote_url
 from .utils.info_utils import get_size_checksum_dict, verify_checksums, verify_splits
 from .utils.logging import WARNING, get_logger

src/datasets/load.py

@@ -27,8 +27,6 @@
 from typing import Dict, List, Optional, Tuple, Union
 from urllib.parse import urlparse
 
-from filelock import FileLock
-
 from .arrow_dataset import Dataset
 from .builder import DatasetBuilder
 from .dataset_dict import DatasetDict
@@ -37,6 +35,7 @@
 from .metric import Metric
 from .splits import Split
 from .utils.download_manager import GenerateMode
+from .utils.filelock import FileLock
 from .utils.file_utils import HF_MODULES_CACHE, DownloadConfig, cached_path, head_hf_s3, hf_bucket_url, hf_github_url
 from .utils.logging import get_logger
 from .utils.version import Version

src/datasets/metric.py

@@ -32,6 +32,7 @@
 from .naming import camelcase_to_snakecase
 from .utils import HF_METRICS_CACHE, copyfunc, temp_seed
 from .utils.download_manager import DownloadManager
+from .utils.filelock import BaseFileLock, FileLock, Timeout
 from .utils.file_utils import DownloadConfig
 from .utils.logging import get_logger
 

src/datasets/utils/file_utils.py

@@ -22,10 +22,10 @@
 
 import numpy as np
 import requests
-from filelock import FileLock
 from tqdm.auto import tqdm
 
 from .. import __version__
+from .filelock import FileLock
 from .logging import WARNING, get_logger