Integrate file_lock inside the lib for better logging control (#859)

* integrate file_lock in the lib for better logging control

* remove file_lock from setup.py

* style

* minor

Author: lhoestq

setup.py

@@ -78,8 +78,6 @@
     "tqdm>=4.27,<4.50.0",
     # dataclasses for Python versions that don't have it
     "dataclasses;python_version<'3.7'",
-    # filesystem locks e.g. to prevent parallel downloads
-    "filelock",
     # for fast hashing
     "xxhash",
     # for better multiprocessing

src/datasets/builder.py

@@ -26,8 +26,6 @@
 from functools import partial
 from typing import Dict, List, Optional, Union
 
-from filelock import FileLock
-
 from . import utils
 from .arrow_dataset import Dataset
 from .arrow_reader import HF_GCP_BASE_URL, ArrowReader, DatasetNotOnHfGcs, MissingFilesOnHfGcs
@@ -46,6 +44,7 @@
 from .splits import Split, SplitDict, SplitGenerator
 from .utils.download_manager import DownloadManager, GenerateMode
 from .utils.file_utils import HF_DATASETS_CACHE, DownloadConfig, is_remote_url
+from .utils.filelock import FileLock
 from .utils.info_utils import get_size_checksum_dict, verify_checksums, verify_splits
 from .utils.logging import WARNING, get_logger
 

src/datasets/load.py

@@ -27,8 +27,6 @@
 from typing import Dict, List, Optional, Tuple, Union
 from urllib.parse import urlparse
 
-from filelock import FileLock
-
 from .arrow_dataset import Dataset
 from .builder import DatasetBuilder
 from .dataset_dict import DatasetDict
@@ -38,6 +36,7 @@
 from .splits import Split
 from .utils.download_manager import GenerateMode
 from .utils.file_utils import HF_MODULES_CACHE, DownloadConfig, cached_path, head_hf_s3, hf_bucket_url, hf_github_url
+from .utils.filelock import FileLock
 from .utils.logging import get_logger
 from .utils.version import Version
 

src/datasets/metric.py

@@ -22,7 +22,6 @@
 
 import numpy as np
 import pyarrow as pa
-from filelock import BaseFileLock, FileLock, Timeout
 
 from .arrow_dataset import Dataset
 from .arrow_reader import ArrowReader
@@ -33,6 +32,7 @@
 from .utils import HF_METRICS_CACHE, copyfunc, temp_seed
 from .utils.download_manager import DownloadManager
 from .utils.file_utils import DownloadConfig
+from .utils.filelock import BaseFileLock, FileLock, Timeout
 from .utils.logging import get_logger
 
 

src/datasets/utils/file_utils.py

@@ -22,10 +22,10 @@
 
 import numpy as np
 import requests
-from filelock import FileLock
 from tqdm.auto import tqdm
 
 from .. import __version__
+from .filelock import FileLock
 from .logging import WARNING, get_logger