From e471baee8c2ebb2c384f29fb0d4121e47150e56a Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Tue, 25 Oct 2022 11:40:46 +0200 Subject: [PATCH 1/2] fsspec lock reset --- .../dataset_wrappers/torch_iterable_dataset.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py b/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py index 78b0803f2f7..4de61e64262 100644 --- a/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py +++ b/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py @@ -1,4 +1,5 @@ import fsspec.asyn +import threading import torch from ...iterable_dataset import IterableDataset, _apply_feature_types @@ -15,8 +16,13 @@ def _set_fsspec_for_multiprocess() -> None: Only required for fsspec >= 0.9.0 See https://github.com/fsspec/gcsfs/issues/379 """ - fsspec.asyn.iothread[0] = None - fsspec.asyn.loop[0] = None + if hasattr(fsspec.asyn, "reset_lock"): + # for future fsspec>2022.05.0 + fsspec.asyn.reset_lock() + else: + fsspec.asyn.iothread[0] = None + fsspec.asyn.loop[0] = None + fsspec.asyn.lock = threading.Lock() class TorchIterableDataset(IterableDataset, torch.utils.data.IterableDataset): From 794ce3a1a6931a34dedf4f4da7d7e9e8dbec1e26 Mon Sep 17 00:00:00 2001 From: Quentin Lhoest Date: Wed, 26 Oct 2022 10:59:55 +0200 Subject: [PATCH 2/2] style --- .../formatting/dataset_wrappers/torch_iterable_dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py b/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py index 4de61e64262..aa3d4887d7c 100644 --- a/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py +++ b/src/datasets/formatting/dataset_wrappers/torch_iterable_dataset.py @@ -1,5 +1,6 @@ -import fsspec.asyn import threading + +import fsspec.asyn import torch from ...iterable_dataset import IterableDataset, _apply_feature_types