From 89327e36145e8de50f30cc55c29062ab7ab621c3 Mon Sep 17 00:00:00 2001 From: mariosasko Date: Tue, 27 Feb 2024 23:54:36 +0100 Subject: [PATCH] Faster `xlistdir` --- src/datasets/download/streaming_download_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datasets/download/streaming_download_manager.py b/src/datasets/download/streaming_download_manager.py index 66f769e1460..0b347216a9c 100644 --- a/src/datasets/download/streaming_download_manager.py +++ b/src/datasets/download/streaming_download_manager.py @@ -550,8 +550,8 @@ def xlistdir(path: str, download_config: Optional[DownloadConfig] = None) -> Lis inner_path = main_hop.split("://")[-1] if inner_path.strip("/") and not fs.isdir(inner_path): raise FileNotFoundError(f"Directory doesn't exist: {path}") - objects = fs.listdir(inner_path) - return [os.path.basename(obj["name"].strip("/")) for obj in objects] + paths = fs.listdir(inner_path, detail=False) + return [os.path.basename(path.rstrip("/")) for path in paths] def xglob(urlpath, *, recursive=False, download_config: Optional[DownloadConfig] = None):