Use unstrip_protocol to merge protocol and path

Author: mariosasko

src/datasets/arrow_writer.py

@@ -313,9 +313,10 @@ def __init__(
         if stream is None:
             fs_token_paths = fsspec.get_fs_token_paths(path, storage_options=storage_options)
             self._fs: fsspec.AbstractFileSystem = fs_token_paths[0]
-            protocol = self._fs.protocol if isinstance(self._fs.protocol, str) else self._fs.protocol[-1]
             self._path = (
-                fs_token_paths[2][0] if not is_remote_filesystem(self._fs) else protocol + "://" + fs_token_paths[2][0]
+                fs_token_paths[2][0]
+                if not is_remote_filesystem(self._fs)
+                else self._fs.unstrip_protocol(fs_token_paths[2][0])
             )
             self.stream = self._fs.open(fs_token_paths[2][0], "wb")
             self._closable_stream = True

src/datasets/builder.py

@@ -329,8 +329,7 @@ def __init__(
         is_local = not is_remote_filesystem(self._fs)
         path_join = os.path.join if is_local else posixpath.join
 
-        protocol = self._fs.protocol if isinstance(self._fs.protocol, str) else self._fs.protocol[-1]
-        self._cache_dir_root = fs_token_paths[2][0] if is_local else protocol + "://" + fs_token_paths[2][0]
+        self._cache_dir_root = fs_token_paths[2][0] if is_local else self._fs.unstrip_protocol(fs_token_paths[2][0])
         self._cache_dir = self._build_cache_dir()
         self._cache_downloaded_dir = (
             path_join(self._cache_dir_root, config.DOWNLOADED_DATASETS_DIR)