Support huggingface_hub v0.x and v1.x

.github/workflows/ci.yml

@@ -72,10 +72,16 @@ jobs:
         run: uv pip install --system "datasets[tests] @ ."
       - name: Install dependencies (latest versions)
         if: ${{ matrix.deps_versions == 'deps-latest' }}
-        run: uv pip install --system --upgrade pyarrow huggingface-hub "dill<0.3.9"
+        run: |
+          uv pip install --system --upgrade pyarrow huggingface-hub "dill<0.3.9"
+          # TODO: remove once transformers v5 / huggingface_hub v1 are released officially
+          uv pip uninstall --system transformers huggingface_hub
+          uv pip install --system --prerelease=allow git+https://github.com/huggingface/transformers.git
       - name: Install dependencies (minimum versions)
         if: ${{ matrix.deps_versions != 'deps-latest' }}
-        run: uv pip install --system pyarrow==21.0.0 huggingface-hub==0.24.7 transformers dill==0.3.1.1
+        run: uv pip install --system pyarrow==21.0.0 huggingface-hub==0.25.0 transformers dill==0.3.1.1
+      - name: Print dependencies
+        run: uv pip list
       - name: Test with pytest
         run: |
           python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
@@ -119,6 +125,8 @@ jobs:
         run: pip install --upgrade uv
       - name: Install dependencies
         run: uv pip install --system "datasets[tests] @ ."
+      - name: Print dependencies
+        run: uv pip list
       - name: Test with pytest
         run: |
           python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
@@ -161,7 +169,14 @@ jobs:
       - name: Install uv
         run: pip install --upgrade uv
       - name: Install dependencies
-        run: uv pip install --system "datasets[tests_numpy2] @ ."
+        run: |
+          uv pip install --system "datasets[tests_numpy2] @ ."
+          # TODO: remove once transformers v5 / huggingface_hub v1 are released officially
+          uv pip uninstall --system transformers huggingface_hub
+          uv pip install --system --prerelease=allow git+https://github.com/huggingface/transformers.git
+      - name: Print dependencies
+        run: pip list
+
       - name: Test with pytest
         run: |
           python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

setup.py

@@ -118,6 +118,7 @@
     "pandas",
     # for downloading datasets over HTTPS
     "requests>=2.32.2",
+    "httpx<1.0.0",
     # progress bars in downloads and data operations
     "tqdm>=4.66.3",
     # for fast hashing
@@ -128,7 +129,7 @@
     # minimum 2023.1.0 to support protocol=kwargs in fsspec's `open`, `get_fs_token_paths`, etc.: see https://github.com/fsspec/filesystem_spec/pull/1143
     "fsspec[http]>=2023.1.0,<=2025.9.0",
     # To get datasets from the Datasets Hub on huggingface.co
-    "huggingface-hub>=0.24.0",
+    "huggingface-hub>=0.25.0,<2.0",
     # Utilities from PyPA to e.g., compare versions
     "packaging",
     # To parse YAML metadata from dataset cards

src/datasets/arrow_dataset.py

@@ -68,9 +68,9 @@
     DatasetCardData,
     HfApi,
 )
-from huggingface_hub.hf_api import HfHubHTTPError, RepoFile, RepositoryNotFoundError
+from huggingface_hub.hf_api import RepoFile
+from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
 from multiprocess import Pool
-from requests import HTTPError
 from tqdm.contrib.concurrent import thread_map
 
 from . import config
@@ -5990,7 +5990,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
                     except HfHubHTTPError as err:
                         if (
                             err.__context__
-                            and isinstance(err.__context__, HTTPError)
+                            and isinstance(err.__context__, HfHubHTTPError)
                             and err.__context__.response.status_code == 409
                         ):
                             # 409 is Conflict (another commit is in progress)
@@ -6040,7 +6040,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
             except HfHubHTTPError as err:
                 if (
                     err.__context__
-                    and isinstance(err.__context__, HTTPError)
+                    and isinstance(err.__context__, HfHubHTTPError)
                     and err.__context__.response.status_code in (412, 409)
                 ):
                     # 412 is Precondition failed (parent_commit isn't satisfied)

src/datasets/data_files.py

@@ -352,7 +352,7 @@ def resolve_pattern(
     protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]
     protocol_prefix = protocol + "://" if protocol != "file" else ""
     glob_kwargs = {}
-    if protocol == "hf" and config.HF_HUB_VERSION >= version.parse("0.20.0"):
+    if protocol == "hf":
         # 10 times faster glob with detail=True (ignores costly info like lastCommit)
         glob_kwargs["expand_info"] = False
     matched_paths = [

src/datasets/dataset_dict.py

@@ -26,7 +26,6 @@
 )
 from huggingface_hub.hf_api import RepoFile
 from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
-from requests import HTTPError
 
 from . import config
 from .arrow_dataset import (
@@ -1917,7 +1916,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
                     except HfHubHTTPError as err:
                         if (
                             err.__context__
-                            and isinstance(err.__context__, HTTPError)
+                            and isinstance(err.__context__, HfHubHTTPError)
                             and err.__context__.response.status_code == 409
                         ):
                             # 409 is Conflict (another commit is in progress)
@@ -1967,7 +1966,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
             except HfHubHTTPError as err:
                 if (
                     err.__context__
-                    and isinstance(err.__context__, HTTPError)
+                    and isinstance(err.__context__, HfHubHTTPError)
                     and err.__context__.response.status_code in (412, 409)
                 ):
                     # 412 is Precondition failed (parent_commit isn't satisfied)
@@ -2786,7 +2785,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
                     except HfHubHTTPError as err:
                         if (
                             err.__context__
-                            and isinstance(err.__context__, HTTPError)
+                            and isinstance(err.__context__, HfHubHTTPError)
                             and err.__context__.response.status_code == 409
                         ):
                             # 409 is Conflict (another commit is in progress)
@@ -2836,7 +2835,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
             except HfHubHTTPError as err:
                 if (
                     err.__context__
-                    and isinstance(err.__context__, HTTPError)
+                    and isinstance(err.__context__, HfHubHTTPError)
                     and err.__context__.response.status_code in (412, 409)
                 ):
                     # 412 is Precondition failed (parent_commit isn't satisfied)

src/datasets/iterable_dataset.py

@@ -30,7 +30,6 @@
 from huggingface_hub.hf_api import RepoFile
 from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
 from multiprocess import Pool
-from requests import HTTPError
 
 from . import config
 from .arrow_dataset import PUSH_TO_HUB_WITHOUT_METADATA_CONFIGS_SPLIT_PATTERN_SHARDED, Dataset, DatasetInfoMixin
@@ -4332,7 +4331,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
                     except HfHubHTTPError as err:
                         if (
                             err.__context__
-                            and isinstance(err.__context__, HTTPError)
+                            and isinstance(err.__context__, HfHubHTTPError)
                             and err.__context__.response.status_code == 409
                         ):
                             # 409 is Conflict (another commit is in progress)
@@ -4382,7 +4381,7 @@ def get_deletions_and_dataset_card() -> tuple[str, list[CommitOperationDelete],
             except HfHubHTTPError as err:
                 if (
                     err.__context__
-                    and isinstance(err.__context__, HTTPError)
+                    and isinstance(err.__context__, HfHubHTTPError)
                     and err.__context__.response.status_code in (412, 409)
                 ):
                     # 412 is Precondition failed (parent_commit isn't satisfied)

src/datasets/load.py

@@ -28,6 +28,7 @@
 from typing import Any, Optional, Union
 
 import fsspec
+import httpx
 import requests
 import yaml
 from fsspec.core import url_to_fs
@@ -948,6 +949,8 @@ def dataset_module_factory(
                         OfflineModeIsEnabled,
                         requests.exceptions.Timeout,
                         requests.exceptions.ConnectionError,
+                        httpx.ConnectError,
+                        httpx.TimeoutException,
                     ),
                 ):
                     raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
@@ -963,6 +966,8 @@ def dataset_module_factory(
                 OfflineModeIsEnabled,
                 requests.exceptions.Timeout,
                 requests.exceptions.ConnectionError,
+                httpx.ConnectError,
+                httpx.TimeoutException,
             ) as e:
                 raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({e.__class__.__name__})") from e
             except GatedRepoError as e:

src/datasets/utils/file_utils.py

@@ -27,12 +27,13 @@
 from xml.etree import ElementTree as ET
 
 import fsspec
+import httpx
 import huggingface_hub
 import huggingface_hub.errors
 import requests
 from fsspec.core import strip_protocol, url_to_fs
 from fsspec.utils import can_be_local
-from huggingface_hub.utils import EntryNotFoundError, get_session, insecure_hashlib
+from huggingface_hub.utils import get_session, insecure_hashlib
 from packaging import version
 
 from .. import __version__, config
@@ -140,7 +141,7 @@ def cached_path(
         ConnectionError: in case of unreachable url
             and no cache on disk
         ValueError: if it couldn't parse the url or filename correctly
-        requests.exceptions.ConnectionError: in case of internet connection issue
+        httpx.NetworkError or requests.exceptions.ConnectionError: in case of internet connection issue
     """
     if download_config is None:
         download_config = DownloadConfig(**download_kwargs)
@@ -246,7 +247,7 @@ def cached_path(
 def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str:
     ua = f"datasets/{__version__}"
     ua += f"; python/{config.PY_VERSION}"
-    ua += f"; huggingface_hub/{huggingface_hub.__version__}"
+    ua += f"; hf_hub/{huggingface_hub.__version__}"
     ua += f"; pyarrow/{config.PYARROW_VERSION}"
     if config.TORCH_AVAILABLE:
         ua += f"; torch/{config.TORCH_VERSION}"
@@ -753,7 +754,7 @@ def xgetsize(path, download_config: Optional[DownloadConfig] = None) -> int:
         fs, *_ = fs, *_ = url_to_fs(path, **storage_options)
         try:
             size = fs.size(main_hop)
-        except EntryNotFoundError:
+        except huggingface_hub.utils.EntryNotFoundError:
             raise FileNotFoundError(f"No such file: {path}")
         if size is None:
             # use xopen instead of fs.open to make data fetching more robust
@@ -817,6 +818,7 @@ def read_with_retries(*args, **kwargs):
                 asyncio.TimeoutError,
                 requests.exceptions.ConnectionError,
                 requests.exceptions.Timeout,
+                httpx.RequestError,
             ) as err:
                 disconnect_err = err
                 logger.warning(
@@ -897,9 +899,6 @@ def _prepare_single_hop_path_and_storage_options(
             "endpoint": config.HF_ENDPOINT,
             **storage_options,
         }
-        # streaming with block_size=0 is only implemented in 0.21 (see https://github.com/huggingface/huggingface_hub/pull/1967)
-        if config.HF_HUB_VERSION < version.parse("0.21.0"):
-            storage_options["block_size"] = "default"
     if storage_options:
         storage_options = {protocol: storage_options}
     return urlpath, storage_options

src/datasets/utils/hub.py

@@ -1,14 +1,6 @@
 from functools import partial
 
 from huggingface_hub import hf_hub_url
-from huggingface_hub.utils import get_session, hf_raise_for_status
 
 
 hf_dataset_url = partial(hf_hub_url, repo_type="dataset")
-
-
-def check_auth(hf_api, repo_id, token=None):
-    headers = hf_api._build_hf_headers(token=token)
-    path = f"{hf_api.endpoint}/api/datasets/{repo_id}/auth-check"
-    r = get_session().get(path, headers=headers)
-    hf_raise_for_status(r)

tests/fixtures/hub.py

@@ -5,9 +5,8 @@
 from typing import Optional
 
 import pytest
-import requests
-from huggingface_hub.hf_api import HfApi, RepositoryNotFoundError
-from huggingface_hub.utils import hf_raise_for_status
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError
 from huggingface_hub.utils._headers import _http_user_agent
 
 
@@ -24,9 +23,14 @@
 def ci_hub_config(monkeypatch):
     monkeypatch.setattr("datasets.config.HF_ENDPOINT", CI_HUB_ENDPOINT)
     monkeypatch.setattr("datasets.config.HUB_DATASETS_URL", CI_HUB_DATASETS_URL)
-    monkeypatch.setattr(
-        "huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE", CI_HFH_HUGGINGFACE_CO_URL_TEMPLATE
-    )
+    monkeypatch.setattr("huggingface_hub.constants.HUGGINGFACE_CO_URL_TEMPLATE", CI_HFH_HUGGINGFACE_CO_URL_TEMPLATE)
+    try:
+        # for backward compatibility with huggingface_hub 0.x
+        monkeypatch.setattr(
+            "huggingface_hub.file_download.HUGGINGFACE_CO_URL_TEMPLATE", CI_HFH_HUGGINGFACE_CO_URL_TEMPLATE
+        )
+    except AttributeError:
+        pass
     old_environ = dict(os.environ)
     os.environ["HF_ENDPOINT"] = CI_HUB_ENDPOINT
     yield
@@ -107,18 +111,11 @@ def _hf_gated_dataset_repo_txt_data(hf_api: HfApi, hf_token, text_file_content):
         repo_id=repo_id,
         repo_type="dataset",
     )
-    path = f"{hf_api.endpoint}/api/datasets/{repo_id}/settings"
-    repo_settings = {"gated": "auto"}
-    r = requests.put(
-        path,
-        headers={"authorization": f"Bearer {hf_token}"},
-        json=repo_settings,
-    )
-    hf_raise_for_status(r)
+    hf_api.update_repo_settings(repo_id, token=hf_token, repo_type="dataset", gated="auto")
     yield repo_id
     try:
         hf_api.delete_repo(repo_id, token=hf_token, repo_type="dataset")
-    except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
+    except (HfHubHTTPError, ValueError):  # catch http error and token invalid error
         pass
 
 
@@ -142,7 +139,7 @@ def hf_private_dataset_repo_txt_data_(hf_api: HfApi, hf_token, text_file_content
     yield repo_id
     try:
         hf_api.delete_repo(repo_id, token=hf_token, repo_type="dataset")
-    except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
+    except (HfHubHTTPError, ValueError):  # catch http error and token invalid error
         pass
 
 
@@ -166,7 +163,7 @@ def hf_private_dataset_repo_zipped_txt_data_(hf_api: HfApi, hf_token, zip_csv_wi
     yield repo_id
     try:
         hf_api.delete_repo(repo_id, token=hf_token, repo_type="dataset")
-    except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
+    except (HfHubHTTPError, ValueError):  # catch http error and token invalid error
         pass
 
 
@@ -190,7 +187,7 @@ def hf_private_dataset_repo_zipped_img_data_(hf_api: HfApi, hf_token, zip_image_
     yield repo_id
     try:
         hf_api.delete_repo(repo_id, token=hf_token, repo_type="dataset")
-    except (requests.exceptions.HTTPError, ValueError):  # catch http error and token invalid error
+    except (HfHubHTTPError, ValueError):  # catch http error and token invalid error
         pass
 
 

tests/test_load.py

@@ -11,7 +11,6 @@
 import dill
 import pyarrow as pa
 import pytest
-import requests
 
 import datasets
 from datasets import config, load_dataset
@@ -767,10 +766,7 @@ def test_load_dataset_from_hub(self):
     def test_load_dataset_namespace(self):
         with self.assertRaises(DatasetNotFoundError) as context:
             datasets.load_dataset("hf-internal-testing/_dummy")
-        self.assertIn(
-            "hf-internal-testing/_dummy",
-            str(context.exception),
-        )
+        self.assertIn("hf-internal-testing/_dummy", str(context.exception))
         for offline_simulation_mode in list(OfflineSimulationMode):
             with offline(offline_simulation_mode):
                 with self.assertRaises(ConnectionError) as context:
@@ -1050,19 +1046,16 @@ def test_load_dataset_with_unsupported_extensions(text_dir_with_unsupported_exte
 
 @pytest.mark.integration
 def test_loading_from_the_datasets_hub_with_token():
-    true_request = requests.Session().request
-
-    def assert_auth(method, url, *args, headers, **kwargs):
-        assert headers["authorization"] == "Bearer foo"
-        return true_request(method, url, *args, headers=headers, **kwargs)
+    class CustomException(Exception):
+        pass
 
-    with patch("requests.Session.request") as mock_request:
-        mock_request.side_effect = assert_auth
+    with patch("huggingface_hub.file_download._get_metadata_or_catch_error") as mock_request:
+        mock_request.side_effect = CustomException()
         with tempfile.TemporaryDirectory() as tmp_dir:
-            with offline():
-                with pytest.raises((ConnectionError, requests.exceptions.ConnectionError)):
-                    load_dataset(SAMPLE_NOT_EXISTING_DATASET_IDENTIFIER, cache_dir=tmp_dir, token="foo")
-        mock_request.assert_called()
+            with pytest.raises(CustomException):
+                load_dataset(SAMPLE_NOT_EXISTING_DATASET_IDENTIFIER, cache_dir=tmp_dir, token="foo")
+        mock_request.assert_called_once()
+        assert mock_request.call_args_list[0][1]["headers"]["authorization"] == "Bearer foo"
 
 
 @pytest.mark.integration