Skip to content

Commit b4e8fc0

Browse files
lhoestqalbertvillanova
authored andcommitted
[Streaming] retry on requests errors (#6963)
* [Streaming] retry on requests errors * lucain's comment
1 parent 97d6582 commit b4e8fc0

File tree

1 file changed

+8
-3
lines changed

1 file changed

+8
-3
lines changed

src/datasets/utils/file_utils.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
Copyright by the AllenNLP authors.
55
"""
66

7+
import asyncio
78
import copy
89
import glob
910
import io
@@ -20,7 +21,6 @@
2021
import warnings
2122
import xml.dom.minidom
2223
import zipfile
23-
from asyncio import TimeoutError
2424
from contextlib import closing, contextmanager
2525
from functools import partial
2626
from io import BytesIO
@@ -31,10 +31,10 @@
3131
from urllib.parse import urljoin, urlparse
3232
from xml.etree import ElementTree as ET
3333

34+
import aiohttp.client_exceptions
3435
import fsspec
3536
import huggingface_hub
3637
import requests
37-
from aiohttp.client_exceptions import ClientError
3838
from fsspec.core import strip_protocol, url_to_fs
3939
from fsspec.utils import can_be_local
4040
from huggingface_hub.utils import EntryNotFoundError, insecure_hashlib
@@ -1103,7 +1103,12 @@ def read_with_retries(*args, **kwargs):
11031103
try:
11041104
out = read(*args, **kwargs)
11051105
break
1106-
except (ClientError, TimeoutError) as err:
1106+
except (
1107+
aiohttp.client_exceptions.ClientError,
1108+
asyncio.TimeoutError,
1109+
requests.exceptions.ConnectTimeout,
1110+
requests.exceptions.ConnectionError,
1111+
) as err:
11071112
disconnect_err = err
11081113
logger.warning(
11091114
f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]"

0 commit comments

Comments
 (0)