Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/datasets/data_files.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import re
from functools import partial
from glob import has_magic
from pathlib import Path, PurePath
from typing import Callable, Dict, List, Optional, Set, Tuple, Union

Expand Down Expand Up @@ -320,7 +321,7 @@ def resolve_pattern(
allowed_extensions (Optional[list], optional): White-list of file extensions to use. Defaults to None (all extensions).
For example: allowed_extensions=[".csv", ".json", ".txt", ".parquet"]
Returns:
List[Union[Path, Url]]: List of paths or URLs to the local or remote files that match the patterns.
List[str]: List of paths or URLs to the local or remote files that match the patterns.
"""
if is_relative_path(pattern):
pattern = xjoin(base_path, pattern)
Expand Down Expand Up @@ -583,7 +584,8 @@ def from_patterns(
)
)
except FileNotFoundError:
pass
if not has_magic(pattern):
raise
origin_metadata = _get_origin_metadata(data_files, download_config=download_config)
return cls(data_files, origin_metadata)

Expand Down
5 changes: 5 additions & 0 deletions tests/test_data_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,11 @@ def test_DataFilesList_from_patterns_locally_with_extra_files(complex_data_dir,
assert len(data_files_list.origin_metadata) == 2


def test_DataFilesList_from_patterns_raises_FileNotFoundError(complex_data_dir):
with pytest.raises(FileNotFoundError):
DataFilesList.from_patterns(["file_that_doesnt_exist.txt"], complex_data_dir)


@pytest.mark.parametrize("pattern", _TEST_PATTERNS)
def test_DataFilesDict_from_patterns_in_dataset_repository(
hub_dataset_repo_path, hub_dataset_repo_patterns_results, pattern
Expand Down