Skip to content

Fix missing info when loading some datasets from Parquet export #6635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 7, 2024
Merged

Fix missing info when loading some datasets from Parquet export #6635

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a068fb9
set default config when single config parquet export
lhoestq Jan 31, 2024
b1006f4
Merge branch 'main' into fix-info-from-single-config-parquet-export
lhoestq Feb 6, 2024
ac5a947
fix tests
lhoestq Feb 6, 2024
ad16ae2
fix test
lhoestq Feb 6, 2024
0487bf3
Update src/datasets/utils/metadata.py
lhoestq Feb 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/datasets/utils/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,8 @@ def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None:

def get_default_config_name(self) -> Optional[str]:
default_config_name = None
if len(self) == 1:
return next(iter(self))
for config_name, metadata_config in self.items():
if config_name == "default" or metadata_config.get("default"):
if default_config_name is None:
Expand Down
4 changes: 2 additions & 2 deletions tests/test_load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ def test_LocalDatasetModuleFactoryWithoutScript_with_single_config_in_metadata(s
assert module_builder_configs[0].drop_labels is True # parameter is passed from metadata

# config named "default" is automatically considered to be a default config
assert module_factory_result.builder_configs_parameters.default_config_name is None
assert module_factory_result.builder_configs_parameters.default_config_name == "custom"

# we don't pass config params to builder in builder_kwargs, they are stored in builder_configs directly
assert "drop_labels" not in module_factory_result.builder_kwargs
Expand Down Expand Up @@ -695,7 +695,7 @@ def test_HubDatasetModuleFactoryWithoutScript_with_one_default_config_in_metadat
assert module_builder_configs[0].drop_labels is True # parameter is passed from metadata

# config named "default" is automatically considered to be a default config
assert module_factory_result.builder_configs_parameters.default_config_name is None
assert module_factory_result.builder_configs_parameters.default_config_name == "custom"

# we don't pass config params to builder in builder_kwargs, they are stored in builder_configs directly
assert "drop_labels" not in module_factory_result.builder_kwargs
Expand Down
2 changes: 1 addition & 1 deletion tests/test_metadata_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ def test_from_yaml_string(self):
@pytest.mark.parametrize(
"readme_content, expected_metadata_configs_dict, expected_default_config_name",
[
(README_METADATA_SINGLE_CONFIG, EXPECTED_METADATA_SINGLE_CONFIG, None),
(README_METADATA_SINGLE_CONFIG, EXPECTED_METADATA_SINGLE_CONFIG, "custom"),
(README_METADATA_TWO_CONFIGS_WITH_DEFAULT_FLAG, EXPECTED_METADATA_TWO_CONFIGS_DEFAULT_FLAG, "v2"),
(README_METADATA_TWO_CONFIGS_WITH_DEFAULT_NAME, EXPECTED_METADATA_TWO_CONFIGS_DEFAULT_NAME, "default"),
],
Expand Down