Skip to content

Commit 0bc709a

Browse files
authored
Fix parquet export infos (#6822)
fix parquet export infos
1 parent 2a14271 commit 0bc709a

File tree

1 file changed

+14
-11
lines changed

1 file changed

+14
-11
lines changed

src/datasets/load.py

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1235,17 +1235,20 @@ def get_module(self) -> DatasetModule:
12351235
pass
12361236
metadata_configs = MetadataConfigs.from_dataset_card_data(dataset_card_data)
12371237
dataset_infos = DatasetInfosDict.from_dataset_card_data(dataset_card_data)
1238-
try:
1239-
exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos(
1240-
dataset=self.name, revision=self.revision, token=self.download_config.token
1241-
)
1242-
exported_dataset_infos = DatasetInfosDict(
1243-
{
1244-
config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name])
1245-
for config_name in exported_dataset_infos
1246-
}
1247-
)
1248-
except _dataset_viewer.DatasetViewerError:
1238+
if config.USE_PARQUET_EXPORT: # maybe don't use the infos from the parquet export
1239+
try:
1240+
exported_dataset_infos = _dataset_viewer.get_exported_dataset_infos(
1241+
dataset=self.name, revision=self.revision, token=self.download_config.token
1242+
)
1243+
exported_dataset_infos = DatasetInfosDict(
1244+
{
1245+
config_name: DatasetInfo.from_dict(exported_dataset_infos[config_name])
1246+
for config_name in exported_dataset_infos
1247+
}
1248+
)
1249+
except _dataset_viewer.DatasetViewerError:
1250+
exported_dataset_infos = None
1251+
else:
12491252
exported_dataset_infos = None
12501253
if exported_dataset_infos:
12511254
exported_dataset_infos.update(dataset_infos)

0 commit comments

Comments
 (0)