fix

Author: lhoestq

src/datasets/arrow_dataset.py

@@ -2491,7 +2491,7 @@ def catch_non_existent_error(func, kwargs):
             ), "All shards have to be defined Datasets, none should still be missing."
 
             logger.info(f"Concatenating {num_proc} shards")
-            result = concatenate_datasets(transformed_shards)
+            result = _concatenate_map_style_datasets(transformed_shards)
             if new_fingerprint is not None:
                 result._fingerprint = new_fingerprint
             return result

tests/test_iterable_dataset.py

@@ -17,7 +17,6 @@
     IterableDataset,
     MappedExamplesIterable,
     RandomlyCyclingMultiSourcesExamplesIterable,
-    ShardShuffledExamplesIterable,
     ShufflingConfig,
     SkipExamplesIterable,
     TakeExamplesIterable,