Reduce default max writer_batch_size (#5163)

Author: mariosasko

src/datasets/arrow_dataset.py

@@ -1617,11 +1617,11 @@ def flatten(self, new_fingerprint: Optional[str] = None, max_depth=16) -> "Datas
     def cast(
         self,
         features: Features,
-        batch_size: Optional[int] = 10_000,
+        batch_size: Optional[int] = 1000,
         keep_in_memory: bool = False,
         load_from_cache_file: bool = True,
         cache_file_name: Optional[str] = None,
-        writer_batch_size: Optional[int] = 10_000,
+        writer_batch_size: Optional[int] = 1000,
         num_proc: Optional[int] = None,
     ) -> "Dataset":
         """

src/datasets/config.py

@@ -169,7 +169,7 @@
 
 # Batch size constants. For more info, see:
 # https://github.com/apache/arrow/blob/master/docs/source/cpp/arrays.rst#size-limitations-and-recommendations)
-DEFAULT_MAX_BATCH_SIZE = 10_000
+DEFAULT_MAX_BATCH_SIZE = 1000
 
 # Size of the preloaded record batch in `Dataset.__iter__`
 ARROW_READER_BATCH_SIZE_IN_DATASET_ITER = 10