Address comments

Author: mariosasko

src/datasets/packaged_modules/csv/csv.py

@@ -68,10 +68,6 @@ def __post_init__(self):
         if self.column_names is not None:
             self.names = self.column_names
 
-    @property
-    def schema(self):
-        return self.features.arrow_schema if self.features is not None else None
-
     @property
     def read_csv_kwargs(self):
         read_csv_kwargs = dict(
@@ -154,7 +150,7 @@ def _split_generators(self, dl_manager):
 
     def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         if self.config.features is not None:
-            schema = self.config.schema
+            schema = self.config.features.arrow_schema
             if all(not require_storage_cast(feature) for feature in self.config.features.values()):
                 # cheaper cast
                 pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
@@ -164,14 +160,14 @@ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         return pa_table
 
     def _generate_tables(self, files):
-        schema = self.config.schema
+        schema = self.config.features.arrow_schema if self.config.features else None
         # dtype allows reading an int column as str
         dtype = (
             {
                 name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object
                 for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values())
             }
-            if schema
+            if schema is not None
             else None
         )
         for file_idx, file in enumerate(files):

src/datasets/packaged_modules/json/json.py

@@ -7,7 +7,6 @@
 import pyarrow.json as paj
 
 import datasets
-from datasets.features.features import require_storage_cast
 from datasets.table import table_cast
 from datasets.utils.file_utils import readline
 
@@ -26,10 +25,6 @@ class JsonConfig(datasets.BuilderConfig):
     chunksize: int = 10 << 20  # 10MB
     newlines_in_values: Optional[bool] = None
 
-    @property
-    def schema(self):
-        return self.features.arrow_schema if self.features is not None else None
-
 
 class Json(datasets.ArrowBasedBuilder):
     BUILDER_CONFIG_CLASS = JsonConfig
@@ -67,13 +62,9 @@ def _split_generators(self, dl_manager):
 
     def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         if self.config.features is not None:
-            schema = self.config.schema
-            if all(not require_storage_cast(feature) for feature in self.config.features.values()):
-                # cheaper cast
-                pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
-            else:
-                # more expensive cast; allows str <-> int/float or str to Audio for example
-                pa_table = table_cast(pa_table, schema)
+            # more expensive cast to support nested structures with keys in a different order
+            # allows str <-> int/float or str to Audio for example
+            pa_table = table_cast(pa_table, self.config.features.arrow_schema)
         return pa_table
 
     def _generate_tables(self, files):

src/datasets/packaged_modules/pandas/pandas.py

@@ -15,10 +15,6 @@ class PandasConfig(datasets.BuilderConfig):
 
     features: Optional[datasets.Features] = None
 
-    @property
-    def schema(self):
-        return self.features.arrow_schema if self.features is not None else None
-
 
 class Pandas(datasets.ArrowBasedBuilder):
     BUILDER_CONFIG_CLASS = PandasConfig
@@ -45,7 +41,7 @@ def _split_generators(self, dl_manager):
 
     def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         if self.config.features is not None:
-            schema = self.config.schema
+            schema = self.config.features.arrow_schema
             if all(not require_storage_cast(feature) for feature in self.config.features.values()):
                 # cheaper cast
                 pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)

src/datasets/packaged_modules/parquet/parquet.py

@@ -20,10 +20,6 @@ class ParquetConfig(datasets.BuilderConfig):
     columns: Optional[List[str]] = None
     features: Optional[datasets.Features] = None
 
-    @property
-    def schema(self):
-        return self.features.arrow_schema if self.features is not None else None
-
 
 class Parquet(datasets.ArrowBasedBuilder):
     BUILDER_CONFIG_CLASS = ParquetConfig
@@ -50,7 +46,7 @@ def _split_generators(self, dl_manager):
 
     def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         if self.config.features is not None:
-            schema = self.config.schema
+            schema = self.config.features.arrow_schema
             if all(not require_storage_cast(feature) for feature in self.config.features.values()):
                 # cheaper cast
                 pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema)
@@ -60,7 +56,7 @@ def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         return pa_table
 
     def _generate_tables(self, files):
-        schema = pa.schema(self.config.features.type) if self.config.features is not None else None
+        schema = self.config.features.arrow_schema if self.config.features is not None else None
         if self.config.features is not None and self.config.columns is not None:
             if sorted(field.name for field in schema) != sorted(self.config.columns):
                 raise ValueError(

src/datasets/packaged_modules/text/text.py

@@ -22,10 +22,6 @@ class TextConfig(datasets.BuilderConfig):
     keep_linebreaks: bool = False
     sample_by: str = "line"
 
-    @property
-    def schema(self):
-        return self.features.arrow_schema if self.features is not None else None
-
 
 class Text(datasets.ArrowBasedBuilder):
     BUILDER_CONFIG_CLASS = TextConfig
@@ -58,7 +54,7 @@ def _split_generators(self, dl_manager):
 
     def _cast_table(self, pa_table: pa.Table) -> pa.Table:
         if self.config.features is not None:
-            schema = self.config.schema
+            schema = self.config.features.arrow_schema
             if all(not require_storage_cast(feature) for feature in self.config.features.values()):
                 # cheaper cast
                 pa_table = pa_table.cast(schema)