fix: 🐛 allow streaming=False in get_rows (#207)

it fixes #206.

Author: severo

src/datasets_preview_backend/config.py

@@ -12,6 +12,7 @@
     DEFAULT_LOG_LEVEL,
     DEFAULT_MAX_AGE_LONG_SECONDS,
     DEFAULT_MAX_AGE_SHORT_SECONDS,
+    DEFAULT_MAX_SIZE_FALLBACK,
     DEFAULT_MONGO_CACHE_DATABASE,
     DEFAULT_MONGO_QUEUE_DATABASE,
     DEFAULT_MONGO_URL,
@@ -50,6 +51,7 @@
 os.environ["HF_SCRIPTS_VERSION"] = DATASETS_REVISION
 
 # for tests - to be removed
+MAX_SIZE_FALLBACK = get_int_value(os.environ, "MAX_SIZE_FALLBACK", DEFAULT_MAX_SIZE_FALLBACK)
 ROWS_MAX_BYTES = get_int_value(d=os.environ, key="ROWS_MAX_BYTES", default=DEFAULT_ROWS_MAX_BYTES)
 ROWS_MAX_NUMBER = get_int_value(d=os.environ, key="ROWS_MAX_NUMBER", default=DEFAULT_ROWS_MAX_NUMBER)
 ROWS_MIN_NUMBER = get_int_value(d=os.environ, key="ROWS_MIN_NUMBER", default=DEFAULT_ROWS_MIN_NUMBER)

src/datasets_preview_backend/models/row.py

@@ -28,7 +28,7 @@ def get_rows(
         dataset_name,
         name=config_name,
         split=split_name,
-        streaming=True,
+        streaming=streaming,
         download_mode=DownloadMode.FORCE_REDOWNLOAD,
         use_auth_token=hf_token,
     )

tests/models/test_split.py

@@ -1,4 +1,4 @@
-from datasets_preview_backend.config import HF_TOKEN, ROWS_MAX_NUMBER
+from datasets_preview_backend.config import HF_TOKEN, MAX_SIZE_FALLBACK, ROWS_MAX_NUMBER
 from datasets_preview_backend.models.split import get_split
 
 # TODO: test fallback
@@ -24,4 +24,21 @@ def test_gated() -> None:
     assert split["rows_response"]["rows"][0]["row"]["year"] == "1855"
 
 
+def test_fallback() -> None:
+    # https://github.com/huggingface/datasets/issues/3185
+    dataset_name = "samsum"
+    config_name = "samsum"
+    split_name = "train"
+    split = get_split(
+        dataset_name,
+        config_name,
+        split_name,
+        HF_TOKEN,
+        rows_max_number=ROWS_MAX_NUMBER,
+        max_size_fallback=MAX_SIZE_FALLBACK,
+    )
+
+    assert len(split["rows_response"]["rows"]) == ROWS_MAX_NUMBER
+
+
 # TODO: test the truncation