Bump max range of dill to 0.3.8 (#6630)

* Bump max range of dill to 0.3.8

Release on Jan 27, 2024: https://pypi.org/project/dill/0.3.8/#history

* Update dill utils with latest dill version

* Create constant to avoid code duplication

* Nit

---------

Co-authored-by: mariosasko <[email protected]>

Author: Matthew Hoffman

.github/workflows/ci.yml

@@ -61,7 +61,7 @@ jobs:
           python -m spacy download fr_core_news_sm
       - name: Install dependencies (latest versions)
         if: ${{ matrix.deps_versions == 'deps-latest' }}
-        run: pip install --upgrade pyarrow huggingface-hub "dill<0.3.8"
+        run: pip install --upgrade pyarrow huggingface-hub dill
       - name: Install dependencies (minimum versions)
         if: ${{ matrix.deps_versions != 'deps-latest' }}
         run: pip install pyarrow==8.0.0 huggingface-hub==0.19.4 transformers dill==0.3.1.1

setup.py

@@ -118,7 +118,7 @@
     # As long as we allow pyarrow < 14.0.1, to fix vulnerability CVE-2023-47248
     "pyarrow-hotfix",
     # For smart caching dataset processing
-    "dill>=0.3.0,<0.3.8",  # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19
+    "dill>=0.3.0,<0.3.9",  # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19
     # For performance gains with apache arrow
     "pandas",
     # for downloading datasets over HTTPS

src/datasets/utils/_dill.py

@@ -114,7 +114,11 @@ def dumps(obj):
     def log(pickler, msg):
         dill._dill.log.info(msg)
 
-elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]:
+elif config.DILL_VERSION.release[:3] in [
+    version.parse("0.3.6").release,
+    version.parse("0.3.7").release,
+    version.parse("0.3.8").release,
+]:
 
     def log(pickler, msg):
         dill._dill.logger.trace(pickler, msg)
@@ -301,7 +305,11 @@ def _save_code(pickler, obj):
         dill._dill.log.info("# Co")
         return
 
-elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]:
+elif config.DILL_VERSION.release[:3] in [
+    version.parse("0.3.6").release,
+    version.parse("0.3.7").release,
+    version.parse("0.3.8").release,
+]:
     # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104
     @pklregister(CodeType)
     def save_code(pickler, obj):