diff --git a/setup.py b/setup.py index b560d367688..c58c5d271df 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ # Minimum 21.0.0 to support `use_content_defined_chunking` in ParquetWriter "pyarrow>=21.0.0", # For smart caching dataset processing - "dill>=0.3.0,<0.3.9", # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19 + "dill>=0.3.0,<0.4.1", # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19 # For performance gains with apache arrow "pandas", # for downloading datasets over HTTPS diff --git a/src/datasets/utils/_dill.py b/src/datasets/utils/_dill.py index 0876a00d9db..fad95f7edf5 100644 --- a/src/datasets/utils/_dill.py +++ b/src/datasets/utils/_dill.py @@ -98,6 +98,17 @@ def proxy(func): return proxy +def _is_supported_dill_version(): + """Check if the current dill version is in the supported range.""" + return config.DILL_VERSION.release[:3] in [ + version.parse("0.3.6").release, + version.parse("0.3.7").release, + version.parse("0.3.8").release, + version.parse("0.3.9").release, + version.parse("0.4.0").release, + ] + + def dump(obj, file): """Pickle an object to a file.""" Pickler(file, recurse=True).dump(obj) @@ -115,11 +126,7 @@ def dumps(obj): def log(pickler, msg): dill._dill.log.info(msg) -elif config.DILL_VERSION.release[:3] in [ - version.parse("0.3.6").release, - version.parse("0.3.7").release, - version.parse("0.3.8").release, -]: +elif _is_supported_dill_version(): def log(pickler, msg): dill._dill.logger.trace(pickler, msg) @@ -312,11 +319,7 @@ def _save_code(pickler, obj): dill._dill.log.info("# Co") return -elif config.DILL_VERSION.release[:3] in [ - version.parse("0.3.6").release, - version.parse("0.3.7").release, - version.parse("0.3.8").release, -]: +elif _is_supported_dill_version(): # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104 @pklregister(CodeType) def save_code(pickler, obj):