From f8187e90aed7e1b96ffaae85cdf4b37108c75d3f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 26 Oct 2017 20:17:28 +0200 Subject: [PATCH 1/5] Some cleanups, fix memoryview support --- cloudpickle/cloudpickle.py | 68 +++++++++++--------------------------- tests/cloudpickle_test.py | 4 +++ 2 files changed, 24 insertions(+), 48 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 4a823b96f..b9e862fc5 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -264,15 +264,12 @@ def dump(self, obj): raise pickle.PicklingError(msg) def save_memoryview(self, obj): - """Fallback to save_string""" - Pickler.save_string(self, str(obj)) + self.save(obj.tobytes()) + dispatch[memoryview] = save_memoryview - def save_buffer(self, obj): - """Fallback to save_string""" - Pickler.save_string(self,str(obj)) - if PY3: - dispatch[memoryview] = save_memoryview - else: + if not PY3: + def save_buffer(self, obj): + self.save(str(obj)) dispatch[buffer] = save_buffer def save_unsupported(self, obj): @@ -387,7 +384,7 @@ def save_function(self, obj, name=None): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) - return Pickler.save_reduce(self, obj=obj, *rv) + return self.save_reduce(obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a @@ -477,18 +474,12 @@ def save_dynamic_class(self, obj): # Push the rehydration function. save(_rehydrate_skeleton_class) - # Mark the start of the args for the rehydration function. + # Mark the start of the args tuple for the rehydration function. write(pickle.MARK) - # Create and memoize an empty class with obj's name and bases. - save(type(obj)) - save(( - obj.__name__, - obj.__bases__, - type_kwargs, - )) - write(pickle.REDUCE) - self.memoize(obj) + # Create and memoize an skeleton class with obj's name and bases. + tp = type(obj) + self.save_reduce(tp, (obj.__name__, obj.__bases__, type_kwargs), obj=obj) # Now save the rest of obj's __dict__. Any references to obj # encountered while saving will point to the skeleton class. @@ -627,37 +618,18 @@ def save_global(self, obj, name=None, pack=struct.pack): The name of this method is somewhat misleading: all types get dispatched here. """ - if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": - if obj in _BUILTIN_TYPE_NAMES: - return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - - if name is None: - name = obj.__name__ - - modname = getattr(obj, "__module__", None) - if modname is None: - try: - # whichmodule() could fail, see - # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling - modname = pickle.whichmodule(obj, name) - except Exception: - modname = '__main__' - - if modname == '__main__': - themodule = None - else: - __import__(modname) - themodule = sys.modules[modname] - self.modules.add(themodule) + try: + return Pickler.save_global(self, obj, name=name) + except Exception: + if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": + if obj in _BUILTIN_TYPE_NAMES: + return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - if hasattr(themodule, name) and getattr(themodule, name) is obj: - return Pickler.save_global(self, obj, name) + typ = type(obj) + if typ is not obj and isinstance(obj, (type, types.ClassType)): + return self.save_dynamic_class(obj) - typ = type(obj) - if typ is not obj and isinstance(obj, (type, types.ClassType)): - self.save_dynamic_class(obj) - else: - raise pickle.PicklingError("Can't pickle %r" % obj) + raise dispatch[type] = save_global dispatch[types.ClassType] = save_global diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 06d0460a1..aa0a66ef7 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -130,6 +130,10 @@ def test_buffer(self): except NameError: # Python 3 does no longer support buffers pass + def test_memoryview(self): + buffer_obj = memoryview(b"Hello") + self.assertEqual(pickle_depickle(buffer_obj), buffer_obj.tobytes()) + def test_lambda(self): self.assertEqual(pickle_depickle(lambda: 1)(), 1) From ca4661b3a20b635f4c240ef763f5759267d74cb9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 26 Oct 2017 20:34:40 +0200 Subject: [PATCH 2/5] Close StringIO timely on exception --- cloudpickle/cloudpickle.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b9e862fc5..caafe2c61 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -854,11 +854,12 @@ def dump(obj, file, protocol=2): def dumps(obj, protocol=2): file = StringIO() - - cp = CloudPickler(file,protocol) - cp.dump(obj) - - return file.getvalue() + try: + cp = CloudPickler(file,protocol) + cp.dump(obj) + return file.getvalue() + finally: + file.close() # including pickles unloading functions in this namespace load = pickle.load From 980ccf5658ecb00281c9a874ad0c2d65a0266649 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 26 Oct 2017 20:35:42 +0200 Subject: [PATCH 3/5] Remove pandas from CI config since it's not used in testing --- .travis.yml | 8 +------- ci/before_install.sh | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index afdba9cb2..7da9dba0e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,7 @@ python: - "3.5" - "3.6" - "pypy" -matrix: - include: - # 0.14.0 is the last version with the old categorical system - - python: 3.3 - env: PANDAS_VERSION_STR="=0.14.0" - - python: 2.7 - env: PANDAS_VERSION_STR="=0.14.0" + # This disables sudo, but makes builds start much faster # See http://blog.travis-ci.com/2014-12-17-faster-builds-with-container-based-infrastructure/ sudo: false diff --git a/ci/before_install.sh b/ci/before_install.sh index 45f7f415b..83a41d44d 100644 --- a/ci/before_install.sh +++ b/ci/before_install.sh @@ -13,5 +13,5 @@ then conda config --set always_yes yes --set changeps1 no conda update -q conda conda info -a - conda create -q -n testenv python=$TRAVIS_PYTHON_VERSION numpy scipy pip pandas + conda create -q -n testenv python=$TRAVIS_PYTHON_VERSION numpy scipy pip fi \ No newline at end of file From 1ccc871a7ae130339b921de93e7422e177e2b4f0 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 27 Oct 2017 12:09:39 +0200 Subject: [PATCH 4/5] Get rid of 2.6 and 3.3 CI builds --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7da9dba0e..099e4c3a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,7 @@ language: python sudo: false python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" From c91aaf110441991307f5097f950764079d0f9652 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 27 Oct 2017 12:18:27 +0200 Subject: [PATCH 5/5] Further cleanups --- cloudpickle/cloudpickle.py | 40 ++++++++++------------------------ tests/cloudpickle_file_test.py | 6 +---- tests/cloudpickle_test.py | 2 -- 3 files changed, 12 insertions(+), 36 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index caafe2c61..e8c278020 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -375,10 +375,7 @@ def save_function(self, obj, name=None): # for different python versions. if not hasattr(obj, '__code__'): if PY3: - if sys.version_info < (3, 4): - raise pickle.PicklingError("Can't pickle %r" % obj) - else: - rv = obj.__reduce_ex__(self.proto) + rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) @@ -542,7 +539,7 @@ def save_function_tuple(self, func): _extract_code_globals_cache = ( weakref.WeakKeyDictionary() - if sys.version_info >= (2, 7) and not hasattr(sys, "pypy_version_info") + if not hasattr(sys, "pypy_version_info") else {}) @classmethod @@ -700,12 +697,7 @@ def save_property(self, obj): dispatch[property] = save_property def save_classmethod(self, obj): - try: - orig_func = obj.__func__ - except AttributeError: # Python 2.6 - orig_func = obj.__get__(None, object) - if isinstance(obj, classmethod): - orig_func = orig_func.__func__ # Unbind + orig_func = obj.__func__ self.save_reduce(type(obj), (orig_func,), obj=obj) dispatch[classmethod] = save_classmethod dispatch[staticmethod] = save_classmethod @@ -745,14 +737,6 @@ def __getattribute__(self, item): if type(operator.attrgetter) is type: dispatch[operator.attrgetter] = save_attrgetter - def save_partial(self, obj): - """Partial objects do not serialize correctly in python2.x -- this fixes the bugs""" - self.save_reduce(_genpartial, (obj.func, obj.args, obj.keywords)) - - if sys.version_info < (2,7): # 2.7 supports partial pickling - dispatch[partial] = save_partial - - def save_file(self, obj): """Save a file""" try: @@ -808,23 +792,21 @@ def save_not_implemented(self, obj): dispatch[type(Ellipsis)] = save_ellipsis dispatch[type(NotImplemented)] = save_not_implemented - # WeakSet was added in 2.7. - if hasattr(weakref, 'WeakSet'): - def save_weakset(self, obj): - self.save_reduce(weakref.WeakSet, (list(obj),)) + def save_weakset(self, obj): + self.save_reduce(weakref.WeakSet, (list(obj),)) - dispatch[weakref.WeakSet] = save_weakset - - """Special functions for Add-on libraries""" - def inject_addons(self): - """Plug in system. Register additional pickling functions if modules already loaded""" - pass + dispatch[weakref.WeakSet] = save_weakset def save_logger(self, obj): self.save_reduce(logging.getLogger, (obj.name,), obj=obj) dispatch[logging.Logger] = save_logger + """Special functions for Add-on libraries""" + def inject_addons(self): + """Plug in system. Register additional pickling functions if modules already loaded""" + pass + # Tornado support diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 4799359fc..7c27fed42 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -4,11 +4,7 @@ import shutil import pickle import sys -try: - from io import StringIO -except ImportError: - # compat for Python 2.6 - from StringIO import StringIO +from io import StringIO import pytest from mock import patch, mock_open diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index aa0a66ef7..70424311d 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -420,8 +420,6 @@ def test_Ellipsis(self): def test_NotImplemented(self): self.assertEqual(NotImplemented, pickle_depickle(NotImplemented)) - @pytest.mark.skipif((3, 0) < sys.version_info < (3, 4), - reason="fails due to pickle behavior in Python 3.0-3.3") def test_builtin_function_without_module(self): on = object.__new__ on_depickled = pickle_depickle(on)