diff --git a/.travis.yml b/.travis.yml index afdba9cb2..099e4c3a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,12 @@ language: python sudo: false python: - - "2.6" - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" - "pypy" -matrix: - include: - # 0.14.0 is the last version with the old categorical system - - python: 3.3 - env: PANDAS_VERSION_STR="=0.14.0" - - python: 2.7 - env: PANDAS_VERSION_STR="=0.14.0" + # This disables sudo, but makes builds start much faster # See http://blog.travis-ci.com/2014-12-17-faster-builds-with-container-based-infrastructure/ sudo: false diff --git a/ci/before_install.sh b/ci/before_install.sh index 45f7f415b..83a41d44d 100644 --- a/ci/before_install.sh +++ b/ci/before_install.sh @@ -13,5 +13,5 @@ then conda config --set always_yes yes --set changeps1 no conda update -q conda conda info -a - conda create -q -n testenv python=$TRAVIS_PYTHON_VERSION numpy scipy pip pandas + conda create -q -n testenv python=$TRAVIS_PYTHON_VERSION numpy scipy pip fi \ No newline at end of file diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 4a823b96f..e8c278020 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -264,15 +264,12 @@ def dump(self, obj): raise pickle.PicklingError(msg) def save_memoryview(self, obj): - """Fallback to save_string""" - Pickler.save_string(self, str(obj)) + self.save(obj.tobytes()) + dispatch[memoryview] = save_memoryview - def save_buffer(self, obj): - """Fallback to save_string""" - Pickler.save_string(self,str(obj)) - if PY3: - dispatch[memoryview] = save_memoryview - else: + if not PY3: + def save_buffer(self, obj): + self.save(str(obj)) dispatch[buffer] = save_buffer def save_unsupported(self, obj): @@ -378,16 +375,13 @@ def save_function(self, obj, name=None): # for different python versions. if not hasattr(obj, '__code__'): if PY3: - if sys.version_info < (3, 4): - raise pickle.PicklingError("Can't pickle %r" % obj) - else: - rv = obj.__reduce_ex__(self.proto) + rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) - return Pickler.save_reduce(self, obj=obj, *rv) + return self.save_reduce(obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a @@ -477,18 +471,12 @@ def save_dynamic_class(self, obj): # Push the rehydration function. save(_rehydrate_skeleton_class) - # Mark the start of the args for the rehydration function. + # Mark the start of the args tuple for the rehydration function. write(pickle.MARK) - # Create and memoize an empty class with obj's name and bases. - save(type(obj)) - save(( - obj.__name__, - obj.__bases__, - type_kwargs, - )) - write(pickle.REDUCE) - self.memoize(obj) + # Create and memoize an skeleton class with obj's name and bases. + tp = type(obj) + self.save_reduce(tp, (obj.__name__, obj.__bases__, type_kwargs), obj=obj) # Now save the rest of obj's __dict__. Any references to obj # encountered while saving will point to the skeleton class. @@ -551,7 +539,7 @@ def save_function_tuple(self, func): _extract_code_globals_cache = ( weakref.WeakKeyDictionary() - if sys.version_info >= (2, 7) and not hasattr(sys, "pypy_version_info") + if not hasattr(sys, "pypy_version_info") else {}) @classmethod @@ -627,37 +615,18 @@ def save_global(self, obj, name=None, pack=struct.pack): The name of this method is somewhat misleading: all types get dispatched here. """ - if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": - if obj in _BUILTIN_TYPE_NAMES: - return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - - if name is None: - name = obj.__name__ - - modname = getattr(obj, "__module__", None) - if modname is None: - try: - # whichmodule() could fail, see - # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling - modname = pickle.whichmodule(obj, name) - except Exception: - modname = '__main__' - - if modname == '__main__': - themodule = None - else: - __import__(modname) - themodule = sys.modules[modname] - self.modules.add(themodule) + try: + return Pickler.save_global(self, obj, name=name) + except Exception: + if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": + if obj in _BUILTIN_TYPE_NAMES: + return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - if hasattr(themodule, name) and getattr(themodule, name) is obj: - return Pickler.save_global(self, obj, name) + typ = type(obj) + if typ is not obj and isinstance(obj, (type, types.ClassType)): + return self.save_dynamic_class(obj) - typ = type(obj) - if typ is not obj and isinstance(obj, (type, types.ClassType)): - self.save_dynamic_class(obj) - else: - raise pickle.PicklingError("Can't pickle %r" % obj) + raise dispatch[type] = save_global dispatch[types.ClassType] = save_global @@ -728,12 +697,7 @@ def save_property(self, obj): dispatch[property] = save_property def save_classmethod(self, obj): - try: - orig_func = obj.__func__ - except AttributeError: # Python 2.6 - orig_func = obj.__get__(None, object) - if isinstance(obj, classmethod): - orig_func = orig_func.__func__ # Unbind + orig_func = obj.__func__ self.save_reduce(type(obj), (orig_func,), obj=obj) dispatch[classmethod] = save_classmethod dispatch[staticmethod] = save_classmethod @@ -773,14 +737,6 @@ def __getattribute__(self, item): if type(operator.attrgetter) is type: dispatch[operator.attrgetter] = save_attrgetter - def save_partial(self, obj): - """Partial objects do not serialize correctly in python2.x -- this fixes the bugs""" - self.save_reduce(_genpartial, (obj.func, obj.args, obj.keywords)) - - if sys.version_info < (2,7): # 2.7 supports partial pickling - dispatch[partial] = save_partial - - def save_file(self, obj): """Save a file""" try: @@ -836,23 +792,21 @@ def save_not_implemented(self, obj): dispatch[type(Ellipsis)] = save_ellipsis dispatch[type(NotImplemented)] = save_not_implemented - # WeakSet was added in 2.7. - if hasattr(weakref, 'WeakSet'): - def save_weakset(self, obj): - self.save_reduce(weakref.WeakSet, (list(obj),)) - - dispatch[weakref.WeakSet] = save_weakset + def save_weakset(self, obj): + self.save_reduce(weakref.WeakSet, (list(obj),)) - """Special functions for Add-on libraries""" - def inject_addons(self): - """Plug in system. Register additional pickling functions if modules already loaded""" - pass + dispatch[weakref.WeakSet] = save_weakset def save_logger(self, obj): self.save_reduce(logging.getLogger, (obj.name,), obj=obj) dispatch[logging.Logger] = save_logger + """Special functions for Add-on libraries""" + def inject_addons(self): + """Plug in system. Register additional pickling functions if modules already loaded""" + pass + # Tornado support @@ -882,11 +836,12 @@ def dump(obj, file, protocol=2): def dumps(obj, protocol=2): file = StringIO() - - cp = CloudPickler(file,protocol) - cp.dump(obj) - - return file.getvalue() + try: + cp = CloudPickler(file,protocol) + cp.dump(obj) + return file.getvalue() + finally: + file.close() # including pickles unloading functions in this namespace load = pickle.load diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 4799359fc..7c27fed42 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -4,11 +4,7 @@ import shutil import pickle import sys -try: - from io import StringIO -except ImportError: - # compat for Python 2.6 - from StringIO import StringIO +from io import StringIO import pytest from mock import patch, mock_open diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 06d0460a1..70424311d 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -130,6 +130,10 @@ def test_buffer(self): except NameError: # Python 3 does no longer support buffers pass + def test_memoryview(self): + buffer_obj = memoryview(b"Hello") + self.assertEqual(pickle_depickle(buffer_obj), buffer_obj.tobytes()) + def test_lambda(self): self.assertEqual(pickle_depickle(lambda: 1)(), 1) @@ -416,8 +420,6 @@ def test_Ellipsis(self): def test_NotImplemented(self): self.assertEqual(NotImplemented, pickle_depickle(NotImplemented)) - @pytest.mark.skipif((3, 0) < sys.version_info < (3, 4), - reason="fails due to pickle behavior in Python 3.0-3.3") def test_builtin_function_without_module(self): on = object.__new__ on_depickled = pickle_depickle(on)