From fb217c9bbe6d93c4a16f5d3cef5467ef09cc4aa7 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 4 Feb 2019 17:47:14 +0100 Subject: [PATCH 01/25] ENH enable custom reduction callbacks in _pickle --- Lib/test/test_pickle.py | 2 +- Modules/_pickle.c | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index b4bce7e6acebcd..2cbf52d8663eb7 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -258,7 +258,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('6P2n3i2n3iP') + basesize = support.calcobjsize('6P2n3i2n3i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 897bbe1f24e46a..5dcf39c62157f9 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -616,6 +616,9 @@ typedef struct PicklerObject { PyObject *pers_func_self; /* borrowed reference to self if pers_func is an unbound method, NULL otherwise */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ + PyObject *global_hook; /* hook for invoking user-defined callbacks + instead of save_global when pickling + functions and classes*/ PyObject *write; /* write() method of the output stream. */ PyObject *output_buffer; /* Write into a local bytearray buffer before @@ -1110,6 +1113,7 @@ _Pickler_New(void) self->fast_memo = NULL; self->max_output_len = WRITE_BUF_SIZE; self->output_len = 0; + self->global_hook = NULL; self->memo = PyMemoTable_New(); self->output_buffer = PyBytes_FromStringAndSize(NULL, @@ -4058,7 +4062,28 @@ save(PicklerObject *self, PyObject *obj, int pers_save) status = save_tuple(self, obj); goto done; } - else if (type == &PyType_Type) { + /* The switch-on-type statement ends here because the next three + * conditions are not exclusive anymore. If global_hook returns + * NotImplementedError, then we must fallback to save_type or save_global + * */ + if (self->global_hook != NULL){ + PyObject *reduce_value = NULL; + reduce_value = PyObject_CallFunctionObjArgs(self->global_hook, self, obj, + NULL); + if (reduce_value == NULL){ + goto error; + } + + if (reduce_value != PyExc_NotImplementedError){ + status = save_reduce(self, reduce_value, obj); + Py_DECREF(reduce_value); + if (status < 0) + goto error; + goto done; + } + } + + if (type == &PyType_Type) { status = save_type(self, obj); goto done; } @@ -4700,6 +4725,7 @@ static PyMemberDef Pickler_members[] = { {"bin", T_INT, offsetof(PicklerObject, bin)}, {"fast", T_INT, offsetof(PicklerObject, fast)}, {"dispatch_table", T_OBJECT_EX, offsetof(PicklerObject, dispatch_table)}, + {"global_hook", T_OBJECT_EX, offsetof(PicklerObject, global_hook)}, {NULL} }; From 7c89f8e8099a1ed791fa6b6dd79a02168886a407 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 22 Mar 2019 22:37:34 +0100 Subject: [PATCH 02/25] TST add a test for custom reduction callbacks --- Lib/test/pickletester.py | 54 ++++++++++++++++++++++++++++++++++++++++ Lib/test/test_pickle.py | 5 ++++ 2 files changed, 59 insertions(+) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 19e8823a731035..50526f9e3e3b2a 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4,6 +4,7 @@ import io import functools import os +import math import pickle import pickletools import shutil @@ -3013,6 +3014,59 @@ def setstate_bbb(obj, state): obj.a = "custom state_setter" + +class AbstractHookTests(unittest.TestCase): + def test_pickler_hook(self): + # test the CPickler ability to register custom, user-defined reduction + # callbacks to pickle functions and classes. + + def custom_reduction_callback(pickler, obj): + obj_name = obj.__name__ + + if obj_name == 'f': + # asking the pickler to save f as 5 + return int, (5, ) + + if obj_name == 'MyClass': + return str, ('NewClass', ) + + elif obj_name == 'log': + # letting the pickler falling back to buitlin save_global to + # pickle functions named 'log' by attribute. + return NotImplementedError + + elif obj_name == 'g': + # in this case, the callback returns an invalid result (not a + # 2-5 tuple), the pickler should raise a proper error. + return False + return NotImplementedError + + def f(): + pass + + def g(): + pass + + class MyClass: + pass + + for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + bio = io.BytesIO() + p = self.pickler_class(bio, proto) + p.global_hook = custom_reduction_callback + + p.dump([f, MyClass, math.log]) + new_f, NewClass, math_log = pickle.loads(bio.getvalue()) + + self.assertEqual(new_f, 5) + self.assertEqual(NewClass, 'NewClass') + self.assertIs(math_log, math.log) + + with self.assertRaises(pickle.PicklingError): + p.dump(g) + + class AbstractDispatchTableTests(unittest.TestCase): def test_default_dispatch_table(self): diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 2cbf52d8663eb7..68fa9f130a45ac 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -11,6 +11,7 @@ import unittest from test import support +from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests @@ -253,6 +254,9 @@ class CChainDispatchTableTests(AbstractDispatchTableTests): def get_dispatch_table(self): return collections.ChainMap({}, pickle.dispatch_table) + class CPicklerHookTests(AbstractHookTests): + pickler_class = _pickle.Pickler + @support.cpython_only class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof @@ -506,6 +510,7 @@ def test_main(): PyPicklerUnpicklerObjectTests, CPicklerUnpicklerObjectTests, CDispatchTableTests, CChainDispatchTableTests, + CPicklerHookTests, InMemoryPickleTests, SizeofTests]) support.run_unittest(*tests) support.run_doctest(pickle) From 3ed729345b0a74f26169872ea703faaf97f1db23 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 22 Mar 2019 22:43:21 +0100 Subject: [PATCH 03/25] MNT news entry --- .../next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst diff --git a/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst b/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst new file mode 100644 index 00000000000000..e03da865ce69d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst @@ -0,0 +1,2 @@ +enable custom reduction callback registration for functions and classes in +_pickle.c From b04ca7d13181661863824e96a04898bf86869011 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 16 Apr 2019 14:09:40 +0200 Subject: [PATCH 04/25] TST enrich the tests --- Lib/test/pickletester.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 50526f9e3e3b2a..966c6031146a61 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3028,7 +3028,7 @@ def custom_reduction_callback(pickler, obj): return int, (5, ) if obj_name == 'MyClass': - return str, ('NewClass', ) + return str, ('some str',) elif obj_name == 'log': # letting the pickler falling back to buitlin save_global to @@ -3039,6 +3039,10 @@ def custom_reduction_callback(pickler, obj): # in this case, the callback returns an invalid result (not a # 2-5 tuple), the pickler should raise a proper error. return False + elif obj_name == 'h': + # Simulate a case when the reducer fails. The error should be + # propagated to the original ``dump`` call. + raise ValueError('The reducer just failed') return NotImplementedError def f(): @@ -3047,6 +3051,9 @@ def f(): def g(): pass + def h(): + pass + class MyClass: pass @@ -3057,15 +3064,19 @@ class MyClass: p.global_hook = custom_reduction_callback p.dump([f, MyClass, math.log]) - new_f, NewClass, math_log = pickle.loads(bio.getvalue()) + new_f, some_str, math_log = pickle.loads(bio.getvalue()) self.assertEqual(new_f, 5) - self.assertEqual(NewClass, 'NewClass') + self.assertEqual(some_str, 'some str') self.assertIs(math_log, math.log) with self.assertRaises(pickle.PicklingError): p.dump(g) + with self.assertRaisesRegex( + ValueError, 'The reducer just failed'): + p.dump(h) + class AbstractDispatchTableTests(unittest.TestCase): From d0ebc9be370daf1d28826f6209de04dbb0d9adaf Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 12:07:58 +0200 Subject: [PATCH 05/25] CLN rename global_hook to reducer_override --- Lib/test/pickletester.py | 2 +- .../Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst | 2 +- Modules/_pickle.c | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 966c6031146a61..dc25d9e451965b 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3061,7 +3061,7 @@ class MyClass: with self.subTest(proto=proto): bio = io.BytesIO() p = self.pickler_class(bio, proto) - p.global_hook = custom_reduction_callback + p.reducer_override = custom_reduction_callback p.dump([f, MyClass, math.log]) new_f, some_str, math_log = pickle.loads(bio.getvalue()) diff --git a/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst b/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst index e03da865ce69d1..641572649694ae 100644 --- a/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst +++ b/Misc/NEWS.d/next/Library/2019-03-22-22-40-00.bpo-35900.oiee0o.rst @@ -1,2 +1,2 @@ enable custom reduction callback registration for functions and classes in -_pickle.c +_pickle.c, using the new Pickler's attribute ``reducer_override`` diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 5dcf39c62157f9..9cafda917f0bad 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -616,7 +616,7 @@ typedef struct PicklerObject { PyObject *pers_func_self; /* borrowed reference to self if pers_func is an unbound method, NULL otherwise */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ - PyObject *global_hook; /* hook for invoking user-defined callbacks + PyObject *reducer_override; /* hook for invoking user-defined callbacks instead of save_global when pickling functions and classes*/ @@ -1113,7 +1113,7 @@ _Pickler_New(void) self->fast_memo = NULL; self->max_output_len = WRITE_BUF_SIZE; self->output_len = 0; - self->global_hook = NULL; + self->reducer_override = NULL; self->memo = PyMemoTable_New(); self->output_buffer = PyBytes_FromStringAndSize(NULL, @@ -4063,12 +4063,12 @@ save(PicklerObject *self, PyObject *obj, int pers_save) goto done; } /* The switch-on-type statement ends here because the next three - * conditions are not exclusive anymore. If global_hook returns + * conditions are not exclusive anymore. If reducer_override returns * NotImplementedError, then we must fallback to save_type or save_global * */ - if (self->global_hook != NULL){ + if (self->reducer_override != NULL){ PyObject *reduce_value = NULL; - reduce_value = PyObject_CallFunctionObjArgs(self->global_hook, self, obj, + reduce_value = PyObject_CallFunctionObjArgs(self->reducer_override, self, obj, NULL); if (reduce_value == NULL){ goto error; @@ -4725,7 +4725,7 @@ static PyMemberDef Pickler_members[] = { {"bin", T_INT, offsetof(PicklerObject, bin)}, {"fast", T_INT, offsetof(PicklerObject, fast)}, {"dispatch_table", T_OBJECT_EX, offsetof(PicklerObject, dispatch_table)}, - {"global_hook", T_OBJECT_EX, offsetof(PicklerObject, global_hook)}, + {"reducer_override", T_OBJECT_EX, offsetof(PicklerObject, reducer_override)}, {NULL} }; From 70798baa08f9cc163ec7cc777c07fec74c449011 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 12:11:46 +0200 Subject: [PATCH 06/25] CLN NotImplementedError -> NotImplemented --- Lib/test/pickletester.py | 4 ++-- Modules/_pickle.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index dc25d9e451965b..08d61f4754563f 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3033,7 +3033,7 @@ def custom_reduction_callback(pickler, obj): elif obj_name == 'log': # letting the pickler falling back to buitlin save_global to # pickle functions named 'log' by attribute. - return NotImplementedError + return NotImplemented elif obj_name == 'g': # in this case, the callback returns an invalid result (not a @@ -3043,7 +3043,7 @@ def custom_reduction_callback(pickler, obj): # Simulate a case when the reducer fails. The error should be # propagated to the original ``dump`` call. raise ValueError('The reducer just failed') - return NotImplementedError + return NotImplemented def f(): pass diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 9cafda917f0bad..0a354a8d8ffdb1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4074,7 +4074,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) goto error; } - if (reduce_value != PyExc_NotImplementedError){ + if (reduce_value != Py_NotImplemented){ status = save_reduce(self, reduce_value, obj); Py_DECREF(reduce_value); if (status < 0) From 3f845410daa56ad81040936da7eb7db6a4a92828 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 16:33:18 +0200 Subject: [PATCH 07/25] FIX decref NotImplemented --- Modules/_pickle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0a354a8d8ffdb1..d21807a646a248 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4081,6 +4081,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) goto error; goto done; } + Py_DECREF(reduce_value); } if (type == &PyType_Type) { From 98e98be6f03e3ab788a5dd2cff1285cb6d0c9a02 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 16:35:37 +0200 Subject: [PATCH 08/25] FIX make reducer_override have (obj) signature --- Lib/test/pickletester.py | 2 +- Modules/_pickle.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 08d61f4754563f..ce4200f9cdf710 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3020,7 +3020,7 @@ def test_pickler_hook(self): # test the CPickler ability to register custom, user-defined reduction # callbacks to pickle functions and classes. - def custom_reduction_callback(pickler, obj): + def custom_reduction_callback(obj): obj_name = obj.__name__ if obj_name == 'f': diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d21807a646a248..4eb13624cb46cd 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4068,8 +4068,8 @@ save(PicklerObject *self, PyObject *obj, int pers_save) * */ if (self->reducer_override != NULL){ PyObject *reduce_value = NULL; - reduce_value = PyObject_CallFunctionObjArgs(self->reducer_override, self, obj, - NULL); + reduce_value = PyObject_CallFunctionObjArgs(self->reducer_override, + obj, NULL); if (reduce_value == NULL){ goto error; } From 17b020c75f30c6c6f5d19035d14fe66ce4ac225a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 16:39:50 +0200 Subject: [PATCH 09/25] TST remove some redundancy in tests --- Lib/test/pickletester.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index ce4200f9cdf710..f106f0dfc60b60 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3030,11 +3030,6 @@ def custom_reduction_callback(obj): if obj_name == 'MyClass': return str, ('some str',) - elif obj_name == 'log': - # letting the pickler falling back to buitlin save_global to - # pickle functions named 'log' by attribute. - return NotImplemented - elif obj_name == 'g': # in this case, the callback returns an invalid result (not a # 2-5 tuple), the pickler should raise a proper error. @@ -3068,6 +3063,10 @@ class MyClass: self.assertEqual(new_f, 5) self.assertEqual(some_str, 'some str') + # math.log does not have its usual reducer overriden, so the + # custom reduction callback should silently direct the pickler + # to the default pickling by attribute, by returning + # NotImplemented self.assertIs(math_log, math.log) with self.assertRaises(pickle.PicklingError): From ad9b0e56eb681958cbdb68c741c247f9d3aa7793 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 17:56:40 +0200 Subject: [PATCH 10/25] ENH reducer_override should now be a method --- Lib/test/pickletester.py | 26 +++----------------------- Lib/test/test_pickle.py | 24 +++++++++++++++++++++++- Modules/_pickle.c | 21 ++++++++++++++++----- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index f106f0dfc60b60..7fec56aa2036c4 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3017,28 +3017,9 @@ def setstate_bbb(obj, state): class AbstractHookTests(unittest.TestCase): def test_pickler_hook(self): - # test the CPickler ability to register custom, user-defined reduction - # callbacks to pickle functions and classes. - - def custom_reduction_callback(obj): - obj_name = obj.__name__ - - if obj_name == 'f': - # asking the pickler to save f as 5 - return int, (5, ) - - if obj_name == 'MyClass': - return str, ('some str',) - - elif obj_name == 'g': - # in this case, the callback returns an invalid result (not a - # 2-5 tuple), the pickler should raise a proper error. - return False - elif obj_name == 'h': - # Simulate a case when the reducer fails. The error should be - # propagated to the original ``dump`` call. - raise ValueError('The reducer just failed') - return NotImplemented + # test the ability of a custom, user-defined CPickler subclass to + # override the default reducing routines of any type using the method + # reducer_override def f(): pass @@ -3056,7 +3037,6 @@ class MyClass: with self.subTest(proto=proto): bio = io.BytesIO() p = self.pickler_class(bio, proto) - p.reducer_override = custom_reduction_callback p.dump([f, MyClass, math.log]) new_f, some_str, math_log = pickle.loads(bio.getvalue()) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 68fa9f130a45ac..446da39f27ec17 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -255,7 +255,29 @@ def get_dispatch_table(self): return collections.ChainMap({}, pickle.dispatch_table) class CPicklerHookTests(AbstractHookTests): - pickler_class = _pickle.Pickler + class CustomPicklerClass(_pickle.Pickler): + """Pickler implementing a reducing hook using reducer_override.""" + def reducer_override(self, obj): + obj_name = obj.__name__ + + if obj_name == 'f': + # asking the pickler to save f as 5 + return int, (5, ) + + if obj_name == 'MyClass': + return str, ('some str',) + + elif obj_name == 'g': + # in this case, the callback returns an invalid result (not + # a 2-5 tuple), the pickler should raise a proper error. + return False + elif obj_name == 'h': + # Simulate a case when the reducer fails. The error should + # be propagated to the original ``dump`` call. + raise ValueError('The reducer just failed') + return NotImplemented + + pickler_class = CustomPicklerClass @support.cpython_only class SizeofTests(unittest.TestCase): diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 4eb13624cb46cd..21b1fbeb88a0f5 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -616,7 +616,7 @@ typedef struct PicklerObject { PyObject *pers_func_self; /* borrowed reference to self if pers_func is an unbound method, NULL otherwise */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ - PyObject *reducer_override; /* hook for invoking user-defined callbacks + PyObject *_reducer_override;/* hook for invoking user-defined callbacks instead of save_global when pickling functions and classes*/ @@ -1113,7 +1113,7 @@ _Pickler_New(void) self->fast_memo = NULL; self->max_output_len = WRITE_BUF_SIZE; self->output_len = 0; - self->reducer_override = NULL; + self->_reducer_override = NULL; self->memo = PyMemoTable_New(); self->output_buffer = PyBytes_FromStringAndSize(NULL, @@ -4066,9 +4066,9 @@ save(PicklerObject *self, PyObject *obj, int pers_save) * conditions are not exclusive anymore. If reducer_override returns * NotImplementedError, then we must fallback to save_type or save_global * */ - if (self->reducer_override != NULL){ + if (self->_reducer_override != NULL) { PyObject *reduce_value = NULL; - reduce_value = PyObject_CallFunctionObjArgs(self->reducer_override, + reduce_value = PyObject_CallFunctionObjArgs(self->_reducer_override, obj, NULL); if (reduce_value == NULL){ goto error; @@ -4206,7 +4206,19 @@ static int dump(PicklerObject *self, PyObject *obj) { const char stop_op = STOP; + PyObject *tmp; + _Py_IDENTIFIER(reducer_override); + if (_PyObject_LookupAttrId((PyObject *)self, &PyId_reducer_override, + &tmp) < 0) { + return -1; + } + /* The private _reducer_override attribute of the pickler acts as a cache + * of a potential reducer_override method. This cache is updated at each + * Pickler.dump call*/ + if (tmp != NULL) { + Py_XSETREF(self->_reducer_override, tmp); + } if (self->proto >= 2) { char header[2]; @@ -4726,7 +4738,6 @@ static PyMemberDef Pickler_members[] = { {"bin", T_INT, offsetof(PicklerObject, bin)}, {"fast", T_INT, offsetof(PicklerObject, fast)}, {"dispatch_table", T_OBJECT_EX, offsetof(PicklerObject, dispatch_table)}, - {"reducer_override", T_OBJECT_EX, offsetof(PicklerObject, reducer_override)}, {NULL} }; From 49b5c15345c4e60559811906d23b1e8f5a4e62fa Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 17:59:34 +0200 Subject: [PATCH 11/25] CLN typo --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 21b1fbeb88a0f5..11d444bc01a31b 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4064,7 +4064,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } /* The switch-on-type statement ends here because the next three * conditions are not exclusive anymore. If reducer_override returns - * NotImplementedError, then we must fallback to save_type or save_global + * NotImplemented, then we must fallback to save_type or save_global * */ if (self->_reducer_override != NULL) { PyObject *reduce_value = NULL; From fa71b80938731ef2c63eb3dca43105e1c96cb84c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 19 Apr 2019 18:42:22 +0200 Subject: [PATCH 12/25] ENH add reducer_override semantics on pickle.py --- Lib/pickle.py | 6 ++++++ Lib/test/pickletester.py | 24 +++++++++++++++++++++++- Lib/test/test_pickle.py | 35 +++++++++++------------------------ 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index 47f0d280efc945..f5438785df69be 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -497,6 +497,12 @@ def save(self, obj, save_persistent_id=True): self.write(self.get(x[0])) return + reducer_override = getattr(self, "reducer_override", None) + if reducer_override is not None: + rv = reducer_override(obj) + if rv is not NotImplemented: + return self.save_reduce(obj=obj, *rv) + # Check the type dispatch table t = type(obj) f = self.dispatch.get(t) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 7fec56aa2036c4..d8afb9be602e7d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3015,6 +3015,28 @@ def setstate_bbb(obj, state): +class AbstractCustomPicklerClass: + """Pickler implementing a reducing hook using reducer_override.""" + def reducer_override(self, obj): + obj_name = getattr(obj, "__name__", None) + + if obj_name == 'f': + # asking the pickler to save f as 5 + return int, (5, ) + + if obj_name == 'MyClass': + return str, ('some str',) + + elif obj_name == 'g': + # in this case, the callback returns an invalid result (not + # a 2-5 tuple), the pickler should raise a proper error. + return False + elif obj_name == 'h': + # Simulate a case when the reducer fails. The error should + # be propagated to the original ``dump`` call. + raise ValueError('The reducer just failed') + return NotImplemented + class AbstractHookTests(unittest.TestCase): def test_pickler_hook(self): # test the ability of a custom, user-defined CPickler subclass to @@ -3049,7 +3071,7 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(Exception): p.dump(g) with self.assertRaisesRegex( diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 446da39f27ec17..435c248802d3d7 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -19,6 +19,7 @@ from test.pickletester import AbstractIdentityPersistentPicklerTests from test.pickletester import AbstractPicklerUnpicklerObjectTests from test.pickletester import AbstractDispatchTableTests +from test.pickletester import AbstractCustomPicklerClass from test.pickletester import BigmemPickleTests try: @@ -254,30 +255,16 @@ class CChainDispatchTableTests(AbstractDispatchTableTests): def get_dispatch_table(self): return collections.ChainMap({}, pickle.dispatch_table) + class PyPicklerHookTests(AbstractHookTests): + class CustomPyPicklerClass(pickle._Pickler, + AbstractCustomPicklerClass): + pass + pickler_class = CustomPyPicklerClass + class CPicklerHookTests(AbstractHookTests): - class CustomPicklerClass(_pickle.Pickler): - """Pickler implementing a reducing hook using reducer_override.""" - def reducer_override(self, obj): - obj_name = obj.__name__ - - if obj_name == 'f': - # asking the pickler to save f as 5 - return int, (5, ) - - if obj_name == 'MyClass': - return str, ('some str',) - - elif obj_name == 'g': - # in this case, the callback returns an invalid result (not - # a 2-5 tuple), the pickler should raise a proper error. - return False - elif obj_name == 'h': - # Simulate a case when the reducer fails. The error should - # be propagated to the original ``dump`` call. - raise ValueError('The reducer just failed') - return NotImplemented - - pickler_class = CustomPicklerClass + class CustomCPicklerClass(_pickle.Pickler, AbstractCustomPicklerClass): + pass + pickler_class = CustomCPicklerClass @support.cpython_only class SizeofTests(unittest.TestCase): @@ -524,7 +511,7 @@ def test_main(): tests = [PyPickleTests, PyUnpicklerTests, PyPicklerTests, PyPersPicklerTests, PyIdPersPicklerTests, PyDispatchTableTests, PyChainDispatchTableTests, - CompatPickleTests] + CompatPickleTests, PyPicklerHookTests] if has_c_implementation: tests.extend([CPickleTests, CUnpicklerTests, CPicklerTests, CPersPicklerTests, CIdPersPicklerTests, From 2e920682af8ad1fb5ae6b4b66f7393aab5d5c97f Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Sat, 20 Apr 2019 18:00:29 +0200 Subject: [PATCH 13/25] Update Modules/_pickle.c Co-Authored-By: pierreglaser --- Modules/_pickle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 11d444bc01a31b..f4ee98c39e2e31 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4074,7 +4074,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) goto error; } - if (reduce_value != Py_NotImplemented){ + if (reduce_value != Py_NotImplemented) { status = save_reduce(self, reduce_value, obj); Py_DECREF(reduce_value); if (status < 0) From c22d6df03e7b54500ac7133f4279aed9453cd980 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 24 Apr 2019 16:20:33 +0200 Subject: [PATCH 14/25] CLN style --- Modules/_pickle.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f4ee98c39e2e31..28fcdf59a4dc9e 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2224,7 +2224,7 @@ save_bytes(PicklerObject *self, PyObject *obj) Python 2 *and* the appropriate 'bytes' object when unpickled using Python 3. Again this is a hack and we don't need to do this with newer protocols. */ - PyObject *reduce_value = NULL; + PyObject *reduce_value; int status; if (PyBytes_GET_SIZE(obj) == 0) { @@ -4070,15 +4070,16 @@ save(PicklerObject *self, PyObject *obj, int pers_save) PyObject *reduce_value = NULL; reduce_value = PyObject_CallFunctionObjArgs(self->_reducer_override, obj, NULL); - if (reduce_value == NULL){ + if (reduce_value == NULL) { goto error; } if (reduce_value != Py_NotImplemented) { status = save_reduce(self, reduce_value, obj); Py_DECREF(reduce_value); - if (status < 0) + if (status < 0) { goto error; + } goto done; } Py_DECREF(reduce_value); From b2642229b8f38c7a43a990e06fa44d414a9119be Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 24 Apr 2019 18:02:04 +0200 Subject: [PATCH 15/25] ensure high-level errors for invalid reduce_values --- Lib/pickle.py | 55 +++++++------ Lib/test/pickletester.py | 6 +- Modules/_pickle.c | 165 ++++++++++++++++++--------------------- 3 files changed, 110 insertions(+), 116 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index f5438785df69be..41eb6749b09f72 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -497,40 +497,43 @@ def save(self, obj, save_persistent_id=True): self.write(self.get(x[0])) return - reducer_override = getattr(self, "reducer_override", None) - if reducer_override is not None: - rv = reducer_override(obj) - if rv is not NotImplemented: - return self.save_reduce(obj=obj, *rv) - - # Check the type dispatch table - t = type(obj) - f = self.dispatch.get(t) - if f is not None: - f(self, obj) # Call unbound method with explicit self - return - - # Check private dispatch table if any, or else copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) + rv = NotImplemented + reduce = getattr(self, "reducer_override", None) if reduce is not None: rv = reduce(obj) - else: - # Check for a class with a custom metaclass; treat as regular class - if issubclass(t, type): - self.save_global(obj) + + if rv is NotImplemented: + + # Check the type dispatch table + t = type(obj) + f = self.dispatch.get(t) + if f is not None: + f(self, obj) # Call unbound method with explicit self return - # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) + # Check private dispatch table if any, or else + # copyreg.dispatch_table + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) if reduce is not None: - rv = reduce(self.proto) + rv = reduce(obj) else: - reduce = getattr(obj, "__reduce__", None) + # Check for a class with a custom metaclass; treat as regular + # class + if issubclass(t, type): + self.save_global(obj) + return + + # Check for a __reduce_ex__ method, fall back to __reduce__ + reduce = getattr(obj, "__reduce_ex__", None) if reduce is not None: - rv = reduce() + rv = reduce(self.proto) else: - raise PicklingError("Can't pickle %r object: %r" % - (t.__name__, obj)) + reduce = getattr(obj, "__reduce__", None) + if reduce is not None: + rv = reduce() + else: + raise PicklingError("Can't pickle %r object: %r" % + (t.__name__, obj)) # Check for string returned by reduce(), meaning "save as global" if isinstance(rv, str): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index d8afb9be602e7d..5118b0d01cf0f7 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3028,8 +3028,8 @@ def reducer_override(self, obj): return str, ('some str',) elif obj_name == 'g': - # in this case, the callback returns an invalid result (not - # a 2-5 tuple), the pickler should raise a proper error. + # in this case, the callback returns an invalid result (not a 2-5 + # tuple or a string), the pickler should raise a proper error. return False elif obj_name == 'h': # Simulate a case when the reducer fails. The error should @@ -3071,7 +3071,7 @@ class MyClass: # NotImplemented self.assertIs(math_log, math.log) - with self.assertRaises(Exception): + with self.assertRaises(pickle.PicklingError): p.dump(g) with self.assertRaisesRegex( diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 28fcdf59a4dc9e..abe5c7f884cd3a 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4066,113 +4066,104 @@ save(PicklerObject *self, PyObject *obj, int pers_save) * conditions are not exclusive anymore. If reducer_override returns * NotImplemented, then we must fallback to save_type or save_global * */ + reduce_value = Py_NotImplemented; + Py_INCREF(reduce_value); if (self->_reducer_override != NULL) { - PyObject *reduce_value = NULL; reduce_value = PyObject_CallFunctionObjArgs(self->_reducer_override, obj, NULL); - if (reduce_value == NULL) { - goto error; - } + } - if (reduce_value != Py_NotImplemented) { - status = save_reduce(self, reduce_value, obj); - Py_DECREF(reduce_value); - if (status < 0) { - goto error; - } + if (reduce_value == Py_NotImplemented) { + if (type == &PyType_Type) { + status = save_type(self, obj); + goto done; + } + else if (type == &PyFunction_Type) { + status = save_global(self, obj, NULL); goto done; } - Py_DECREF(reduce_value); - } - - if (type == &PyType_Type) { - status = save_type(self, obj); - goto done; - } - else if (type == &PyFunction_Type) { - status = save_global(self, obj, NULL); - goto done; - } - /* XXX: This part needs some unit tests. */ + /* XXX: This part needs some unit tests. */ - /* Get a reduction callable, and call it. This may come from - * self.dispatch_table, copyreg.dispatch_table, the object's - * __reduce_ex__ method, or the object's __reduce__ method. - */ - if (self->dispatch_table == NULL) { - PickleState *st = _Pickle_GetGlobalState(); - reduce_func = PyDict_GetItemWithError(st->dispatch_table, - (PyObject *)type); - if (reduce_func == NULL) { - if (PyErr_Occurred()) { - goto error; + /* Get a reduction callable, and call it. This may come from + * self.dispatch_table, copyreg.dispatch_table, the object's + * __reduce_ex__ method, or the object's __reduce__ method. + */ + if (self->dispatch_table == NULL) { + PickleState *st = _Pickle_GetGlobalState(); + reduce_func = PyDict_GetItemWithError(st->dispatch_table, + (PyObject *)type); + if (reduce_func == NULL) { + if (PyErr_Occurred()) { + goto error; + } + } else { + /* PyDict_GetItemWithError() returns a borrowed reference. + Increase the reference count to be consistent with + PyObject_GetItem and _PyObject_GetAttrId used below. */ + Py_INCREF(reduce_func); } } else { - /* PyDict_GetItemWithError() returns a borrowed reference. - Increase the reference count to be consistent with - PyObject_GetItem and _PyObject_GetAttrId used below. */ - Py_INCREF(reduce_func); - } - } else { - reduce_func = PyObject_GetItem(self->dispatch_table, - (PyObject *)type); - if (reduce_func == NULL) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) - PyErr_Clear(); - else - goto error; - } - } - if (reduce_func != NULL) { - Py_INCREF(obj); - reduce_value = _Pickle_FastCall(reduce_func, obj); - } - else if (PyType_IsSubtype(type, &PyType_Type)) { - status = save_global(self, obj, NULL); - goto done; - } - else { - _Py_IDENTIFIER(__reduce__); - _Py_IDENTIFIER(__reduce_ex__); - - - /* XXX: If the __reduce__ method is defined, __reduce_ex__ is - automatically defined as __reduce__. While this is convenient, this - make it impossible to know which method was actually called. Of - course, this is not a big deal. But still, it would be nice to let - the user know which method was called when something go - wrong. Incidentally, this means if __reduce_ex__ is not defined, we - don't actually have to check for a __reduce__ method. */ - - /* Check for a __reduce_ex__ method. */ - if (_PyObject_LookupAttrId(obj, &PyId___reduce_ex__, &reduce_func) < 0) { - goto error; + reduce_func = PyObject_GetItem(self->dispatch_table, + (PyObject *)type); + if (reduce_func == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) + PyErr_Clear(); + else + goto error; + } } if (reduce_func != NULL) { - PyObject *proto; - proto = PyLong_FromLong(self->proto); - if (proto != NULL) { - reduce_value = _Pickle_FastCall(reduce_func, proto); - } + Py_INCREF(obj); + reduce_value = _Pickle_FastCall(reduce_func, obj); + } + else if (PyType_IsSubtype(type, &PyType_Type)) { + status = save_global(self, obj, NULL); + goto done; } else { - PickleState *st = _Pickle_GetGlobalState(); - - /* Check for a __reduce__ method. */ - reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__); + _Py_IDENTIFIER(__reduce__); + _Py_IDENTIFIER(__reduce_ex__); + + + /* XXX: If the __reduce__ method is defined, __reduce_ex__ is + automatically defined as __reduce__. While this is convenient, + this make it impossible to know which method was actually + called. Of course, this is not a big deal. But still, it would + be nice to let the user know which method was called when + something go wrong. Incidentally, this means if __reduce_ex__ is + not defined, we don't actually have to check for a __reduce__ + method. */ + + /* Check for a __reduce_ex__ method. */ + if (_PyObject_LookupAttrId(obj, &PyId___reduce_ex__, + &reduce_func) < 0) { + goto error; + } if (reduce_func != NULL) { - reduce_value = _PyObject_CallNoArg(reduce_func); + PyObject *proto; + proto = PyLong_FromLong(self->proto); + if (proto != NULL) { + reduce_value = _Pickle_FastCall(reduce_func, proto); + } } else { - PyErr_Format(st->PicklingError, - "can't pickle '%.200s' object: %R", - type->tp_name, obj); - goto error; + PickleState *st = _Pickle_GetGlobalState(); + + /* Check for a __reduce__ method. */ + reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__); + if (reduce_func != NULL) { + reduce_value = _PyObject_CallNoArg(reduce_func); + } + else { + PyErr_Format(st->PicklingError, + "can't pickle '%.200s' object: %R", + type->tp_name, obj); + goto error; + } } } } - if (reduce_value == NULL) goto error; From 91a9042337a0f074b4cb43eb7dc3eafab6fbe174 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 24 Apr 2019 19:52:31 +0200 Subject: [PATCH 16/25] DOC --- Doc/library/pickle.rst | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 3d89536d7d1182..14369ecfd8c622 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -725,6 +725,69 @@ share the same dispatch table. The equivalent code using the f = io.BytesIO() p = pickle.Pickler(f) +.. _reducer_override: + +Subclassing the Pickler class +----------------------------- + +For most use-cases, it is recommended to simply use the ``dispatch_table`` of a +``Pickler`` instance to customize its behavior, as explained above. + +However, using the ``dispatch_table`` may not be flexible enough. in particular +if we want to customize the pickling logic based on another criterion than the +object's type, or if we want to customize the pickling of functions and +classes. + +For those cases, it is possible to subclass from the ``Pickler`` class and +implement a `reducer_override` method. This method can return an arbitrary +reduction tuple (see :meth:`__reduce__`). It can alternatively return +``NotImplemented`` to fallback to the traditional behavior. + +If both the ``dispatch_table`` and ``reducer_override`` are defined, then +``reducer_override`` method takes priority. + +.. Note:: + For performance reasons, the C implementation of pickle does not allow to + override the pickling of the following objects: ``None``, ``True``, + ``False``, and instances of ``long``, ``float``, ``bytes``, ``str``, + ``dict``, ``set``, ``frozenset``, ``list``, ``tuple``, + + +Here is a simple example:: + + import io + import pickle + + + class MyClass: + my_attribute = 1 + + + class MyPickler(pickle.Pickler): + def reducer_override(self, obj): + """Custom reducer for MyClass.""" + + if getattr(obj, "__name__", None) == "MyClass": + return type, (obj.__name__, obj.__bases__, + {'my_attribute': obj.my_attribute}) + else: + # For any other object, fallback to the usual pickling routines + # (builtin or dispatch_table) + return NotImplemented + + + f = io.BytesIO() + p = MyPickler(f) + p.dump(MyClass) + + del MyClass + + my_depickled_class = pickle.loads(f.getvalue()) + + assert my_depickled_class.__name__ == "MyClass" + assert my_depickled_class.my_attribute == 1 + + .. _pickle-state: Handling Stateful Objects From c48192ba80e1c07d97afa2d9711d589b90e08edf Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 10:57:18 +0200 Subject: [PATCH 17/25] DOC style and phrasing --- Doc/library/pickle.rst | 17 ++++++++--------- Lib/pickle.py | 1 - 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 14369ecfd8c622..d20062d4dbf9c6 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -727,30 +727,30 @@ share the same dispatch table. The equivalent code using the .. _reducer_override: -Subclassing the Pickler class ------------------------------ +Subclassing the ``Pickler`` class +--------------------------------- For most use-cases, it is recommended to simply use the ``dispatch_table`` of a ``Pickler`` instance to customize its behavior, as explained above. -However, using the ``dispatch_table`` may not be flexible enough. in particular -if we want to customize the pickling logic based on another criterion than the -object's type, or if we want to customize the pickling of functions and +However, using the ``dispatch_table`` may not be flexible enough. In particular +we may want to customize the pickling logic based on another criterion than the +object's type, or we may want to customize the pickling of functions and classes. For those cases, it is possible to subclass from the ``Pickler`` class and -implement a `reducer_override` method. This method can return an arbitrary +implement a ``reducer_override`` method. This method can return an arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return ``NotImplemented`` to fallback to the traditional behavior. If both the ``dispatch_table`` and ``reducer_override`` are defined, then -``reducer_override`` method takes priority. +the ``reducer_override`` method takes priority. .. Note:: For performance reasons, the C implementation of pickle does not allow to override the pickling of the following objects: ``None``, ``True``, ``False``, and instances of ``long``, ``float``, ``bytes``, ``str``, - ``dict``, ``set``, ``frozenset``, ``list``, ``tuple``, + ``dict``, ``set``, ``frozenset``, ``list`` and ``tuple``. Here is a simple example:: @@ -766,7 +766,6 @@ Here is a simple example:: class MyPickler(pickle.Pickler): def reducer_override(self, obj): """Custom reducer for MyClass.""" - if getattr(obj, "__name__", None) == "MyClass": return type, (obj.__name__, obj.__bases__, {'my_attribute': obj.my_attribute}) diff --git a/Lib/pickle.py b/Lib/pickle.py index 41eb6749b09f72..595beda4765afe 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -503,7 +503,6 @@ def save(self, obj, save_persistent_id=True): rv = reduce(obj) if rv is NotImplemented: - # Check the type dispatch table t = type(obj) f = self.dispatch.get(t) From 229b81c27e8dc58e010a879a9848aaeadf79e779 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 12:08:07 +0200 Subject: [PATCH 18/25] reference reducer_override in Pickler class doc --- Doc/library/pickle.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index d20062d4dbf9c6..ebe4055dd447d4 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -366,6 +366,18 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and Use :func:`pickletools.optimize` if you need more compact pickles. + .. method:: reducer_override(self, obj) + + Special reducer that can be defined in ``Pickler`` subclasses. This method has + priority over any reducer in the ``dispatch_table``. + It should conform to the same interface as a :meth:`__reduce__` method, + and can optionally return ``NotImplemented`` to fallback on + ``dispatch_table``-registered reducers to pickle ``obj``. + + For a detailed example on how to use ``reducer_override``, see: `Subclassing the Pickler class`_ + + .. versionadded:: 3.8 + .. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict") @@ -727,7 +739,7 @@ share the same dispatch table. The equivalent code using the .. _reducer_override: -Subclassing the ``Pickler`` class +Subclassing the Pickler class --------------------------------- For most use-cases, it is recommended to simply use the ``dispatch_table`` of a From ea11b2e2380bdfca83a83c227606d9a38a3b5e6a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 12:19:18 +0200 Subject: [PATCH 19/25] CLN style --- Doc/library/pickle.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index ebe4055dd447d4..ea93f7ca9fb096 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -368,13 +368,14 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and .. method:: reducer_override(self, obj) - Special reducer that can be defined in ``Pickler`` subclasses. This method has - priority over any reducer in the ``dispatch_table``. - It should conform to the same interface as a :meth:`__reduce__` method, - and can optionally return ``NotImplemented`` to fallback on + Special reducer that can be defined in ``Pickler`` subclasses. This + method has priority over any reducer in the ``dispatch_table``. It + should conform to the same interface as a :meth:`__reduce__` method, and + can optionally return ``NotImplemented`` to fallback on ``dispatch_table``-registered reducers to pickle ``obj``. - For a detailed example on how to use ``reducer_override``, see: `Subclassing the Pickler class`_ + For a detailed example on how to use ``reducer_override``, see: + `Subclassing the Pickler class`_ .. versionadded:: 3.8 From 34bb2861e6f7aa387348ad37c22e1e186656e8e8 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 12:32:55 +0200 Subject: [PATCH 20/25] CLN use proper rst directive --- Doc/library/pickle.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index ea93f7ca9fb096..6620eaadf6b6e5 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -375,7 +375,7 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and ``dispatch_table``-registered reducers to pickle ``obj``. For a detailed example on how to use ``reducer_override``, see: - `Subclassing the Pickler class`_ + :ref:`reducer_override` .. versionadded:: 3.8 @@ -740,7 +740,7 @@ share the same dispatch table. The equivalent code using the .. _reducer_override: -Subclassing the Pickler class +Subclassing the ``Pickler`` class --------------------------------- For most use-cases, it is recommended to simply use the ``dispatch_table`` of a From 1120c10bd71f87977e1874cd05c16268d051ddba Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 12:46:34 +0200 Subject: [PATCH 21/25] CLN period --- Doc/library/pickle.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 6620eaadf6b6e5..710696aa0dd6ea 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -375,7 +375,7 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and ``dispatch_table``-registered reducers to pickle ``obj``. For a detailed example on how to use ``reducer_override``, see: - :ref:`reducer_override` + :ref:`reducer_override`. .. versionadded:: 3.8 From dc8f2769adaa57f00a282d6f87b959861461f24c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 12:47:28 +0200 Subject: [PATCH 22/25] CLN style --- Lib/test/pickletester.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 5118b0d01cf0f7..4f8c2942df93dd 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3031,10 +3031,12 @@ def reducer_override(self, obj): # in this case, the callback returns an invalid result (not a 2-5 # tuple or a string), the pickler should raise a proper error. return False + elif obj_name == 'h': # Simulate a case when the reducer fails. The error should # be propagated to the original ``dump`` call. raise ValueError('The reducer just failed') + return NotImplemented class AbstractHookTests(unittest.TestCase): From 7df275166846e8405147ddedf3ffc2bbfd383182 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 13:08:08 +0200 Subject: [PATCH 23/25] DOC more rst references --- Doc/library/pickle.rst | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 710696aa0dd6ea..c2a55432782b5f 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -368,13 +368,13 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and .. method:: reducer_override(self, obj) - Special reducer that can be defined in ``Pickler`` subclasses. This - method has priority over any reducer in the ``dispatch_table``. It + Special reducer that can be defined in :class:`Pickler` subclasses. This + method has priority over any reducer in the :attr:`dispatch_table`. It should conform to the same interface as a :meth:`__reduce__` method, and can optionally return ``NotImplemented`` to fallback on - ``dispatch_table``-registered reducers to pickle ``obj``. + :attr:`dispatch_table`-registered reducers to pickle ``obj``. - For a detailed example on how to use ``reducer_override``, see: + For a detailed example on how to use :meth:`~Pickler.reducer_override`, see: :ref:`reducer_override`. .. versionadded:: 3.8 @@ -743,27 +743,28 @@ share the same dispatch table. The equivalent code using the Subclassing the ``Pickler`` class --------------------------------- -For most use-cases, it is recommended to simply use the ``dispatch_table`` of a -``Pickler`` instance to customize its behavior, as explained above. +For most use-cases, it is recommended to simply use the :attr:`~Pickler.dispatch_table` of a +:class:`Pickler` instance to customize its behavior, as explained above. -However, using the ``dispatch_table`` may not be flexible enough. In particular +However, using the :attr:`~Pickler.dispatch_table` may not be flexible enough. In particular we may want to customize the pickling logic based on another criterion than the object's type, or we may want to customize the pickling of functions and classes. -For those cases, it is possible to subclass from the ``Pickler`` class and -implement a ``reducer_override`` method. This method can return an arbitrary +For those cases, it is possible to subclass from the :class:`Pickler` class and +implement a :meth:`~Pickler.reducer_override` method. This method can return an arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return ``NotImplemented`` to fallback to the traditional behavior. -If both the ``dispatch_table`` and ``reducer_override`` are defined, then -the ``reducer_override`` method takes priority. +If both the :attr:`~Pickler.dispatch_table` and :meth:`~Pickler.reducer_override` are defined, then +the :meth:`~Pickler.reducer_override` method takes priority. .. Note:: For performance reasons, the C implementation of pickle does not allow to override the pickling of the following objects: ``None``, ``True``, - ``False``, and instances of ``long``, ``float``, ``bytes``, ``str``, - ``dict``, ``set``, ``frozenset``, ``list`` and ``tuple``. + ``False``, and instances of :class:`int`, :class:`float`, :class:`bytes`, + :class:`str`, :class:`dict`, :class:`set`, :class:`frozenset`, :class:`list` + and :class:`tuple`. Here is a simple example:: From 2de069eedb1d37268b63943c9bf1745230fd1a1a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 26 Apr 2019 13:09:26 +0200 Subject: [PATCH 24/25] CLN style --- Doc/library/pickle.rst | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index c2a55432782b5f..b73a054b62afff 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -374,8 +374,8 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and can optionally return ``NotImplemented`` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. - For a detailed example on how to use :meth:`~Pickler.reducer_override`, see: - :ref:`reducer_override`. + For a detailed example on how to use :meth:`~Pickler.reducer_override`, + see: :ref:`reducer_override`. .. versionadded:: 3.8 @@ -743,21 +743,23 @@ share the same dispatch table. The equivalent code using the Subclassing the ``Pickler`` class --------------------------------- -For most use-cases, it is recommended to simply use the :attr:`~Pickler.dispatch_table` of a -:class:`Pickler` instance to customize its behavior, as explained above. +For most use-cases, it is recommended to simply use the +:attr:`~Pickler.dispatch_table` of a :class:`Pickler` instance to customize its +behavior, as explained above. -However, using the :attr:`~Pickler.dispatch_table` may not be flexible enough. In particular -we may want to customize the pickling logic based on another criterion than the -object's type, or we may want to customize the pickling of functions and -classes. +However, using the :attr:`~Pickler.dispatch_table` may not be flexible enough. +In particular we may want to customize the pickling logic based on another +criterion than the object's type, or we may want to customize the pickling of +functions and classes. For those cases, it is possible to subclass from the :class:`Pickler` class and -implement a :meth:`~Pickler.reducer_override` method. This method can return an arbitrary -reduction tuple (see :meth:`__reduce__`). It can alternatively return +implement a :meth:`~Pickler.reducer_override` method. This method can return an +arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return ``NotImplemented`` to fallback to the traditional behavior. -If both the :attr:`~Pickler.dispatch_table` and :meth:`~Pickler.reducer_override` are defined, then -the :meth:`~Pickler.reducer_override` method takes priority. +If both the :attr:`~Pickler.dispatch_table` and +:meth:`~Pickler.reducer_override` are defined, then the +:meth:`~Pickler.reducer_override` method takes priority. .. Note:: For performance reasons, the C implementation of pickle does not allow to From e6a6b49b5bf071f5365f883248d594d8e00f9fe7 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2019 22:19:24 +0200 Subject: [PATCH 25/25] - Fix reference leaks - Some readibility changes --- Doc/library/pickle.rst | 147 +++++++++++++++---------------- Modules/_pickle.c | 192 ++++++++++++++++++++++------------------- 2 files changed, 171 insertions(+), 168 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index b73a054b62afff..55005f009431d0 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -356,16 +356,6 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and .. versionadded:: 3.3 - .. attribute:: fast - - Deprecated. Enable fast mode if set to a true value. The fast mode - disables the usage of memo, therefore speeding the pickling process by not - generating superfluous PUT opcodes. It should not be used with - self-referential objects, doing otherwise will cause :class:`Pickler` to - recurse infinitely. - - Use :func:`pickletools.optimize` if you need more compact pickles. - .. method:: reducer_override(self, obj) Special reducer that can be defined in :class:`Pickler` subclasses. This @@ -374,11 +364,20 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and can optionally return ``NotImplemented`` to fallback on :attr:`dispatch_table`-registered reducers to pickle ``obj``. - For a detailed example on how to use :meth:`~Pickler.reducer_override`, - see: :ref:`reducer_override`. + For a detailed example, see :ref:`reducer_override`. .. versionadded:: 3.8 + .. attribute:: fast + + Deprecated. Enable fast mode if set to a true value. The fast mode + disables the usage of memo, therefore speeding the pickling process by not + generating superfluous PUT opcodes. It should not be used with + self-referential objects, doing otherwise will cause :class:`Pickler` to + recurse infinitely. + + Use :func:`pickletools.optimize` if you need more compact pickles. + .. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict") @@ -738,71 +737,6 @@ share the same dispatch table. The equivalent code using the f = io.BytesIO() p = pickle.Pickler(f) -.. _reducer_override: - -Subclassing the ``Pickler`` class ---------------------------------- - -For most use-cases, it is recommended to simply use the -:attr:`~Pickler.dispatch_table` of a :class:`Pickler` instance to customize its -behavior, as explained above. - -However, using the :attr:`~Pickler.dispatch_table` may not be flexible enough. -In particular we may want to customize the pickling logic based on another -criterion than the object's type, or we may want to customize the pickling of -functions and classes. - -For those cases, it is possible to subclass from the :class:`Pickler` class and -implement a :meth:`~Pickler.reducer_override` method. This method can return an -arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return -``NotImplemented`` to fallback to the traditional behavior. - -If both the :attr:`~Pickler.dispatch_table` and -:meth:`~Pickler.reducer_override` are defined, then the -:meth:`~Pickler.reducer_override` method takes priority. - -.. Note:: - For performance reasons, the C implementation of pickle does not allow to - override the pickling of the following objects: ``None``, ``True``, - ``False``, and instances of :class:`int`, :class:`float`, :class:`bytes`, - :class:`str`, :class:`dict`, :class:`set`, :class:`frozenset`, :class:`list` - and :class:`tuple`. - - -Here is a simple example:: - - import io - import pickle - - - class MyClass: - my_attribute = 1 - - - class MyPickler(pickle.Pickler): - def reducer_override(self, obj): - """Custom reducer for MyClass.""" - if getattr(obj, "__name__", None) == "MyClass": - return type, (obj.__name__, obj.__bases__, - {'my_attribute': obj.my_attribute}) - else: - # For any other object, fallback to the usual pickling routines - # (builtin or dispatch_table) - return NotImplemented - - - f = io.BytesIO() - p = MyPickler(f) - p.dump(MyClass) - - del MyClass - - my_depickled_class = pickle.loads(f.getvalue()) - - assert my_depickled_class.__name__ == "MyClass" - assert my_depickled_class.my_attribute == 1 - - .. _pickle-state: Handling Stateful Objects @@ -869,6 +803,65 @@ A sample usage might be something like this:: >>> new_reader.readline() '3: Goodbye!' +.. _reducer_override: + +Custom Reduction for Types, Functions, and Other Objects +-------------------------------------------------------- + +.. versionadded:: 3.8 + +Sometimes, :attr:`~Pickler.dispatch_table` may not be flexible enough. +In particular we may want to customize pickling based on another criterion +than the object's type, or we may want to customize the pickling of +functions and classes. + +For those cases, it is possible to subclass from the :class:`Pickler` class and +implement a :meth:`~Pickler.reducer_override` method. This method can return an +arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return +``NotImplemented`` to fallback to the traditional behavior. + +If both the :attr:`~Pickler.dispatch_table` and +:meth:`~Pickler.reducer_override` are defined, then +:meth:`~Pickler.reducer_override` method takes priority. + +.. Note:: + For performance reasons, :meth:`~Pickler.reducer_override` may not be + called for the following objects: ``None``, ``True``, ``False``, and + exact instances of :class:`int`, :class:`float`, :class:`bytes`, + :class:`str`, :class:`dict`, :class:`set`, :class:`frozenset`, :class:`list` + and :class:`tuple`. + +Here is a simple example where we allow pickling and reconstructing +a given class:: + + import io + import pickle + + class MyClass: + my_attribute = 1 + + class MyPickler(pickle.Pickler): + def reducer_override(self, obj): + """Custom reducer for MyClass.""" + if getattr(obj, "__name__", None) == "MyClass": + return type, (obj.__name__, obj.__bases__, + {'my_attribute': obj.my_attribute}) + else: + # For any other object, fallback to usual reduction + return NotImplemented + + f = io.BytesIO() + p = MyPickler(f) + p.dump(MyClass) + + del MyClass + + unpickled_class = pickle.loads(f.getvalue()) + + assert isinstance(unpickled_class, type) + assert unpickled_class.__name__ == "MyClass" + assert unpickled_class.my_attribute == 1 + .. _pickle-restrict: diff --git a/Modules/_pickle.c b/Modules/_pickle.c index abe5c7f884cd3a..87f3cf7b614aab 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -616,7 +616,7 @@ typedef struct PicklerObject { PyObject *pers_func_self; /* borrowed reference to self if pers_func is an unbound method, NULL otherwise */ PyObject *dispatch_table; /* private dispatch_table, can be NULL */ - PyObject *_reducer_override;/* hook for invoking user-defined callbacks + PyObject *reducer_override; /* hook for invoking user-defined callbacks instead of save_global when pickling functions and classes*/ @@ -1113,7 +1113,7 @@ _Pickler_New(void) self->fast_memo = NULL; self->max_output_len = WRITE_BUF_SIZE; self->output_len = 0; - self->_reducer_override = NULL; + self->reducer_override = NULL; self->memo = PyMemoTable_New(); self->output_buffer = PyBytes_FromStringAndSize(NULL, @@ -4062,111 +4062,116 @@ save(PicklerObject *self, PyObject *obj, int pers_save) status = save_tuple(self, obj); goto done; } - /* The switch-on-type statement ends here because the next three - * conditions are not exclusive anymore. If reducer_override returns - * NotImplemented, then we must fallback to save_type or save_global - * */ - reduce_value = Py_NotImplemented; - Py_INCREF(reduce_value); - if (self->_reducer_override != NULL) { - reduce_value = PyObject_CallFunctionObjArgs(self->_reducer_override, - obj, NULL); - } - if (reduce_value == Py_NotImplemented) { - if (type == &PyType_Type) { - status = save_type(self, obj); - goto done; + /* Now, check reducer_override. If it returns NotImplemented, + * fallback to save_type or save_global, and then perhaps to the + * regular reduction mechanism. + */ + if (self->reducer_override != NULL) { + reduce_value = PyObject_CallFunctionObjArgs(self->reducer_override, + obj, NULL); + if (reduce_value == NULL) { + goto error; } - else if (type == &PyFunction_Type) { - status = save_global(self, obj, NULL); - goto done; + if (reduce_value != Py_NotImplemented) { + goto reduce; } + Py_DECREF(reduce_value); + reduce_value = NULL; + } - /* XXX: This part needs some unit tests. */ + if (type == &PyType_Type) { + status = save_type(self, obj); + goto done; + } + else if (type == &PyFunction_Type) { + status = save_global(self, obj, NULL); + goto done; + } - /* Get a reduction callable, and call it. This may come from - * self.dispatch_table, copyreg.dispatch_table, the object's - * __reduce_ex__ method, or the object's __reduce__ method. - */ - if (self->dispatch_table == NULL) { - PickleState *st = _Pickle_GetGlobalState(); - reduce_func = PyDict_GetItemWithError(st->dispatch_table, - (PyObject *)type); - if (reduce_func == NULL) { - if (PyErr_Occurred()) { - goto error; - } - } else { - /* PyDict_GetItemWithError() returns a borrowed reference. - Increase the reference count to be consistent with - PyObject_GetItem and _PyObject_GetAttrId used below. */ - Py_INCREF(reduce_func); + /* XXX: This part needs some unit tests. */ + + /* Get a reduction callable, and call it. This may come from + * self.dispatch_table, copyreg.dispatch_table, the object's + * __reduce_ex__ method, or the object's __reduce__ method. + */ + if (self->dispatch_table == NULL) { + PickleState *st = _Pickle_GetGlobalState(); + reduce_func = PyDict_GetItemWithError(st->dispatch_table, + (PyObject *)type); + if (reduce_func == NULL) { + if (PyErr_Occurred()) { + goto error; } } else { - reduce_func = PyObject_GetItem(self->dispatch_table, - (PyObject *)type); - if (reduce_func == NULL) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) - PyErr_Clear(); - else - goto error; - } + /* PyDict_GetItemWithError() returns a borrowed reference. + Increase the reference count to be consistent with + PyObject_GetItem and _PyObject_GetAttrId used below. */ + Py_INCREF(reduce_func); + } + } else { + reduce_func = PyObject_GetItem(self->dispatch_table, + (PyObject *)type); + if (reduce_func == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) + PyErr_Clear(); + else + goto error; } - if (reduce_func != NULL) { - Py_INCREF(obj); - reduce_value = _Pickle_FastCall(reduce_func, obj); + } + if (reduce_func != NULL) { + Py_INCREF(obj); + reduce_value = _Pickle_FastCall(reduce_func, obj); + } + else if (PyType_IsSubtype(type, &PyType_Type)) { + status = save_global(self, obj, NULL); + goto done; + } + else { + _Py_IDENTIFIER(__reduce__); + _Py_IDENTIFIER(__reduce_ex__); + + + /* XXX: If the __reduce__ method is defined, __reduce_ex__ is + automatically defined as __reduce__. While this is convenient, this + make it impossible to know which method was actually called. Of + course, this is not a big deal. But still, it would be nice to let + the user know which method was called when something go + wrong. Incidentally, this means if __reduce_ex__ is not defined, we + don't actually have to check for a __reduce__ method. */ + + /* Check for a __reduce_ex__ method. */ + if (_PyObject_LookupAttrId(obj, &PyId___reduce_ex__, &reduce_func) < 0) { + goto error; } - else if (PyType_IsSubtype(type, &PyType_Type)) { - status = save_global(self, obj, NULL); - goto done; + if (reduce_func != NULL) { + PyObject *proto; + proto = PyLong_FromLong(self->proto); + if (proto != NULL) { + reduce_value = _Pickle_FastCall(reduce_func, proto); + } } else { - _Py_IDENTIFIER(__reduce__); - _Py_IDENTIFIER(__reduce_ex__); - - - /* XXX: If the __reduce__ method is defined, __reduce_ex__ is - automatically defined as __reduce__. While this is convenient, - this make it impossible to know which method was actually - called. Of course, this is not a big deal. But still, it would - be nice to let the user know which method was called when - something go wrong. Incidentally, this means if __reduce_ex__ is - not defined, we don't actually have to check for a __reduce__ - method. */ - - /* Check for a __reduce_ex__ method. */ - if (_PyObject_LookupAttrId(obj, &PyId___reduce_ex__, - &reduce_func) < 0) { - goto error; - } + PickleState *st = _Pickle_GetGlobalState(); + + /* Check for a __reduce__ method. */ + reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__); if (reduce_func != NULL) { - PyObject *proto; - proto = PyLong_FromLong(self->proto); - if (proto != NULL) { - reduce_value = _Pickle_FastCall(reduce_func, proto); - } + reduce_value = _PyObject_CallNoArg(reduce_func); } else { - PickleState *st = _Pickle_GetGlobalState(); - - /* Check for a __reduce__ method. */ - reduce_func = _PyObject_GetAttrId(obj, &PyId___reduce__); - if (reduce_func != NULL) { - reduce_value = _PyObject_CallNoArg(reduce_func); - } - else { - PyErr_Format(st->PicklingError, - "can't pickle '%.200s' object: %R", - type->tp_name, obj); - goto error; - } + PyErr_Format(st->PicklingError, + "can't pickle '%.200s' object: %R", + type->tp_name, obj); + goto error; } } } + if (reduce_value == NULL) goto error; + reduce: if (PyUnicode_Check(reduce_value)) { status = save_global(self, obj, reduce_value); goto done; @@ -4205,12 +4210,14 @@ dump(PicklerObject *self, PyObject *obj) &tmp) < 0) { return -1; } - /* The private _reducer_override attribute of the pickler acts as a cache - * of a potential reducer_override method. This cache is updated at each - * Pickler.dump call*/ + /* Cache the reducer_override method, if it exists. */ if (tmp != NULL) { - Py_XSETREF(self->_reducer_override, tmp); + Py_XSETREF(self->reducer_override, tmp); } + else { + Py_CLEAR(self->reducer_override); + } + if (self->proto >= 2) { char header[2]; @@ -4334,6 +4341,7 @@ Pickler_dealloc(PicklerObject *self) Py_XDECREF(self->pers_func); Py_XDECREF(self->dispatch_table); Py_XDECREF(self->fast_memo); + Py_XDECREF(self->reducer_override); PyMemoTable_Del(self->memo); @@ -4347,6 +4355,7 @@ Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) Py_VISIT(self->pers_func); Py_VISIT(self->dispatch_table); Py_VISIT(self->fast_memo); + Py_VISIT(self->reducer_override); return 0; } @@ -4358,6 +4367,7 @@ Pickler_clear(PicklerObject *self) Py_CLEAR(self->pers_func); Py_CLEAR(self->dispatch_table); Py_CLEAR(self->fast_memo); + Py_CLEAR(self->reducer_override); if (self->memo != NULL) { PyMemoTable *memo = self->memo;