BUG: Fix bug pickling namedtuple.

cloudpickle/cloudpickle.py

@@ -180,6 +180,32 @@ def _builtin_type(name):
     return getattr(types, name)
 
 
+def _make__new__factory(type_):
+    def _factory():
+        return type_.__new__
+    return _factory
+
+
+# NOTE: These need to be module globals so that they're pickleable as globals.
+_get_dict_new = _make__new__factory(dict)
+_get_frozenset_new = _make__new__factory(frozenset)
+_get_list_new = _make__new__factory(list)
+_get_set_new = _make__new__factory(set)
+_get_tuple_new = _make__new__factory(tuple)
+_get_object_new = _make__new__factory(object)
+
+# Pre-defined set of builtin_function_or_method instances that can be
+# serialized.
+_BUILTIN_TYPE_CONSTRUCTORS = {
+    dict.__new__: _get_dict_new,
+    frozenset.__new__: _get_frozenset_new,
+    set.__new__: _get_set_new,
+    list.__new__: _get_list_new,
+    tuple.__new__: _get_tuple_new,
+    object.__new__: _get_object_new,
+}
+
+
 if sys.version_info < (3, 4):
     def _walk_global_ops(code):
         """
@@ -306,6 +332,18 @@ def save_function(self, obj, name=None):
         Determines what kind of function obj is (e.g. lambda, defined at
         interactive prompt, etc) and handles the pickling appropriately.
         """
+        if obj in _BUILTIN_TYPE_CONSTRUCTORS:
+            # We keep a special-cased cache of built-in type constructors at
+            # global scope, because these functions are structured very
+            # differently in different python versions and implementations (for
+            # example, they're instances of types.BuiltinFunctionType in
+            # CPython, but they're ordinary types.FunctionType instances in
+            # PyPy).
+            #
+            # If the function we've received is in that cache, we just
+            # serialize it as a lookup into the cache.
+            return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj)
+
         write = self.write
 
         if name is None:
@@ -332,7 +370,7 @@ def save_function(self, obj, name=None):
                 return self.save_global(obj, name)
 
         # a builtin_function_or_method which comes in as an attribute of some
-        # object (e.g., object.__new__, itertools.chain.from_iterable) will end
+        # object (e.g., itertools.chain.from_iterable) will end
         # up with modname "__main__" and so end up here. But these functions
         # have no __code__ attribute in CPython, so the handling for
         # user-defined functions below will fail.
@@ -408,15 +446,18 @@ def save_dynamic_class(self, obj):
         from global modules.
         """
         clsdict = dict(obj.__dict__)  # copy dict proxy to a dict
-        if not isinstance(clsdict.get('__dict__', None), property):
-            # don't extract dict that are properties
-            clsdict.pop('__dict__', None)
-            clsdict.pop('__weakref__', None)
+        clsdict.pop('__weakref__', None)
 
-        # hack as __new__ is stored differently in the __dict__
-        new_override = clsdict.get('__new__', None)
-        if new_override:
-            clsdict['__new__'] = obj.__new__
+        # On PyPy, __doc__ is a readonly attribute, so we need to include it in
+        # the initial skeleton class.  This is safe because we know that the
+        # doc can't participate in a cycle with the original class.
+        type_kwargs = {'__doc__': clsdict.pop('__doc__', None)}
+
+        # If type overrides __dict__ as a property, include it in the type kwargs.
+        # In Python 2, we can't set this attribute after construction.
+        __dict__ = clsdict.pop('__dict__', None)
+        if isinstance(__dict__, property):
+            type_kwargs['__dict__'] = __dict__
 
         save = self.save
         write = self.write
@@ -439,17 +480,12 @@ def save_dynamic_class(self, obj):
         # Mark the start of the args for the rehydration function.
         write(pickle.MARK)
 
-        # On PyPy, __doc__ is a readonly attribute, so we need to include it in
-        # the initial skeleton class.  This is safe because we know that the
-        # doc can't participate in a cycle with the original class.
-        doc_dict = {'__doc__': clsdict.pop('__doc__', None)}
-
         # Create and memoize an empty class with obj's name and bases.
         save(type(obj))
         save((
             obj.__name__,
             obj.__bases__,
-            doc_dict,
+            type_kwargs,
         ))
         write(pickle.REDUCE)
         self.memoize(obj)

tests/cloudpickle_test.py

@@ -1,7 +1,7 @@
 from __future__ import division
 
 import abc
-
+import collections
 import base64
 import functools
 import imp
@@ -701,5 +701,27 @@ def test_function_module_name(self):
         func = lambda x: x
         self.assertEqual(pickle_depickle(func).__module__, func.__module__)
 
+    def test_namedtuple(self):
+
+        MyTuple = collections.namedtuple('MyTuple', ['a', 'b', 'c'])
+        t = MyTuple(1, 2, 3)
+
+        depickled_t, depickled_MyTuple = pickle_depickle([t, MyTuple])
+        self.assertTrue(isinstance(depickled_t, depickled_MyTuple))
+
+        self.assertEqual((depickled_t.a, depickled_t.b, depickled_t.c), (1, 2, 3))
+        self.assertEqual((depickled_t[0], depickled_t[1], depickled_t[2]), (1, 2, 3))
+
+        self.assertEqual(depickled_MyTuple.__name__, 'MyTuple')
+        self.assertTrue(issubclass(depickled_MyTuple, tuple))
+
+    def test_builtin_type__new__(self):
+        # Functions occasionally take the __new__ of these types as default
+        # parameters for factories.  For example, on Python 3.3,
+        # `tuple.__new__` is a default value for some methods of namedtuple.
+        for t in list, tuple, set, frozenset, dict, object:
+            self.assertTrue(pickle_depickle(t.__new__) is t.__new__)
+
+
 if __name__ == '__main__':
     unittest.main()