From 2f2b69855d6524e15d12c15ddc0adce629e7de84 Mon Sep 17 00:00:00 2001 From: larryhastings Date: Thu, 29 Apr 2021 20:09:08 -0700 Subject: [PATCH] bpo-43901: Lazy-create an empty annotations dict in all unannotated user classes and modules (#25623) Change class and module objects to lazy-create empty annotations dicts on demand. The annotations dicts are stored in the object's `__dict__` for backwards compatibility. --- Lib/test/ann_module4.py | 5 + Lib/test/test_grammar.py | 3 +- Lib/test/test_module.py | 54 +++++++++ Lib/test/test_opcodes.py | 5 +- Lib/test/test_type_annotations.py | 103 ++++++++++++++++++ Lib/typing.py | 2 + .../2021-04-25-22-50-47.bpo-43901.oKjG5E.rst | 3 + Objects/moduleobject.c | 72 +++++++++++- Objects/typeobject.c | 69 ++++++++++++ 9 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 Lib/test/ann_module4.py create mode 100644 Lib/test/test_type_annotations.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-04-25-22-50-47.bpo-43901.oKjG5E.rst diff --git a/Lib/test/ann_module4.py b/Lib/test/ann_module4.py new file mode 100644 index 00000000000000..13e9aee54c98b6 --- /dev/null +++ b/Lib/test/ann_module4.py @@ -0,0 +1,5 @@ +# This ann_module isn't for test_typing, +# it's for test_module + +a:int=3 +b:str=4 diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 6f79e19a54435b..46f70e5d176fcd 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -382,8 +382,7 @@ class CC(metaclass=CMeta): self.assertEqual(CC.__annotations__['xx'], 'ANNOT') def test_var_annot_module_semantics(self): - with self.assertRaises(AttributeError): - print(test.__annotations__) + self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int]}) self.assertEqual(ann_module.M.__annotations__, diff --git a/Lib/test/test_module.py b/Lib/test/test_module.py index 1d44563579fd2f..aa5ee49854059c 100644 --- a/Lib/test/test_module.py +++ b/Lib/test/test_module.py @@ -286,6 +286,60 @@ class M(ModuleType): melon = Descr() self.assertRaises(RuntimeError, getattr, M("mymod"), "melon") + def test_lazy_create_annotations(self): + # module objects lazy create their __annotations__ dict on demand. + # the annotations dict is stored in module.__dict__. + # a freshly created module shouldn't have an annotations dict yet. + foo = ModuleType("foo") + for i in range(4): + self.assertFalse("__annotations__" in foo.__dict__) + d = foo.__annotations__ + self.assertTrue("__annotations__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations__'], d) + if i % 2: + del foo.__annotations__ + else: + del foo.__dict__['__annotations__'] + + def test_setting_annotations(self): + foo = ModuleType("foo") + for i in range(4): + self.assertFalse("__annotations__" in foo.__dict__) + d = {'a': int} + foo.__annotations__ = d + self.assertTrue("__annotations__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations__'], d) + if i % 2: + del foo.__annotations__ + else: + del foo.__dict__['__annotations__'] + + def test_annotations_getset_raises(self): + # module has no dict, all operations fail + foo = ModuleType.__new__(ModuleType) + with self.assertRaises(TypeError): + print(foo.__annotations__) + with self.assertRaises(TypeError): + foo.__annotations__ = {} + with self.assertRaises(TypeError): + del foo.__annotations__ + + # double delete + foo = ModuleType("foo") + foo.__annotations__ = {} + del foo.__annotations__ + with self.assertRaises(AttributeError): + del foo.__annotations__ + + def test_annotations_are_created_correctly(self): + from test import ann_module4 + self.assertTrue("__annotations__" in ann_module4.__dict__) + del ann_module4.__annotations__ + self.assertFalse("__annotations__" in ann_module4.__dict__) + + # frozen and namespace module reprs are tested in importlib. diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index d43a8303b17109..e880c3f1ac875e 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -31,10 +31,9 @@ def test_setup_annotations_line(self): except OSError: pass - def test_no_annotations_if_not_needed(self): + def test_default_annotations_exist(self): class C: pass - with self.assertRaises(AttributeError): - C.__annotations__ + self.assertEqual(C.__annotations__, {}) def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py new file mode 100644 index 00000000000000..f6c99bda3aa6f6 --- /dev/null +++ b/Lib/test/test_type_annotations.py @@ -0,0 +1,103 @@ +import unittest + +class TypeAnnotationTests(unittest.TestCase): + + def test_lazy_create_annotations(self): + # type objects lazy create their __annotations__ dict on demand. + # the annotations dict is stored in type.__dict__. + # a freshly created type shouldn't have an annotations dict yet. + foo = type("Foo", (), {}) + for i in range(3): + self.assertFalse("__annotations__" in foo.__dict__) + d = foo.__annotations__ + self.assertTrue("__annotations__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations__'], d) + del foo.__annotations__ + + def test_setting_annotations(self): + foo = type("Foo", (), {}) + for i in range(3): + self.assertFalse("__annotations__" in foo.__dict__) + d = {'a': int} + foo.__annotations__ = d + self.assertTrue("__annotations__" in foo.__dict__) + self.assertEqual(foo.__annotations__, d) + self.assertEqual(foo.__dict__['__annotations__'], d) + del foo.__annotations__ + + def test_annotations_getset_raises(self): + # builtin types don't have __annotations__ (yet!) + with self.assertRaises(AttributeError): + print(float.__annotations__) + with self.assertRaises(TypeError): + float.__annotations__ = {} + with self.assertRaises(TypeError): + del float.__annotations__ + + # double delete + foo = type("Foo", (), {}) + foo.__annotations__ = {} + del foo.__annotations__ + with self.assertRaises(AttributeError): + del foo.__annotations__ + + def test_annotations_are_created_correctly(self): + class C: + a:int=3 + b:str=4 + self.assertTrue("__annotations__" in C.__dict__) + del C.__annotations__ + self.assertFalse("__annotations__" in C.__dict__) + + def test_descriptor_still_works(self): + class C: + def __init__(self, name=None, bases=None, d=None): + self.my_annotations = None + + @property + def __annotations__(self): + if not hasattr(self, 'my_annotations'): + self.my_annotations = {} + if not isinstance(self.my_annotations, dict): + self.my_annotations = {} + return self.my_annotations + + @__annotations__.setter + def __annotations__(self, value): + if not isinstance(value, dict): + raise ValueError("can only set __annotations__ to a dict") + self.my_annotations = value + + @__annotations__.deleter + def __annotations__(self): + if hasattr(self, 'my_annotations') and self.my_annotations == None: + raise AttributeError('__annotations__') + self.my_annotations = None + + c = C() + self.assertEqual(c.__annotations__, {}) + d = {'a':'int'} + c.__annotations__ = d + self.assertEqual(c.__annotations__, d) + with self.assertRaises(ValueError): + c.__annotations__ = 123 + del c.__annotations__ + with self.assertRaises(AttributeError): + del c.__annotations__ + self.assertEqual(c.__annotations__, {}) + + + class D(metaclass=C): + pass + + self.assertEqual(D.__annotations__, {}) + d = {'a':'int'} + D.__annotations__ = d + self.assertEqual(D.__annotations__, d) + with self.assertRaises(ValueError): + D.__annotations__ = 123 + del D.__annotations__ + with self.assertRaises(AttributeError): + del D.__annotations__ + self.assertEqual(D.__annotations__, {}) diff --git a/Lib/typing.py b/Lib/typing.py index d409517ff58e9a..ff964343c5336a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1677,6 +1677,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): else: base_globals = globalns ann = base.__dict__.get('__annotations__', {}) + if isinstance(ann, types.GetSetDescriptorType): + ann = {} base_locals = dict(vars(base)) if localns is None else localns if localns is None and globalns is None: # This is surprising, but required. Before Python 3.10, diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-04-25-22-50-47.bpo-43901.oKjG5E.rst b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-22-50-47.bpo-43901.oKjG5E.rst new file mode 100644 index 00000000000000..2ab93d1f756f6e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-04-25-22-50-47.bpo-43901.oKjG5E.rst @@ -0,0 +1,3 @@ +Change class and module objects to lazy-create empty annotations dicts on +demand. The annotations dicts are stored in the object's __dict__ for +backwards compatibility. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index a6eb85bdc2a962..cdb365d29a914d 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -12,6 +12,9 @@ static Py_ssize_t max_module_number; _Py_IDENTIFIER(__doc__); _Py_IDENTIFIER(__name__); _Py_IDENTIFIER(__spec__); +_Py_IDENTIFIER(__dict__); +_Py_IDENTIFIER(__dir__); +_Py_IDENTIFIER(__annotations__); static PyMemberDef module_members[] = { {"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY}, @@ -807,8 +810,6 @@ module_clear(PyModuleObject *m) static PyObject * module_dir(PyObject *self, PyObject *args) { - _Py_IDENTIFIER(__dict__); - _Py_IDENTIFIER(__dir__); PyObject *result = NULL; PyObject *dict = _PyObject_GetAttrId(self, &PyId___dict__); @@ -841,6 +842,71 @@ static PyMethodDef module_methods[] = { {0} }; +static PyObject * +module_get_annotations(PyModuleObject *m, void *Py_UNUSED(ignored)) +{ + PyObject *dict = _PyObject_GetAttrId((PyObject *)m, &PyId___dict__); + + if ((dict == NULL) || !PyDict_Check(dict)) { + PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); + return NULL; + } + + PyObject *annotations; + /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ + if (_PyDict_ContainsId(dict, &PyId___annotations__)) { + annotations = _PyDict_GetItemIdWithError(dict, &PyId___annotations__); + /* + ** _PyDict_GetItemIdWithError could still fail, + ** for instance with a well-timed Ctrl-C or a MemoryError. + ** so let's be totally safe. + */ + if (annotations) { + Py_INCREF(annotations); + } + } else { + annotations = PyDict_New(); + if (annotations) { + int result = _PyDict_SetItemId(dict, &PyId___annotations__, annotations); + if (result) { + Py_CLEAR(annotations); + } + } + } + Py_DECREF(dict); + return annotations; +} + +static int +module_set_annotations(PyModuleObject *m, PyObject *value, void *Py_UNUSED(ignored)) +{ + PyObject *dict = _PyObject_GetAttrId((PyObject *)m, &PyId___dict__); + + if ((dict == NULL) || !PyDict_Check(dict)) { + PyErr_Format(PyExc_TypeError, ".__dict__ is not a dictionary"); + return -1; + } + + if (value != NULL) { + /* set */ + return _PyDict_SetItemId(dict, &PyId___annotations__, value); + } + + /* delete */ + if (!_PyDict_ContainsId(dict, &PyId___annotations__)) { + PyErr_Format(PyExc_AttributeError, "__annotations__"); + return -1; + } + + return _PyDict_DelItemId(dict, &PyId___annotations__); +} + + +static PyGetSetDef module_getsets[] = { + {"__annotations__", (getter)module_get_annotations, (setter)module_set_annotations}, + {NULL} +}; + PyTypeObject PyModule_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) "module", /* tp_name */ @@ -872,7 +938,7 @@ PyTypeObject PyModule_Type = { 0, /* tp_iternext */ module_methods, /* tp_methods */ module_members, /* tp_members */ - 0, /* tp_getset */ + module_getsets, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index e1c8be4b815452..ac4dc1da4411dc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -52,6 +52,7 @@ typedef struct PySlot_Offset { /* alphabetical order */ _Py_IDENTIFIER(__abstractmethods__); +_Py_IDENTIFIER(__annotations__); _Py_IDENTIFIER(__class__); _Py_IDENTIFIER(__class_getitem__); _Py_IDENTIFIER(__classcell__); @@ -930,6 +931,73 @@ type_set_doc(PyTypeObject *type, PyObject *value, void *context) return _PyDict_SetItemId(type->tp_dict, &PyId___doc__, value); } +static PyObject * +type_get_annotations(PyTypeObject *type, void *context) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotations__'", type->tp_name); + return NULL; + } + + PyObject *annotations; + /* there's no _PyDict_GetItemId without WithError, so let's LBYL. */ + if (_PyDict_ContainsId(type->tp_dict, &PyId___annotations__)) { + annotations = _PyDict_GetItemIdWithError(type->tp_dict, &PyId___annotations__); + /* + ** _PyDict_GetItemIdWithError could still fail, + ** for instance with a well-timed Ctrl-C or a MemoryError. + ** so let's be totally safe. + */ + if (annotations) { + if (Py_TYPE(annotations)->tp_descr_get) { + annotations = Py_TYPE(annotations)->tp_descr_get(annotations, NULL, + (PyObject *)type); + } else { + Py_INCREF(annotations); + } + } + } else { + annotations = PyDict_New(); + if (annotations) { + int result = _PyDict_SetItemId(type->tp_dict, &PyId___annotations__, annotations); + if (result) { + Py_CLEAR(annotations); + } else { + PyType_Modified(type); + } + } + } + return annotations; +} + +static int +type_set_annotations(PyTypeObject *type, PyObject *value, void *context) +{ + if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + PyErr_Format(PyExc_TypeError, "can't set attributes of built-in/extension type '%s'", type->tp_name); + return -1; + } + + int result; + if (value != NULL) { + /* set */ + result = _PyDict_SetItemId(type->tp_dict, &PyId___annotations__, value); + } else { + /* delete */ + if (!_PyDict_ContainsId(type->tp_dict, &PyId___annotations__)) { + PyErr_Format(PyExc_AttributeError, "__annotations__"); + return -1; + } + result = _PyDict_DelItemId(type->tp_dict, &PyId___annotations__); + } + + if (result == 0) { + PyType_Modified(type); + } + return result; +} + + /*[clinic input] type.__instancecheck__ -> bool @@ -973,6 +1041,7 @@ static PyGetSetDef type_getsets[] = { {"__dict__", (getter)type_dict, NULL, NULL}, {"__doc__", (getter)type_get_doc, (setter)type_set_doc, NULL}, {"__text_signature__", (getter)type_get_text_signature, NULL, NULL}, + {"__annotations__", (getter)type_get_annotations, (setter)type_set_annotations, NULL}, {0} };